diff --git a/apps/n8n/manifests/cnpg-cluster.yaml b/apps/n8n/manifests/cnpg-cluster.yaml index d8ccdbd..b64df8a 100644 --- a/apps/n8n/manifests/cnpg-cluster.yaml +++ b/apps/n8n/manifests/cnpg-cluster.yaml @@ -47,5 +47,25 @@ spec: database: n8n owner: n8n + # Backup to the VPS MinIO backup receiver (Phase 1b). Previously had no + # backup block at all. VPS_MINIO_ENDPOINT placeholder matches + # infrastructure/vault/manifests/raft-snapshot-cronjob.yaml - replace with + # the VPS's actual Netbird address once bootstrapped. + backup: + barmanObjectStore: + destinationPath: s3://cnpg-backups/pg-n8n + endpointURL: http://vps-minio.netbird.internal:30900 + s3Credentials: + accessKeyId: + name: vps-minio-secret + key: accesskey + secretAccessKey: + name: vps-minio-secret + key: secretkey + wal: + compression: gzip + maxParallel: 2 + retentionPolicy: "30d" + monitoring: enablePodMonitor: true diff --git a/apps/n8n/manifests/vps-minio-secret.yaml b/apps/n8n/manifests/vps-minio-secret.yaml new file mode 100644 index 0000000..330209e --- /dev/null +++ b/apps/n8n/manifests/vps-minio-secret.yaml @@ -0,0 +1,26 @@ +--- +# VPS MinIO credentials for CNPG's barmanObjectStore backup target. +# Same Vault source as infrastructure/vault/manifests/raft-snapshot-cronjob.yaml +# (secret/vps-minio-credentials) - populated once, manually, after VPS bootstrap. +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: vps-minio-credentials + namespace: n8n +spec: + refreshInterval: 1h + secretStoreRef: + kind: ClusterSecretStore + name: vault-backend + target: + name: vps-minio-secret + creationPolicy: Owner + data: + - secretKey: accesskey + remoteRef: + key: vps-minio-credentials + property: access-key + - secretKey: secretkey + remoteRef: + key: vps-minio-credentials + property: secret-key diff --git a/apps/nextcloud/manifests/cnpg-cluster.yaml b/apps/nextcloud/manifests/cnpg-cluster.yaml new file mode 100644 index 0000000..ece9be6 --- /dev/null +++ b/apps/nextcloud/manifests/cnpg-cluster.yaml @@ -0,0 +1,68 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: pg-nextcloud + namespace: nextcloud +spec: + imageName: ghcr.io/cloudnative-pg/postgresql:16 + instances: 2 + + resources: + requests: + memory: "512Mi" + cpu: "50m" + limits: + memory: "2Gi" + cpu: "500m" + + affinity: + topologyKey: kubernetes.io/hostname + + storage: + size: 20Gi + storageClass: rook-ceph-block + + primaryUpdateStrategy: unsupervised + + postgresql: + parameters: + max_connections: "200" + shared_buffers: "512MB" + effective_cache_size: "1536MB" + maintenance_work_mem: "128MB" + checkpoint_completion_target: "0.9" + wal_buffers: "16MB" + default_statistics_target: "100" + random_page_cost: "1.1" + effective_io_concurrency: "200" + work_mem: "2621kB" + min_wal_size: "1GB" + max_wal_size: "4GB" + + bootstrap: + initdb: + database: nextcloud + owner: nextcloud + + # Backup to the VPS MinIO backup receiver. VPS_MINIO_ENDPOINT placeholder + # matches infrastructure/vault/manifests/raft-snapshot-cronjob.yaml - replace + # with the VPS's actual Netbird address once bootstrapped. This covers the + # DB only - file PVC content is separate, see nextcloud-pvc-sync-cronjob.yaml. + backup: + barmanObjectStore: + destinationPath: s3://cnpg-backups/pg-nextcloud + endpointURL: http://vps-minio.netbird.internal:30900 + s3Credentials: + accessKeyId: + name: vps-minio-secret + key: accesskey + secretAccessKey: + name: vps-minio-secret + key: secretkey + wal: + compression: gzip + maxParallel: 2 + retentionPolicy: "30d" + + monitoring: + enablePodMonitor: true diff --git a/apps/nextcloud/manifests/cnpg-secrets.yaml b/apps/nextcloud/manifests/cnpg-secrets.yaml new file mode 100644 index 0000000..1231d1b --- /dev/null +++ b/apps/nextcloud/manifests/cnpg-secrets.yaml @@ -0,0 +1,42 @@ +--- +# Mirrors apps/n8n/manifests/cnpg-secrets.yaml: CNPG creates pg-nextcloud-app +# with a generated password on cluster init; ESO overrides it with a +# generator-managed password so it's rotated/tracked the same way as every +# other app secret in this repo. +apiVersion: generators.external-secrets.io/v1alpha1 +kind: Password +metadata: + name: nextcloud-cnpg-secret + namespace: nextcloud +spec: + length: 42 + digits: 5 + symbols: 5 + symbolCharacters: "-_$@" + noUpper: false + allowRepeat: true + +--- +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: nextcloud-cnpg-secret + namespace: nextcloud +spec: + refreshInterval: "24h" + target: + # Merges into the pg-nextcloud-app secret CNPG creates automatically + name: pg-nextcloud-app + creationPolicy: Merge + template: + metadata: + labels: + cnpg.io/reload: "true" + data: + password: "{{ .password }}" + dataFrom: + - sourceRef: + generatorRef: + apiVersion: generators.external-secrets.io/v1alpha1 + kind: Password + name: nextcloud-cnpg-secret diff --git a/apps/nextcloud/manifests/mariadb-deployment.yaml b/apps/nextcloud/manifests/mariadb-deployment.yaml deleted file mode 100644 index 5b5b092..0000000 --- a/apps/nextcloud/manifests/mariadb-deployment.yaml +++ /dev/null @@ -1,92 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: mariadb-nextcloud - namespace: nextcloud - labels: - app: mariadb-nextcloud -spec: - replicas: 1 - selector: - matchLabels: - app: mariadb-nextcloud - template: - metadata: - labels: - app: mariadb-nextcloud - spec: - securityContext: - runAsUser: 999 - runAsGroup: 999 - fsGroup: 999 - containers: - - name: mariadb - image: mariadb:11.4 - securityContext: - runAsUser: 999 - runAsGroup: 999 - runAsNonRoot: true - allowPrivilegeEscalation: false - capabilities: - drop: ["ALL"] - args: - - --character-set-server=utf8mb4 - - --collation-server=utf8mb4_unicode_ci - - --init-connect=SET NAMES utf8mb4 COLLATE utf8mb4_unicode_ci - env: - - name: MARIADB_ROOT_PASSWORD - valueFrom: - secretKeyRef: - name: mariadb-nextcloud - key: root-password - - name: MARIADB_PASSWORD - valueFrom: - secretKeyRef: - name: mariadb-nextcloud - key: password - - name: MARIADB_USER - value: nextcloud - - name: MARIADB_DATABASE - value: nextcloud - ports: - - containerPort: 3306 - volumeMounts: - - name: mariadb-data - mountPath: /var/lib/mysql - - name: tmp - mountPath: /tmp - - name: run-mysqld - mountPath: /run/mysqld - volumes: - - name: tmp - emptyDir: {} - - name: run-mysqld - emptyDir: {} - - name: mariadb-data - persistentVolumeClaim: - claimName: mariadb-nextcloud-pvc ---- -apiVersion: v1 -kind: Service -metadata: - name: mariadb-nextcloud - namespace: nextcloud -spec: - selector: - app: mariadb-nextcloud - ports: - - port: 3306 - targetPort: 3306 ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: mariadb-nextcloud-pvc - namespace: nextcloud -spec: - accessModes: - - ReadWriteOnce - storageClassName: rook-ceph-block - resources: - requests: - storage: 10Gi \ No newline at end of file diff --git a/apps/nextcloud/manifests/mariadb-secrets.yaml b/apps/nextcloud/manifests/mariadb-secrets.yaml deleted file mode 100644 index 3302daa..0000000 --- a/apps/nextcloud/manifests/mariadb-secrets.yaml +++ /dev/null @@ -1,35 +0,0 @@ ---- -apiVersion: generators.external-secrets.io/v1alpha1 -kind: Password -metadata: - name: nextcloud-mariadb-password - namespace: nextcloud -spec: - length: 32 - digits: 5 - symbols: 3 - symbolCharacters: "-_$" - noUpper: false - allowRepeat: true ---- -apiVersion: external-secrets.io/v1 -kind: ExternalSecret -metadata: - name: nextcloud-mariadb-secret - namespace: nextcloud -spec: - refreshInterval: "24h" - target: - name: mariadb-nextcloud - creationPolicy: Owner - template: - data: - password: "{{ .password }}" - root-password: "{{ .password }}" - user: nextcloud - dataFrom: - - sourceRef: - generatorRef: - apiVersion: generators.external-secrets.io/v1alpha1 - kind: Password - name: nextcloud-mariadb-password \ No newline at end of file diff --git a/apps/nextcloud/manifests/pvc-sync-cronjob.yaml b/apps/nextcloud/manifests/pvc-sync-cronjob.yaml new file mode 100644 index 0000000..bf006b9 --- /dev/null +++ b/apps/nextcloud/manifests/pvc-sync-cronjob.yaml @@ -0,0 +1,124 @@ +--- +# Syncs Nextcloud's file PVC content (nextcloud-nextcloud, 200Gi) to the VPS +# MinIO backup receiver via restic. The one piece of the Nextcloud backup story +# no operator covers - actual file blobs, not database rows (DB is handled by +# CNPG's barmanObjectStore in cnpg-cluster.yaml). +# +# Caveat: this backs up the live-mounted volume with no application-level +# quiesce (no Nextcloud maintenance-mode pause around the snapshot). Acceptable +# for a no-real-data test environment; for anything with real user data, pair +# this with `occ maintenance:mode --on` before / `--off` after. +# +# Uses podAffinity to land on the same node as a running Nextcloud pod, since +# the PVC is RWO (Ceph RBD) - RWO allows multiple pods to mount it concurrently +# only when co-located on the same node. Verify the label selector below +# matches the actual Nextcloud chart's pod labels before relying on this. +apiVersion: generators.external-secrets.io/v1alpha1 +kind: Password +metadata: + name: nextcloud-restic-password + namespace: nextcloud +spec: + length: 48 + digits: 5 + symbols: 5 + symbolCharacters: "-_$@" + noUpper: false + allowRepeat: true +--- +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: nextcloud-restic-password + namespace: nextcloud +spec: + refreshInterval: "0" # generate once - changing this after the repo is initialized would lock you out of existing backups + target: + name: nextcloud-restic-password + creationPolicy: Owner + data: [] + dataFrom: + - sourceRef: + generatorRef: + apiVersion: generators.external-secrets.io/v1alpha1 + kind: Password + name: nextcloud-restic-password +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: nextcloud-pvc-sync + namespace: nextcloud +spec: + schedule: "0 2 * * *" # daily at 02:00 - bulky, don't run it as often as the DB backups + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 2 + template: + spec: + restartPolicy: Never + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: nextcloud + topologyKey: kubernetes.io/hostname + containers: + - name: restic-backup + image: restic/restic:0.16.4 + command: + - /bin/sh + - -c + - | + set -eu + export RESTIC_REPOSITORY="s3:http://${VPS_MINIO_ENDPOINT}/nextcloud-files/restic-repo" + export RESTIC_PASSWORD_FILE=/restic-secret/password + export AWS_ACCESS_KEY_ID="${MINIO_ACCESS_KEY}" + export AWS_SECRET_ACCESS_KEY="${MINIO_SECRET_KEY}" + + restic snapshots >/dev/null 2>&1 || restic init + + echo "==> Backing up /data..." + restic backup /data --tag nextcloud --host nextcloud-k8s + + echo "==> Pruning: keep 7 daily, 4 weekly, 6 monthly..." + restic forget --tag nextcloud --host nextcloud-k8s \ + --keep-daily 7 --keep-weekly 4 --keep-monthly 6 --prune + + echo "==> Done." + env: + # VPS's Netbird address - replace once bootstrapped, matches + # infrastructure/vault/manifests/raft-snapshot-cronjob.yaml + - name: VPS_MINIO_ENDPOINT + value: "vps-minio.netbird.internal:30900" + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: vps-minio-secret + key: accesskey + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: vps-minio-secret + key: secretkey + volumeMounts: + - name: nextcloud-data + mountPath: /data + readOnly: true + - name: restic-secret + mountPath: /restic-secret + readOnly: true + volumes: + - name: nextcloud-data + persistentVolumeClaim: + claimName: nextcloud-nextcloud + - name: restic-secret + secret: + secretName: nextcloud-restic-password + items: + - key: password + path: password diff --git a/apps/nextcloud/manifests/redis-cr.yaml b/apps/nextcloud/manifests/redis-cr.yaml new file mode 100644 index 0000000..6cb3653 --- /dev/null +++ b/apps/nextcloud/manifests/redis-cr.yaml @@ -0,0 +1,72 @@ +--- +# Standalone Redis via the ot-container-kit redis-operator that's already +# deployed as infra (argocd-apps/infrastructure/redis-operator.yaml) but +# currently unused - GitLab's Redis is a hand-rolled StatefulSet instead +# (apps/gitlab/manifests/redis-cluster.yaml), not this operator. This is the +# first real consumer of it in the repo. +# +# NOTE: verify this CR against `kubectl explain redis.spec` (or the chart's +# CRD source) for the redis-operator 0.15.0 actually deployed before first +# apply - the ot-container-kit CRD schema has shifted across versions and +# this wasn't checked against a live cluster. +apiVersion: generators.external-secrets.io/v1alpha1 +kind: Password +metadata: + name: nextcloud-redis-password + namespace: nextcloud +spec: + length: 32 + digits: 5 + symbols: 0 + noUpper: false + allowRepeat: true +--- +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: nextcloud-redis-password + namespace: nextcloud +spec: + refreshInterval: "0" # generate once, don't rotate (Redis CR reads this at pod start only) + target: + name: redis-nextcloud-secret + creationPolicy: Owner + template: + data: + password: "{{ .password }}" + dataFrom: + - sourceRef: + generatorRef: + apiVersion: generators.external-secrets.io/v1alpha1 + kind: Password + name: nextcloud-redis-password +--- +apiVersion: redis.redis.opstreelabs.in/v1beta2 +kind: Redis +metadata: + name: redis-nextcloud + namespace: nextcloud +spec: + kubernetesConfig: + image: quay.io/opstree/redis:v7.0.12 + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 25m + memory: 128Mi + limits: + cpu: 250m + memory: 256Mi + redisExporter: + enabled: false + redisSecret: + name: redis-nextcloud-secret + key: password + storage: + volumeClaimTemplate: + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: rook-ceph-block + resources: + requests: + storage: 2Gi diff --git a/apps/nextcloud/manifests/vps-minio-secret.yaml b/apps/nextcloud/manifests/vps-minio-secret.yaml new file mode 100644 index 0000000..2a98e06 --- /dev/null +++ b/apps/nextcloud/manifests/vps-minio-secret.yaml @@ -0,0 +1,27 @@ +--- +# VPS MinIO credentials for CNPG's barmanObjectStore backup target, and reused +# by the Phase 1b PVC-content sync CronJob. Same Vault source as +# infrastructure/vault/manifests/raft-snapshot-cronjob.yaml +# (secret/vps-minio-credentials) - populated once, manually, after VPS bootstrap. +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: vps-minio-credentials + namespace: nextcloud +spec: + refreshInterval: 1h + secretStoreRef: + kind: ClusterSecretStore + name: vault-backend + target: + name: vps-minio-secret + creationPolicy: Owner + data: + - secretKey: accesskey + remoteRef: + key: vps-minio-credentials + property: access-key + - secretKey: secretkey + remoteRef: + key: vps-minio-credentials + property: secret-key diff --git a/apps/nextcloud/values.yaml b/apps/nextcloud/values.yaml index 69e1d1b..d2f1c01 100644 --- a/apps/nextcloud/values.yaml +++ b/apps/nextcloud/values.yaml @@ -73,16 +73,27 @@ nextcloud: externalDatabase: enabled: true - type: mysql - host: mariadb-nextcloud - port: 3306 + # Moved from MariaDB to CNPG-backed Postgres (apps/nextcloud/manifests/cnpg-cluster.yaml) + # for backup/DR consistency with the rest of the stack (n8n, Authentik, GitLab + # all use CNPG's native barmanObjectStore backup - MariaDB needed a bespoke + # mysqldump job instead). pg-nextcloud-rw is CNPG's generated read-write + # Service name for the "pg-nextcloud" Cluster. + # + # IMPORTANT: switching type here does NOT migrate existing data. Nextcloud + # requires an explicit `occ db:convert-type pgsql ...` run before cutover if + # there's real data in the old MariaDB instance. usernameKey/passwordKey below + # match CNPG's generated app-secret keys (confirmed against the existing, + # already-working infrastructure/authentik/values.yaml pg-authentik-app usage). + type: postgresql + host: pg-nextcloud-rw + port: 5432 user: nextcloud database: nextcloud existingSecret: enabled: true - secretName: mariadb-nextcloud + secretName: pg-nextcloud-app passwordKey: password - usernameKey: user + usernameKey: username # matches CNPG's generated app-secret key, confirmed against infrastructure/authentik/values.yaml's existing pg-authentik-app usage persistence: enabled: true @@ -99,7 +110,16 @@ resources: memory: 2Gi redis: - enabled: false + # Was disabled; now points at the standalone Redis CR (redis-operator) in + # apps/nextcloud/manifests/redis-cr.yaml instead of the chart's bundled + # subchart - fixes Nextcloud's file-locking/caching as a side benefit of + # this migration. Verify these key names against `helm show values + # nextcloud/nextcloud` for the deployed chart version before applying. + enabled: true + host: redis-nextcloud + port: 6379 + existingSecret: redis-nextcloud-secret + existingSecretPasswordKey: password metrics: enabled: false diff --git a/argocd-apps/projects.yaml b/argocd-apps/projects.yaml index 72bf973..37d8d24 100644 --- a/argocd-apps/projects.yaml +++ b/argocd-apps/projects.yaml @@ -57,3 +57,26 @@ spec: namespaceResourceWhitelist: - group: '*' kind: '*' +--- +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: vps-standby + namespace: argocd +spec: + # VPS warm-standby / backup-site services. Deployed to the "vps-standby" cluster + # registered via `argocd cluster add ... --name vps-standby` (see + # scripts/vps-bootstrap.sh). Scoped to that cluster only — nothing in this + # project should ever target the home cluster. + description: VPS warm-standby / backup-site services + sourceRepos: + - '*' + destinations: + - namespace: '*' + name: vps-standby + clusterResourceWhitelist: + - group: '*' + kind: '*' + namespaceResourceWhitelist: + - group: '*' + kind: '*' diff --git a/argocd-apps/vps-standby/minio.yaml b/argocd-apps/vps-standby/minio.yaml new file mode 100644 index 0000000..4eaca76 --- /dev/null +++ b/argocd-apps/vps-standby/minio.yaml @@ -0,0 +1,30 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: vps-minio + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "1" # first thing on the vps-standby cluster - everything else backs up to it + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: vps-standby + sources: + - repoURL: https://charts.min.io/ + chart: minio + targetRevision: 5.4.0 + helm: + valueFiles: + - $values/infrastructure/vps-standby/minio/values.yaml + - repoURL: https://gitlab.kube.huskypup.net/Scooby/Homelabv4.git + targetRevision: main + ref: values + destination: + name: vps-standby + namespace: minio + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/infrastructure/authentik/cnpg-cluster.yaml b/infrastructure/authentik/manifests/cnpg-cluster.yaml similarity index 63% rename from infrastructure/authentik/cnpg-cluster.yaml rename to infrastructure/authentik/manifests/cnpg-cluster.yaml index 2e9b1c4..55f5982 100644 --- a/infrastructure/authentik/cnpg-cluster.yaml +++ b/infrastructure/authentik/manifests/cnpg-cluster.yaml @@ -47,7 +47,25 @@ spec: database: app owner: app + # Backup to the VPS MinIO backup receiver (Phase 1b). VPS_MINIO_ENDPOINT + # placeholder matches infrastructure/vault/manifests/raft-snapshot-cronjob.yaml - + # replace with the VPS's actual Netbird address once bootstrapped. + backup: + barmanObjectStore: + destinationPath: s3://cnpg-backups/pg-authentik + endpointURL: http://vps-minio.netbird.internal:30900 + s3Credentials: + accessKeyId: + name: vps-minio-secret + key: accesskey + secretAccessKey: + name: vps-minio-secret + key: secretkey + wal: + compression: gzip + maxParallel: 2 + retentionPolicy: "30d" + monitoring: enablePodMonitor: true - diff --git a/infrastructure/authentik/cnpg-podmonitor.yaml b/infrastructure/authentik/manifests/cnpg-podmonitor.yaml similarity index 100% rename from infrastructure/authentik/cnpg-podmonitor.yaml rename to infrastructure/authentik/manifests/cnpg-podmonitor.yaml diff --git a/infrastructure/authentik/cnpg-secrets.yaml b/infrastructure/authentik/manifests/cnpg-secrets.yaml similarity index 100% rename from infrastructure/authentik/cnpg-secrets.yaml rename to infrastructure/authentik/manifests/cnpg-secrets.yaml diff --git a/infrastructure/authentik/manifests/vps-minio-secret.yaml b/infrastructure/authentik/manifests/vps-minio-secret.yaml new file mode 100644 index 0000000..a7e4a85 --- /dev/null +++ b/infrastructure/authentik/manifests/vps-minio-secret.yaml @@ -0,0 +1,26 @@ +--- +# VPS MinIO credentials for CNPG's barmanObjectStore backup target. +# Same Vault source as infrastructure/vault/manifests/raft-snapshot-cronjob.yaml +# (secret/vps-minio-credentials) - populated once, manually, after VPS bootstrap. +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: vps-minio-credentials + namespace: authentik +spec: + refreshInterval: 1h + secretStoreRef: + kind: ClusterSecretStore + name: vault-backend + target: + name: vps-minio-secret + creationPolicy: Owner + data: + - secretKey: accesskey + remoteRef: + key: vps-minio-credentials + property: access-key + - secretKey: secretkey + remoteRef: + key: vps-minio-credentials + property: secret-key diff --git a/infrastructure/cert-manager/manifests/home-ip-ddns-cronjob.yaml b/infrastructure/cert-manager/manifests/home-ip-ddns-cronjob.yaml new file mode 100644 index 0000000..e1caf2f --- /dev/null +++ b/infrastructure/cert-manager/manifests/home-ip-ddns-cronjob.yaml @@ -0,0 +1,102 @@ +--- +# Keeps home.kube.huskypup.net pointed at this cluster's current public IP in +# Cloudflare. This is the health-check target the VPS's DNS failover watcher +# (scripts/vps-dns-failover.sh) uses to decide whether home is reachable - +# it only needs to run while home is up, which is exactly when it can run. +# +# Reuses the cloudflare-token-secret already wired via ExternalSecret for +# cert-manager's DNS-01 solver (see secret-cf-token.yaml) - same zone, same +# token, no new secret plumbing. +apiVersion: v1 +kind: ConfigMap +metadata: + name: home-ip-ddns-script + namespace: cert-manager +data: + update.sh: | + #!/bin/sh + set -eu + + ZONE_NAME="kube.huskypup.net" + RECORD_NAME="home.kube.huskypup.net" + TOKEN="$(cat /etc/cf/cloudflare-token)" + + CURRENT_IP="$(curl -sf https://cloudflare.com/cdn-cgi/trace | grep -o '^ip=.*' | cut -d= -f2)" + if [ -z "$CURRENT_IP" ]; then + echo "ERROR: could not determine public IP" + exit 1 + fi + + ZONE_ID="$(curl -sf -H "Authorization: Bearer ${TOKEN}" \ + "https://api.cloudflare.com/client/v4/zones?name=${ZONE_NAME}" \ + | jq -r '.result[0].id')" + + RECORD_JSON="$(curl -sf -H "Authorization: Bearer ${TOKEN}" \ + "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records?name=${RECORD_NAME}&type=A")" + RECORD_ID="$(echo "$RECORD_JSON" | jq -r '.result[0].id // empty')" + EXISTING_IP="$(echo "$RECORD_JSON" | jq -r '.result[0].content // empty')" + + if [ "$EXISTING_IP" = "$CURRENT_IP" ]; then + echo "home.kube.huskypup.net already up to date (${CURRENT_IP})" + exit 0 + fi + + BODY="{\"type\":\"A\",\"name\":\"${RECORD_NAME}\",\"content\":\"${CURRENT_IP}\",\"ttl\":120,\"proxied\":false}" + + if [ -n "$RECORD_ID" ]; then + echo "Updating ${RECORD_NAME}: ${EXISTING_IP} -> ${CURRENT_IP}" + curl -sf -X PATCH -H "Authorization: Bearer ${TOKEN}" -H "Content-Type: application/json" \ + -d "$BODY" \ + "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records/${RECORD_ID}" >/dev/null + else + echo "Creating ${RECORD_NAME} -> ${CURRENT_IP}" + curl -sf -X POST -H "Authorization: Bearer ${TOKEN}" -H "Content-Type: application/json" \ + -d "$BODY" \ + "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records" >/dev/null + fi + echo "done" +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: home-ip-ddns + namespace: cert-manager +spec: + schedule: "*/10 * * * *" # every 10 minutes; cheap, and only matters while home is up + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 2 + template: + spec: + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsUser: 65534 + containers: + - name: ddns-update + image: alpine/k8s:1.32.13 # already has curl + jq (see MEMORY.md kubectl image note) + command: ["/bin/sh", "/scripts/update.sh"] + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + volumeMounts: + - name: script + mountPath: /scripts + - name: cf-token + mountPath: /etc/cf + readOnly: true + volumes: + - name: script + configMap: + name: home-ip-ddns-script + defaultMode: 0755 + - name: cf-token + secret: + secretName: cloudflare-token-secret + items: + - key: cloudflare-token + path: cloudflare-token diff --git a/infrastructure/cert-manager/manifests/secret-cf-token.yaml b/infrastructure/cert-manager/manifests/secret-cf-token.yaml index a3d44d1..bfa1628 100644 --- a/infrastructure/cert-manager/manifests/secret-cf-token.yaml +++ b/infrastructure/cert-manager/manifests/secret-cf-token.yaml @@ -1,9 +1,22 @@ --- -apiVersion: v1 -kind: Secret +# Was previously a plain Secret with the Cloudflare token committed in git. +# Rotated to ExternalSecret+Vault (see scripts/rotate-cloudflare-token.sh) - +# never commit a live token here again. +apiVersion: external-secrets.io/v1 +kind: ExternalSecret metadata: name: cloudflare-token-secret namespace: cert-manager -type: Opaque -stringData: - cloudflare-token: Mg9Yx1ku3-rIn7VR7Wf_PZ1uir7AqUsx3IZVpuRX # be sure you are generating an API token and not a global API key https://cert-manager.io/docs/configuration/acme/dns01/cloudflare/#api-tokens +spec: + refreshInterval: 1h + secretStoreRef: + kind: ClusterSecretStore + name: vault-backend + target: + name: cloudflare-token-secret + creationPolicy: Owner + data: + - secretKey: cloudflare-token + remoteRef: + key: cloudflare-dns-token + property: token diff --git a/infrastructure/vault/manifests/raft-snapshot-cronjob.yaml b/infrastructure/vault/manifests/raft-snapshot-cronjob.yaml new file mode 100644 index 0000000..3ff6f17 --- /dev/null +++ b/infrastructure/vault/manifests/raft-snapshot-cronjob.yaml @@ -0,0 +1,116 @@ +--- +# Periodic Vault raft snapshot, shipped to the VPS MinIO backup receiver. +# This is the DR path for Vault's data independent of the unseal-key custody +# story - a Ceph/cluster-loss disaster is recovered by standing up a fresh +# Vault and `vault operator raft snapshot restore`ing the latest one of these, +# not by anything to do with the unseal key itself. +# +# Requires a one-time manual step after VPS bootstrap: store the VPS MinIO +# root credentials (see infrastructure/vps-standby/minio/values.yaml) into +# this cluster's Vault so ESO can hand them to the CronJob: +# vault kv put secret/vps-minio-credentials \ +# access-key= secret-key= +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: vps-minio-credentials + namespace: vault +spec: + refreshInterval: 1h + secretStoreRef: + kind: ClusterSecretStore + name: vault-backend + target: + name: vps-minio-credentials + creationPolicy: Owner + data: + - secretKey: access-key + remoteRef: + key: vps-minio-credentials + property: access-key + - secretKey: secret-key + remoteRef: + key: vps-minio-credentials + property: secret-key +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vault-raft-snapshot-script + namespace: vault +data: + snapshot.sh: | + #!/bin/bash + set -euo pipefail + + # VPS's Netbird address - replace with the actual peer IP/hostname once + # the VPS is bootstrapped and joined to the mesh (scripts/vps-bootstrap.sh). + VPS_MINIO_ENDPOINT="${VPS_MINIO_ENDPOINT:-vps-minio.netbird.internal:30900}" + BUCKET="vault-raft-snapshots" + SNAP_NAME="vault-raft-$(date -u +%Y%m%dT%H%M%SZ).snap" + + ROOT_TOKEN="$(kubectl -n vault get secret vault-init-keys -o jsonpath='{.data.VAULT_ROOT_TOKEN}' | base64 -d)" + + echo "==> Taking raft snapshot from vault-0..." + kubectl -n vault exec vault-0 -- env VAULT_TOKEN="$ROOT_TOKEN" \ + vault operator raft snapshot save "/tmp/${SNAP_NAME}" + + echo "==> Copying snapshot out of vault-0..." + kubectl -n vault cp "vault-0:/tmp/${SNAP_NAME}" "/tmp/${SNAP_NAME}" + kubectl -n vault exec vault-0 -- rm -f "/tmp/${SNAP_NAME}" + + echo "==> Installing mc (MinIO client)..." + curl -sf https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc + chmod +x /usr/local/bin/mc + mc alias set vps-minio "http://${VPS_MINIO_ENDPOINT}" \ + "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}" >/dev/null + + echo "==> Uploading ${SNAP_NAME} to vps-minio/${BUCKET}..." + mc cp "/tmp/${SNAP_NAME}" "vps-minio/${BUCKET}/${SNAP_NAME}" + rm -f "/tmp/${SNAP_NAME}" + + echo "==> Pruning snapshots older than 30 days..." + mc find "vps-minio/${BUCKET}" --older-than 30d --exec "mc rm {}" || true + + echo "==> Done: ${SNAP_NAME}" +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: vault-raft-snapshot + namespace: vault +spec: + schedule: "0 */6 * * *" # every 6 hours + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 2 + template: + spec: + serviceAccountName: argocd-hook-sa # already has kubectl exec rights in this namespace (see vault-init-job.yaml) + restartPolicy: Never + containers: + - name: raft-snapshot + image: alpine/k8s:1.32.13 + command: ["/bin/bash", "/scripts/snapshot.sh"] + env: + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: vps-minio-credentials + key: access-key + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: vps-minio-credentials + key: secret-key + volumeMounts: + - name: scripts + mountPath: /scripts + volumes: + - name: scripts + configMap: + name: vault-raft-snapshot-script + defaultMode: 0755 diff --git a/infrastructure/vps-standby/minio/values.yaml b/infrastructure/vps-standby/minio/values.yaml new file mode 100644 index 0000000..d080769 --- /dev/null +++ b/infrastructure/vps-standby/minio/values.yaml @@ -0,0 +1,55 @@ +# MinIO on the VPS - shared backup-receiver bucket for the vps-standby site. +# Standalone mode: single node, single VPS disk, no erasure coding needed here - +# this is a backup *copy*, not itself something requiring HA. +# +# rootUser/rootPassword are intentionally left unset: the chart auto-generates +# a random root password and stores it in a Secret (minio/minio chart default +# behavior) rather than committing credentials to git. Retrieve after first +# deploy with: +# kubectl --context vps-standby -n minio get secret minio -o jsonpath='{.data.rootPassword}' | base64 -d + +mode: standalone + +persistence: + enabled: true + storageClass: local-path + size: 150Gi # budget out of the VPS's 360GB disk; leaves room for Phase 2 standby PVCs + +resources: + requests: + cpu: 50m + memory: 256Mi + limits: + cpu: 500m + memory: 1Gi + +# Buckets used by the backup/standby plumbing (Phase 1b / Phase 2). Created on +# first deploy; safe to append to as later phases land. +buckets: + - name: vault-raft-snapshots + policy: none + purge: false + - name: cnpg-backups + policy: none + purge: false + - name: nextcloud-files + policy: none + purge: false + +# No ingress here - MinIO is reached over Netbird (while home is up) or from +# workloads inside the vps-standby cluster itself. It never needs to be public. +ingress: + enabled: false + +# NodePort so home-cluster CronJobs (vault raft snapshots, CNPG barman backups, +# Nextcloud PVC sync) can reach this over the Netbird tunnel at +# :30900 - single-node cluster, so NodePort is simplest here. +# Verify the exact key path against `helm show values minio/minio` for the +# deployed chart version (5.4.0) before applying - not confirmed live. +service: + type: NodePort + nodePort: 30900 + +metrics: + serviceMonitor: + enabled: false # no Prometheus on the VPS cluster (out of scope for this build) diff --git a/scripts/rotate-cloudflare-token.sh b/scripts/rotate-cloudflare-token.sh new file mode 100755 index 0000000..f1d3651 --- /dev/null +++ b/scripts/rotate-cloudflare-token.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# rotate-cloudflare-token.sh - Rotate the Cloudflare DNS-01 token out of git and into Vault +# +# infrastructure/cert-manager/manifests/secret-cf-token.yaml used to contain a live +# Cloudflare API token committed in plaintext. It's now an ExternalSecret pulling +# from Vault at secret/cloudflare-dns-token#token - this script populates that path. +# +# This does NOT create or revoke the Cloudflare token itself - that's a manual step +# in the Cloudflare dashboard, deliberately not automated here since it's a live, +# outward-facing credential change: +# +# 1. Cloudflare dashboard -> My Profile -> API Tokens -> Create Token +# Scope: Zone:DNS:Edit, restricted to the kube.huskypup.net zone only +# 2. Run this script with the new token +# 3. Confirm cert-manager can still issue certs (kubectl get certificaterequests -A) +# 4. THEN go back to Cloudflare and revoke the old token +# (the one that was committed in git - assume it's compromised) +# +# Usage: +# ./scripts/rotate-cloudflare-token.sh +# +# Prerequisites: +# - Vault initialized and unsealed +# - kubectl configured for the home cluster + +set -euo pipefail + +NEW_TOKEN="${1:?Usage: $0 }" + +echo "=== Cloudflare DNS-01 Token Rotation ===" + +echo "Storing new token in Vault at secret/cloudflare-dns-token..." +ROOT_TOKEN=$(kubectl -n vault get secret vault-init-keys -o jsonpath='{.data.VAULT_ROOT_TOKEN}' | base64 -d) +kubectl exec -n vault vault-0 -- env "VAULT_TOKEN=${ROOT_TOKEN}" \ + vault kv put secret/cloudflare-dns-token token="${NEW_TOKEN}" + +echo "Forcing ExternalSecret refresh..." +kubectl -n cert-manager annotate externalsecret cloudflare-token-secret \ + force-sync="$(date +%s)" --overwrite + +echo "" +echo "=== Done ===" +echo "Next:" +echo " 1. Verify: kubectl -n cert-manager get secret cloudflare-token-secret -o jsonpath='{.data.cloudflare-token}' | base64 -d" +echo " 2. Verify a cert still renews cleanly (or delete one Certificate to force a test issuance)" +echo " 3. Revoke the OLD token in the Cloudflare dashboard once confirmed working" diff --git a/scripts/vps-bootstrap.sh b/scripts/vps-bootstrap.sh new file mode 100755 index 0000000..b7871ee --- /dev/null +++ b/scripts/vps-bootstrap.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +# vps-bootstrap.sh - Phase 0: turn a bare VPS into the vps-standby ArgoCD destination +# +# Run this ON THE VPS itself (as root, or via sudo), not against the home cluster. +# Installs k3s (single node), joins the existing self-hosted Netbird mesh, and +# installs cert-manager with the same Cloudflare DNS-01 ClusterIssuer pattern used +# at home — so TLS issuance works identically regardless of which site is "live" +# (DNS-01 only needs DNS control, not public HTTP reachability). +# +# This script does NOT register the cluster with ArgoCD — that's a one-time manual +# step run from your workstation/home cluster once this script prints the kubeconfig +# (ArgoCD can't reach the VPS until it exists, and shouldn't hold cluster-admin creds +# for a box it doesn't manage yet). +# +# Usage: +# sudo ./scripts/vps-bootstrap.sh +# +# Prerequisites: +# - A Netbird setup key (Netbird dashboard → Settings → Setup Keys → create +# a reusable, non-ephemeral key) +# - A Cloudflare API token scoped to Zone:DNS:Edit for kube.huskypup.net only +# (create a NEW token for this — do not reuse the one from +# infrastructure/cert-manager/manifests/secret-cf-token.yaml, that one is +# being rotated/retired; see Phase 0.5) +# - Ubuntu/Debian VPS with a public IP, run as root + +set -euo pipefail + +NETBIRD_SETUP_KEY="${1:?Usage: $0 }" +CLOUDFLARE_TOKEN="${2:?Usage: $0 }" + +NETBIRD_MGMT_URL="https://netbird.kube.huskypup.net" +LETSENCRYPT_EMAIL="garrettstone499@gmail.com" +DNS_ZONE="kube.huskypup.net" +CERT_MANAGER_VERSION="v1.13.2" # matches infrastructure/cert-manager chart version at home + +if [ "$(id -u)" -ne 0 ]; then + echo "ERROR: run as root (sudo $0 ...)" >&2 + exit 1 +fi + +echo "=================================================" +echo "VPS Standby Bootstrap - Phase 0" +echo "=================================================" +echo "" + +# --- k3s ----------------------------------------------------------------- +if command -v k3s >/dev/null 2>&1; then + echo "✅ k3s already installed, skipping install" +else + echo "Installing k3s (single node)..." + # Keep the built-in Traefik ingress controller — this is a lean standby box, + # not a mirror of home's Istio/Envoy-Gateway mesh. servicelb is fine too + # since this is a single node with a real public IP. + curl -sfL https://get.k3s.io | sh -s - \ + --write-kubeconfig-mode 644 \ + --disable metrics-server + echo "✅ k3s installed" +fi + +echo "Waiting for k3s node to be Ready..." +for i in $(seq 1 30); do + if k3s kubectl get nodes 2>/dev/null | grep -q " Ready"; then + echo "✅ node is Ready" + break + fi + sleep 5 +done +k3s kubectl get nodes + +export KUBECONFIG=/etc/rancher/k3s/k3s.yaml + +# --- Netbird --------------------------------------------------------------- +if command -v netbird >/dev/null 2>&1 && netbird status 2>/dev/null | grep -q "Management: Connected"; then + echo "✅ Netbird already connected, skipping" +else + echo "Installing Netbird client..." + curl -fsSL https://pkgs.netbird.io/install.sh | sh + echo "Joining Netbird mesh (${NETBIRD_MGMT_URL})..." + netbird up --management-url "${NETBIRD_MGMT_URL}" --setup-key "${NETBIRD_SETUP_KEY}" + echo "✅ Netbird joined" +fi +echo "Note: this Netbird session is used opportunistically for backup/sync traffic" +echo "while home is up. It is NOT the path used to reach this VPS when home is down —" +echo "that's direct SSH on this box's public IP. See plan doc, decision #2." + +# --- cert-manager ------------------------------------------------------------ +echo "" +echo "Installing cert-manager ${CERT_MANAGER_VERSION}..." +if ! command -v helm >/dev/null 2>&1; then + curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +fi + +helm repo add jetstack https://charts.jetstack.io 2>/dev/null || true +helm repo update jetstack + +if helm -n cert-manager status cert-manager >/dev/null 2>&1; then + echo "✅ cert-manager already installed" +else + helm install cert-manager jetstack/cert-manager \ + --namespace cert-manager \ + --create-namespace \ + --version "${CERT_MANAGER_VERSION}" \ + --set installCRDs=true \ + --wait --timeout 300s + echo "✅ cert-manager installed" +fi + +echo "Waiting for cert-manager webhook to be ready..." +k3s kubectl -n cert-manager rollout status deployment/cert-manager-webhook --timeout=120s + +# --- Cloudflare DNS-01 ClusterIssuer (same pattern as home) ----------------- +echo "" +echo "Applying Cloudflare token Secret + ClusterIssuer..." +cat <:/etc/rancher/k3s/k3s.yaml ~/vps-standby-kubeconfig.yaml" +echo " Then edit the 'server:' line inside it to use this VPS's public IP" +echo " instead of 127.0.0.1." +echo "" +echo " 2. Register it with ArgoCD:" +echo " KUBECONFIG=~/vps-standby-kubeconfig.yaml argocd cluster add default --name vps-standby" +echo "" +echo " 3. Confirm registration:" +echo " argocd cluster list" +echo "" +echo "This box is otherwise reachable via:" +echo " - Netbird (while home's self-hosted mesh is up)" +echo " - Direct SSH on its public IP (always, break-glass path)" diff --git a/scripts/vps-dns-failover/vps-dns-failover.service b/scripts/vps-dns-failover/vps-dns-failover.service new file mode 100644 index 0000000..cd5e070 --- /dev/null +++ b/scripts/vps-dns-failover/vps-dns-failover.service @@ -0,0 +1,10 @@ +[Unit] +Description=VPS dual-site DNS failover check (Homelabv4 vps-standby) +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/vps-dns-failover.sh +# Deliberately no dependency on k3s/docker being up - this must keep working +# even if the VPS's own cluster is unhealthy. diff --git a/scripts/vps-dns-failover/vps-dns-failover.sh b/scripts/vps-dns-failover/vps-dns-failover.sh new file mode 100755 index 0000000..5eabb8e --- /dev/null +++ b/scripts/vps-dns-failover/vps-dns-failover.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# vps-dns-failover.sh - Phase 0.5 dual-site DNS failover watcher +# +# Runs ON THE VPS as a systemd timer (see vps-dns-failover.timer/.service in this +# directory) - deliberately plain systemd, not a k8s CronJob, so it works even if +# the VPS's own k3s is unhealthy. It must never depend on anything inside the home +# cluster (that's the thing it's watching) or the VPS's own k3s (that's a separate +# failure domain from this box's basic OS-level networking). +# +# Health-checks home.kube.huskypup.net (kept current by an in-cluster CronJob while +# home is up - see infrastructure/cert-manager/manifests/home-ip-ddns-cronjob.yaml) +# and flips Cloudflare A records for the standby-service hostnames between home's +# public IP and this VPS's own public IP, with a consecutive-check threshold so a +# single blip doesn't cause a flap. +# +# State (current active site + streak counters) persists in $STATE_DIR between +# runs since each systemd timer firing is a fresh process. +# +# Install: +# sudo mkdir -p /etc/vps-dns-failover +# sudo sh -c 'echo "" > /etc/vps-dns-failover/cloudflare-token' +# sudo chmod 600 /etc/vps-dns-failover/cloudflare-token +# sudo cp vps-dns-failover.sh /usr/local/bin/vps-dns-failover.sh +# sudo chmod +x /usr/local/bin/vps-dns-failover.sh +# sudo cp vps-dns-failover.service vps-dns-failover.timer /etc/systemd/system/ +# sudo systemctl daemon-reload +# sudo systemctl enable --now vps-dns-failover.timer + +set -euo pipefail + +TOKEN_FILE="/etc/vps-dns-failover/cloudflare-token" +STATE_DIR="/var/lib/vps-dns-failover" +ZONE_NAME="kube.huskypup.net" +HOME_CHECK_HOST="home.kube.huskypup.net" +HOME_CHECK_PORT=443 +FAILURE_THRESHOLD=3 # consecutive failed checks before flipping to the VPS +SUCCESS_THRESHOLD=3 # consecutive successful checks before flipping back to home +STANDBY_HOSTNAMES=( + vault.kube.huskypup.net + auth.kube.huskypup.net + gitea.kube.huskypup.net + n8n.kube.huskypup.net + nextcloud.kube.huskypup.net +) + +mkdir -p "$STATE_DIR" +TOKEN="$(cat "$TOKEN_FILE")" +STATE_FILE="${STATE_DIR}/state" # format: " " + +if [ -f "$STATE_FILE" ]; then + read -r ACTIVE FAIL_STREAK SUCCESS_STREAK < "$STATE_FILE" +else + ACTIVE="home" + FAIL_STREAK=0 + SUCCESS_STREAK=0 +fi + +# --- health check ------------------------------------------------------------ +HOME_IP="$(dig +short A "$HOME_CHECK_HOST" @1.1.1.1 | tail -n1)" +if [ -n "$HOME_IP" ] && timeout 5 bash -c "cat < /dev/null > /dev/tcp/${HOME_IP}/${HOME_CHECK_PORT}" 2>/dev/null; then + HEALTHY=1 +else + HEALTHY=0 +fi + +if [ "$HEALTHY" = 1 ]; then + FAIL_STREAK=0 + SUCCESS_STREAK=$((SUCCESS_STREAK + 1)) +else + SUCCESS_STREAK=0 + FAIL_STREAK=$((FAIL_STREAK + 1)) +fi + +echo "$(date -u +%FT%TZ) active=${ACTIVE} healthy=${HEALTHY} fail_streak=${FAIL_STREAK} success_streak=${SUCCESS_STREAK} home_ip=${HOME_IP:-none}" + +# --- Cloudflare helpers -------------------------------------------------------- +cf_zone_id() { + curl -sf -H "Authorization: Bearer ${TOKEN}" \ + "https://api.cloudflare.com/client/v4/zones?name=${ZONE_NAME}" | jq -r '.result[0].id' +} + +cf_set_record() { + local zone_id="$1" hostname="$2" target_ip="$3" + local record_json record_id + record_json="$(curl -sf -H "Authorization: Bearer ${TOKEN}" \ + "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records?name=${hostname}&type=A")" + record_id="$(echo "$record_json" | jq -r '.result[0].id // empty')" + local body="{\"type\":\"A\",\"name\":\"${hostname}\",\"content\":\"${target_ip}\",\"ttl\":60,\"proxied\":false}" + if [ -n "$record_id" ]; then + curl -sf -X PATCH -H "Authorization: Bearer ${TOKEN}" -H "Content-Type: application/json" \ + -d "$body" "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records/${record_id}" >/dev/null + else + curl -sf -X POST -H "Authorization: Bearer ${TOKEN}" -H "Content-Type: application/json" \ + -d "$body" "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records" >/dev/null + fi + echo " ${hostname} -> ${target_ip}" +} + +flip_to() { + local target="$1" + local target_ip + if [ "$target" = "vps" ]; then + target_ip="$(curl -sf https://cloudflare.com/cdn-cgi/trace | grep -o '^ip=.*' | cut -d= -f2)" + else + target_ip="$HOME_IP" + fi + if [ -z "$target_ip" ]; then + echo "ERROR: could not determine target IP for '${target}', not flipping" + return 1 + fi + echo "Flipping standby hostnames to ${target} (${target_ip})..." + local zone_id + zone_id="$(cf_zone_id)" + for h in "${STANDBY_HOSTNAMES[@]}"; do + cf_set_record "$zone_id" "$h" "$target_ip" + done +} + +# --- decide ------------------------------------------------------------------ +if [ "$ACTIVE" = "home" ] && [ "$FAIL_STREAK" -ge "$FAILURE_THRESHOLD" ]; then + flip_to "vps" + ACTIVE="vps" + FAIL_STREAK=0 + SUCCESS_STREAK=0 +elif [ "$ACTIVE" = "vps" ] && [ "$SUCCESS_STREAK" -ge "$SUCCESS_THRESHOLD" ]; then + flip_to "home" + ACTIVE="home" + FAIL_STREAK=0 + SUCCESS_STREAK=0 +fi + +echo "${ACTIVE} ${FAIL_STREAK} ${SUCCESS_STREAK}" > "$STATE_FILE" diff --git a/scripts/vps-dns-failover/vps-dns-failover.timer b/scripts/vps-dns-failover/vps-dns-failover.timer new file mode 100644 index 0000000..220b9ec --- /dev/null +++ b/scripts/vps-dns-failover/vps-dns-failover.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Run vps-dns-failover check every 2 minutes + +[Timer] +OnBootSec=1min +OnUnitActiveSec=2min +AccuracySec=10s + +[Install] +WantedBy=timers.target