mirror of
https://gitlab.kube.huskypup.net/Scooby/Homelabv4.git
synced 2026-08-20 23:16:49 +00:00
Add VPS warm-standby/backup site (Phase 0-1b)
Foundation for a DR/backup path using an always-on VPS as a second ArgoCD-managed cluster, plus DB/backup standardization work that fell out of it: - vps-standby ArgoCD cluster destination + AppProject, MinIO backup receiver, VPS bootstrap script (k3s, Netbird, cert-manager) - Dual-site DNS failover watcher + home-IP DDNS CronJob, Cloudflare token moved out of git into Vault+ExternalSecret - Nextcloud migrated from ad-hoc MariaDB to CNPG + redis-operator (matches n8n/Authentik/GitLab's backup-native pattern) - Authentik's CNPG manifests moved into the actual ArgoCD-synced manifests/ path (were present but never wired into the sync path) - Vault raft-snapshot CronJob, CNPG barmanObjectStore backups (Authentik/n8n/Nextcloud), Nextcloud file-PVC restic sync - all targeting the new VPS MinIO receiver See VPS Warm-Standby plan doc for full design rationale.
This commit is contained in:
Scooby Husky
@@ -47,5 +47,25 @@ spec:
|
||||
database: n8n
|
||||
owner: n8n
|
||||
|
||||
# Backup to the VPS MinIO backup receiver (Phase 1b). Previously had no
|
||||
# backup block at all. VPS_MINIO_ENDPOINT placeholder matches
|
||||
# infrastructure/vault/manifests/raft-snapshot-cronjob.yaml - replace with
|
||||
# the VPS's actual Netbird address once bootstrapped.
|
||||
backup:
|
||||
barmanObjectStore:
|
||||
destinationPath: s3://cnpg-backups/pg-n8n
|
||||
endpointURL: http://vps-minio.netbird.internal:30900
|
||||
s3Credentials:
|
||||
accessKeyId:
|
||||
name: vps-minio-secret
|
||||
key: accesskey
|
||||
secretAccessKey:
|
||||
name: vps-minio-secret
|
||||
key: secretkey
|
||||
wal:
|
||||
compression: gzip
|
||||
maxParallel: 2
|
||||
retentionPolicy: "30d"
|
||||
|
||||
monitoring:
|
||||
enablePodMonitor: true
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
---
|
||||
# VPS MinIO credentials for CNPG's barmanObjectStore backup target.
|
||||
# Same Vault source as infrastructure/vault/manifests/raft-snapshot-cronjob.yaml
|
||||
# (secret/vps-minio-credentials) - populated once, manually, after VPS bootstrap.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: vps-minio-credentials
|
||||
namespace: n8n
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: vault-backend
|
||||
target:
|
||||
name: vps-minio-secret
|
||||
creationPolicy: Owner
|
||||
data:
|
||||
- secretKey: accesskey
|
||||
remoteRef:
|
||||
key: vps-minio-credentials
|
||||
property: access-key
|
||||
- secretKey: secretkey
|
||||
remoteRef:
|
||||
key: vps-minio-credentials
|
||||
property: secret-key
|
||||
@@ -0,0 +1,68 @@
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: Cluster
|
||||
metadata:
|
||||
name: pg-nextcloud
|
||||
namespace: nextcloud
|
||||
spec:
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16
|
||||
instances: 2
|
||||
|
||||
resources:
|
||||
requests:
|
||||
memory: "512Mi"
|
||||
cpu: "50m"
|
||||
limits:
|
||||
memory: "2Gi"
|
||||
cpu: "500m"
|
||||
|
||||
affinity:
|
||||
topologyKey: kubernetes.io/hostname
|
||||
|
||||
storage:
|
||||
size: 20Gi
|
||||
storageClass: rook-ceph-block
|
||||
|
||||
primaryUpdateStrategy: unsupervised
|
||||
|
||||
postgresql:
|
||||
parameters:
|
||||
max_connections: "200"
|
||||
shared_buffers: "512MB"
|
||||
effective_cache_size: "1536MB"
|
||||
maintenance_work_mem: "128MB"
|
||||
checkpoint_completion_target: "0.9"
|
||||
wal_buffers: "16MB"
|
||||
default_statistics_target: "100"
|
||||
random_page_cost: "1.1"
|
||||
effective_io_concurrency: "200"
|
||||
work_mem: "2621kB"
|
||||
min_wal_size: "1GB"
|
||||
max_wal_size: "4GB"
|
||||
|
||||
bootstrap:
|
||||
initdb:
|
||||
database: nextcloud
|
||||
owner: nextcloud
|
||||
|
||||
# Backup to the VPS MinIO backup receiver. VPS_MINIO_ENDPOINT placeholder
|
||||
# matches infrastructure/vault/manifests/raft-snapshot-cronjob.yaml - replace
|
||||
# with the VPS's actual Netbird address once bootstrapped. This covers the
|
||||
# DB only - file PVC content is separate, see nextcloud-pvc-sync-cronjob.yaml.
|
||||
backup:
|
||||
barmanObjectStore:
|
||||
destinationPath: s3://cnpg-backups/pg-nextcloud
|
||||
endpointURL: http://vps-minio.netbird.internal:30900
|
||||
s3Credentials:
|
||||
accessKeyId:
|
||||
name: vps-minio-secret
|
||||
key: accesskey
|
||||
secretAccessKey:
|
||||
name: vps-minio-secret
|
||||
key: secretkey
|
||||
wal:
|
||||
compression: gzip
|
||||
maxParallel: 2
|
||||
retentionPolicy: "30d"
|
||||
|
||||
monitoring:
|
||||
enablePodMonitor: true
|
||||
@@ -0,0 +1,42 @@
|
||||
---
|
||||
# Mirrors apps/n8n/manifests/cnpg-secrets.yaml: CNPG creates pg-nextcloud-app
|
||||
# with a generated password on cluster init; ESO overrides it with a
|
||||
# generator-managed password so it's rotated/tracked the same way as every
|
||||
# other app secret in this repo.
|
||||
apiVersion: generators.external-secrets.io/v1alpha1
|
||||
kind: Password
|
||||
metadata:
|
||||
name: nextcloud-cnpg-secret
|
||||
namespace: nextcloud
|
||||
spec:
|
||||
length: 42
|
||||
digits: 5
|
||||
symbols: 5
|
||||
symbolCharacters: "-_$@"
|
||||
noUpper: false
|
||||
allowRepeat: true
|
||||
|
||||
---
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: nextcloud-cnpg-secret
|
||||
namespace: nextcloud
|
||||
spec:
|
||||
refreshInterval: "24h"
|
||||
target:
|
||||
# Merges into the pg-nextcloud-app secret CNPG creates automatically
|
||||
name: pg-nextcloud-app
|
||||
creationPolicy: Merge
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
cnpg.io/reload: "true"
|
||||
data:
|
||||
password: "{{ .password }}"
|
||||
dataFrom:
|
||||
- sourceRef:
|
||||
generatorRef:
|
||||
apiVersion: generators.external-secrets.io/v1alpha1
|
||||
kind: Password
|
||||
name: nextcloud-cnpg-secret
|
||||
@@ -1,92 +0,0 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: mariadb-nextcloud
|
||||
namespace: nextcloud
|
||||
labels:
|
||||
app: mariadb-nextcloud
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: mariadb-nextcloud
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: mariadb-nextcloud
|
||||
spec:
|
||||
securityContext:
|
||||
runAsUser: 999
|
||||
runAsGroup: 999
|
||||
fsGroup: 999
|
||||
containers:
|
||||
- name: mariadb
|
||||
image: mariadb:11.4
|
||||
securityContext:
|
||||
runAsUser: 999
|
||||
runAsGroup: 999
|
||||
runAsNonRoot: true
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
args:
|
||||
- --character-set-server=utf8mb4
|
||||
- --collation-server=utf8mb4_unicode_ci
|
||||
- --init-connect=SET NAMES utf8mb4 COLLATE utf8mb4_unicode_ci
|
||||
env:
|
||||
- name: MARIADB_ROOT_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: mariadb-nextcloud
|
||||
key: root-password
|
||||
- name: MARIADB_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: mariadb-nextcloud
|
||||
key: password
|
||||
- name: MARIADB_USER
|
||||
value: nextcloud
|
||||
- name: MARIADB_DATABASE
|
||||
value: nextcloud
|
||||
ports:
|
||||
- containerPort: 3306
|
||||
volumeMounts:
|
||||
- name: mariadb-data
|
||||
mountPath: /var/lib/mysql
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
- name: run-mysqld
|
||||
mountPath: /run/mysqld
|
||||
volumes:
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
- name: run-mysqld
|
||||
emptyDir: {}
|
||||
- name: mariadb-data
|
||||
persistentVolumeClaim:
|
||||
claimName: mariadb-nextcloud-pvc
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: mariadb-nextcloud
|
||||
namespace: nextcloud
|
||||
spec:
|
||||
selector:
|
||||
app: mariadb-nextcloud
|
||||
ports:
|
||||
- port: 3306
|
||||
targetPort: 3306
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: mariadb-nextcloud-pvc
|
||||
namespace: nextcloud
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: rook-ceph-block
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
@@ -1,35 +0,0 @@
|
||||
---
|
||||
apiVersion: generators.external-secrets.io/v1alpha1
|
||||
kind: Password
|
||||
metadata:
|
||||
name: nextcloud-mariadb-password
|
||||
namespace: nextcloud
|
||||
spec:
|
||||
length: 32
|
||||
digits: 5
|
||||
symbols: 3
|
||||
symbolCharacters: "-_$"
|
||||
noUpper: false
|
||||
allowRepeat: true
|
||||
---
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: nextcloud-mariadb-secret
|
||||
namespace: nextcloud
|
||||
spec:
|
||||
refreshInterval: "24h"
|
||||
target:
|
||||
name: mariadb-nextcloud
|
||||
creationPolicy: Owner
|
||||
template:
|
||||
data:
|
||||
password: "{{ .password }}"
|
||||
root-password: "{{ .password }}"
|
||||
user: nextcloud
|
||||
dataFrom:
|
||||
- sourceRef:
|
||||
generatorRef:
|
||||
apiVersion: generators.external-secrets.io/v1alpha1
|
||||
kind: Password
|
||||
name: nextcloud-mariadb-password
|
||||
@@ -0,0 +1,124 @@
|
||||
---
|
||||
# Syncs Nextcloud's file PVC content (nextcloud-nextcloud, 200Gi) to the VPS
|
||||
# MinIO backup receiver via restic. The one piece of the Nextcloud backup story
|
||||
# no operator covers - actual file blobs, not database rows (DB is handled by
|
||||
# CNPG's barmanObjectStore in cnpg-cluster.yaml).
|
||||
#
|
||||
# Caveat: this backs up the live-mounted volume with no application-level
|
||||
# quiesce (no Nextcloud maintenance-mode pause around the snapshot). Acceptable
|
||||
# for a no-real-data test environment; for anything with real user data, pair
|
||||
# this with `occ maintenance:mode --on` before / `--off` after.
|
||||
#
|
||||
# Uses podAffinity to land on the same node as a running Nextcloud pod, since
|
||||
# the PVC is RWO (Ceph RBD) - RWO allows multiple pods to mount it concurrently
|
||||
# only when co-located on the same node. Verify the label selector below
|
||||
# matches the actual Nextcloud chart's pod labels before relying on this.
|
||||
apiVersion: generators.external-secrets.io/v1alpha1
|
||||
kind: Password
|
||||
metadata:
|
||||
name: nextcloud-restic-password
|
||||
namespace: nextcloud
|
||||
spec:
|
||||
length: 48
|
||||
digits: 5
|
||||
symbols: 5
|
||||
symbolCharacters: "-_$@"
|
||||
noUpper: false
|
||||
allowRepeat: true
|
||||
---
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: nextcloud-restic-password
|
||||
namespace: nextcloud
|
||||
spec:
|
||||
refreshInterval: "0" # generate once - changing this after the repo is initialized would lock you out of existing backups
|
||||
target:
|
||||
name: nextcloud-restic-password
|
||||
creationPolicy: Owner
|
||||
data: []
|
||||
dataFrom:
|
||||
- sourceRef:
|
||||
generatorRef:
|
||||
apiVersion: generators.external-secrets.io/v1alpha1
|
||||
kind: Password
|
||||
name: nextcloud-restic-password
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: nextcloud-pvc-sync
|
||||
namespace: nextcloud
|
||||
spec:
|
||||
schedule: "0 2 * * *" # daily at 02:00 - bulky, don't run it as often as the DB backups
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
affinity:
|
||||
podAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- labelSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: nextcloud
|
||||
topologyKey: kubernetes.io/hostname
|
||||
containers:
|
||||
- name: restic-backup
|
||||
image: restic/restic:0.16.4
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
export RESTIC_REPOSITORY="s3:http://${VPS_MINIO_ENDPOINT}/nextcloud-files/restic-repo"
|
||||
export RESTIC_PASSWORD_FILE=/restic-secret/password
|
||||
export AWS_ACCESS_KEY_ID="${MINIO_ACCESS_KEY}"
|
||||
export AWS_SECRET_ACCESS_KEY="${MINIO_SECRET_KEY}"
|
||||
|
||||
restic snapshots >/dev/null 2>&1 || restic init
|
||||
|
||||
echo "==> Backing up /data..."
|
||||
restic backup /data --tag nextcloud --host nextcloud-k8s
|
||||
|
||||
echo "==> Pruning: keep 7 daily, 4 weekly, 6 monthly..."
|
||||
restic forget --tag nextcloud --host nextcloud-k8s \
|
||||
--keep-daily 7 --keep-weekly 4 --keep-monthly 6 --prune
|
||||
|
||||
echo "==> Done."
|
||||
env:
|
||||
# VPS's Netbird address - replace once bootstrapped, matches
|
||||
# infrastructure/vault/manifests/raft-snapshot-cronjob.yaml
|
||||
- name: VPS_MINIO_ENDPOINT
|
||||
value: "vps-minio.netbird.internal:30900"
|
||||
- name: MINIO_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: vps-minio-secret
|
||||
key: accesskey
|
||||
- name: MINIO_SECRET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: vps-minio-secret
|
||||
key: secretkey
|
||||
volumeMounts:
|
||||
- name: nextcloud-data
|
||||
mountPath: /data
|
||||
readOnly: true
|
||||
- name: restic-secret
|
||||
mountPath: /restic-secret
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: nextcloud-data
|
||||
persistentVolumeClaim:
|
||||
claimName: nextcloud-nextcloud
|
||||
- name: restic-secret
|
||||
secret:
|
||||
secretName: nextcloud-restic-password
|
||||
items:
|
||||
- key: password
|
||||
path: password
|
||||
@@ -0,0 +1,72 @@
|
||||
---
|
||||
# Standalone Redis via the ot-container-kit redis-operator that's already
|
||||
# deployed as infra (argocd-apps/infrastructure/redis-operator.yaml) but
|
||||
# currently unused - GitLab's Redis is a hand-rolled StatefulSet instead
|
||||
# (apps/gitlab/manifests/redis-cluster.yaml), not this operator. This is the
|
||||
# first real consumer of it in the repo.
|
||||
#
|
||||
# NOTE: verify this CR against `kubectl explain redis.spec` (or the chart's
|
||||
# CRD source) for the redis-operator 0.15.0 actually deployed before first
|
||||
# apply - the ot-container-kit CRD schema has shifted across versions and
|
||||
# this wasn't checked against a live cluster.
|
||||
apiVersion: generators.external-secrets.io/v1alpha1
|
||||
kind: Password
|
||||
metadata:
|
||||
name: nextcloud-redis-password
|
||||
namespace: nextcloud
|
||||
spec:
|
||||
length: 32
|
||||
digits: 5
|
||||
symbols: 0
|
||||
noUpper: false
|
||||
allowRepeat: true
|
||||
---
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: nextcloud-redis-password
|
||||
namespace: nextcloud
|
||||
spec:
|
||||
refreshInterval: "0" # generate once, don't rotate (Redis CR reads this at pod start only)
|
||||
target:
|
||||
name: redis-nextcloud-secret
|
||||
creationPolicy: Owner
|
||||
template:
|
||||
data:
|
||||
password: "{{ .password }}"
|
||||
dataFrom:
|
||||
- sourceRef:
|
||||
generatorRef:
|
||||
apiVersion: generators.external-secrets.io/v1alpha1
|
||||
kind: Password
|
||||
name: nextcloud-redis-password
|
||||
---
|
||||
apiVersion: redis.redis.opstreelabs.in/v1beta2
|
||||
kind: Redis
|
||||
metadata:
|
||||
name: redis-nextcloud
|
||||
namespace: nextcloud
|
||||
spec:
|
||||
kubernetesConfig:
|
||||
image: quay.io/opstree/redis:v7.0.12
|
||||
imagePullPolicy: IfNotPresent
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 250m
|
||||
memory: 256Mi
|
||||
redisExporter:
|
||||
enabled: false
|
||||
redisSecret:
|
||||
name: redis-nextcloud-secret
|
||||
key: password
|
||||
storage:
|
||||
volumeClaimTemplate:
|
||||
spec:
|
||||
accessModes: ["ReadWriteOnce"]
|
||||
storageClassName: rook-ceph-block
|
||||
resources:
|
||||
requests:
|
||||
storage: 2Gi
|
||||
@@ -0,0 +1,27 @@
|
||||
---
|
||||
# VPS MinIO credentials for CNPG's barmanObjectStore backup target, and reused
|
||||
# by the Phase 1b PVC-content sync CronJob. Same Vault source as
|
||||
# infrastructure/vault/manifests/raft-snapshot-cronjob.yaml
|
||||
# (secret/vps-minio-credentials) - populated once, manually, after VPS bootstrap.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: vps-minio-credentials
|
||||
namespace: nextcloud
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: vault-backend
|
||||
target:
|
||||
name: vps-minio-secret
|
||||
creationPolicy: Owner
|
||||
data:
|
||||
- secretKey: accesskey
|
||||
remoteRef:
|
||||
key: vps-minio-credentials
|
||||
property: access-key
|
||||
- secretKey: secretkey
|
||||
remoteRef:
|
||||
key: vps-minio-credentials
|
||||
property: secret-key
|
||||
@@ -73,16 +73,27 @@ nextcloud:
|
||||
|
||||
externalDatabase:
|
||||
enabled: true
|
||||
type: mysql
|
||||
host: mariadb-nextcloud
|
||||
port: 3306
|
||||
# Moved from MariaDB to CNPG-backed Postgres (apps/nextcloud/manifests/cnpg-cluster.yaml)
|
||||
# for backup/DR consistency with the rest of the stack (n8n, Authentik, GitLab
|
||||
# all use CNPG's native barmanObjectStore backup - MariaDB needed a bespoke
|
||||
# mysqldump job instead). pg-nextcloud-rw is CNPG's generated read-write
|
||||
# Service name for the "pg-nextcloud" Cluster.
|
||||
#
|
||||
# IMPORTANT: switching type here does NOT migrate existing data. Nextcloud
|
||||
# requires an explicit `occ db:convert-type pgsql ...` run before cutover if
|
||||
# there's real data in the old MariaDB instance. usernameKey/passwordKey below
|
||||
# match CNPG's generated app-secret keys (confirmed against the existing,
|
||||
# already-working infrastructure/authentik/values.yaml pg-authentik-app usage).
|
||||
type: postgresql
|
||||
host: pg-nextcloud-rw
|
||||
port: 5432
|
||||
user: nextcloud
|
||||
database: nextcloud
|
||||
existingSecret:
|
||||
enabled: true
|
||||
secretName: mariadb-nextcloud
|
||||
secretName: pg-nextcloud-app
|
||||
passwordKey: password
|
||||
usernameKey: user
|
||||
usernameKey: username # matches CNPG's generated app-secret key, confirmed against infrastructure/authentik/values.yaml's existing pg-authentik-app usage
|
||||
|
||||
persistence:
|
||||
enabled: true
|
||||
@@ -99,7 +110,16 @@ resources:
|
||||
memory: 2Gi
|
||||
|
||||
redis:
|
||||
enabled: false
|
||||
# Was disabled; now points at the standalone Redis CR (redis-operator) in
|
||||
# apps/nextcloud/manifests/redis-cr.yaml instead of the chart's bundled
|
||||
# subchart - fixes Nextcloud's file-locking/caching as a side benefit of
|
||||
# this migration. Verify these key names against `helm show values
|
||||
# nextcloud/nextcloud` for the deployed chart version before applying.
|
||||
enabled: true
|
||||
host: redis-nextcloud
|
||||
port: 6379
|
||||
existingSecret: redis-nextcloud-secret
|
||||
existingSecretPasswordKey: password
|
||||
|
||||
metrics:
|
||||
enabled: false
|
||||
|
||||
@@ -57,3 +57,26 @@ spec:
|
||||
namespaceResourceWhitelist:
|
||||
- group: '*'
|
||||
kind: '*'
|
||||
---
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: AppProject
|
||||
metadata:
|
||||
name: vps-standby
|
||||
namespace: argocd
|
||||
spec:
|
||||
# VPS warm-standby / backup-site services. Deployed to the "vps-standby" cluster
|
||||
# registered via `argocd cluster add ... --name vps-standby` (see
|
||||
# scripts/vps-bootstrap.sh). Scoped to that cluster only — nothing in this
|
||||
# project should ever target the home cluster.
|
||||
description: VPS warm-standby / backup-site services
|
||||
sourceRepos:
|
||||
- '*'
|
||||
destinations:
|
||||
- namespace: '*'
|
||||
name: vps-standby
|
||||
clusterResourceWhitelist:
|
||||
- group: '*'
|
||||
kind: '*'
|
||||
namespaceResourceWhitelist:
|
||||
- group: '*'
|
||||
kind: '*'
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: vps-minio
|
||||
namespace: argocd
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "1" # first thing on the vps-standby cluster - everything else backs up to it
|
||||
finalizers:
|
||||
- resources-finalizer.argocd.argoproj.io
|
||||
spec:
|
||||
project: vps-standby
|
||||
sources:
|
||||
- repoURL: https://charts.min.io/
|
||||
chart: minio
|
||||
targetRevision: 5.4.0
|
||||
helm:
|
||||
valueFiles:
|
||||
- $values/infrastructure/vps-standby/minio/values.yaml
|
||||
- repoURL: https://gitlab.kube.huskypup.net/Scooby/Homelabv4.git
|
||||
targetRevision: main
|
||||
ref: values
|
||||
destination:
|
||||
name: vps-standby
|
||||
namespace: minio
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
+19
-1
@@ -47,7 +47,25 @@ spec:
|
||||
database: app
|
||||
owner: app
|
||||
|
||||
# Backup to the VPS MinIO backup receiver (Phase 1b). VPS_MINIO_ENDPOINT
|
||||
# placeholder matches infrastructure/vault/manifests/raft-snapshot-cronjob.yaml -
|
||||
# replace with the VPS's actual Netbird address once bootstrapped.
|
||||
backup:
|
||||
barmanObjectStore:
|
||||
destinationPath: s3://cnpg-backups/pg-authentik
|
||||
endpointURL: http://vps-minio.netbird.internal:30900
|
||||
s3Credentials:
|
||||
accessKeyId:
|
||||
name: vps-minio-secret
|
||||
key: accesskey
|
||||
secretAccessKey:
|
||||
name: vps-minio-secret
|
||||
key: secretkey
|
||||
wal:
|
||||
compression: gzip
|
||||
maxParallel: 2
|
||||
retentionPolicy: "30d"
|
||||
|
||||
monitoring:
|
||||
enablePodMonitor: true
|
||||
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
---
|
||||
# VPS MinIO credentials for CNPG's barmanObjectStore backup target.
|
||||
# Same Vault source as infrastructure/vault/manifests/raft-snapshot-cronjob.yaml
|
||||
# (secret/vps-minio-credentials) - populated once, manually, after VPS bootstrap.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: vps-minio-credentials
|
||||
namespace: authentik
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: vault-backend
|
||||
target:
|
||||
name: vps-minio-secret
|
||||
creationPolicy: Owner
|
||||
data:
|
||||
- secretKey: accesskey
|
||||
remoteRef:
|
||||
key: vps-minio-credentials
|
||||
property: access-key
|
||||
- secretKey: secretkey
|
||||
remoteRef:
|
||||
key: vps-minio-credentials
|
||||
property: secret-key
|
||||
@@ -0,0 +1,102 @@
|
||||
---
|
||||
# Keeps home.kube.huskypup.net pointed at this cluster's current public IP in
|
||||
# Cloudflare. This is the health-check target the VPS's DNS failover watcher
|
||||
# (scripts/vps-dns-failover.sh) uses to decide whether home is reachable -
|
||||
# it only needs to run while home is up, which is exactly when it can run.
|
||||
#
|
||||
# Reuses the cloudflare-token-secret already wired via ExternalSecret for
|
||||
# cert-manager's DNS-01 solver (see secret-cf-token.yaml) - same zone, same
|
||||
# token, no new secret plumbing.
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: home-ip-ddns-script
|
||||
namespace: cert-manager
|
||||
data:
|
||||
update.sh: |
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
ZONE_NAME="kube.huskypup.net"
|
||||
RECORD_NAME="home.kube.huskypup.net"
|
||||
TOKEN="$(cat /etc/cf/cloudflare-token)"
|
||||
|
||||
CURRENT_IP="$(curl -sf https://cloudflare.com/cdn-cgi/trace | grep -o '^ip=.*' | cut -d= -f2)"
|
||||
if [ -z "$CURRENT_IP" ]; then
|
||||
echo "ERROR: could not determine public IP"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ZONE_ID="$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
|
||||
"https://api.cloudflare.com/client/v4/zones?name=${ZONE_NAME}" \
|
||||
| jq -r '.result[0].id')"
|
||||
|
||||
RECORD_JSON="$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
|
||||
"https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records?name=${RECORD_NAME}&type=A")"
|
||||
RECORD_ID="$(echo "$RECORD_JSON" | jq -r '.result[0].id // empty')"
|
||||
EXISTING_IP="$(echo "$RECORD_JSON" | jq -r '.result[0].content // empty')"
|
||||
|
||||
if [ "$EXISTING_IP" = "$CURRENT_IP" ]; then
|
||||
echo "home.kube.huskypup.net already up to date (${CURRENT_IP})"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
BODY="{\"type\":\"A\",\"name\":\"${RECORD_NAME}\",\"content\":\"${CURRENT_IP}\",\"ttl\":120,\"proxied\":false}"
|
||||
|
||||
if [ -n "$RECORD_ID" ]; then
|
||||
echo "Updating ${RECORD_NAME}: ${EXISTING_IP} -> ${CURRENT_IP}"
|
||||
curl -sf -X PATCH -H "Authorization: Bearer ${TOKEN}" -H "Content-Type: application/json" \
|
||||
-d "$BODY" \
|
||||
"https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records/${RECORD_ID}" >/dev/null
|
||||
else
|
||||
echo "Creating ${RECORD_NAME} -> ${CURRENT_IP}"
|
||||
curl -sf -X POST -H "Authorization: Bearer ${TOKEN}" -H "Content-Type: application/json" \
|
||||
-d "$BODY" \
|
||||
"https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records" >/dev/null
|
||||
fi
|
||||
echo "done"
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: home-ip-ddns
|
||||
namespace: cert-manager
|
||||
spec:
|
||||
schedule: "*/10 * * * *" # every 10 minutes; cheap, and only matters while home is up
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65534
|
||||
containers:
|
||||
- name: ddns-update
|
||||
image: alpine/k8s:1.32.13 # already has curl + jq (see MEMORY.md kubectl image note)
|
||||
command: ["/bin/sh", "/scripts/update.sh"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
volumeMounts:
|
||||
- name: script
|
||||
mountPath: /scripts
|
||||
- name: cf-token
|
||||
mountPath: /etc/cf
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: script
|
||||
configMap:
|
||||
name: home-ip-ddns-script
|
||||
defaultMode: 0755
|
||||
- name: cf-token
|
||||
secret:
|
||||
secretName: cloudflare-token-secret
|
||||
items:
|
||||
- key: cloudflare-token
|
||||
path: cloudflare-token
|
||||
@@ -1,9 +1,22 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
# Was previously a plain Secret with the Cloudflare token committed in git.
|
||||
# Rotated to ExternalSecret+Vault (see scripts/rotate-cloudflare-token.sh) -
|
||||
# never commit a live token here again.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: cloudflare-token-secret
|
||||
namespace: cert-manager
|
||||
type: Opaque
|
||||
stringData:
|
||||
cloudflare-token: Mg9Yx1ku3-rIn7VR7Wf_PZ1uir7AqUsx3IZVpuRX # be sure you are generating an API token and not a global API key https://cert-manager.io/docs/configuration/acme/dns01/cloudflare/#api-tokens
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: vault-backend
|
||||
target:
|
||||
name: cloudflare-token-secret
|
||||
creationPolicy: Owner
|
||||
data:
|
||||
- secretKey: cloudflare-token
|
||||
remoteRef:
|
||||
key: cloudflare-dns-token
|
||||
property: token
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
---
|
||||
# Periodic Vault raft snapshot, shipped to the VPS MinIO backup receiver.
|
||||
# This is the DR path for Vault's data independent of the unseal-key custody
|
||||
# story - a Ceph/cluster-loss disaster is recovered by standing up a fresh
|
||||
# Vault and `vault operator raft snapshot restore`ing the latest one of these,
|
||||
# not by anything to do with the unseal key itself.
|
||||
#
|
||||
# Requires a one-time manual step after VPS bootstrap: store the VPS MinIO
|
||||
# root credentials (see infrastructure/vps-standby/minio/values.yaml) into
|
||||
# this cluster's Vault so ESO can hand them to the CronJob:
|
||||
# vault kv put secret/vps-minio-credentials \
|
||||
# access-key=<minio-root-user> secret-key=<minio-root-password>
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: vps-minio-credentials
|
||||
namespace: vault
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: vault-backend
|
||||
target:
|
||||
name: vps-minio-credentials
|
||||
creationPolicy: Owner
|
||||
data:
|
||||
- secretKey: access-key
|
||||
remoteRef:
|
||||
key: vps-minio-credentials
|
||||
property: access-key
|
||||
- secretKey: secret-key
|
||||
remoteRef:
|
||||
key: vps-minio-credentials
|
||||
property: secret-key
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: vault-raft-snapshot-script
|
||||
namespace: vault
|
||||
data:
|
||||
snapshot.sh: |
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# VPS's Netbird address - replace with the actual peer IP/hostname once
|
||||
# the VPS is bootstrapped and joined to the mesh (scripts/vps-bootstrap.sh).
|
||||
VPS_MINIO_ENDPOINT="${VPS_MINIO_ENDPOINT:-vps-minio.netbird.internal:30900}"
|
||||
BUCKET="vault-raft-snapshots"
|
||||
SNAP_NAME="vault-raft-$(date -u +%Y%m%dT%H%M%SZ).snap"
|
||||
|
||||
ROOT_TOKEN="$(kubectl -n vault get secret vault-init-keys -o jsonpath='{.data.VAULT_ROOT_TOKEN}' | base64 -d)"
|
||||
|
||||
echo "==> Taking raft snapshot from vault-0..."
|
||||
kubectl -n vault exec vault-0 -- env VAULT_TOKEN="$ROOT_TOKEN" \
|
||||
vault operator raft snapshot save "/tmp/${SNAP_NAME}"
|
||||
|
||||
echo "==> Copying snapshot out of vault-0..."
|
||||
kubectl -n vault cp "vault-0:/tmp/${SNAP_NAME}" "/tmp/${SNAP_NAME}"
|
||||
kubectl -n vault exec vault-0 -- rm -f "/tmp/${SNAP_NAME}"
|
||||
|
||||
echo "==> Installing mc (MinIO client)..."
|
||||
curl -sf https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc
|
||||
chmod +x /usr/local/bin/mc
|
||||
mc alias set vps-minio "http://${VPS_MINIO_ENDPOINT}" \
|
||||
"${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}" >/dev/null
|
||||
|
||||
echo "==> Uploading ${SNAP_NAME} to vps-minio/${BUCKET}..."
|
||||
mc cp "/tmp/${SNAP_NAME}" "vps-minio/${BUCKET}/${SNAP_NAME}"
|
||||
rm -f "/tmp/${SNAP_NAME}"
|
||||
|
||||
echo "==> Pruning snapshots older than 30 days..."
|
||||
mc find "vps-minio/${BUCKET}" --older-than 30d --exec "mc rm {}" || true
|
||||
|
||||
echo "==> Done: ${SNAP_NAME}"
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: vault-raft-snapshot
|
||||
namespace: vault
|
||||
spec:
|
||||
schedule: "0 */6 * * *" # every 6 hours
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
template:
|
||||
spec:
|
||||
serviceAccountName: argocd-hook-sa # already has kubectl exec rights in this namespace (see vault-init-job.yaml)
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: raft-snapshot
|
||||
image: alpine/k8s:1.32.13
|
||||
command: ["/bin/bash", "/scripts/snapshot.sh"]
|
||||
env:
|
||||
- name: MINIO_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: vps-minio-credentials
|
||||
key: access-key
|
||||
- name: MINIO_SECRET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: vps-minio-credentials
|
||||
key: secret-key
|
||||
volumeMounts:
|
||||
- name: scripts
|
||||
mountPath: /scripts
|
||||
volumes:
|
||||
- name: scripts
|
||||
configMap:
|
||||
name: vault-raft-snapshot-script
|
||||
defaultMode: 0755
|
||||
@@ -0,0 +1,55 @@
|
||||
# MinIO on the VPS - shared backup-receiver bucket for the vps-standby site.
|
||||
# Standalone mode: single node, single VPS disk, no erasure coding needed here -
|
||||
# this is a backup *copy*, not itself something requiring HA.
|
||||
#
|
||||
# rootUser/rootPassword are intentionally left unset: the chart auto-generates
|
||||
# a random root password and stores it in a Secret (minio/minio chart default
|
||||
# behavior) rather than committing credentials to git. Retrieve after first
|
||||
# deploy with:
|
||||
# kubectl --context vps-standby -n minio get secret minio -o jsonpath='{.data.rootPassword}' | base64 -d
|
||||
|
||||
mode: standalone
|
||||
|
||||
persistence:
|
||||
enabled: true
|
||||
storageClass: local-path
|
||||
size: 150Gi # budget out of the VPS's 360GB disk; leaves room for Phase 2 standby PVCs
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
|
||||
# Buckets used by the backup/standby plumbing (Phase 1b / Phase 2). Created on
|
||||
# first deploy; safe to append to as later phases land.
|
||||
buckets:
|
||||
- name: vault-raft-snapshots
|
||||
policy: none
|
||||
purge: false
|
||||
- name: cnpg-backups
|
||||
policy: none
|
||||
purge: false
|
||||
- name: nextcloud-files
|
||||
policy: none
|
||||
purge: false
|
||||
|
||||
# No ingress here - MinIO is reached over Netbird (while home is up) or from
|
||||
# workloads inside the vps-standby cluster itself. It never needs to be public.
|
||||
ingress:
|
||||
enabled: false
|
||||
|
||||
# NodePort so home-cluster CronJobs (vault raft snapshots, CNPG barman backups,
|
||||
# Nextcloud PVC sync) can reach this over the Netbird tunnel at
|
||||
# <VPS_NETBIRD_IP>:30900 - single-node cluster, so NodePort is simplest here.
|
||||
# Verify the exact key path against `helm show values minio/minio` for the
|
||||
# deployed chart version (5.4.0) before applying - not confirmed live.
|
||||
service:
|
||||
type: NodePort
|
||||
nodePort: 30900
|
||||
|
||||
metrics:
|
||||
serviceMonitor:
|
||||
enabled: false # no Prometheus on the VPS cluster (out of scope for this build)
|
||||
Executable
+46
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env bash
|
||||
# rotate-cloudflare-token.sh - Rotate the Cloudflare DNS-01 token out of git and into Vault
|
||||
#
|
||||
# infrastructure/cert-manager/manifests/secret-cf-token.yaml used to contain a live
|
||||
# Cloudflare API token committed in plaintext. It's now an ExternalSecret pulling
|
||||
# from Vault at secret/cloudflare-dns-token#token - this script populates that path.
|
||||
#
|
||||
# This does NOT create or revoke the Cloudflare token itself - that's a manual step
|
||||
# in the Cloudflare dashboard, deliberately not automated here since it's a live,
|
||||
# outward-facing credential change:
|
||||
#
|
||||
# 1. Cloudflare dashboard -> My Profile -> API Tokens -> Create Token
|
||||
# Scope: Zone:DNS:Edit, restricted to the kube.huskypup.net zone only
|
||||
# 2. Run this script with the new token
|
||||
# 3. Confirm cert-manager can still issue certs (kubectl get certificaterequests -A)
|
||||
# 4. THEN go back to Cloudflare and revoke the old token
|
||||
# (the one that was committed in git - assume it's compromised)
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/rotate-cloudflare-token.sh <new-cloudflare-token>
|
||||
#
|
||||
# Prerequisites:
|
||||
# - Vault initialized and unsealed
|
||||
# - kubectl configured for the home cluster
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
NEW_TOKEN="${1:?Usage: $0 <new-cloudflare-token>}"
|
||||
|
||||
echo "=== Cloudflare DNS-01 Token Rotation ==="
|
||||
|
||||
echo "Storing new token in Vault at secret/cloudflare-dns-token..."
|
||||
ROOT_TOKEN=$(kubectl -n vault get secret vault-init-keys -o jsonpath='{.data.VAULT_ROOT_TOKEN}' | base64 -d)
|
||||
kubectl exec -n vault vault-0 -- env "VAULT_TOKEN=${ROOT_TOKEN}" \
|
||||
vault kv put secret/cloudflare-dns-token token="${NEW_TOKEN}"
|
||||
|
||||
echo "Forcing ExternalSecret refresh..."
|
||||
kubectl -n cert-manager annotate externalsecret cloudflare-token-secret \
|
||||
force-sync="$(date +%s)" --overwrite
|
||||
|
||||
echo ""
|
||||
echo "=== Done ==="
|
||||
echo "Next:"
|
||||
echo " 1. Verify: kubectl -n cert-manager get secret cloudflare-token-secret -o jsonpath='{.data.cloudflare-token}' | base64 -d"
|
||||
echo " 2. Verify a cert still renews cleanly (or delete one Certificate to force a test issuance)"
|
||||
echo " 3. Revoke the OLD token in the Cloudflare dashboard once confirmed working"
|
||||
Executable
+169
@@ -0,0 +1,169 @@
|
||||
#!/usr/bin/env bash
|
||||
# vps-bootstrap.sh - Phase 0: turn a bare VPS into the vps-standby ArgoCD destination
|
||||
#
|
||||
# Run this ON THE VPS itself (as root, or via sudo), not against the home cluster.
|
||||
# Installs k3s (single node), joins the existing self-hosted Netbird mesh, and
|
||||
# installs cert-manager with the same Cloudflare DNS-01 ClusterIssuer pattern used
|
||||
# at home — so TLS issuance works identically regardless of which site is "live"
|
||||
# (DNS-01 only needs DNS control, not public HTTP reachability).
|
||||
#
|
||||
# This script does NOT register the cluster with ArgoCD — that's a one-time manual
|
||||
# step run from your workstation/home cluster once this script prints the kubeconfig
|
||||
# (ArgoCD can't reach the VPS until it exists, and shouldn't hold cluster-admin creds
|
||||
# for a box it doesn't manage yet).
|
||||
#
|
||||
# Usage:
|
||||
# sudo ./scripts/vps-bootstrap.sh <netbird-setup-key> <cloudflare-dns-edit-token>
|
||||
#
|
||||
# Prerequisites:
|
||||
# - A Netbird setup key (Netbird dashboard → Settings → Setup Keys → create
|
||||
# a reusable, non-ephemeral key)
|
||||
# - A Cloudflare API token scoped to Zone:DNS:Edit for kube.huskypup.net only
|
||||
# (create a NEW token for this — do not reuse the one from
|
||||
# infrastructure/cert-manager/manifests/secret-cf-token.yaml, that one is
|
||||
# being rotated/retired; see Phase 0.5)
|
||||
# - Ubuntu/Debian VPS with a public IP, run as root
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
NETBIRD_SETUP_KEY="${1:?Usage: $0 <netbird-setup-key> <cloudflare-dns-edit-token>}"
|
||||
CLOUDFLARE_TOKEN="${2:?Usage: $0 <netbird-setup-key> <cloudflare-dns-edit-token>}"
|
||||
|
||||
NETBIRD_MGMT_URL="https://netbird.kube.huskypup.net"
|
||||
LETSENCRYPT_EMAIL="garrettstone499@gmail.com"
|
||||
DNS_ZONE="kube.huskypup.net"
|
||||
CERT_MANAGER_VERSION="v1.13.2" # matches infrastructure/cert-manager chart version at home
|
||||
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
echo "ERROR: run as root (sudo $0 ...)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "================================================="
|
||||
echo "VPS Standby Bootstrap - Phase 0"
|
||||
echo "================================================="
|
||||
echo ""
|
||||
|
||||
# --- k3s -----------------------------------------------------------------
|
||||
if command -v k3s >/dev/null 2>&1; then
|
||||
echo "✅ k3s already installed, skipping install"
|
||||
else
|
||||
echo "Installing k3s (single node)..."
|
||||
# Keep the built-in Traefik ingress controller — this is a lean standby box,
|
||||
# not a mirror of home's Istio/Envoy-Gateway mesh. servicelb is fine too
|
||||
# since this is a single node with a real public IP.
|
||||
curl -sfL https://get.k3s.io | sh -s - \
|
||||
--write-kubeconfig-mode 644 \
|
||||
--disable metrics-server
|
||||
echo "✅ k3s installed"
|
||||
fi
|
||||
|
||||
echo "Waiting for k3s node to be Ready..."
|
||||
for i in $(seq 1 30); do
|
||||
if k3s kubectl get nodes 2>/dev/null | grep -q " Ready"; then
|
||||
echo "✅ node is Ready"
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
k3s kubectl get nodes
|
||||
|
||||
export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
|
||||
|
||||
# --- Netbird ---------------------------------------------------------------
|
||||
if command -v netbird >/dev/null 2>&1 && netbird status 2>/dev/null | grep -q "Management: Connected"; then
|
||||
echo "✅ Netbird already connected, skipping"
|
||||
else
|
||||
echo "Installing Netbird client..."
|
||||
curl -fsSL https://pkgs.netbird.io/install.sh | sh
|
||||
echo "Joining Netbird mesh (${NETBIRD_MGMT_URL})..."
|
||||
netbird up --management-url "${NETBIRD_MGMT_URL}" --setup-key "${NETBIRD_SETUP_KEY}"
|
||||
echo "✅ Netbird joined"
|
||||
fi
|
||||
echo "Note: this Netbird session is used opportunistically for backup/sync traffic"
|
||||
echo "while home is up. It is NOT the path used to reach this VPS when home is down —"
|
||||
echo "that's direct SSH on this box's public IP. See plan doc, decision #2."
|
||||
|
||||
# --- cert-manager ------------------------------------------------------------
|
||||
echo ""
|
||||
echo "Installing cert-manager ${CERT_MANAGER_VERSION}..."
|
||||
if ! command -v helm >/dev/null 2>&1; then
|
||||
curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
|
||||
fi
|
||||
|
||||
helm repo add jetstack https://charts.jetstack.io 2>/dev/null || true
|
||||
helm repo update jetstack
|
||||
|
||||
if helm -n cert-manager status cert-manager >/dev/null 2>&1; then
|
||||
echo "✅ cert-manager already installed"
|
||||
else
|
||||
helm install cert-manager jetstack/cert-manager \
|
||||
--namespace cert-manager \
|
||||
--create-namespace \
|
||||
--version "${CERT_MANAGER_VERSION}" \
|
||||
--set installCRDs=true \
|
||||
--wait --timeout 300s
|
||||
echo "✅ cert-manager installed"
|
||||
fi
|
||||
|
||||
echo "Waiting for cert-manager webhook to be ready..."
|
||||
k3s kubectl -n cert-manager rollout status deployment/cert-manager-webhook --timeout=120s
|
||||
|
||||
# --- Cloudflare DNS-01 ClusterIssuer (same pattern as home) -----------------
|
||||
echo ""
|
||||
echo "Applying Cloudflare token Secret + ClusterIssuer..."
|
||||
cat <<EOF | k3s kubectl apply -f -
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: cloudflare-token-secret
|
||||
namespace: cert-manager
|
||||
type: Opaque
|
||||
stringData:
|
||||
cloudflare-token: "${CLOUDFLARE_TOKEN}"
|
||||
---
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: ClusterIssuer
|
||||
metadata:
|
||||
name: letsencrypt-production
|
||||
spec:
|
||||
acme:
|
||||
server: https://acme-v02.api.letsencrypt.org/directory
|
||||
email: ${LETSENCRYPT_EMAIL}
|
||||
privateKeySecretRef:
|
||||
name: letsencrypt-production
|
||||
solvers:
|
||||
- dns01:
|
||||
cloudflare:
|
||||
email: ${LETSENCRYPT_EMAIL}
|
||||
apiTokenSecretRef:
|
||||
name: cloudflare-token-secret
|
||||
key: cloudflare-token
|
||||
selector:
|
||||
dnsZones:
|
||||
- "${DNS_ZONE}"
|
||||
EOF
|
||||
echo "✅ ClusterIssuer letsencrypt-production ready"
|
||||
|
||||
# --- Output for ArgoCD registration -----------------------------------------
|
||||
echo ""
|
||||
echo "================================================="
|
||||
echo "✅ VPS foundation bootstrap complete"
|
||||
echo "================================================="
|
||||
echo ""
|
||||
echo "Next step (run from your workstation, NOT this VPS):"
|
||||
echo ""
|
||||
echo " 1. Copy this VPS's kubeconfig to your workstation, e.g.:"
|
||||
echo " scp root@<vps-ip>:/etc/rancher/k3s/k3s.yaml ~/vps-standby-kubeconfig.yaml"
|
||||
echo " Then edit the 'server:' line inside it to use this VPS's public IP"
|
||||
echo " instead of 127.0.0.1."
|
||||
echo ""
|
||||
echo " 2. Register it with ArgoCD:"
|
||||
echo " KUBECONFIG=~/vps-standby-kubeconfig.yaml argocd cluster add default --name vps-standby"
|
||||
echo ""
|
||||
echo " 3. Confirm registration:"
|
||||
echo " argocd cluster list"
|
||||
echo ""
|
||||
echo "This box is otherwise reachable via:"
|
||||
echo " - Netbird (while home's self-hosted mesh is up)"
|
||||
echo " - Direct SSH on its public IP (always, break-glass path)"
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=VPS dual-site DNS failover check (Homelabv4 vps-standby)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/vps-dns-failover.sh
|
||||
# Deliberately no dependency on k3s/docker being up - this must keep working
|
||||
# even if the VPS's own cluster is unhealthy.
|
||||
Executable
+132
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env bash
|
||||
# vps-dns-failover.sh - Phase 0.5 dual-site DNS failover watcher
|
||||
#
|
||||
# Runs ON THE VPS as a systemd timer (see vps-dns-failover.timer/.service in this
|
||||
# directory) - deliberately plain systemd, not a k8s CronJob, so it works even if
|
||||
# the VPS's own k3s is unhealthy. It must never depend on anything inside the home
|
||||
# cluster (that's the thing it's watching) or the VPS's own k3s (that's a separate
|
||||
# failure domain from this box's basic OS-level networking).
|
||||
#
|
||||
# Health-checks home.kube.huskypup.net (kept current by an in-cluster CronJob while
|
||||
# home is up - see infrastructure/cert-manager/manifests/home-ip-ddns-cronjob.yaml)
|
||||
# and flips Cloudflare A records for the standby-service hostnames between home's
|
||||
# public IP and this VPS's own public IP, with a consecutive-check threshold so a
|
||||
# single blip doesn't cause a flap.
|
||||
#
|
||||
# State (current active site + streak counters) persists in $STATE_DIR between
|
||||
# runs since each systemd timer firing is a fresh process.
|
||||
#
|
||||
# Install:
|
||||
# sudo mkdir -p /etc/vps-dns-failover
|
||||
# sudo sh -c 'echo "<dns-edit-scoped-cloudflare-token>" > /etc/vps-dns-failover/cloudflare-token'
|
||||
# sudo chmod 600 /etc/vps-dns-failover/cloudflare-token
|
||||
# sudo cp vps-dns-failover.sh /usr/local/bin/vps-dns-failover.sh
|
||||
# sudo chmod +x /usr/local/bin/vps-dns-failover.sh
|
||||
# sudo cp vps-dns-failover.service vps-dns-failover.timer /etc/systemd/system/
|
||||
# sudo systemctl daemon-reload
|
||||
# sudo systemctl enable --now vps-dns-failover.timer
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
TOKEN_FILE="/etc/vps-dns-failover/cloudflare-token"
|
||||
STATE_DIR="/var/lib/vps-dns-failover"
|
||||
ZONE_NAME="kube.huskypup.net"
|
||||
HOME_CHECK_HOST="home.kube.huskypup.net"
|
||||
HOME_CHECK_PORT=443
|
||||
FAILURE_THRESHOLD=3 # consecutive failed checks before flipping to the VPS
|
||||
SUCCESS_THRESHOLD=3 # consecutive successful checks before flipping back to home
|
||||
STANDBY_HOSTNAMES=(
|
||||
vault.kube.huskypup.net
|
||||
auth.kube.huskypup.net
|
||||
gitea.kube.huskypup.net
|
||||
n8n.kube.huskypup.net
|
||||
nextcloud.kube.huskypup.net
|
||||
)
|
||||
|
||||
mkdir -p "$STATE_DIR"
|
||||
TOKEN="$(cat "$TOKEN_FILE")"
|
||||
STATE_FILE="${STATE_DIR}/state" # format: "<active-site> <fail-streak> <success-streak>"
|
||||
|
||||
if [ -f "$STATE_FILE" ]; then
|
||||
read -r ACTIVE FAIL_STREAK SUCCESS_STREAK < "$STATE_FILE"
|
||||
else
|
||||
ACTIVE="home"
|
||||
FAIL_STREAK=0
|
||||
SUCCESS_STREAK=0
|
||||
fi
|
||||
|
||||
# --- health check ------------------------------------------------------------
|
||||
HOME_IP="$(dig +short A "$HOME_CHECK_HOST" @1.1.1.1 | tail -n1)"
|
||||
if [ -n "$HOME_IP" ] && timeout 5 bash -c "cat < /dev/null > /dev/tcp/${HOME_IP}/${HOME_CHECK_PORT}" 2>/dev/null; then
|
||||
HEALTHY=1
|
||||
else
|
||||
HEALTHY=0
|
||||
fi
|
||||
|
||||
if [ "$HEALTHY" = 1 ]; then
|
||||
FAIL_STREAK=0
|
||||
SUCCESS_STREAK=$((SUCCESS_STREAK + 1))
|
||||
else
|
||||
SUCCESS_STREAK=0
|
||||
FAIL_STREAK=$((FAIL_STREAK + 1))
|
||||
fi
|
||||
|
||||
echo "$(date -u +%FT%TZ) active=${ACTIVE} healthy=${HEALTHY} fail_streak=${FAIL_STREAK} success_streak=${SUCCESS_STREAK} home_ip=${HOME_IP:-none}"
|
||||
|
||||
# --- Cloudflare helpers --------------------------------------------------------
|
||||
cf_zone_id() {
|
||||
curl -sf -H "Authorization: Bearer ${TOKEN}" \
|
||||
"https://api.cloudflare.com/client/v4/zones?name=${ZONE_NAME}" | jq -r '.result[0].id'
|
||||
}
|
||||
|
||||
cf_set_record() {
|
||||
local zone_id="$1" hostname="$2" target_ip="$3"
|
||||
local record_json record_id
|
||||
record_json="$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
|
||||
"https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records?name=${hostname}&type=A")"
|
||||
record_id="$(echo "$record_json" | jq -r '.result[0].id // empty')"
|
||||
local body="{\"type\":\"A\",\"name\":\"${hostname}\",\"content\":\"${target_ip}\",\"ttl\":60,\"proxied\":false}"
|
||||
if [ -n "$record_id" ]; then
|
||||
curl -sf -X PATCH -H "Authorization: Bearer ${TOKEN}" -H "Content-Type: application/json" \
|
||||
-d "$body" "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records/${record_id}" >/dev/null
|
||||
else
|
||||
curl -sf -X POST -H "Authorization: Bearer ${TOKEN}" -H "Content-Type: application/json" \
|
||||
-d "$body" "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records" >/dev/null
|
||||
fi
|
||||
echo " ${hostname} -> ${target_ip}"
|
||||
}
|
||||
|
||||
flip_to() {
|
||||
local target="$1"
|
||||
local target_ip
|
||||
if [ "$target" = "vps" ]; then
|
||||
target_ip="$(curl -sf https://cloudflare.com/cdn-cgi/trace | grep -o '^ip=.*' | cut -d= -f2)"
|
||||
else
|
||||
target_ip="$HOME_IP"
|
||||
fi
|
||||
if [ -z "$target_ip" ]; then
|
||||
echo "ERROR: could not determine target IP for '${target}', not flipping"
|
||||
return 1
|
||||
fi
|
||||
echo "Flipping standby hostnames to ${target} (${target_ip})..."
|
||||
local zone_id
|
||||
zone_id="$(cf_zone_id)"
|
||||
for h in "${STANDBY_HOSTNAMES[@]}"; do
|
||||
cf_set_record "$zone_id" "$h" "$target_ip"
|
||||
done
|
||||
}
|
||||
|
||||
# --- decide ------------------------------------------------------------------
|
||||
if [ "$ACTIVE" = "home" ] && [ "$FAIL_STREAK" -ge "$FAILURE_THRESHOLD" ]; then
|
||||
flip_to "vps"
|
||||
ACTIVE="vps"
|
||||
FAIL_STREAK=0
|
||||
SUCCESS_STREAK=0
|
||||
elif [ "$ACTIVE" = "vps" ] && [ "$SUCCESS_STREAK" -ge "$SUCCESS_THRESHOLD" ]; then
|
||||
flip_to "home"
|
||||
ACTIVE="home"
|
||||
FAIL_STREAK=0
|
||||
SUCCESS_STREAK=0
|
||||
fi
|
||||
|
||||
echo "${ACTIVE} ${FAIL_STREAK} ${SUCCESS_STREAK}" > "$STATE_FILE"
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Run vps-dns-failover check every 2 minutes
|
||||
|
||||
[Timer]
|
||||
OnBootSec=1min
|
||||
OnUnitActiveSec=2min
|
||||
AccuracySec=10s
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
Reference in New Issue
Block a user