mirror of
https://gitlab.kube.huskypup.net/Scooby/Homelabv4.git
synced 2026-08-21 05:26:49 +00:00
Add VPS warm-standby/backup site (Phase 0-1b)
Foundation for a DR/backup path using an always-on VPS as a second ArgoCD-managed cluster, plus DB/backup standardization work that fell out of it: - vps-standby ArgoCD cluster destination + AppProject, MinIO backup receiver, VPS bootstrap script (k3s, Netbird, cert-manager) - Dual-site DNS failover watcher + home-IP DDNS CronJob, Cloudflare token moved out of git into Vault+ExternalSecret - Nextcloud migrated from ad-hoc MariaDB to CNPG + redis-operator (matches n8n/Authentik/GitLab's backup-native pattern) - Authentik's CNPG manifests moved into the actual ArgoCD-synced manifests/ path (were present but never wired into the sync path) - Vault raft-snapshot CronJob, CNPG barmanObjectStore backups (Authentik/n8n/Nextcloud), Nextcloud file-PVC restic sync - all targeting the new VPS MinIO receiver See VPS Warm-Standby plan doc for full design rationale.
This commit is contained in:
Scooby Husky
Executable
+46
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env bash
|
||||
# rotate-cloudflare-token.sh - Rotate the Cloudflare DNS-01 token out of git and into Vault
|
||||
#
|
||||
# infrastructure/cert-manager/manifests/secret-cf-token.yaml used to contain a live
|
||||
# Cloudflare API token committed in plaintext. It's now an ExternalSecret pulling
|
||||
# from Vault at secret/cloudflare-dns-token#token - this script populates that path.
|
||||
#
|
||||
# This does NOT create or revoke the Cloudflare token itself - that's a manual step
|
||||
# in the Cloudflare dashboard, deliberately not automated here since it's a live,
|
||||
# outward-facing credential change:
|
||||
#
|
||||
# 1. Cloudflare dashboard -> My Profile -> API Tokens -> Create Token
|
||||
# Scope: Zone:DNS:Edit, restricted to the kube.huskypup.net zone only
|
||||
# 2. Run this script with the new token
|
||||
# 3. Confirm cert-manager can still issue certs (kubectl get certificaterequests -A)
|
||||
# 4. THEN go back to Cloudflare and revoke the old token
|
||||
# (the one that was committed in git - assume it's compromised)
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/rotate-cloudflare-token.sh <new-cloudflare-token>
|
||||
#
|
||||
# Prerequisites:
|
||||
# - Vault initialized and unsealed
|
||||
# - kubectl configured for the home cluster
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
NEW_TOKEN="${1:?Usage: $0 <new-cloudflare-token>}"
|
||||
|
||||
echo "=== Cloudflare DNS-01 Token Rotation ==="
|
||||
|
||||
echo "Storing new token in Vault at secret/cloudflare-dns-token..."
|
||||
ROOT_TOKEN=$(kubectl -n vault get secret vault-init-keys -o jsonpath='{.data.VAULT_ROOT_TOKEN}' | base64 -d)
|
||||
kubectl exec -n vault vault-0 -- env "VAULT_TOKEN=${ROOT_TOKEN}" \
|
||||
vault kv put secret/cloudflare-dns-token token="${NEW_TOKEN}"
|
||||
|
||||
echo "Forcing ExternalSecret refresh..."
|
||||
kubectl -n cert-manager annotate externalsecret cloudflare-token-secret \
|
||||
force-sync="$(date +%s)" --overwrite
|
||||
|
||||
echo ""
|
||||
echo "=== Done ==="
|
||||
echo "Next:"
|
||||
echo " 1. Verify: kubectl -n cert-manager get secret cloudflare-token-secret -o jsonpath='{.data.cloudflare-token}' | base64 -d"
|
||||
echo " 2. Verify a cert still renews cleanly (or delete one Certificate to force a test issuance)"
|
||||
echo " 3. Revoke the OLD token in the Cloudflare dashboard once confirmed working"
|
||||
Executable
+169
@@ -0,0 +1,169 @@
|
||||
#!/usr/bin/env bash
|
||||
# vps-bootstrap.sh - Phase 0: turn a bare VPS into the vps-standby ArgoCD destination
|
||||
#
|
||||
# Run this ON THE VPS itself (as root, or via sudo), not against the home cluster.
|
||||
# Installs k3s (single node), joins the existing self-hosted Netbird mesh, and
|
||||
# installs cert-manager with the same Cloudflare DNS-01 ClusterIssuer pattern used
|
||||
# at home — so TLS issuance works identically regardless of which site is "live"
|
||||
# (DNS-01 only needs DNS control, not public HTTP reachability).
|
||||
#
|
||||
# This script does NOT register the cluster with ArgoCD — that's a one-time manual
|
||||
# step run from your workstation/home cluster once this script prints the kubeconfig
|
||||
# (ArgoCD can't reach the VPS until it exists, and shouldn't hold cluster-admin creds
|
||||
# for a box it doesn't manage yet).
|
||||
#
|
||||
# Usage:
|
||||
# sudo ./scripts/vps-bootstrap.sh <netbird-setup-key> <cloudflare-dns-edit-token>
|
||||
#
|
||||
# Prerequisites:
|
||||
# - A Netbird setup key (Netbird dashboard → Settings → Setup Keys → create
|
||||
# a reusable, non-ephemeral key)
|
||||
# - A Cloudflare API token scoped to Zone:DNS:Edit for kube.huskypup.net only
|
||||
# (create a NEW token for this — do not reuse the one from
|
||||
# infrastructure/cert-manager/manifests/secret-cf-token.yaml, that one is
|
||||
# being rotated/retired; see Phase 0.5)
|
||||
# - Ubuntu/Debian VPS with a public IP, run as root
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
NETBIRD_SETUP_KEY="${1:?Usage: $0 <netbird-setup-key> <cloudflare-dns-edit-token>}"
|
||||
CLOUDFLARE_TOKEN="${2:?Usage: $0 <netbird-setup-key> <cloudflare-dns-edit-token>}"
|
||||
|
||||
NETBIRD_MGMT_URL="https://netbird.kube.huskypup.net"
|
||||
LETSENCRYPT_EMAIL="garrettstone499@gmail.com"
|
||||
DNS_ZONE="kube.huskypup.net"
|
||||
CERT_MANAGER_VERSION="v1.13.2" # matches infrastructure/cert-manager chart version at home
|
||||
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
echo "ERROR: run as root (sudo $0 ...)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "================================================="
|
||||
echo "VPS Standby Bootstrap - Phase 0"
|
||||
echo "================================================="
|
||||
echo ""
|
||||
|
||||
# --- k3s -----------------------------------------------------------------
|
||||
if command -v k3s >/dev/null 2>&1; then
|
||||
echo "✅ k3s already installed, skipping install"
|
||||
else
|
||||
echo "Installing k3s (single node)..."
|
||||
# Keep the built-in Traefik ingress controller — this is a lean standby box,
|
||||
# not a mirror of home's Istio/Envoy-Gateway mesh. servicelb is fine too
|
||||
# since this is a single node with a real public IP.
|
||||
curl -sfL https://get.k3s.io | sh -s - \
|
||||
--write-kubeconfig-mode 644 \
|
||||
--disable metrics-server
|
||||
echo "✅ k3s installed"
|
||||
fi
|
||||
|
||||
echo "Waiting for k3s node to be Ready..."
|
||||
for i in $(seq 1 30); do
|
||||
if k3s kubectl get nodes 2>/dev/null | grep -q " Ready"; then
|
||||
echo "✅ node is Ready"
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
k3s kubectl get nodes
|
||||
|
||||
export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
|
||||
|
||||
# --- Netbird ---------------------------------------------------------------
|
||||
if command -v netbird >/dev/null 2>&1 && netbird status 2>/dev/null | grep -q "Management: Connected"; then
|
||||
echo "✅ Netbird already connected, skipping"
|
||||
else
|
||||
echo "Installing Netbird client..."
|
||||
curl -fsSL https://pkgs.netbird.io/install.sh | sh
|
||||
echo "Joining Netbird mesh (${NETBIRD_MGMT_URL})..."
|
||||
netbird up --management-url "${NETBIRD_MGMT_URL}" --setup-key "${NETBIRD_SETUP_KEY}"
|
||||
echo "✅ Netbird joined"
|
||||
fi
|
||||
echo "Note: this Netbird session is used opportunistically for backup/sync traffic"
|
||||
echo "while home is up. It is NOT the path used to reach this VPS when home is down —"
|
||||
echo "that's direct SSH on this box's public IP. See plan doc, decision #2."
|
||||
|
||||
# --- cert-manager ------------------------------------------------------------
|
||||
echo ""
|
||||
echo "Installing cert-manager ${CERT_MANAGER_VERSION}..."
|
||||
if ! command -v helm >/dev/null 2>&1; then
|
||||
curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
|
||||
fi
|
||||
|
||||
helm repo add jetstack https://charts.jetstack.io 2>/dev/null || true
|
||||
helm repo update jetstack
|
||||
|
||||
if helm -n cert-manager status cert-manager >/dev/null 2>&1; then
|
||||
echo "✅ cert-manager already installed"
|
||||
else
|
||||
helm install cert-manager jetstack/cert-manager \
|
||||
--namespace cert-manager \
|
||||
--create-namespace \
|
||||
--version "${CERT_MANAGER_VERSION}" \
|
||||
--set installCRDs=true \
|
||||
--wait --timeout 300s
|
||||
echo "✅ cert-manager installed"
|
||||
fi
|
||||
|
||||
echo "Waiting for cert-manager webhook to be ready..."
|
||||
k3s kubectl -n cert-manager rollout status deployment/cert-manager-webhook --timeout=120s
|
||||
|
||||
# --- Cloudflare DNS-01 ClusterIssuer (same pattern as home) -----------------
|
||||
echo ""
|
||||
echo "Applying Cloudflare token Secret + ClusterIssuer..."
|
||||
cat <<EOF | k3s kubectl apply -f -
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: cloudflare-token-secret
|
||||
namespace: cert-manager
|
||||
type: Opaque
|
||||
stringData:
|
||||
cloudflare-token: "${CLOUDFLARE_TOKEN}"
|
||||
---
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: ClusterIssuer
|
||||
metadata:
|
||||
name: letsencrypt-production
|
||||
spec:
|
||||
acme:
|
||||
server: https://acme-v02.api.letsencrypt.org/directory
|
||||
email: ${LETSENCRYPT_EMAIL}
|
||||
privateKeySecretRef:
|
||||
name: letsencrypt-production
|
||||
solvers:
|
||||
- dns01:
|
||||
cloudflare:
|
||||
email: ${LETSENCRYPT_EMAIL}
|
||||
apiTokenSecretRef:
|
||||
name: cloudflare-token-secret
|
||||
key: cloudflare-token
|
||||
selector:
|
||||
dnsZones:
|
||||
- "${DNS_ZONE}"
|
||||
EOF
|
||||
echo "✅ ClusterIssuer letsencrypt-production ready"
|
||||
|
||||
# --- Output for ArgoCD registration -----------------------------------------
|
||||
echo ""
|
||||
echo "================================================="
|
||||
echo "✅ VPS foundation bootstrap complete"
|
||||
echo "================================================="
|
||||
echo ""
|
||||
echo "Next step (run from your workstation, NOT this VPS):"
|
||||
echo ""
|
||||
echo " 1. Copy this VPS's kubeconfig to your workstation, e.g.:"
|
||||
echo " scp root@<vps-ip>:/etc/rancher/k3s/k3s.yaml ~/vps-standby-kubeconfig.yaml"
|
||||
echo " Then edit the 'server:' line inside it to use this VPS's public IP"
|
||||
echo " instead of 127.0.0.1."
|
||||
echo ""
|
||||
echo " 2. Register it with ArgoCD:"
|
||||
echo " KUBECONFIG=~/vps-standby-kubeconfig.yaml argocd cluster add default --name vps-standby"
|
||||
echo ""
|
||||
echo " 3. Confirm registration:"
|
||||
echo " argocd cluster list"
|
||||
echo ""
|
||||
echo "This box is otherwise reachable via:"
|
||||
echo " - Netbird (while home's self-hosted mesh is up)"
|
||||
echo " - Direct SSH on its public IP (always, break-glass path)"
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=VPS dual-site DNS failover check (Homelabv4 vps-standby)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/vps-dns-failover.sh
|
||||
# Deliberately no dependency on k3s/docker being up - this must keep working
|
||||
# even if the VPS's own cluster is unhealthy.
|
||||
Executable
+132
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env bash
|
||||
# vps-dns-failover.sh - Phase 0.5 dual-site DNS failover watcher
|
||||
#
|
||||
# Runs ON THE VPS as a systemd timer (see vps-dns-failover.timer/.service in this
|
||||
# directory) - deliberately plain systemd, not a k8s CronJob, so it works even if
|
||||
# the VPS's own k3s is unhealthy. It must never depend on anything inside the home
|
||||
# cluster (that's the thing it's watching) or the VPS's own k3s (that's a separate
|
||||
# failure domain from this box's basic OS-level networking).
|
||||
#
|
||||
# Health-checks home.kube.huskypup.net (kept current by an in-cluster CronJob while
|
||||
# home is up - see infrastructure/cert-manager/manifests/home-ip-ddns-cronjob.yaml)
|
||||
# and flips Cloudflare A records for the standby-service hostnames between home's
|
||||
# public IP and this VPS's own public IP, with a consecutive-check threshold so a
|
||||
# single blip doesn't cause a flap.
|
||||
#
|
||||
# State (current active site + streak counters) persists in $STATE_DIR between
|
||||
# runs since each systemd timer firing is a fresh process.
|
||||
#
|
||||
# Install:
|
||||
# sudo mkdir -p /etc/vps-dns-failover
|
||||
# sudo sh -c 'echo "<dns-edit-scoped-cloudflare-token>" > /etc/vps-dns-failover/cloudflare-token'
|
||||
# sudo chmod 600 /etc/vps-dns-failover/cloudflare-token
|
||||
# sudo cp vps-dns-failover.sh /usr/local/bin/vps-dns-failover.sh
|
||||
# sudo chmod +x /usr/local/bin/vps-dns-failover.sh
|
||||
# sudo cp vps-dns-failover.service vps-dns-failover.timer /etc/systemd/system/
|
||||
# sudo systemctl daemon-reload
|
||||
# sudo systemctl enable --now vps-dns-failover.timer
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
TOKEN_FILE="/etc/vps-dns-failover/cloudflare-token"
|
||||
STATE_DIR="/var/lib/vps-dns-failover"
|
||||
ZONE_NAME="kube.huskypup.net"
|
||||
HOME_CHECK_HOST="home.kube.huskypup.net"
|
||||
HOME_CHECK_PORT=443
|
||||
FAILURE_THRESHOLD=3 # consecutive failed checks before flipping to the VPS
|
||||
SUCCESS_THRESHOLD=3 # consecutive successful checks before flipping back to home
|
||||
STANDBY_HOSTNAMES=(
|
||||
vault.kube.huskypup.net
|
||||
auth.kube.huskypup.net
|
||||
gitea.kube.huskypup.net
|
||||
n8n.kube.huskypup.net
|
||||
nextcloud.kube.huskypup.net
|
||||
)
|
||||
|
||||
mkdir -p "$STATE_DIR"
|
||||
TOKEN="$(cat "$TOKEN_FILE")"
|
||||
STATE_FILE="${STATE_DIR}/state" # format: "<active-site> <fail-streak> <success-streak>"
|
||||
|
||||
if [ -f "$STATE_FILE" ]; then
|
||||
read -r ACTIVE FAIL_STREAK SUCCESS_STREAK < "$STATE_FILE"
|
||||
else
|
||||
ACTIVE="home"
|
||||
FAIL_STREAK=0
|
||||
SUCCESS_STREAK=0
|
||||
fi
|
||||
|
||||
# --- health check ------------------------------------------------------------
|
||||
HOME_IP="$(dig +short A "$HOME_CHECK_HOST" @1.1.1.1 | tail -n1)"
|
||||
if [ -n "$HOME_IP" ] && timeout 5 bash -c "cat < /dev/null > /dev/tcp/${HOME_IP}/${HOME_CHECK_PORT}" 2>/dev/null; then
|
||||
HEALTHY=1
|
||||
else
|
||||
HEALTHY=0
|
||||
fi
|
||||
|
||||
if [ "$HEALTHY" = 1 ]; then
|
||||
FAIL_STREAK=0
|
||||
SUCCESS_STREAK=$((SUCCESS_STREAK + 1))
|
||||
else
|
||||
SUCCESS_STREAK=0
|
||||
FAIL_STREAK=$((FAIL_STREAK + 1))
|
||||
fi
|
||||
|
||||
echo "$(date -u +%FT%TZ) active=${ACTIVE} healthy=${HEALTHY} fail_streak=${FAIL_STREAK} success_streak=${SUCCESS_STREAK} home_ip=${HOME_IP:-none}"
|
||||
|
||||
# --- Cloudflare helpers --------------------------------------------------------
|
||||
cf_zone_id() {
|
||||
curl -sf -H "Authorization: Bearer ${TOKEN}" \
|
||||
"https://api.cloudflare.com/client/v4/zones?name=${ZONE_NAME}" | jq -r '.result[0].id'
|
||||
}
|
||||
|
||||
cf_set_record() {
|
||||
local zone_id="$1" hostname="$2" target_ip="$3"
|
||||
local record_json record_id
|
||||
record_json="$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
|
||||
"https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records?name=${hostname}&type=A")"
|
||||
record_id="$(echo "$record_json" | jq -r '.result[0].id // empty')"
|
||||
local body="{\"type\":\"A\",\"name\":\"${hostname}\",\"content\":\"${target_ip}\",\"ttl\":60,\"proxied\":false}"
|
||||
if [ -n "$record_id" ]; then
|
||||
curl -sf -X PATCH -H "Authorization: Bearer ${TOKEN}" -H "Content-Type: application/json" \
|
||||
-d "$body" "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records/${record_id}" >/dev/null
|
||||
else
|
||||
curl -sf -X POST -H "Authorization: Bearer ${TOKEN}" -H "Content-Type: application/json" \
|
||||
-d "$body" "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records" >/dev/null
|
||||
fi
|
||||
echo " ${hostname} -> ${target_ip}"
|
||||
}
|
||||
|
||||
flip_to() {
|
||||
local target="$1"
|
||||
local target_ip
|
||||
if [ "$target" = "vps" ]; then
|
||||
target_ip="$(curl -sf https://cloudflare.com/cdn-cgi/trace | grep -o '^ip=.*' | cut -d= -f2)"
|
||||
else
|
||||
target_ip="$HOME_IP"
|
||||
fi
|
||||
if [ -z "$target_ip" ]; then
|
||||
echo "ERROR: could not determine target IP for '${target}', not flipping"
|
||||
return 1
|
||||
fi
|
||||
echo "Flipping standby hostnames to ${target} (${target_ip})..."
|
||||
local zone_id
|
||||
zone_id="$(cf_zone_id)"
|
||||
for h in "${STANDBY_HOSTNAMES[@]}"; do
|
||||
cf_set_record "$zone_id" "$h" "$target_ip"
|
||||
done
|
||||
}
|
||||
|
||||
# --- decide ------------------------------------------------------------------
|
||||
if [ "$ACTIVE" = "home" ] && [ "$FAIL_STREAK" -ge "$FAILURE_THRESHOLD" ]; then
|
||||
flip_to "vps"
|
||||
ACTIVE="vps"
|
||||
FAIL_STREAK=0
|
||||
SUCCESS_STREAK=0
|
||||
elif [ "$ACTIVE" = "vps" ] && [ "$SUCCESS_STREAK" -ge "$SUCCESS_THRESHOLD" ]; then
|
||||
flip_to "home"
|
||||
ACTIVE="home"
|
||||
FAIL_STREAK=0
|
||||
SUCCESS_STREAK=0
|
||||
fi
|
||||
|
||||
echo "${ACTIVE} ${FAIL_STREAK} ${SUCCESS_STREAK}" > "$STATE_FILE"
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Run vps-dns-failover check every 2 minutes
|
||||
|
||||
[Timer]
|
||||
OnBootSec=1min
|
||||
OnUnitActiveSec=2min
|
||||
AccuracySec=10s
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
Reference in New Issue
Block a user