diff --git a/argocd-apps/vps-standby/cert-manager-issuer.yaml b/argocd-apps/vps-standby/cert-manager-issuer.yaml new file mode 100644 index 0000000..c8b1f4d --- /dev/null +++ b/argocd-apps/vps-standby/cert-manager-issuer.yaml @@ -0,0 +1,22 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: vps-cert-manager-issuer + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "1" # before the per-service Ingress resources (wave 3+) that reference it + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: vps-standby + source: + repoURL: https://gitlab.kube.huskypup.net/Scooby/Homelabv4.git + targetRevision: main + path: infrastructure/vps-standby/cert-manager/manifests + destination: + name: vps-standby + namespace: cert-manager + syncPolicy: + automated: + prune: true + selfHeal: true diff --git a/infrastructure/vps-standby/authentik/manifests/ingress.yaml b/infrastructure/vps-standby/authentik/manifests/ingress.yaml new file mode 100644 index 0000000..0d1a825 --- /dev/null +++ b/infrastructure/vps-standby/authentik/manifests/ingress.yaml @@ -0,0 +1,34 @@ +--- +# See infrastructure/vps-standby/vault/manifests/ingress.yaml for the +# vps.huskypup.net subdomain design rationale. +# +# CAVEAT: Authentik's provider/application configs (redirect_uris, etc.) +# are replicated byte-for-byte from home and point at *.kube.huskypup.net +# - SSO logins between VPS-hosted apps and this VPS Authentik instance +# won't complete correctly (redirect mismatch) until that's addressed +# separately. Direct/local Authentik admin login still works fine for +# browsing and verifying replicated data. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: vps-authentik + namespace: authentik + annotations: + cert-manager.io/cluster-issuer: letsencrypt-production +spec: + ingressClassName: traefik + tls: + - hosts: + - auth.vps.huskypup.net + secretName: vps-authentik-tls + rules: + - host: auth.vps.huskypup.net + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: vps-authentik-server + port: + number: 80 diff --git a/infrastructure/vps-standby/authentik/values.yaml b/infrastructure/vps-standby/authentik/values.yaml index 04a30f1..25e5cff 100644 --- a/infrastructure/vps-standby/authentik/values.yaml +++ b/infrastructure/vps-standby/authentik/values.yaml @@ -1,11 +1,11 @@ # Authentik warm standby on the VPS - Phase 2. The CNPG replica cluster # (manifests/cnpg-cluster.yaml) continuously replays WAL from home in the -# background; the app itself is NOT run in normal operation (replicas: 0) -# since the replica DB is read-only until a deliberate manual promotion - -# an Authentik pod trying to write against a read-only DB would just -# crashloop uselessly. Promotion runbook: flip the CNPG cluster's -# spec.replica.enabled to false (see cnpg-cluster.yaml), then scale -# server/worker up from 0. +# background. The app itself DOES run continuously (replicas: 1, reachable +# at auth.vps.huskypup.net - see manifests/ingress.yaml) so the replicated +# data is browsable/verifiable at all times, even though the underlying DB +# is a read-only CNPG replica - writes (new logins, session creation) will +# error until a deliberate manual promotion. Promotion runbook: flip the +# CNPG cluster's spec.replica.enabled to false (see cnpg-cluster.yaml). # # The `authentik:` block below (secret_key/postgresql "env://" indirection) # is copied VERBATIM from infrastructure/authentik/values.yaml on purpose - @@ -34,7 +34,7 @@ global: prefix: AUTHENTIK_POSTGRESQL__ env: - name: AUTHENTIK_URL - value: "https://auth.kube.huskypup.net" + value: "https://auth.vps.huskypup.net" - name: AUTHENTIK_POSTGRESQL__HOST valueFrom: secretKeyRef: @@ -59,7 +59,7 @@ global: value: "5432" server: - replicas: 0 + replicas: 1 containerSecurityContext: runAsNonRoot: true allowPrivilegeEscalation: false @@ -78,7 +78,7 @@ server: enabled: false worker: - replicas: 0 + replicas: 1 containerSecurityContext: runAsNonRoot: true allowPrivilegeEscalation: false diff --git a/infrastructure/vps-standby/cert-manager/manifests/cluster-issuer.yaml b/infrastructure/vps-standby/cert-manager/manifests/cluster-issuer.yaml new file mode 100644 index 0000000..fe4c92d --- /dev/null +++ b/infrastructure/vps-standby/cert-manager/manifests/cluster-issuer.yaml @@ -0,0 +1,28 @@ +--- +# cert-manager itself is installed directly by scripts/vps-bootstrap.sh +# (Phase 0, out-of-band like k3s/Netbird) - not GitOps-managed here. This +# ClusterIssuer just rides the same GitOps pipeline as everything else in +# vps-standby, DNS-01 via Cloudflare (same pattern as home's +# infrastructure/cert-manager/manifests/letsencrypt-issuer.yaml). +# +# cloudflare-token-secret is a plain Secret copied here manually +# (kubectl, not git - same reasoning as every other VPS secret): +# kubectl -n cert-manager create secret generic cloudflare-token-secret \ +# --from-literal=cloudflare-token= +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-production +spec: + acme: + server: https://acme-v02.api.letsencrypt.org/directory + email: garrettstone499@gmail.com + privateKeySecretRef: + name: letsencrypt-production + solvers: + - dns01: + cloudflare: + email: garrettstone499@gmail.com + apiTokenSecretRef: + name: cloudflare-token-secret + key: cloudflare-token diff --git a/infrastructure/vps-standby/gitea/manifests/ingress.yaml b/infrastructure/vps-standby/gitea/manifests/ingress.yaml new file mode 100644 index 0000000..a0bb9f2 --- /dev/null +++ b/infrastructure/vps-standby/gitea/manifests/ingress.yaml @@ -0,0 +1,27 @@ +--- +# See infrastructure/vps-standby/vault/manifests/ingress.yaml for the +# vps.huskypup.net subdomain design rationale. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: vps-gitea + namespace: gitea + annotations: + cert-manager.io/cluster-issuer: letsencrypt-production +spec: + ingressClassName: traefik + tls: + - hosts: + - gitea.vps.huskypup.net + secretName: vps-gitea-tls + rules: + - host: gitea.vps.huskypup.net + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: vps-gitea-http + port: + number: 3000 diff --git a/infrastructure/vps-standby/n8n/manifests/ingress.yaml b/infrastructure/vps-standby/n8n/manifests/ingress.yaml new file mode 100644 index 0000000..17f09c8 --- /dev/null +++ b/infrastructure/vps-standby/n8n/manifests/ingress.yaml @@ -0,0 +1,27 @@ +--- +# See infrastructure/vps-standby/vault/manifests/ingress.yaml for the +# vps.huskypup.net subdomain design rationale. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: vps-n8n + namespace: n8n + annotations: + cert-manager.io/cluster-issuer: letsencrypt-production +spec: + ingressClassName: traefik + tls: + - hosts: + - n8n.vps.huskypup.net + secretName: vps-n8n-tls + rules: + - host: n8n.vps.huskypup.net + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: vps-n8n + port: + number: 80 diff --git a/infrastructure/vps-standby/n8n/values.yaml b/infrastructure/vps-standby/n8n/values.yaml index ef759ee..20b2b08 100644 --- a/infrastructure/vps-standby/n8n/values.yaml +++ b/infrastructure/vps-standby/n8n/values.yaml @@ -1,8 +1,11 @@ # n8n warm standby on the VPS - Phase 2. Same discipline as # infrastructure/vps-standby/authentik/values.yaml: the CNPG replica -# cluster keeps the DB continuously warm in the background, but the app -# itself stays at replicaCount: 0 until a deliberate manual promotion -# (flip pg-n8n's spec.replica.enabled to false, then scale this up). +# cluster keeps the DB continuously warm in the background, and the app +# runs continuously too (replicaCount: 1, reachable at +# n8n.vps.huskypup.net - see manifests/ingress.yaml) - workflow +# saves/executions will error against the read-only DB until a +# deliberate manual promotion (flip pg-n8n's spec.replica.enabled to +# false), but the UI and existing workflow definitions are browsable. # # N8N_ENCRYPTION_KEY must be byte-identical to home's - it decrypts # stored credentials (API keys, OAuth tokens, etc.) that live encrypted @@ -28,7 +31,7 @@ config: generic: timezone: America/New_York path: / - host: n8n.kube.huskypup.net + host: n8n.vps.huskypup.net port: 5678 protocol: https executions: @@ -41,8 +44,7 @@ config: secret: {} -# Not run in normal operation - see file header. -replicaCount: 0 +replicaCount: 1 service: type: ClusterIP @@ -85,8 +87,8 @@ extraEnvSecrets: key: encryption-key extraEnv: - WEBHOOK_URL: https://n8n.kube.huskypup.net/ - N8N_EDITOR_BASE_URL: https://n8n.kube.huskypup.net + WEBHOOK_URL: https://n8n.vps.huskypup.net/ + N8N_EDITOR_BASE_URL: https://n8n.vps.huskypup.net N8N_LOG_LEVEL: error postgresql: diff --git a/infrastructure/vps-standby/nextcloud/manifests/ingress.yaml b/infrastructure/vps-standby/nextcloud/manifests/ingress.yaml new file mode 100644 index 0000000..ac40c04 --- /dev/null +++ b/infrastructure/vps-standby/nextcloud/manifests/ingress.yaml @@ -0,0 +1,27 @@ +--- +# See infrastructure/vps-standby/vault/manifests/ingress.yaml for the +# vps.huskypup.net subdomain design rationale. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: vps-nextcloud + namespace: nextcloud + annotations: + cert-manager.io/cluster-issuer: letsencrypt-production +spec: + ingressClassName: traefik + tls: + - hosts: + - nextcloud.vps.huskypup.net + secretName: vps-nextcloud-tls + rules: + - host: nextcloud.vps.huskypup.net + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: vps-nextcloud + port: + number: 8080 diff --git a/infrastructure/vps-standby/nextcloud/values.yaml b/infrastructure/vps-standby/nextcloud/values.yaml index 19170f0..d91b7c3 100644 --- a/infrastructure/vps-standby/nextcloud/values.yaml +++ b/infrastructure/vps-standby/nextcloud/values.yaml @@ -1,19 +1,20 @@ # Nextcloud warm standby on the VPS - Phase 2. Same discipline as the # other vps-standby apps: CNPG replica cluster (manifests/cnpg-cluster.yaml) # keeps the DB warm, manifests/pvc-restore-cronjob.yaml keeps file content -# warm, but the app itself stays at replicaCount: 0 until a deliberate -# manual promotion. +# warm, and the app runs continuously too (replicaCount: 1, reachable at +# nextcloud.vps.huskypup.net - see manifests/ingress.yaml) so replicated +# files/users are browsable at all times - uploads/changes will error +# against the read-only DB until a deliberate manual promotion. # # No Redis here - the home instance uses the redis-operator # (infrastructure/vps-standby has no redis-operator deployed, out of -# scope for a standby that isn't actually serving traffic). Nextcloud -# runs fine without Redis (falls back to DB-based locking, just slower) - -# acceptable for an emergency-promotion scenario; add a real Redis at -# promotion time if desired. -replicaCount: 0 +# scope for a standby that isn't actually serving writable traffic). +# Nextcloud runs fine without Redis (falls back to DB-based locking, +# just slower) - add a real Redis at promotion time if desired. +replicaCount: 1 nextcloud: - host: nextcloud.kube.huskypup.net + host: nextcloud.vps.huskypup.net username: "" password: "" @@ -39,7 +40,7 @@ nextcloud: 1 => '172.16.0.0/12', ), 'overwriteprotocol' => 'https', - 'overwrite.cli.url' => 'https://nextcloud.kube.huskypup.net', + 'overwrite.cli.url' => 'https://nextcloud.vps.huskypup.net', 'allow_local_remote_servers' => true, ); temp.config.php: |- diff --git a/infrastructure/vps-standby/vault/manifests/ingress.yaml b/infrastructure/vps-standby/vault/manifests/ingress.yaml new file mode 100644 index 0000000..7ee9d3e --- /dev/null +++ b/infrastructure/vps-standby/vault/manifests/ingress.yaml @@ -0,0 +1,30 @@ +--- +# Public-ish access to the VPS Vault standby, replacing the old +# "reachable over Netbird only" model with a stable, permanent hostname +# (vault.vps.huskypup.net) that doesn't depend on home being up at all - +# see the vps.huskypup.net subdomain design note in +# infrastructure/vps-standby/cert-manager/manifests/cluster-issuer.yaml. +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: vps-vault + namespace: vault + annotations: + cert-manager.io/cluster-issuer: letsencrypt-production +spec: + ingressClassName: traefik + tls: + - hosts: + - vault.vps.huskypup.net + secretName: vps-vault-tls + rules: + - host: vault.vps.huskypup.net + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: vps-vault-ui + port: + number: 8200 diff --git a/scripts/vps-dns-failover/vps-dns-failover.service b/scripts/vps-dns-failover/vps-dns-failover.service deleted file mode 100644 index cd5e070..0000000 --- a/scripts/vps-dns-failover/vps-dns-failover.service +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=VPS dual-site DNS failover check (Homelabv4 vps-standby) -After=network-online.target -Wants=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/local/bin/vps-dns-failover.sh -# Deliberately no dependency on k3s/docker being up - this must keep working -# even if the VPS's own cluster is unhealthy. diff --git a/scripts/vps-dns-failover/vps-dns-failover.sh b/scripts/vps-dns-failover/vps-dns-failover.sh deleted file mode 100755 index e198739..0000000 --- a/scripts/vps-dns-failover/vps-dns-failover.sh +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env bash -# vps-dns-failover.sh - Phase 0.5 dual-site DNS failover watcher -# -# Runs ON THE VPS as a systemd timer (see vps-dns-failover.timer/.service in this -# directory) - deliberately plain systemd, not a k8s CronJob, so it works even if -# the VPS's own k3s is unhealthy. It must never depend on anything inside the home -# cluster (that's the thing it's watching) or the VPS's own k3s (that's a separate -# failure domain from this box's basic OS-level networking). -# -# Health-checks home.kube.huskypup.net (kept current by an in-cluster CronJob while -# home is up - see infrastructure/cert-manager/manifests/home-ip-ddns-cronjob.yaml) -# and flips Cloudflare A records for the standby-service hostnames between home's -# public IP and this VPS's own public IP, with a consecutive-check threshold so a -# single blip doesn't cause a flap. -# -# State (current active site + streak counters) persists in $STATE_DIR between -# runs since each systemd timer firing is a fresh process. -# -# Install: -# sudo mkdir -p /etc/vps-dns-failover -# sudo sh -c 'echo "" > /etc/vps-dns-failover/cloudflare-token' -# sudo chmod 600 /etc/vps-dns-failover/cloudflare-token -# sudo cp vps-dns-failover.sh /usr/local/bin/vps-dns-failover.sh -# sudo chmod +x /usr/local/bin/vps-dns-failover.sh -# sudo cp vps-dns-failover.service vps-dns-failover.timer /etc/systemd/system/ -# sudo systemctl daemon-reload -# sudo systemctl enable --now vps-dns-failover.timer - -set -euo pipefail - -TOKEN_FILE="/etc/vps-dns-failover/cloudflare-token" -STATE_DIR="/var/lib/vps-dns-failover" -ZONE_NAME="huskypup.net" # Cloudflare zone is the parent domain - kube.huskypup.net is just a record within it, not its own zone -HOME_CHECK_HOST="home.kube.huskypup.net" -HOME_CHECK_PORT=443 -FAILURE_THRESHOLD=3 # consecutive failed checks before flipping to the VPS -SUCCESS_THRESHOLD=3 # consecutive successful checks before flipping back to home -STANDBY_HOSTNAMES=( - vault.kube.huskypup.net - auth.kube.huskypup.net - gitea.kube.huskypup.net - n8n.kube.huskypup.net - nextcloud.kube.huskypup.net -) - -mkdir -p "$STATE_DIR" -TOKEN="$(cat "$TOKEN_FILE")" -STATE_FILE="${STATE_DIR}/state" # format: " " - -if [ -f "$STATE_FILE" ]; then - read -r ACTIVE FAIL_STREAK SUCCESS_STREAK < "$STATE_FILE" -else - ACTIVE="home" - FAIL_STREAK=0 - SUCCESS_STREAK=0 -fi - -# --- health check ------------------------------------------------------------ -HOME_IP="$(dig +short A "$HOME_CHECK_HOST" @1.1.1.1 | tail -n1)" -if [ -n "$HOME_IP" ] && timeout 5 bash -c "cat < /dev/null > /dev/tcp/${HOME_IP}/${HOME_CHECK_PORT}" 2>/dev/null; then - HEALTHY=1 -else - HEALTHY=0 -fi - -if [ "$HEALTHY" = 1 ]; then - FAIL_STREAK=0 - SUCCESS_STREAK=$((SUCCESS_STREAK + 1)) -else - SUCCESS_STREAK=0 - FAIL_STREAK=$((FAIL_STREAK + 1)) -fi - -echo "$(date -u +%FT%TZ) active=${ACTIVE} healthy=${HEALTHY} fail_streak=${FAIL_STREAK} success_streak=${SUCCESS_STREAK} home_ip=${HOME_IP:-none}" - -# --- Cloudflare helpers -------------------------------------------------------- -cf_zone_id() { - curl -sf -H "Authorization: Bearer ${TOKEN}" \ - "https://api.cloudflare.com/client/v4/zones?name=${ZONE_NAME}" | jq -r '.result[0].id' -} - -cf_set_record() { - local zone_id="$1" hostname="$2" target_ip="$3" - local record_json record_id - record_json="$(curl -sf -H "Authorization: Bearer ${TOKEN}" \ - "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records?name=${hostname}&type=A")" - record_id="$(echo "$record_json" | jq -r '.result[0].id // empty')" - local body="{\"type\":\"A\",\"name\":\"${hostname}\",\"content\":\"${target_ip}\",\"ttl\":60,\"proxied\":false}" - if [ -n "$record_id" ]; then - curl -sf -X PATCH -H "Authorization: Bearer ${TOKEN}" -H "Content-Type: application/json" \ - -d "$body" "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records/${record_id}" >/dev/null - else - curl -sf -X POST -H "Authorization: Bearer ${TOKEN}" -H "Content-Type: application/json" \ - -d "$body" "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records" >/dev/null - fi - echo " ${hostname} -> ${target_ip}" -} - -flip_to() { - local target="$1" - local target_ip - if [ "$target" = "vps" ]; then - target_ip="$(curl -sf https://cloudflare.com/cdn-cgi/trace | grep -o '^ip=.*' | cut -d= -f2)" - else - target_ip="$HOME_IP" - fi - if [ -z "$target_ip" ]; then - echo "ERROR: could not determine target IP for '${target}', not flipping" - return 1 - fi - echo "Flipping standby hostnames to ${target} (${target_ip})..." - local zone_id - zone_id="$(cf_zone_id)" - for h in "${STANDBY_HOSTNAMES[@]}"; do - cf_set_record "$zone_id" "$h" "$target_ip" - done -} - -# --- decide ------------------------------------------------------------------ -if [ "$ACTIVE" = "home" ] && [ "$FAIL_STREAK" -ge "$FAILURE_THRESHOLD" ]; then - flip_to "vps" - ACTIVE="vps" - FAIL_STREAK=0 - SUCCESS_STREAK=0 -elif [ "$ACTIVE" = "vps" ] && [ "$SUCCESS_STREAK" -ge "$SUCCESS_THRESHOLD" ]; then - flip_to "home" - ACTIVE="home" - FAIL_STREAK=0 - SUCCESS_STREAK=0 -fi - -echo "${ACTIVE} ${FAIL_STREAK} ${SUCCESS_STREAK}" > "$STATE_FILE" diff --git a/scripts/vps-dns-failover/vps-dns-failover.timer b/scripts/vps-dns-failover/vps-dns-failover.timer deleted file mode 100644 index 220b9ec..0000000 --- a/scripts/vps-dns-failover/vps-dns-failover.timer +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Run vps-dns-failover check every 2 minutes - -[Timer] -OnBootSec=1min -OnUnitActiveSec=2min -AccuracySec=10s - -[Install] -WantedBy=timers.target