diff --git a/argocd-apps/vps-standby/vault.yaml b/argocd-apps/vps-standby/vault.yaml new file mode 100644 index 0000000..49d99cb --- /dev/null +++ b/argocd-apps/vps-standby/vault.yaml @@ -0,0 +1,33 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: vps-vault + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "2" # after vps-minio (wave 1) - restore job needs it + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: vps-standby + sources: + - repoURL: https://helm.releases.hashicorp.com + chart: vault + targetRevision: 0.32.0 + helm: + valueFiles: + - $values/infrastructure/vps-standby/vault/values.yaml + - repoURL: https://gitlab.kube.huskypup.net/Scooby/Homelabv4.git + targetRevision: main + ref: values + - repoURL: https://gitlab.kube.huskypup.net/Scooby/Homelabv4.git + targetRevision: main + path: infrastructure/vps-standby/vault/manifests + destination: + name: vps-standby + namespace: vault + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/infrastructure/vps-standby/vault/manifests/restore-cronjob.yaml b/infrastructure/vps-standby/vault/manifests/restore-cronjob.yaml new file mode 100644 index 0000000..a24e0e5 --- /dev/null +++ b/infrastructure/vps-standby/vault/manifests/restore-cronjob.yaml @@ -0,0 +1,187 @@ +--- +# Periodically restores home Vault's latest raft snapshot into this VPS +# standby instance. Runs 30 min after the home-side snapshot job (which +# runs every 6h, see infrastructure/vault/manifests/raft-snapshot-cronjob.yaml) +# to give it time to land in this cluster's own MinIO. +# +# Why this needs a COPY of home's real unseal key (manifests/../README below) +# rather than a fresh one generated here: `vault operator raft snapshot +# restore` replaces the ENTIRE raft storage backend - including the +# encrypted keyring - with home's. After a restore, this Vault is sealed +# with HOME's keyring, not whatever it had before. The only way to unseal +# it afterward is with home's actual unseal key. That key was copied here +# once via: +# kubectl -n vault create secret generic vault-unseal-key \ +# --from-literal=key= +# (kubectl directly, not git - same reasoning as vps-minio-root-secret). +# +# First-run bootstrap: a brand new Vault pod is uninitialized, and Vault +# doesn't let you choose your own keys for a fresh init - so this generates +# a throwaway single Shamir key (threshold 1) purely to get past init and +# unseal long enough to perform the very first restore. That throwaway key +# is discarded immediately after use; it stops mattering the moment the +# first restore completes, since the keyring gets overwritten by home's. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: vault-restore + namespace: vault +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: vault-restore + namespace: vault +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "delete"] + - apiGroups: [""] + resources: ["pods/exec"] + verbs: ["create"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: vault-restore + namespace: vault +subjects: + - kind: ServiceAccount + name: vault-restore + namespace: vault +roleRef: + kind: Role + name: vault-restore + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vault-restore-script + namespace: vault +data: + restore.sh: | + #!/bin/bash + set -euo pipefail + + VAULT_POD="vault-0" + UNSEAL_KEY="$(kubectl -n vault get secret vault-unseal-key -o jsonpath='{.data.key}' | base64 -d)" + + vault_exec() { + kubectl -n vault exec -i "$VAULT_POD" -- env VAULT_ADDR=http://127.0.0.1:8200 vault "$@" + } + + echo "==> Checking Vault status..." + STATUS_JSON="$(vault_exec status -format=json 2>&1 || true)" + INITIALIZED="$(echo "$STATUS_JSON" | jq -r '.initialized // empty' 2>/dev/null || echo "")" + SEALED="$(echo "$STATUS_JSON" | jq -r '.sealed // empty' 2>/dev/null || echo "")" + + if [ "$INITIALIZED" != "true" ]; then + echo "==> First run: initializing with a throwaway single-key seal..." + INIT_JSON="$(vault_exec operator init -key-shares=1 -key-threshold=1 -format=json)" + THROWAWAY_KEY="$(echo "$INIT_JSON" | jq -r '.unseal_keys_b64[0]')" + echo "==> Unsealing with throwaway key for first restore..." + vault_exec operator unseal "$THROWAWAY_KEY" >/dev/null + unset THROWAWAY_KEY INIT_JSON + elif [ "$SEALED" = "true" ]; then + echo "==> Sealed - unsealing with the stored home unseal key..." + vault_exec operator unseal "$UNSEAL_KEY" >/dev/null + else + echo "==> Already unsealed." + fi + + echo "==> Installing mc (MinIO client)..." + curl -fsSL https://dl.min.io/client/mc/release/linux-amd64/mc -o /tmp/mc + chmod +x /tmp/mc + export MC_CONFIG_DIR=/tmp/.mc + + echo "==> Finding the latest snapshot in local MinIO..." + /tmp/mc alias set local-minio "http://vps-minio.minio.svc.cluster.local:9000" \ + "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}" >/dev/null + LATEST="$(/tmp/mc ls local-minio/vault-raft-snapshots --json | jq -rs 'sort_by(.lastModified) | last | .key')" + if [ -z "$LATEST" ] || [ "$LATEST" = "null" ]; then + echo "No snapshots found yet - nothing to restore." + exit 0 + fi + echo "==> Latest snapshot: ${LATEST}" + + echo "==> Downloading snapshot..." + /tmp/mc cp "local-minio/vault-raft-snapshots/${LATEST}" "/tmp/${LATEST}" >/dev/null + + echo "==> Copying snapshot into ${VAULT_POD}..." + kubectl -n vault cp "/tmp/${LATEST}" "${VAULT_POD}:/tmp/${LATEST}" + + echo "==> Restoring raft snapshot (this replaces all data + the keyring)..." + kubectl -n vault exec -i "$VAULT_POD" -- env VAULT_ADDR=http://127.0.0.1:8200 \ + vault operator raft snapshot restore -force "/tmp/${LATEST}" + kubectl -n vault exec "$VAULT_POD" -- rm -f "/tmp/${LATEST}" + + echo "==> Restarting Vault to fully reload post-restore state..." + kubectl -n vault delete pod "$VAULT_POD" + echo "==> Waiting for ${VAULT_POD} to come back..." + kubectl -n vault wait --for=condition=Ready "pod/${VAULT_POD}" --timeout=120s || true + + echo "==> Unsealing with home's real key (restore overwrote the keyring)..." + for i in 1 2 3 4 5; do + if vault_exec operator unseal "$UNSEAL_KEY" >/dev/null 2>&1; then + echo "==> Unsealed." + break + fi + echo " not ready yet, retrying in 5s..." + sleep 5 + done + + echo "==> Done." +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: vault-restore + namespace: vault +spec: + schedule: "30 */6 * * *" # 30 min after home's snapshot job, same 6h cadence + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 1 + template: + spec: + serviceAccountName: vault-restore + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + containers: + - name: restore + image: alpine/k8s:1.32.13 + command: ["/bin/bash", "/scripts/restore.sh"] + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + env: + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: vps-minio-root-secret + key: rootUser + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: vps-minio-root-secret + key: rootPassword + volumeMounts: + - name: scripts + mountPath: /scripts + volumes: + - name: scripts + configMap: + name: vault-restore-script + defaultMode: 0755 diff --git a/infrastructure/vps-standby/vault/values.yaml b/infrastructure/vps-standby/vault/values.yaml new file mode 100644 index 0000000..caa1dc0 --- /dev/null +++ b/infrastructure/vps-standby/vault/values.yaml @@ -0,0 +1,55 @@ +# Vault warm standby on the VPS - Phase 2 of the VPS plan. +# +# This is NOT an independent Vault with its own root of trust. It exists +# purely to periodically absorb home Vault's raft snapshots (already +# flowing to this VPS's own MinIO via infrastructure/vault/manifests/ +# raft-snapshot-cronjob.yaml) so it can be promoted quickly during a real +# home outage. See manifests/restore-cronjob.yaml for the restore loop. +# +# Deliberately scoped-down from home Vault's offline-Shamir-share posture: +# single node, single-key auto-unseal via a key copied to this cluster +# (kubectl, not git - see manifests/restore-cronjob.yaml's header comment +# for why a copy of the real key, not a fresh one, is required here). +# Acceptable because this is a read-only standby, not the root of trust, +# and there's no independent data at stake - restoring a raft snapshot +# overwrites everything here with home's actual state (and keyring) on +# every cycle. +server: + standalone: + enabled: true + config: | + ui = true + listener "tcp" { + address = "0.0.0.0:8200" + tls_disable = 1 + } + storage "raft" { + path = "/vault/data" + } + disable_mlock = true + + ha: + enabled: false + + dataStorage: + enabled: true + size: 5Gi + storageClass: local-path + + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + memory: 512Mi + + # No ingress/public exposure - reached over Netbird from home while it's + # up, or directly on the VPS during an actual promoted-standby incident. + service: + enabled: true + +injector: + enabled: false # no sidecar-injection use case on a restore-only standby + +ui: + enabled: true