--- # Periodically restores home Vault's latest raft snapshot into this VPS # standby instance. Runs 30 min after the home-side snapshot job (which # runs every 6h, see infrastructure/vault/manifests/raft-snapshot-cronjob.yaml) # to give it time to land in this cluster's own MinIO. # # Why this needs a COPY of home's real unseal key (manifests/../README below) # rather than a fresh one generated here: `vault operator raft snapshot # restore` replaces the ENTIRE raft storage backend - including the # encrypted keyring - with home's. After a restore, this Vault is sealed # with HOME's keyring, not whatever it had before. The only way to unseal # it afterward is with home's actual unseal key. That key was copied here # once via: # kubectl -n vault create secret generic vault-unseal-key \ # --from-literal=key= # (kubectl directly, not git - same reasoning as vps-minio-root-secret). # # First-run bootstrap: a brand new Vault pod is uninitialized, and Vault # doesn't let you choose your own keys for a fresh init - so this generates # a throwaway single Shamir key (threshold 1) purely to get past init and # unseal long enough to perform the very first restore. That throwaway key # is discarded immediately after use; it stops mattering the moment the # first restore completes, since the keyring gets overwritten by home's. apiVersion: v1 kind: ServiceAccount metadata: name: vault-restore namespace: vault --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: vault-restore namespace: vault rules: - apiGroups: [""] resources: ["pods"] verbs: ["get", "list", "delete"] - apiGroups: [""] resources: ["pods/exec"] verbs: ["create"] - apiGroups: [""] resources: ["secrets"] verbs: ["get"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: vault-restore namespace: vault subjects: - kind: ServiceAccount name: vault-restore namespace: vault roleRef: kind: Role name: vault-restore apiGroup: rbac.authorization.k8s.io --- apiVersion: v1 kind: ConfigMap metadata: name: vault-restore-script namespace: vault data: restore.sh: | #!/bin/bash set -euo pipefail # StatefulSet pod name follows the Helm release name (vps-vault), not # the chart's default "vault-0". VAULT_POD="vps-vault-0" UNSEAL_KEY="$(kubectl -n vault get secret vault-unseal-key -o jsonpath='{.data.key}' | base64 -d)" vault_exec() { kubectl -n vault exec -i "$VAULT_POD" -- env VAULT_ADDR=http://127.0.0.1:8200 vault "$@" } echo "==> Checking Vault status..." STATUS_JSON="$(vault_exec status -format=json 2>&1 || true)" INITIALIZED="$(echo "$STATUS_JSON" | jq -r '.initialized // empty' 2>/dev/null || echo "")" SEALED="$(echo "$STATUS_JSON" | jq -r '.sealed // empty' 2>/dev/null || echo "")" if [ "$INITIALIZED" != "true" ]; then echo "==> First run: initializing with a throwaway single-key seal..." INIT_JSON="$(vault_exec operator init -key-shares=1 -key-threshold=1 -format=json)" THROWAWAY_KEY="$(echo "$INIT_JSON" | jq -r '.unseal_keys_b64[0]')" echo "==> Unsealing with throwaway key for first restore..." vault_exec operator unseal "$THROWAWAY_KEY" >/dev/null unset THROWAWAY_KEY INIT_JSON elif [ "$SEALED" = "true" ]; then echo "==> Sealed - unsealing with the stored home unseal key..." vault_exec operator unseal "$UNSEAL_KEY" >/dev/null else echo "==> Already unsealed." fi echo "==> Installing mc (MinIO client)..." curl -fsSL https://dl.min.io/client/mc/release/linux-amd64/mc -o /tmp/mc chmod +x /tmp/mc export MC_CONFIG_DIR=/tmp/.mc echo "==> Finding the latest snapshot in local MinIO..." /tmp/mc alias set local-minio "http://vps-minio.minio.svc.cluster.local:9000" \ "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}" >/dev/null LATEST="$(/tmp/mc ls local-minio/vault-raft-snapshots --json | jq -rs 'sort_by(.lastModified) | last | .key')" if [ -z "$LATEST" ] || [ "$LATEST" = "null" ]; then echo "No snapshots found yet - nothing to restore." exit 0 fi echo "==> Latest snapshot: ${LATEST}" echo "==> Downloading snapshot..." /tmp/mc cp "local-minio/vault-raft-snapshots/${LATEST}" "/tmp/${LATEST}" >/dev/null echo "==> Copying snapshot into ${VAULT_POD}..." kubectl -n vault cp "/tmp/${LATEST}" "${VAULT_POD}:/tmp/${LATEST}" echo "==> Restoring raft snapshot (this replaces all data + the keyring)..." kubectl -n vault exec -i "$VAULT_POD" -- env VAULT_ADDR=http://127.0.0.1:8200 \ vault operator raft snapshot restore -force "/tmp/${LATEST}" kubectl -n vault exec "$VAULT_POD" -- rm -f "/tmp/${LATEST}" echo "==> Restarting Vault to fully reload post-restore state..." kubectl -n vault delete pod "$VAULT_POD" echo "==> Waiting for ${VAULT_POD} to come back..." kubectl -n vault wait --for=condition=Ready "pod/${VAULT_POD}" --timeout=120s || true echo "==> Unsealing with home's real key (restore overwrote the keyring)..." for i in 1 2 3 4 5; do if vault_exec operator unseal "$UNSEAL_KEY" >/dev/null 2>&1; then echo "==> Unsealed." break fi echo " not ready yet, retrying in 5s..." sleep 5 done echo "==> Done." --- apiVersion: batch/v1 kind: CronJob metadata: name: vault-restore namespace: vault spec: schedule: "30 */6 * * *" # 30 min after home's snapshot job, same 6h cadence concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 3 jobTemplate: spec: backoffLimit: 1 template: spec: serviceAccountName: vault-restore restartPolicy: Never securityContext: runAsNonRoot: true runAsUser: 65534 seccompProfile: type: RuntimeDefault containers: - name: restore image: alpine/k8s:1.32.13 command: ["/bin/bash", "/scripts/restore.sh"] securityContext: allowPrivilegeEscalation: false capabilities: drop: ["ALL"] env: - name: MINIO_ACCESS_KEY valueFrom: secretKeyRef: name: vps-minio-root-secret key: rootUser - name: MINIO_SECRET_KEY valueFrom: secretKeyRef: name: vps-minio-root-secret key: rootPassword volumeMounts: - name: scripts mountPath: /scripts volumes: - name: scripts configMap: name: vault-restore-script defaultMode: 0755