Homelabv4/infrastructure/vps-standby/vault/manifests/restore-cronjob.yaml

---
# Periodically restores home Vault's latest raft snapshot into this VPS
# standby instance. Runs 30 min after the home-side snapshot job (which
# runs every 6h, see infrastructure/vault/manifests/raft-snapshot-cronjob.yaml)
# to give it time to land in this cluster's own MinIO.
#
# Why this needs a COPY of home's real unseal key (manifests/../README below)
# rather than a fresh one generated here: `vault operator raft snapshot
# restore` replaces the ENTIRE raft storage backend - including the
# encrypted keyring - with home's. After a restore, this Vault is sealed
# with HOME's keyring, not whatever it had before. The only way to unseal
# it afterward is with home's actual unseal key. That key was copied here
# once via:
#   kubectl -n vault create secret generic vault-unseal-key \
#     --from-literal=key=<home's VAULT_UNSEAL_KEY>
# (kubectl directly, not git - same reasoning as vps-minio-root-secret).
#
# First-run bootstrap: a brand new Vault pod is uninitialized, and Vault
# doesn't let you choose your own keys for a fresh init - so this generates
# a throwaway single Shamir key (threshold 1) purely to get past init and
# unseal long enough to perform the very first restore. That throwaway key
# is discarded immediately after use; it stops mattering the moment the
# first restore completes, since the keyring gets overwritten by home's.
apiVersion: v1
kind: ServiceAccount
metadata:
  name: vault-restore
  namespace: vault
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: vault-restore
  namespace: vault
rules:
  - apiGroups: [""]
    resources: ["pods"]
    verbs: ["get", "list", "delete"]
  - apiGroups: [""]
    resources: ["pods/exec"]
    verbs: ["create"]
  - apiGroups: [""]
    resources: ["secrets"]
    verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: vault-restore
  namespace: vault
subjects:
  - kind: ServiceAccount
    name: vault-restore
    namespace: vault
roleRef:
  kind: Role
  name: vault-restore
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: vault-restore-script
  namespace: vault
data:
  restore.sh: |
    #!/bin/bash
    set -euo pipefail

    # StatefulSet pod name follows the Helm release name (vps-vault), not
    # the chart's default "vault-0".
    VAULT_POD="vps-vault-0"
    UNSEAL_KEY="$(kubectl -n vault get secret vault-unseal-key -o jsonpath='{.data.key}' | base64 -d)"
    # Home's real root token - after the first successful restore, this
    # Vault's auth data is byte-for-byte home's (as of that snapshot), so
    # home's root token is valid here too. Needed to authenticate
    # `raft snapshot restore`, which is a privileged operation - unseal
    # alone isn't enough. Stored the same way as the unseal key (kubectl,
    # not git):
    #   kubectl -n vault create secret generic vault-root-token \
    #     --from-literal=token=<home's VAULT_ROOT_TOKEN>
    HOME_ROOT_TOKEN="$(kubectl -n vault get secret vault-root-token -o jsonpath='{.data.token}' | base64 -d)"
    RESTORE_TOKEN=""

    vault_exec() {
      kubectl -n vault exec -i "$VAULT_POD" -- env VAULT_ADDR=http://127.0.0.1:8200 vault "$@"
    }

    vault_exec_auth() {
      kubectl -n vault exec -i "$VAULT_POD" -- env VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN="$RESTORE_TOKEN" vault "$@"
    }

    echo "==> Checking Vault status..."
    STATUS_JSON="$(vault_exec status -format=json 2>&1 || true)"
    INITIALIZED="$(echo "$STATUS_JSON" | jq -r '.initialized // empty' 2>/dev/null || echo "")"
    SEALED="$(echo "$STATUS_JSON" | jq -r '.sealed // empty' 2>/dev/null || echo "")"

    if [ "$INITIALIZED" != "true" ]; then
      echo "==> First run: initializing with a throwaway single-key seal..."
      INIT_JSON="$(vault_exec operator init -key-shares=1 -key-threshold=1 -format=json)"
      THROWAWAY_KEY="$(echo "$INIT_JSON" | jq -r '.unseal_keys_b64[0]')"
      RESTORE_TOKEN="$(echo "$INIT_JSON" | jq -r '.root_token')"
      echo "==> Unsealing with throwaway key for first restore..."
      vault_exec operator unseal "$THROWAWAY_KEY" >/dev/null
      unset THROWAWAY_KEY INIT_JSON
    else
      # Already restored at least once before - home's root token is valid
      # here regardless of seal state, since it's restored FROM home.
      RESTORE_TOKEN="$HOME_ROOT_TOKEN"
      if [ "$SEALED" = "true" ]; then
        echo "==> Sealed - unsealing with the stored home unseal key..."
        vault_exec operator unseal "$UNSEAL_KEY" >/dev/null
      else
        echo "==> Already unsealed."
      fi
    fi

    echo "==> Installing mc (MinIO client)..."
    curl -fsSL https://dl.min.io/client/mc/release/linux-amd64/mc -o /tmp/mc
    chmod +x /tmp/mc
    export MC_CONFIG_DIR=/tmp/.mc

    echo "==> Finding the latest snapshot in local MinIO..."
    /tmp/mc alias set local-minio "http://vps-minio.minio.svc.cluster.local:9000" \
      "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}" >/dev/null
    LATEST="$(/tmp/mc ls local-minio/vault-raft-snapshots --json | jq -rs 'sort_by(.lastModified) | last | .key')"
    if [ -z "$LATEST" ] || [ "$LATEST" = "null" ]; then
      echo "No snapshots found yet - nothing to restore."
      exit 0
    fi
    echo "==> Latest snapshot: ${LATEST}"

    echo "==> Downloading snapshot..."
    /tmp/mc cp "local-minio/vault-raft-snapshots/${LATEST}" "/tmp/${LATEST}" >/dev/null

    echo "==> Copying snapshot into ${VAULT_POD}..."
    kubectl -n vault cp "/tmp/${LATEST}" "${VAULT_POD}:/tmp/${LATEST}"

    echo "==> Restoring raft snapshot (this replaces all data + the keyring)..."
    vault_exec_auth operator raft snapshot restore -force "/tmp/${LATEST}"
    kubectl -n vault exec "$VAULT_POD" -- rm -f "/tmp/${LATEST}"

    echo "==> Restarting Vault to fully reload post-restore state..."
    kubectl -n vault delete pod "$VAULT_POD"
    echo "==> Waiting for ${VAULT_POD} to come back..."
    kubectl -n vault wait --for=condition=Ready "pod/${VAULT_POD}" --timeout=120s || true

    echo "==> Unsealing with home's real key (restore overwrote the keyring)..."
    for i in 1 2 3 4 5; do
      if vault_exec operator unseal "$UNSEAL_KEY" >/dev/null 2>&1; then
        echo "==> Unsealed."
        break
      fi
      echo "    not ready yet, retrying in 5s..."
      sleep 5
    done

    echo "==> Done."
---
apiVersion: batch/v1
kind: CronJob
metadata:
  name: vault-restore
  namespace: vault
spec:
  schedule: "30 */6 * * *"  # 30 min after home's snapshot job, same 6h cadence
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
  jobTemplate:
    spec:
      backoffLimit: 1
      template:
        spec:
          serviceAccountName: vault-restore
          restartPolicy: Never
          securityContext:
            runAsNonRoot: true
            runAsUser: 65534
            seccompProfile:
              type: RuntimeDefault
          containers:
            - name: restore
              image: alpine/k8s:1.32.13
              command: ["/bin/bash", "/scripts/restore.sh"]
              securityContext:
                allowPrivilegeEscalation: false
                capabilities:
                  drop: ["ALL"]
              env:
                - name: MINIO_ACCESS_KEY
                  valueFrom:
                    secretKeyRef:
                      name: vps-minio-root-secret
                      key: rootUser
                - name: MINIO_SECRET_KEY
                  valueFrom:
                    secretKeyRef:
                      name: vps-minio-root-secret
                      key: rootPassword
              volumeMounts:
                - name: scripts
                  mountPath: /scripts
          volumes:
            - name: scripts
              configMap:
                name: vault-restore-script
                defaultMode: 0755