mirror of
https://gitlab.kube.huskypup.net/Scooby/Homelabv4.git
synced 2026-08-20 23:16:49 +00:00
Phase 2: deploy Vault warm standby on the VPS
Single-node Vault (raft storage, standalone mode) plus a restore CronJob that pulls the latest raft snapshot from the VPS's own local MinIO (already receiving snapshots every 6h from home) and restores it every 6h, offset 30 minutes after the home-side snapshot job. Not an independent root of trust: raft snapshot restore replaces the entire storage backend including the keyring, so after every restore this Vault is sealed with HOME's actual keyring - unsealing it needs a copy of home's real unseal key, stored directly on the VPS cluster (kubectl, not git, same pattern as vps-minio-root-secret). First-run bootstrap uses a throwaway single-Shamir-key init just to get through the very first restore, then is irrelevant from then on. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Scooby Husky
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
8aed0bf98e
commit
d0c5563671
@@ -0,0 +1,187 @@
|
||||
---
|
||||
# Periodically restores home Vault's latest raft snapshot into this VPS
|
||||
# standby instance. Runs 30 min after the home-side snapshot job (which
|
||||
# runs every 6h, see infrastructure/vault/manifests/raft-snapshot-cronjob.yaml)
|
||||
# to give it time to land in this cluster's own MinIO.
|
||||
#
|
||||
# Why this needs a COPY of home's real unseal key (manifests/../README below)
|
||||
# rather than a fresh one generated here: `vault operator raft snapshot
|
||||
# restore` replaces the ENTIRE raft storage backend - including the
|
||||
# encrypted keyring - with home's. After a restore, this Vault is sealed
|
||||
# with HOME's keyring, not whatever it had before. The only way to unseal
|
||||
# it afterward is with home's actual unseal key. That key was copied here
|
||||
# once via:
|
||||
# kubectl -n vault create secret generic vault-unseal-key \
|
||||
# --from-literal=key=<home's VAULT_UNSEAL_KEY>
|
||||
# (kubectl directly, not git - same reasoning as vps-minio-root-secret).
|
||||
#
|
||||
# First-run bootstrap: a brand new Vault pod is uninitialized, and Vault
|
||||
# doesn't let you choose your own keys for a fresh init - so this generates
|
||||
# a throwaway single Shamir key (threshold 1) purely to get past init and
|
||||
# unseal long enough to perform the very first restore. That throwaway key
|
||||
# is discarded immediately after use; it stops mattering the moment the
|
||||
# first restore completes, since the keyring gets overwritten by home's.
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: vault-restore
|
||||
namespace: vault
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: vault-restore
|
||||
namespace: vault
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["pods"]
|
||||
verbs: ["get", "list", "delete"]
|
||||
- apiGroups: [""]
|
||||
resources: ["pods/exec"]
|
||||
verbs: ["create"]
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["get"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: vault-restore
|
||||
namespace: vault
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: vault-restore
|
||||
namespace: vault
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: vault-restore
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: vault-restore-script
|
||||
namespace: vault
|
||||
data:
|
||||
restore.sh: |
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
VAULT_POD="vault-0"
|
||||
UNSEAL_KEY="$(kubectl -n vault get secret vault-unseal-key -o jsonpath='{.data.key}' | base64 -d)"
|
||||
|
||||
vault_exec() {
|
||||
kubectl -n vault exec -i "$VAULT_POD" -- env VAULT_ADDR=http://127.0.0.1:8200 vault "$@"
|
||||
}
|
||||
|
||||
echo "==> Checking Vault status..."
|
||||
STATUS_JSON="$(vault_exec status -format=json 2>&1 || true)"
|
||||
INITIALIZED="$(echo "$STATUS_JSON" | jq -r '.initialized // empty' 2>/dev/null || echo "")"
|
||||
SEALED="$(echo "$STATUS_JSON" | jq -r '.sealed // empty' 2>/dev/null || echo "")"
|
||||
|
||||
if [ "$INITIALIZED" != "true" ]; then
|
||||
echo "==> First run: initializing with a throwaway single-key seal..."
|
||||
INIT_JSON="$(vault_exec operator init -key-shares=1 -key-threshold=1 -format=json)"
|
||||
THROWAWAY_KEY="$(echo "$INIT_JSON" | jq -r '.unseal_keys_b64[0]')"
|
||||
echo "==> Unsealing with throwaway key for first restore..."
|
||||
vault_exec operator unseal "$THROWAWAY_KEY" >/dev/null
|
||||
unset THROWAWAY_KEY INIT_JSON
|
||||
elif [ "$SEALED" = "true" ]; then
|
||||
echo "==> Sealed - unsealing with the stored home unseal key..."
|
||||
vault_exec operator unseal "$UNSEAL_KEY" >/dev/null
|
||||
else
|
||||
echo "==> Already unsealed."
|
||||
fi
|
||||
|
||||
echo "==> Installing mc (MinIO client)..."
|
||||
curl -fsSL https://dl.min.io/client/mc/release/linux-amd64/mc -o /tmp/mc
|
||||
chmod +x /tmp/mc
|
||||
export MC_CONFIG_DIR=/tmp/.mc
|
||||
|
||||
echo "==> Finding the latest snapshot in local MinIO..."
|
||||
/tmp/mc alias set local-minio "http://vps-minio.minio.svc.cluster.local:9000" \
|
||||
"${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}" >/dev/null
|
||||
LATEST="$(/tmp/mc ls local-minio/vault-raft-snapshots --json | jq -rs 'sort_by(.lastModified) | last | .key')"
|
||||
if [ -z "$LATEST" ] || [ "$LATEST" = "null" ]; then
|
||||
echo "No snapshots found yet - nothing to restore."
|
||||
exit 0
|
||||
fi
|
||||
echo "==> Latest snapshot: ${LATEST}"
|
||||
|
||||
echo "==> Downloading snapshot..."
|
||||
/tmp/mc cp "local-minio/vault-raft-snapshots/${LATEST}" "/tmp/${LATEST}" >/dev/null
|
||||
|
||||
echo "==> Copying snapshot into ${VAULT_POD}..."
|
||||
kubectl -n vault cp "/tmp/${LATEST}" "${VAULT_POD}:/tmp/${LATEST}"
|
||||
|
||||
echo "==> Restoring raft snapshot (this replaces all data + the keyring)..."
|
||||
kubectl -n vault exec -i "$VAULT_POD" -- env VAULT_ADDR=http://127.0.0.1:8200 \
|
||||
vault operator raft snapshot restore -force "/tmp/${LATEST}"
|
||||
kubectl -n vault exec "$VAULT_POD" -- rm -f "/tmp/${LATEST}"
|
||||
|
||||
echo "==> Restarting Vault to fully reload post-restore state..."
|
||||
kubectl -n vault delete pod "$VAULT_POD"
|
||||
echo "==> Waiting for ${VAULT_POD} to come back..."
|
||||
kubectl -n vault wait --for=condition=Ready "pod/${VAULT_POD}" --timeout=120s || true
|
||||
|
||||
echo "==> Unsealing with home's real key (restore overwrote the keyring)..."
|
||||
for i in 1 2 3 4 5; do
|
||||
if vault_exec operator unseal "$UNSEAL_KEY" >/dev/null 2>&1; then
|
||||
echo "==> Unsealed."
|
||||
break
|
||||
fi
|
||||
echo " not ready yet, retrying in 5s..."
|
||||
sleep 5
|
||||
done
|
||||
|
||||
echo "==> Done."
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: vault-restore
|
||||
namespace: vault
|
||||
spec:
|
||||
schedule: "30 */6 * * *" # 30 min after home's snapshot job, same 6h cadence
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 1
|
||||
template:
|
||||
spec:
|
||||
serviceAccountName: vault-restore
|
||||
restartPolicy: Never
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65534
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: restore
|
||||
image: alpine/k8s:1.32.13
|
||||
command: ["/bin/bash", "/scripts/restore.sh"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
env:
|
||||
- name: MINIO_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: vps-minio-root-secret
|
||||
key: rootUser
|
||||
- name: MINIO_SECRET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: vps-minio-root-secret
|
||||
key: rootPassword
|
||||
volumeMounts:
|
||||
- name: scripts
|
||||
mountPath: /scripts
|
||||
volumes:
|
||||
- name: scripts
|
||||
configMap:
|
||||
name: vault-restore-script
|
||||
defaultMode: 0755
|
||||
Reference in New Issue
Block a user