mirror of
https://gitlab.kube.huskypup.net/Scooby/Homelabv4.git
synced 2026-08-21 11:36:50 +00:00
Scooby HuskyandClaude Sonnet 5
d6e8c2aa2d
Confirmed end-to-end tonight: condition=Ready correctly times out every cycle since Vault can't be Ready while sealed (the unseal step comes right after this wait) - harmless via the existing || true fallback, but wastes up to 2 minutes per restore cycle waiting on a condition that can never be met at this point. Poll for the container process merely being started instead. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
227 lines
8.6 KiB
YAML
227 lines
8.6 KiB
YAML
---
|
|
# Periodically restores home Vault's latest raft snapshot into this VPS
|
|
# standby instance. Runs 30 min after the home-side snapshot job (which
|
|
# runs every 6h, see infrastructure/vault/manifests/raft-snapshot-cronjob.yaml)
|
|
# to give it time to land in this cluster's own MinIO.
|
|
#
|
|
# Why this needs a COPY of home's real unseal key (manifests/../README below)
|
|
# rather than a fresh one generated here: `vault operator raft snapshot
|
|
# restore` replaces the ENTIRE raft storage backend - including the
|
|
# encrypted keyring - with home's. After a restore, this Vault is sealed
|
|
# with HOME's keyring, not whatever it had before. The only way to unseal
|
|
# it afterward is with home's actual unseal key. That key was copied here
|
|
# once via:
|
|
# kubectl -n vault create secret generic vault-unseal-key \
|
|
# --from-literal=key=<home's VAULT_UNSEAL_KEY>
|
|
# (kubectl directly, not git - same reasoning as vps-minio-root-secret).
|
|
#
|
|
# First-run bootstrap: a brand new Vault pod is uninitialized, and Vault
|
|
# doesn't let you choose your own keys for a fresh init - so this generates
|
|
# a throwaway single Shamir key (threshold 1) purely to get past init and
|
|
# unseal long enough to perform the very first restore. That throwaway key
|
|
# is discarded immediately after use; it stops mattering the moment the
|
|
# first restore completes, since the keyring gets overwritten by home's.
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: vault-restore
|
|
namespace: vault
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: Role
|
|
metadata:
|
|
name: vault-restore
|
|
namespace: vault
|
|
rules:
|
|
- apiGroups: [""]
|
|
resources: ["pods"]
|
|
# watch is required by `kubectl wait` - without it the wait's internal
|
|
# watch retry loop never terminates cleanly even with --timeout set
|
|
# (confirmed live 2026-08-18: it just kept retrying past the stated
|
|
# 120s timeout). The restore/unseal steps themselves still succeed
|
|
# without it - this only affects how gracefully the job detects the
|
|
# pod coming back before unsealing.
|
|
verbs: ["get", "list", "watch", "delete"]
|
|
- apiGroups: [""]
|
|
resources: ["pods/exec"]
|
|
verbs: ["create"]
|
|
- apiGroups: [""]
|
|
resources: ["secrets"]
|
|
verbs: ["get"]
|
|
---
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
kind: RoleBinding
|
|
metadata:
|
|
name: vault-restore
|
|
namespace: vault
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: vault-restore
|
|
namespace: vault
|
|
roleRef:
|
|
kind: Role
|
|
name: vault-restore
|
|
apiGroup: rbac.authorization.k8s.io
|
|
---
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: vault-restore-script
|
|
namespace: vault
|
|
data:
|
|
restore.sh: |
|
|
#!/bin/bash
|
|
set -euo pipefail
|
|
|
|
# StatefulSet pod name follows the Helm release name (vps-vault), not
|
|
# the chart's default "vault-0".
|
|
VAULT_POD="vps-vault-0"
|
|
UNSEAL_KEY="$(kubectl -n vault get secret vault-unseal-key -o jsonpath='{.data.key}' | base64 -d)"
|
|
# Home's real root token - after the first successful restore, this
|
|
# Vault's auth data is byte-for-byte home's (as of that snapshot), so
|
|
# home's root token is valid here too. Needed to authenticate
|
|
# `raft snapshot restore`, which is a privileged operation - unseal
|
|
# alone isn't enough. Stored the same way as the unseal key (kubectl,
|
|
# not git):
|
|
# kubectl -n vault create secret generic vault-root-token \
|
|
# --from-literal=token=<home's VAULT_ROOT_TOKEN>
|
|
HOME_ROOT_TOKEN="$(kubectl -n vault get secret vault-root-token -o jsonpath='{.data.token}' | base64 -d)"
|
|
RESTORE_TOKEN=""
|
|
|
|
vault_exec() {
|
|
kubectl -n vault exec -i "$VAULT_POD" -- env VAULT_ADDR=http://127.0.0.1:8200 vault "$@"
|
|
}
|
|
|
|
vault_exec_auth() {
|
|
kubectl -n vault exec -i "$VAULT_POD" -- env VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN="$RESTORE_TOKEN" vault "$@"
|
|
}
|
|
|
|
echo "==> Checking Vault status..."
|
|
STATUS_JSON="$(vault_exec status -format=json 2>&1 || true)"
|
|
INITIALIZED="$(echo "$STATUS_JSON" | jq -r '.initialized // empty' 2>/dev/null || echo "")"
|
|
SEALED="$(echo "$STATUS_JSON" | jq -r '.sealed // empty' 2>/dev/null || echo "")"
|
|
|
|
if [ "$INITIALIZED" != "true" ]; then
|
|
echo "==> First run: initializing with a throwaway single-key seal..."
|
|
INIT_JSON="$(vault_exec operator init -key-shares=1 -key-threshold=1 -format=json)"
|
|
THROWAWAY_KEY="$(echo "$INIT_JSON" | jq -r '.unseal_keys_b64[0]')"
|
|
RESTORE_TOKEN="$(echo "$INIT_JSON" | jq -r '.root_token')"
|
|
echo "==> Unsealing with throwaway key for first restore..."
|
|
vault_exec operator unseal "$THROWAWAY_KEY" >/dev/null
|
|
unset THROWAWAY_KEY INIT_JSON
|
|
else
|
|
# Already restored at least once before - home's root token is valid
|
|
# here regardless of seal state, since it's restored FROM home.
|
|
RESTORE_TOKEN="$HOME_ROOT_TOKEN"
|
|
if [ "$SEALED" = "true" ]; then
|
|
echo "==> Sealed - unsealing with the stored home unseal key..."
|
|
vault_exec operator unseal "$UNSEAL_KEY" >/dev/null
|
|
else
|
|
echo "==> Already unsealed."
|
|
fi
|
|
fi
|
|
|
|
echo "==> Installing mc (MinIO client)..."
|
|
curl -fsSL https://dl.min.io/client/mc/release/linux-amd64/mc -o /tmp/mc
|
|
chmod +x /tmp/mc
|
|
export MC_CONFIG_DIR=/tmp/.mc
|
|
|
|
echo "==> Finding the latest snapshot in local MinIO..."
|
|
/tmp/mc alias set local-minio "http://vps-minio.minio.svc.cluster.local:9000" \
|
|
"${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}" >/dev/null
|
|
LATEST="$(/tmp/mc ls local-minio/vault-raft-snapshots --json | jq -rs 'sort_by(.lastModified) | last | .key')"
|
|
if [ -z "$LATEST" ] || [ "$LATEST" = "null" ]; then
|
|
echo "No snapshots found yet - nothing to restore."
|
|
exit 0
|
|
fi
|
|
echo "==> Latest snapshot: ${LATEST}"
|
|
|
|
echo "==> Downloading snapshot..."
|
|
/tmp/mc cp "local-minio/vault-raft-snapshots/${LATEST}" "/tmp/${LATEST}" >/dev/null
|
|
|
|
echo "==> Copying snapshot into ${VAULT_POD}..."
|
|
kubectl -n vault cp "/tmp/${LATEST}" "${VAULT_POD}:/tmp/${LATEST}"
|
|
|
|
echo "==> Restoring raft snapshot (this replaces all data + the keyring)..."
|
|
vault_exec_auth operator raft snapshot restore -force "/tmp/${LATEST}"
|
|
kubectl -n vault exec "$VAULT_POD" -- rm -f "/tmp/${LATEST}"
|
|
|
|
echo "==> Restarting Vault to fully reload post-restore state..."
|
|
kubectl -n vault delete pod "$VAULT_POD"
|
|
echo "==> Waiting for ${VAULT_POD} to come back..."
|
|
# Not `kubectl wait --for=condition=Ready` - Vault's readiness probe
|
|
# requires unsealed state, which only happens in the step AFTER this
|
|
# wait (chicken-and-egg: it would never report Ready before we've had
|
|
# a chance to unseal it). Poll for the container process merely being
|
|
# started instead, which doesn't depend on any readiness probe.
|
|
for i in $(seq 1 24); do
|
|
PHASE="$(kubectl -n vault get "pod/${VAULT_POD}" -o jsonpath='{.status.containerStatuses[0].state.running}' 2>/dev/null || echo "")"
|
|
if [ -n "$PHASE" ]; then
|
|
echo "==> ${VAULT_POD} container is running."
|
|
break
|
|
fi
|
|
sleep 5
|
|
done
|
|
|
|
echo "==> Unsealing with home's real key (restore overwrote the keyring)..."
|
|
for i in 1 2 3 4 5; do
|
|
if vault_exec operator unseal "$UNSEAL_KEY" >/dev/null 2>&1; then
|
|
echo "==> Unsealed."
|
|
break
|
|
fi
|
|
echo " not ready yet, retrying in 5s..."
|
|
sleep 5
|
|
done
|
|
|
|
echo "==> Done."
|
|
---
|
|
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: vault-restore
|
|
namespace: vault
|
|
spec:
|
|
schedule: "30 */6 * * *" # 30 min after home's snapshot job, same 6h cadence
|
|
concurrencyPolicy: Forbid
|
|
successfulJobsHistoryLimit: 3
|
|
failedJobsHistoryLimit: 3
|
|
jobTemplate:
|
|
spec:
|
|
backoffLimit: 1
|
|
template:
|
|
spec:
|
|
serviceAccountName: vault-restore
|
|
restartPolicy: Never
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 65534
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
containers:
|
|
- name: restore
|
|
image: alpine/k8s:1.32.13
|
|
command: ["/bin/bash", "/scripts/restore.sh"]
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop: ["ALL"]
|
|
env:
|
|
- name: MINIO_ACCESS_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: vps-minio-root-secret
|
|
key: rootUser
|
|
- name: MINIO_SECRET_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: vps-minio-root-secret
|
|
key: rootPassword
|
|
volumeMounts:
|
|
- name: scripts
|
|
mountPath: /scripts
|
|
volumes:
|
|
- name: scripts
|
|
configMap:
|
|
name: vault-restore-script
|
|
defaultMode: 0755
|