From 07a59d3b345279047fad2a405fce361586021542 Mon Sep 17 00:00:00 2001 From: Scooby Husky Date: Mon, 17 Aug 2026 19:56:12 -0500 Subject: [PATCH] Fix vault-raft-snapshot: detect active leader, suspend until VPS egress exists vault operator raft snapshot save fails against a standby Vault node with 'incomplete snapshot, unable to read SHA256SUMS.sealed file' - it must run directly against the active HA leader. The job was hardcoded to vault-0, which has been a standby since boot, so it had been failing on every run. Fixed to detect the actual leader at runtime via vault status is_self. Also suspended the CronJob: the upload step targets vps-minio.netbird.internal, unreachable from any pod in this cluster (same missing egress-route gap as tonight's CrowdSec/CNPG-backup fixes). The leader-detection fix is kept since it's correct and independent - just unsuspend once real egress routing exists. Co-Authored-By: Claude Sonnet 5 --- .../manifests/raft-snapshot-cronjob.yaml | 37 ++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/infrastructure/vault/manifests/raft-snapshot-cronjob.yaml b/infrastructure/vault/manifests/raft-snapshot-cronjob.yaml index f987da7..1636d19 100644 --- a/infrastructure/vault/manifests/raft-snapshot-cronjob.yaml +++ b/infrastructure/vault/manifests/raft-snapshot-cronjob.yaml @@ -51,13 +51,33 @@ data: ROOT_TOKEN="$(kubectl -n vault get secret vault-init-keys -o jsonpath='{.data.VAULT_ROOT_TOKEN}' | base64 -d)" - echo "==> Taking raft snapshot from vault-0..." - kubectl -n vault exec vault-0 -- env VAULT_TOKEN="$ROOT_TOKEN" \ + # `vault operator raft snapshot save` fails against a standby node with + # "incomplete snapshot, unable to read SHA256SUMS.sealed file" - it must + # run directly against the active HA leader, not a proxied standby. + # Found 2026-08-17: this job had been failing every run since it was + # added, hardcoded to vault-0 (frequently a standby). Detect the actual + # leader at runtime instead of hardcoding a pod name. + echo "==> Finding the active Vault leader..." + LEADER_POD="" + for pod in vault-0 vault-1 vault-2; do + is_leader="$(kubectl -n vault exec "$pod" -- env VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN="$ROOT_TOKEN" \ + vault status -format=json 2>/dev/null | jq -r '.is_self // false')" + if [ "$is_leader" = "true" ]; then + LEADER_POD="$pod" + break + fi + done + if [ -z "$LEADER_POD" ]; then + echo "ERROR: could not find an active Vault leader among vault-0/1/2" >&2 + exit 1 + fi + echo "==> Taking raft snapshot from ${LEADER_POD} (active leader)..." + kubectl -n vault exec "$LEADER_POD" -- env VAULT_TOKEN="$ROOT_TOKEN" \ vault operator raft snapshot save "/tmp/${SNAP_NAME}" - echo "==> Copying snapshot out of vault-0..." - kubectl -n vault cp "vault-0:/tmp/${SNAP_NAME}" "/tmp/${SNAP_NAME}" - kubectl -n vault exec vault-0 -- rm -f "/tmp/${SNAP_NAME}" + echo "==> Copying snapshot out of ${LEADER_POD}..." + kubectl -n vault cp "${LEADER_POD}:/tmp/${SNAP_NAME}" "/tmp/${SNAP_NAME}" + kubectl -n vault exec "$LEADER_POD" -- rm -f "/tmp/${SNAP_NAME}" echo "==> Installing mc (MinIO client)..." curl -sf https://dl.min.io/client/mc/release/linux-amd64/mc -o /tmp/mc @@ -82,6 +102,13 @@ metadata: namespace: vault spec: schedule: "0 */6 * * *" # every 6 hours + # Suspended 2026-08-17: the upload step targets vps-minio.netbird.internal, + # which no pod in this cluster can currently reach - no node has an + # egress route into the Netbird mesh (see commit history around this + # date for the full investigation). The leader-detection fix above is + # real and independent of this, so it's kept fixed rather than reverted. + # Unsuspend once real pod-egress routing to the VPS exists. + suspend: true concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 3