gitea_admin/Homelabv4
1
0
Fork 0
mirror of https://gitlab.kube.huskypup.net/Scooby/Homelabv4.git synced 2026-08-20 23:16:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix vault-raft-snapshot: detect active leader, suspend until VPS egress exists

Browse Source 
vault operator raft snapshot save fails against a standby Vault node with
'incomplete snapshot, unable to read SHA256SUMS.sealed file' - it must run
directly against the active HA leader. The job was hardcoded to vault-0,
which has been a standby since boot, so it had been failing on every run.

Fixed to detect the actual leader at runtime via vault status is_self.
Also suspended the CronJob: the upload step targets
vps-minio.netbird.internal, unreachable from any pod in this cluster (same
missing egress-route gap as tonight's CrowdSec/CNPG-backup fixes). The
leader-detection fix is kept since it's correct and independent - just
unsuspend once real egress routing exists.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Scooby Husky
2026-08-17 19:56:12 -05:00
co-authored by Claude Sonnet 5
parent e18308b750
commit 07a59d3b34
1 changed files with 32 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
infrastructure/vault/manifests/raft-snapshot-cronjob.yaml
+32 -5
View File
@@ -51,13 +51,33 @@ data:
ROOT_TOKEN="$(kubectl -n vault get secret vault-init-keys -o jsonpath='{.data.VAULT_ROOT_TOKEN}' | base64 -d)"
echo "==> Taking raft snapshot from vault-0..."
kubectl -n vault exec vault-0 -- env VAULT_TOKEN="$ROOT_TOKEN" \
# `vault operator raft snapshot save` fails against a standby node with
# "incomplete snapshot, unable to read SHA256SUMS.sealed file" - it must
# run directly against the active HA leader, not a proxied standby.
# Found 2026-08-17: this job had been failing every run since it was
# added, hardcoded to vault-0 (frequently a standby). Detect the actual
# leader at runtime instead of hardcoding a pod name.
echo "==> Finding the active Vault leader..."
LEADER_POD=""
for pod in vault-0 vault-1 vault-2; do
is_leader="$(kubectl -n vault exec "$pod" -- env VAULT_ADDR=http://127.0.0.1:8200 VAULT_TOKEN="$ROOT_TOKEN" \
vault status -format=json 2>/dev/null | jq -r '.is_self // false')"
if [ "$is_leader" = "true" ]; then
LEADER_POD="$pod"
break
fi
done
if [ -z "$LEADER_POD" ]; then
echo "ERROR: could not find an active Vault leader among vault-0/1/2" >&2
exit 1
fi
echo "==> Taking raft snapshot from ${LEADER_POD} (active leader)..."
kubectl -n vault exec "$LEADER_POD" -- env VAULT_TOKEN="$ROOT_TOKEN" \
vault operator raft snapshot save "/tmp/${SNAP_NAME}"
echo "==> Copying snapshot out of vault-0..."
kubectl -n vault cp "vault-0:/tmp/${SNAP_NAME}" "/tmp/${SNAP_NAME}"
kubectl -n vault exec vault-0 -- rm -f "/tmp/${SNAP_NAME}"
echo "==> Copying snapshot out of ${LEADER_POD}..."
kubectl -n vault cp "${LEADER_POD}:/tmp/${SNAP_NAME}" "/tmp/${SNAP_NAME}"
kubectl -n vault exec "$LEADER_POD" -- rm -f "/tmp/${SNAP_NAME}"
echo "==> Installing mc (MinIO client)..."
curl -sf https://dl.min.io/client/mc/release/linux-amd64/mc -o /tmp/mc
@@ -82,6 +102,13 @@ metadata:
namespace: vault
spec:
schedule: "0 */6 * * *" # every 6 hours
# Suspended 2026-08-17: the upload step targets vps-minio.netbird.internal,
# which no pod in this cluster can currently reach - no node has an
# egress route into the Netbird mesh (see commit history around this
# date for the full investigation). The leader-detection fix above is
# real and independent of this, so it's kept fixed rather than reverted.
# Unsuspend once real pod-egress routing to the VPS exists.
suspend: true
concurrencyPolicy: Forbid
successfulJobsHistoryLimit: 3
failedJobsHistoryLimit: 3