apiVersion: v1 kind: ConfigMap metadata: name: vault-init-script namespace: vault data: vault-init-and-store.sh: | #!/bin/bash set -euo pipefail # === Vault Auto-Init and Store Script (ArgoCD Job Version) === # Adapted from Homelabv5/scripts/vault-init-and-store.sh # Runs inside a Job pod with kubectl access via ServiceAccount. # Uses kubectl exec to interact with the vault-0 pod directly # instead of port-forwarding + local vault CLI. VAULT_NS="${VAULT_NS:-vault}" SECRET_NAME="${SECRET_NAME:-vault-init-keys}" KV_MOUNT="${KV_MOUNT:-secret}" JOB_NS_FOR_ROLE="${JOB_NS_FOR_ROLE:-auth-proxy}" JOB_SA_FOR_ROLE="${JOB_SA_FOR_ROLE:-oauth2-bootstrap}" POLICY_NAME="${POLICY_NAME:-oauth2-writer}" ROLE_NAME="${ROLE_NAME:-eso-writer}" VAULT_POD="vault-0" # Helper: run vault CLI inside the vault pod vault_exec() { # -i forwards stdin - needed for the heredoc-piped `vault policy write # NAME -` call below. Without it, `kubectl exec` never passes stdin # through and vault sees an empty policy body. This was unreachable # until the init-detection fix above (the script always failed earlier, # every run, before ever getting this far). kubectl -n "$VAULT_NS" exec -i "$VAULT_POD" -- env VAULT_ADDR=http://127.0.0.1:8200 vault "$@" } # --- Wait for vault-0 pod to be ready --- echo "==> Waiting for vault-0 pod to be running..." for i in $(seq 1 60); do PHASE=$(kubectl -n "$VAULT_NS" get pod "$VAULT_POD" -o jsonpath='{.status.phase}' 2>/dev/null || echo "") if [ "$PHASE" = "Running" ]; then echo " vault-0 is running" break fi echo " waiting for vault-0... (attempt $i/60, phase=$PHASE)" sleep 5 done # Give Vault a moment to start its listener sleep 5 # --- Check init/seal status --- # `vault status` legitimately exits non-zero for normal states (2 = sealed, # still prints valid JSON) as well as real failures (1 = can't connect, no # output). The old `|| echo '{}'` fallback couldn't tell those apart - it # discarded valid "already initialized, just sealed" JSON on exit 2 too, # which made this script wrongly conclude "not initialized" and attempt to # re-init an already-initialized Vault (always fails: "Vault is already # initialized"). Only fall back to {} when there's truly no output. echo "==> Checking Vault status..." status_json="$(vault_exec status -format=json 2>/dev/null)" if [ -z "$status_json" ]; then status_json="{}" fi # jq, not grep/cut - `vault status -format=json` pretty-prints with a space # after each colon ("initialized": true), which the old grep -o # '"initialized":[a-z]*' pattern never matched (no space in the pattern) - # initialized/sealed silently parsed as empty strings on every run # regardless of actual Vault state, which is the real reason this script # always tried to re-init an already-initialized Vault. initialized="$(echo "$status_json" | jq -r '.initialized // empty')" sealed="$(echo "$status_json" | jq -r '.sealed // empty')" root_token="" unseal_key="" # Pull existing secret if present if kubectl -n "$VAULT_NS" get secret "$SECRET_NAME" >/dev/null 2>&1; then root_token="$(kubectl -n "$VAULT_NS" get secret "$SECRET_NAME" -o jsonpath='{.data.VAULT_ROOT_TOKEN}' | base64 -d || true)" unseal_key="$(kubectl -n "$VAULT_NS" get secret "$SECRET_NAME" -o jsonpath='{.data.VAULT_UNSEAL_KEY}' | base64 -d || true)" fi # Initialize if needed if [ "$initialized" != "true" ]; then echo "==> Vault not initialized; initializing..." init_json="$(vault_exec operator init -key-shares=1 -key-threshold=1 -format=json)" root_token="$(echo "$init_json" | grep -o '"root_token":"[^"]*"' | cut -d'"' -f4)" unseal_key="$(echo "$init_json" | grep -o '"unseal_keys_b64":\["[^"]*"\]' | grep -o '\["[^"]*"\]' | tr -d '[]"')" sealed="true" echo "==> Storing root token & unseal key in Secret ${VAULT_NS}/${SECRET_NAME}" kubectl -n "$VAULT_NS" create secret generic "$SECRET_NAME" \ --from-literal=VAULT_ROOT_TOKEN="$root_token" \ --from-literal=VAULT_UNSEAL_KEY="$unseal_key" \ --dry-run=client -o yaml | kubectl apply -f - else echo "==> Vault already initialized." fi # Unseal if needed if [ "$sealed" = "true" ]; then if [ -z "$unseal_key" ]; then echo "ERROR: Vault is sealed and no unseal key available" exit 1 fi echo "==> Unsealing..." vault_exec operator unseal "$unseal_key" >/dev/null fi # Login if [ -z "$root_token" ]; then echo "==> Reading root token from Secret..." root_token="$(kubectl -n "$VAULT_NS" get secret "$SECRET_NAME" -o jsonpath='{.data.VAULT_ROOT_TOKEN}' | base64 -d)" fi vault_exec login "$root_token" >/dev/null # Ensure KV v2 enabled if ! vault_exec secrets list -format=json 2>/dev/null | grep -q "\"${KV_MOUNT}/\""; then echo "==> Enabling KV v2 at ${KV_MOUNT}/" vault_exec secrets enable -path="$KV_MOUNT" -version=2 kv >/dev/null fi # Configure Kubernetes auth using Vault's own pod identity as the # TokenReview reviewer, rather than a manually-minted static token. # # BUG (found 2026-08-17, live cluster ~2hrs after boot): this used to do # `kubectl create token vault-auth` with no --duration, which defaults to # a 1-hour TTL, then wrote that JWT into auth/kubernetes/config as a # static token_reviewer_jwt. ~1hr after every cluster boot / hook rerun, # that token silently expired, so Vault's TokenReview calls (used by # EVERY kubernetes-auth login, including ESO's) started failing k8s-side # with 401 - which Vault surfaces to callers as a generic, unhelpful # "permission denied" 403 on /auth/kubernetes/login, with nothing logged # at INFO/ERROR. This cascaded into ClusterSecretStore vault-backend # going InvalidProviderConfig and every ExternalSecret in the cluster # failing to sync - the real cause behind a broad ArgoCD "Degraded" wave # that had nothing to do with Vault's seal state (which was fine). # # Fix: leave token_reviewer_jwt unset (explicitly cleared below). # disable_local_ca_jwt defaults to false, so Vault falls back to reading # its own pod's projected SA token from disk on every TokenReview call - # that token is auto-refreshed by kubelet for the life of the pod, so # there's nothing to expire. The vault pods' own SA ("vault", not # "vault-auth") already carries system:auth-delegator via the existing # vault-server-binding ClusterRoleBinding, so no separate reviewer SA is # needed at all - the vault-auth SA/binding below is now unused, kept # only so an old cluster doesn't need manual cleanup. kube_ca="$(kubectl -n kube-system get configmap kube-root-ca.crt -o jsonpath='{.data.ca\.crt}')" kube_host="https://kubernetes.default.svc:443" vault_exec auth enable kubernetes >/dev/null 2>&1 || true vault_exec write auth/kubernetes/config \ token_reviewer_jwt="" \ kubernetes_host="$kube_host" \ kubernetes_ca_cert="$kube_ca" \ disable_local_ca_jwt=false >/dev/null # Policy + role for ESO/oauth2 job vault_exec policy write "$POLICY_NAME" - >/dev/null <<'HCL' path "secret/data/*" { capabilities = ["create", "update", "read", "list"] } path "secret/metadata/*" { capabilities = ["create", "update", "read", "list"] } HCL vault_exec write "auth/kubernetes/role/${ROLE_NAME}" \ bound_service_account_names="${JOB_SA_FOR_ROLE},external-secrets" \ bound_service_account_namespaces="${JOB_NS_FOR_ROLE},external-secrets" \ policies="${POLICY_NAME}" \ ttl="24h" >/dev/null # --- OIDC Auth (Authentik SSO) --- OIDC_CLIENT_ID="9816a5ae7e7914b5d18f4ab939d011a98f8c8d6b3bb6777c46431afa06ac4a85" OIDC_CLIENT_SECRET="ed2ba1c6378c7a46341b5162f39a7fab80e37596b01ed387c3719e8e0040344cf1daa307476c2e7a7f75041b3979275b1ebf00bb8bad94c864b4a38ded544f7b" OIDC_DISCOVERY_URL="https://auth.kube.huskypup.net/application/o/vault/" echo "==> Configuring OIDC auth (Authentik)..." vault_exec auth enable oidc >/dev/null 2>&1 || true vault_exec write auth/oidc/config \ oidc_discovery_url="$OIDC_DISCOVERY_URL" \ oidc_client_id="$OIDC_CLIENT_ID" \ oidc_client_secret="$OIDC_CLIENT_SECRET" \ default_role="default" >/dev/null vault_exec policy write vault-admin - >/dev/null <<'HCL' path "*" { capabilities = ["create", "read", "update", "delete", "list", "sudo"] } HCL vault_exec write auth/oidc/role/default \ user_claim="sub" \ allowed_redirect_uris="https://vault.kube.huskypup.net/ui/vault/auth/oidc/oidc/callback,http://localhost:8250/oidc/callback" \ policies="vault-admin" \ oidc_scopes="openid,email,profile" \ token_ttl="1h" >/dev/null echo "==> Done." echo "K8s Secret with init creds: ${VAULT_NS}/${SECRET_NAME}" echo "IMPORTANT: back these up securely and delete the Secret when you're comfortable:" echo " kubectl -n ${VAULT_NS} delete secret ${SECRET_NAME}"