Fix Vault kubernetes-auth reviewer token expiring 1hr after every boot

vault-init-and-store.sh minted the auth/kubernetes/config reviewer JWT via
'kubectl create token vault-auth' with no --duration, defaulting to a 1hr
TTL, then wrote it into Vault as a static token_reviewer_jwt. ~1hr after
every cluster boot/hook rerun it silently expired, breaking TokenReview
for every kubernetes-auth login (including ESO's) with a generic 403 that
logged nothing at INFO/ERROR. This was the actual root cause of today's
broad ArgoCD Degraded wave across ~14 apps - not Vault's seal state, which
was fine the whole time.

Fix: clear token_reviewer_jwt and rely on Vault's local-JWT auto-detection
(disable_local_ca_jwt=false default), which reads the vault pod's own
kubelet-refreshed SA token from disk on every call instead of a static
copy. The vault SA already has system:auth-delegator via the existing
vault-server-binding ClusterRoleBinding.

Applied live directly against Vault to unblock immediately; confirmed
ClusterSecretStore vault-backend flipped to Ready=True and previously
broken ExternalSecrets (guacamole, teslamate, unpoller, netbird x3) all
resynced successfully.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

infrastructure/vault/manifests/vault-init-configmap.yaml

@@ -120,23 +120,40 @@ data:
       vault_exec secrets enable -path="$KV_MOUNT" -version=2 kv >/dev/null
     fi
 
-    # Configure Kubernetes auth with a reviewer token from a local SA
+    # Configure Kubernetes auth using Vault's own pod identity as the
-    echo "==> Ensuring reviewer SA + binding"
+    # TokenReview reviewer, rather than a manually-minted static token.
-    kubectl -n "$VAULT_NS" get sa vault-auth >/dev/null 2>&1 || kubectl -n "$VAULT_NS" create sa vault-auth
+    #
-    kubectl get clusterrolebinding vault-auth-delegator >/dev/null 2>&1 || \
+    # BUG (found 2026-08-17, live cluster ~2hrs after boot): this used to do
-      kubectl create clusterrolebinding vault-auth-delegator \
+    # `kubectl create token vault-auth` with no --duration, which defaults to
-        --clusterrole=system:auth-delegator \
+    # a 1-hour TTL, then wrote that JWT into auth/kubernetes/config as a
-        --serviceaccount="${VAULT_NS}:vault-auth"
+    # static token_reviewer_jwt. ~1hr after every cluster boot / hook rerun,
-
+    # that token silently expired, so Vault's TokenReview calls (used by
-    reviewer_jwt="$(kubectl -n "$VAULT_NS" create token vault-auth)"
+    # EVERY kubernetes-auth login, including ESO's) started failing k8s-side
+    # with 401 - which Vault surfaces to callers as a generic, unhelpful
+    # "permission denied" 403 on /auth/kubernetes/login, with nothing logged
+    # at INFO/ERROR. This cascaded into ClusterSecretStore vault-backend
+    # going InvalidProviderConfig and every ExternalSecret in the cluster
+    # failing to sync - the real cause behind a broad ArgoCD "Degraded" wave
+    # that had nothing to do with Vault's seal state (which was fine).
+    #
+    # Fix: leave token_reviewer_jwt unset (explicitly cleared below).
+    # disable_local_ca_jwt defaults to false, so Vault falls back to reading
+    # its own pod's projected SA token from disk on every TokenReview call -
+    # that token is auto-refreshed by kubelet for the life of the pod, so
+    # there's nothing to expire. The vault pods' own SA ("vault", not
+    # "vault-auth") already carries system:auth-delegator via the existing
+    # vault-server-binding ClusterRoleBinding, so no separate reviewer SA is
+    # needed at all - the vault-auth SA/binding below is now unused, kept
+    # only so an old cluster doesn't need manual cleanup.
     kube_ca="$(kubectl -n kube-system get configmap kube-root-ca.crt -o jsonpath='{.data.ca\.crt}')"
     kube_host="https://kubernetes.default.svc:443"
 
     vault_exec auth enable kubernetes >/dev/null 2>&1 || true
     vault_exec write auth/kubernetes/config \
-      token_reviewer_jwt="$reviewer_jwt" \
+      token_reviewer_jwt="" \
       kubernetes_host="$kube_host" \
-      kubernetes_ca_cert="$kube_ca" >/dev/null
+      kubernetes_ca_cert="$kube_ca" \
+      disable_local_ca_jwt=false >/dev/null
 
     # Policy + role for ESO/oauth2 job
     vault_exec policy write "$POLICY_NAME" - >/dev/null <<'HCL'
@@ -155,6 +172,33 @@ data:
       policies="${POLICY_NAME}" \
       ttl="24h" >/dev/null
 
+    # --- OIDC Auth (Authentik SSO) ---
+    OIDC_CLIENT_ID="9816a5ae7e7914b5d18f4ab939d011a98f8c8d6b3bb6777c46431afa06ac4a85"
+    OIDC_CLIENT_SECRET="ed2ba1c6378c7a46341b5162f39a7fab80e37596b01ed387c3719e8e0040344cf1daa307476c2e7a7f75041b3979275b1ebf00bb8bad94c864b4a38ded544f7b"
+    OIDC_DISCOVERY_URL="https://auth.kube.huskypup.net/application/o/vault/"
+
+    echo "==> Configuring OIDC auth (Authentik)..."
+    vault_exec auth enable oidc >/dev/null 2>&1 || true
+
+    vault_exec write auth/oidc/config \
+      oidc_discovery_url="$OIDC_DISCOVERY_URL" \
+      oidc_client_id="$OIDC_CLIENT_ID" \
+      oidc_client_secret="$OIDC_CLIENT_SECRET" \
+      default_role="default" >/dev/null
+
+    vault_exec policy write vault-admin - >/dev/null <<'HCL'
+    path "*" {
+      capabilities = ["create", "read", "update", "delete", "list", "sudo"]
+    }
+    HCL
+
+    vault_exec write auth/oidc/role/default \
+      user_claim="sub" \
+      allowed_redirect_uris="https://vault.kube.huskypup.net/ui/vault/auth/oidc/oidc/callback,http://localhost:8250/oidc/callback" \
+      policies="vault-admin" \
+      oidc_scopes="openid,email,profile" \
+      token_ttl="1h" >/dev/null
+
     echo "==> Done."
     echo "K8s Secret with init creds: ${VAULT_NS}/${SECRET_NAME}"
     echo "IMPORTANT: back these up securely and delete the Secret when you're comfortable:"