Homelabv4/infrastructure/vault/manifests/vault-init-configmap.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: vault-init-script
  namespace: vault
data:
  vault-init-and-store.sh: |
    #!/bin/bash
    set -euo pipefail

    # === Vault Auto-Init and Store Script (ArgoCD Job Version) ===
    # Adapted from Homelabv5/scripts/vault-init-and-store.sh
    # Runs inside a Job pod with kubectl access via ServiceAccount.
    # Uses kubectl exec to interact with the vault-0 pod directly
    # instead of port-forwarding + local vault CLI.

    VAULT_NS="${VAULT_NS:-vault}"
    SECRET_NAME="${SECRET_NAME:-vault-init-keys}"
    KV_MOUNT="${KV_MOUNT:-secret}"
    JOB_NS_FOR_ROLE="${JOB_NS_FOR_ROLE:-auth-proxy}"
    JOB_SA_FOR_ROLE="${JOB_SA_FOR_ROLE:-oauth2-bootstrap}"
    POLICY_NAME="${POLICY_NAME:-oauth2-writer}"
    ROLE_NAME="${ROLE_NAME:-eso-writer}"

    VAULT_POD="vault-0"

    # Helper: run vault CLI inside the vault pod
    vault_exec() {
      # -i forwards stdin - needed for the heredoc-piped `vault policy write
      # NAME -` call below. Without it, `kubectl exec` never passes stdin
      # through and vault sees an empty policy body. This was unreachable
      # until the init-detection fix above (the script always failed earlier,
      # every run, before ever getting this far).
      kubectl -n "$VAULT_NS" exec -i "$VAULT_POD" -- env VAULT_ADDR=http://127.0.0.1:8200 vault "$@"
    }

    # --- Wait for vault-0 pod to be ready ---
    echo "==> Waiting for vault-0 pod to be running..."
    for i in $(seq 1 60); do
      PHASE=$(kubectl -n "$VAULT_NS" get pod "$VAULT_POD" -o jsonpath='{.status.phase}' 2>/dev/null || echo "")
      if [ "$PHASE" = "Running" ]; then
        echo "  vault-0 is running"
        break
      fi
      echo "  waiting for vault-0... (attempt $i/60, phase=$PHASE)"
      sleep 5
    done

    # Give Vault a moment to start its listener
    sleep 5

    # --- Check init/seal status ---
    # `vault status` legitimately exits non-zero for normal states (2 = sealed,
    # still prints valid JSON) as well as real failures (1 = can't connect, no
    # output). The old `|| echo '{}'` fallback couldn't tell those apart - it
    # discarded valid "already initialized, just sealed" JSON on exit 2 too,
    # which made this script wrongly conclude "not initialized" and attempt to
    # re-init an already-initialized Vault (always fails: "Vault is already
    # initialized"). Only fall back to {} when there's truly no output.
    echo "==> Checking Vault status..."
    status_json="$(vault_exec status -format=json 2>/dev/null)"
    if [ -z "$status_json" ]; then
      status_json="{}"
    fi
    # jq, not grep/cut - `vault status -format=json` pretty-prints with a space
    # after each colon ("initialized": true), which the old grep -o
    # '"initialized":[a-z]*' pattern never matched (no space in the pattern) -
    # initialized/sealed silently parsed as empty strings on every run
    # regardless of actual Vault state, which is the real reason this script
    # always tried to re-init an already-initialized Vault.
    initialized="$(echo "$status_json" | jq -r '.initialized // empty')"
    sealed="$(echo "$status_json" | jq -r '.sealed // empty')"

    root_token=""
    unseal_key=""

    # Pull existing secret if present
    if kubectl -n "$VAULT_NS" get secret "$SECRET_NAME" >/dev/null 2>&1; then
      root_token="$(kubectl -n "$VAULT_NS" get secret "$SECRET_NAME" -o jsonpath='{.data.VAULT_ROOT_TOKEN}' | base64 -d || true)"
      unseal_key="$(kubectl -n "$VAULT_NS" get secret "$SECRET_NAME" -o jsonpath='{.data.VAULT_UNSEAL_KEY}' | base64 -d || true)"
    fi

    # Initialize if needed
    if [ "$initialized" != "true" ]; then
      echo "==> Vault not initialized; initializing..."
      init_json="$(vault_exec operator init -key-shares=1 -key-threshold=1 -format=json)"
      root_token="$(echo "$init_json" | grep -o '"root_token":"[^"]*"' | cut -d'"' -f4)"
      unseal_key="$(echo "$init_json" | grep -o '"unseal_keys_b64":\["[^"]*"\]' | grep -o '\["[^"]*"\]' | tr -d '[]"')"
      sealed="true"

      echo "==> Storing root token & unseal key in Secret ${VAULT_NS}/${SECRET_NAME}"
      kubectl -n "$VAULT_NS" create secret generic "$SECRET_NAME" \
        --from-literal=VAULT_ROOT_TOKEN="$root_token" \
        --from-literal=VAULT_UNSEAL_KEY="$unseal_key" \
        --dry-run=client -o yaml | kubectl apply -f -
    else
      echo "==> Vault already initialized."
    fi

    # Unseal if needed
    if [ "$sealed" = "true" ]; then
      if [ -z "$unseal_key" ]; then
        echo "ERROR: Vault is sealed and no unseal key available"
        exit 1
      fi
      echo "==> Unsealing..."
      vault_exec operator unseal "$unseal_key" >/dev/null
    fi

    # Login
    if [ -z "$root_token" ]; then
      echo "==> Reading root token from Secret..."
      root_token="$(kubectl -n "$VAULT_NS" get secret "$SECRET_NAME" -o jsonpath='{.data.VAULT_ROOT_TOKEN}' | base64 -d)"
    fi
    vault_exec login "$root_token" >/dev/null

    # Ensure KV v2 enabled
    if ! vault_exec secrets list -format=json 2>/dev/null | grep -q "\"${KV_MOUNT}/\""; then
      echo "==> Enabling KV v2 at ${KV_MOUNT}/"
      vault_exec secrets enable -path="$KV_MOUNT" -version=2 kv >/dev/null
    fi

    # Configure Kubernetes auth using Vault's own pod identity as the
    # TokenReview reviewer, rather than a manually-minted static token.
    #
    # BUG (found 2026-08-17, live cluster ~2hrs after boot): this used to do
    # `kubectl create token vault-auth` with no --duration, which defaults to
    # a 1-hour TTL, then wrote that JWT into auth/kubernetes/config as a
    # static token_reviewer_jwt. ~1hr after every cluster boot / hook rerun,
    # that token silently expired, so Vault's TokenReview calls (used by
    # EVERY kubernetes-auth login, including ESO's) started failing k8s-side
    # with 401 - which Vault surfaces to callers as a generic, unhelpful
    # "permission denied" 403 on /auth/kubernetes/login, with nothing logged
    # at INFO/ERROR. This cascaded into ClusterSecretStore vault-backend
    # going InvalidProviderConfig and every ExternalSecret in the cluster
    # failing to sync - the real cause behind a broad ArgoCD "Degraded" wave
    # that had nothing to do with Vault's seal state (which was fine).
    #
    # Fix: leave token_reviewer_jwt unset (explicitly cleared below).
    # disable_local_ca_jwt defaults to false, so Vault falls back to reading
    # its own pod's projected SA token from disk on every TokenReview call -
    # that token is auto-refreshed by kubelet for the life of the pod, so
    # there's nothing to expire. The vault pods' own SA ("vault", not
    # "vault-auth") already carries system:auth-delegator via the existing
    # vault-server-binding ClusterRoleBinding, so no separate reviewer SA is
    # needed at all - the vault-auth SA/binding below is now unused, kept
    # only so an old cluster doesn't need manual cleanup.
    kube_ca="$(kubectl -n kube-system get configmap kube-root-ca.crt -o jsonpath='{.data.ca\.crt}')"
    kube_host="https://kubernetes.default.svc:443"

    vault_exec auth enable kubernetes >/dev/null 2>&1 || true
    vault_exec write auth/kubernetes/config \
      token_reviewer_jwt="" \
      kubernetes_host="$kube_host" \
      kubernetes_ca_cert="$kube_ca" \
      disable_local_ca_jwt=false >/dev/null

    # Policy + role for ESO/oauth2 job
    vault_exec policy write "$POLICY_NAME" - >/dev/null <<'HCL'
    path "secret/data/*" {
      capabilities = ["create", "update", "read", "list"]
    }

    path "secret/metadata/*" {
      capabilities = ["create", "update", "read", "list"]
    }
    HCL

    vault_exec write "auth/kubernetes/role/${ROLE_NAME}" \
      bound_service_account_names="${JOB_SA_FOR_ROLE},external-secrets" \
      bound_service_account_namespaces="${JOB_NS_FOR_ROLE},external-secrets" \
      policies="${POLICY_NAME}" \
      ttl="24h" >/dev/null

    # --- OIDC Auth (Authentik SSO) ---
    OIDC_CLIENT_ID="9816a5ae7e7914b5d18f4ab939d011a98f8c8d6b3bb6777c46431afa06ac4a85"
    OIDC_CLIENT_SECRET="ed2ba1c6378c7a46341b5162f39a7fab80e37596b01ed387c3719e8e0040344cf1daa307476c2e7a7f75041b3979275b1ebf00bb8bad94c864b4a38ded544f7b"
    OIDC_DISCOVERY_URL="https://auth.kube.huskypup.net/application/o/vault/"

    echo "==> Configuring OIDC auth (Authentik)..."
    vault_exec auth enable oidc >/dev/null 2>&1 || true

    vault_exec write auth/oidc/config \
      oidc_discovery_url="$OIDC_DISCOVERY_URL" \
      oidc_client_id="$OIDC_CLIENT_ID" \
      oidc_client_secret="$OIDC_CLIENT_SECRET" \
      default_role="default" >/dev/null

    vault_exec policy write vault-admin - >/dev/null <<'HCL'
    path "*" {
      capabilities = ["create", "read", "update", "delete", "list", "sudo"]
    }
    HCL

    vault_exec write auth/oidc/role/default \
      user_claim="sub" \
      allowed_redirect_uris="https://vault.kube.huskypup.net/ui/vault/auth/oidc/oidc/callback,http://localhost:8250/oidc/callback" \
      policies="vault-admin" \
      oidc_scopes="openid,email,profile" \
      token_ttl="1h" >/dev/null

    echo "==> Done."
    echo "K8s Secret with init creds: ${VAULT_NS}/${SECRET_NAME}"
    echo "IMPORTANT: back these up securely and delete the Secret when you're comfortable:"
    echo "  kubectl -n ${VAULT_NS} delete secret ${SECRET_NAME}"