mirror of
https://gitlab.kube.huskypup.net/Scooby/Homelabv4.git
synced 2026-08-21 11:36:50 +00:00
Scooby HuskyandClaude Sonnet 5
d9a4c16481
Completes the 'make the repo proper' cleanup from earlier this session -
these were flagged but deliberately not touched in 185e9c2 given the
blast radius (live SSO for 8 apps). User confirmed: fix all 8 now.
infrastructure/authentik/{argocd,gitlab,nextcloud,grafana,n8n,guacamole,
rancher,vault}-blueprint.yaml: converted from plain ConfigMap (client_secret
hardcoded) to ExternalSecret with a templated blueprint body
(client_secret: "{{ .clientSecret }}") pulling from Vault. Chart already
supports mounting blueprints from Secrets (blueprints.secrets, alongside
blueprints.configMaps) - infrastructure/authentik/values.yaml updated to
route these 8 there instead.
For argocd/nextcloud/n8n/guacamole/rancher: Vault already had the matching
value at secret/<app>-oauth (the APP side was already Vault-backed via its
own ExternalSecret) - the blueprint was the only remaining plaintext copy.
For gitlab/grafana/vault: Vault had no copy at all yet - created
secret/{gitlab,grafana,vault}-oauth with the EXISTING live values (not
rotated - these are the actual working credentials right now, rotating
would break login until every consumer is updated in lockstep, which is
out of scope for a cleanup pass). Also fixed the OTHER plaintext copies
that existed for these three specifically:
- apps/gitlab/manifests/external-secret-oidc.yaml (new): replaces a
manually kubectl-created, never-git-tracked gitlab-oidc-secret.
- infrastructure/grafana/manifests/grafana-oauth-secret.yaml: was a
plain Secret whose own comment said 'hardcoded from blueprint'.
- infrastructure/vault/manifests/vault-init-{configmap,job}.yaml: this
one COULDN'T be converted to the same ExternalSecret-from-Vault
pattern - it's the PostSync Job that grants ESO's own Kubernetes-auth
role in Vault, so ESO can't yet authenticate to pull anything from
Vault at the point this script runs (genuinely circular). Sourced
from a new vault-oidc-bootstrap Secret instead - created once
manually (kubectl, not git, matching how Vault's own root/unseal
material is already handled), independent of the ESO pipeline.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
220 lines
10 KiB
YAML
220 lines
10 KiB
YAML
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: vault-init-script
|
|
namespace: vault
|
|
data:
|
|
vault-init-and-store.sh: |
|
|
#!/bin/bash
|
|
set -euo pipefail
|
|
|
|
# === Vault Auto-Init and Store Script (ArgoCD Job Version) ===
|
|
# Adapted from Homelabv5/scripts/vault-init-and-store.sh
|
|
# Runs inside a Job pod with kubectl access via ServiceAccount.
|
|
# Uses kubectl exec to interact with the vault-0 pod directly
|
|
# instead of port-forwarding + local vault CLI.
|
|
|
|
VAULT_NS="${VAULT_NS:-vault}"
|
|
SECRET_NAME="${SECRET_NAME:-vault-init-keys}"
|
|
KV_MOUNT="${KV_MOUNT:-secret}"
|
|
JOB_NS_FOR_ROLE="${JOB_NS_FOR_ROLE:-auth-proxy}"
|
|
JOB_SA_FOR_ROLE="${JOB_SA_FOR_ROLE:-oauth2-bootstrap}"
|
|
POLICY_NAME="${POLICY_NAME:-oauth2-writer}"
|
|
ROLE_NAME="${ROLE_NAME:-eso-writer}"
|
|
|
|
VAULT_POD="vault-0"
|
|
|
|
# Helper: run vault CLI inside the vault pod
|
|
vault_exec() {
|
|
# -i forwards stdin - needed for the heredoc-piped `vault policy write
|
|
# NAME -` call below. Without it, `kubectl exec` never passes stdin
|
|
# through and vault sees an empty policy body. This was unreachable
|
|
# until the init-detection fix above (the script always failed earlier,
|
|
# every run, before ever getting this far).
|
|
kubectl -n "$VAULT_NS" exec -i "$VAULT_POD" -- env VAULT_ADDR=http://127.0.0.1:8200 vault "$@"
|
|
}
|
|
|
|
# --- Wait for vault-0 pod to be ready ---
|
|
echo "==> Waiting for vault-0 pod to be running..."
|
|
for i in $(seq 1 60); do
|
|
PHASE=$(kubectl -n "$VAULT_NS" get pod "$VAULT_POD" -o jsonpath='{.status.phase}' 2>/dev/null || echo "")
|
|
if [ "$PHASE" = "Running" ]; then
|
|
echo " vault-0 is running"
|
|
break
|
|
fi
|
|
echo " waiting for vault-0... (attempt $i/60, phase=$PHASE)"
|
|
sleep 5
|
|
done
|
|
|
|
# Give Vault a moment to start its listener
|
|
sleep 5
|
|
|
|
# --- Check init/seal status ---
|
|
# `vault status` legitimately exits non-zero for normal states (2 = sealed,
|
|
# still prints valid JSON) as well as real failures (1 = can't connect, no
|
|
# output). The old `|| echo '{}'` fallback couldn't tell those apart - it
|
|
# discarded valid "already initialized, just sealed" JSON on exit 2 too,
|
|
# which made this script wrongly conclude "not initialized" and attempt to
|
|
# re-init an already-initialized Vault (always fails: "Vault is already
|
|
# initialized"). Only fall back to {} when there's truly no output.
|
|
echo "==> Checking Vault status..."
|
|
status_json="$(vault_exec status -format=json 2>/dev/null)"
|
|
if [ -z "$status_json" ]; then
|
|
status_json="{}"
|
|
fi
|
|
# jq, not grep/cut - `vault status -format=json` pretty-prints with a space
|
|
# after each colon ("initialized": true), which the old grep -o
|
|
# '"initialized":[a-z]*' pattern never matched (no space in the pattern) -
|
|
# initialized/sealed silently parsed as empty strings on every run
|
|
# regardless of actual Vault state, which is the real reason this script
|
|
# always tried to re-init an already-initialized Vault.
|
|
initialized="$(echo "$status_json" | jq -r '.initialized // empty')"
|
|
sealed="$(echo "$status_json" | jq -r '.sealed // empty')"
|
|
|
|
root_token=""
|
|
unseal_key=""
|
|
|
|
# Pull existing secret if present
|
|
if kubectl -n "$VAULT_NS" get secret "$SECRET_NAME" >/dev/null 2>&1; then
|
|
root_token="$(kubectl -n "$VAULT_NS" get secret "$SECRET_NAME" -o jsonpath='{.data.VAULT_ROOT_TOKEN}' | base64 -d || true)"
|
|
unseal_key="$(kubectl -n "$VAULT_NS" get secret "$SECRET_NAME" -o jsonpath='{.data.VAULT_UNSEAL_KEY}' | base64 -d || true)"
|
|
fi
|
|
|
|
# Initialize if needed
|
|
if [ "$initialized" != "true" ]; then
|
|
echo "==> Vault not initialized; initializing..."
|
|
init_json="$(vault_exec operator init -key-shares=1 -key-threshold=1 -format=json)"
|
|
root_token="$(echo "$init_json" | grep -o '"root_token":"[^"]*"' | cut -d'"' -f4)"
|
|
unseal_key="$(echo "$init_json" | grep -o '"unseal_keys_b64":\["[^"]*"\]' | grep -o '\["[^"]*"\]' | tr -d '[]"')"
|
|
sealed="true"
|
|
|
|
echo "==> Storing root token & unseal key in Secret ${VAULT_NS}/${SECRET_NAME}"
|
|
kubectl -n "$VAULT_NS" create secret generic "$SECRET_NAME" \
|
|
--from-literal=VAULT_ROOT_TOKEN="$root_token" \
|
|
--from-literal=VAULT_UNSEAL_KEY="$unseal_key" \
|
|
--dry-run=client -o yaml | kubectl apply -f -
|
|
else
|
|
echo "==> Vault already initialized."
|
|
fi
|
|
|
|
# Unseal if needed
|
|
if [ "$sealed" = "true" ]; then
|
|
if [ -z "$unseal_key" ]; then
|
|
echo "ERROR: Vault is sealed and no unseal key available"
|
|
exit 1
|
|
fi
|
|
echo "==> Unsealing..."
|
|
vault_exec operator unseal "$unseal_key" >/dev/null
|
|
fi
|
|
|
|
# Login
|
|
if [ -z "$root_token" ]; then
|
|
echo "==> Reading root token from Secret..."
|
|
root_token="$(kubectl -n "$VAULT_NS" get secret "$SECRET_NAME" -o jsonpath='{.data.VAULT_ROOT_TOKEN}' | base64 -d)"
|
|
fi
|
|
vault_exec login "$root_token" >/dev/null
|
|
|
|
# Ensure KV v2 enabled
|
|
if ! vault_exec secrets list -format=json 2>/dev/null | grep -q "\"${KV_MOUNT}/\""; then
|
|
echo "==> Enabling KV v2 at ${KV_MOUNT}/"
|
|
vault_exec secrets enable -path="$KV_MOUNT" -version=2 kv >/dev/null
|
|
fi
|
|
|
|
# Configure Kubernetes auth using Vault's own pod identity as the
|
|
# TokenReview reviewer, rather than a manually-minted static token.
|
|
#
|
|
# BUG (found 2026-08-17, live cluster ~2hrs after boot): this used to do
|
|
# `kubectl create token vault-auth` with no --duration, which defaults to
|
|
# a 1-hour TTL, then wrote that JWT into auth/kubernetes/config as a
|
|
# static token_reviewer_jwt. ~1hr after every cluster boot / hook rerun,
|
|
# that token silently expired, so Vault's TokenReview calls (used by
|
|
# EVERY kubernetes-auth login, including ESO's) started failing k8s-side
|
|
# with 401 - which Vault surfaces to callers as a generic, unhelpful
|
|
# "permission denied" 403 on /auth/kubernetes/login, with nothing logged
|
|
# at INFO/ERROR. This cascaded into ClusterSecretStore vault-backend
|
|
# going InvalidProviderConfig and every ExternalSecret in the cluster
|
|
# failing to sync - the real cause behind a broad ArgoCD "Degraded" wave
|
|
# that had nothing to do with Vault's seal state (which was fine).
|
|
#
|
|
# Fix: leave token_reviewer_jwt unset (explicitly cleared below).
|
|
# disable_local_ca_jwt defaults to false, so Vault falls back to reading
|
|
# its own pod's projected SA token from disk on every TokenReview call -
|
|
# that token is auto-refreshed by kubelet for the life of the pod, so
|
|
# there's nothing to expire. The vault pods' own SA ("vault", not
|
|
# "vault-auth") already carries system:auth-delegator via the existing
|
|
# vault-server-binding ClusterRoleBinding, so no separate reviewer SA is
|
|
# needed at all - the vault-auth SA/binding below is now unused, kept
|
|
# only so an old cluster doesn't need manual cleanup.
|
|
kube_ca="$(kubectl -n kube-system get configmap kube-root-ca.crt -o jsonpath='{.data.ca\.crt}')"
|
|
kube_host="https://kubernetes.default.svc:443"
|
|
|
|
vault_exec auth enable kubernetes >/dev/null 2>&1 || true
|
|
vault_exec write auth/kubernetes/config \
|
|
token_reviewer_jwt="" \
|
|
kubernetes_host="$kube_host" \
|
|
kubernetes_ca_cert="$kube_ca" \
|
|
disable_local_ca_jwt=false >/dev/null
|
|
|
|
# Policy + role for ESO/oauth2 job
|
|
vault_exec policy write "$POLICY_NAME" - >/dev/null <<'HCL'
|
|
path "secret/data/*" {
|
|
capabilities = ["create", "update", "read", "list"]
|
|
}
|
|
|
|
path "secret/metadata/*" {
|
|
capabilities = ["create", "update", "read", "list"]
|
|
}
|
|
HCL
|
|
|
|
vault_exec write "auth/kubernetes/role/${ROLE_NAME}" \
|
|
bound_service_account_names="${JOB_SA_FOR_ROLE},external-secrets" \
|
|
bound_service_account_namespaces="${JOB_NS_FOR_ROLE},external-secrets" \
|
|
policies="${POLICY_NAME}" \
|
|
ttl="24h" >/dev/null
|
|
|
|
# --- OIDC Auth (Authentik SSO) ---
|
|
# OIDC_CLIENT_ID/SECRET used to be hardcoded here in plaintext - found
|
|
# and fixed 2026-08-20. Can't source this from Vault itself the way
|
|
# every other app's OIDC secret now does (infrastructure/authentik/
|
|
# vault-blueprint.yaml, secret/vault-oauth) - this job is what GRANTS
|
|
# ESO's own Kubernetes-auth role a few lines above
|
|
# (bound_service_account_names="...,external-secrets"), so ESO can't
|
|
# yet authenticate to Vault at the point this script runs - genuinely
|
|
# circular. Sourced from vault-oidc-bootstrap instead, a plain Secret
|
|
# created once manually (kubectl, not git):
|
|
# kubectl -n vault create secret generic vault-oidc-bootstrap \
|
|
# --from-literal=client_id=<same value as secret/vault-oauth> \
|
|
# --from-literal=client_secret=<same value as secret/vault-oauth>
|
|
# envFrom on the Job (vault-init-job.yaml) injects these as
|
|
# VAULT_OIDC_CLIENT_ID/VAULT_OIDC_CLIENT_SECRET.
|
|
OIDC_CLIENT_ID="${VAULT_OIDC_CLIENT_ID:?VAULT_OIDC_CLIENT_ID not set - see vault-oidc-bootstrap secret}"
|
|
OIDC_CLIENT_SECRET="${VAULT_OIDC_CLIENT_SECRET:?VAULT_OIDC_CLIENT_SECRET not set - see vault-oidc-bootstrap secret}"
|
|
OIDC_DISCOVERY_URL="https://auth.kube.huskypup.net/application/o/vault/"
|
|
|
|
echo "==> Configuring OIDC auth (Authentik)..."
|
|
vault_exec auth enable oidc >/dev/null 2>&1 || true
|
|
|
|
vault_exec write auth/oidc/config \
|
|
oidc_discovery_url="$OIDC_DISCOVERY_URL" \
|
|
oidc_client_id="$OIDC_CLIENT_ID" \
|
|
oidc_client_secret="$OIDC_CLIENT_SECRET" \
|
|
default_role="default" >/dev/null
|
|
|
|
vault_exec policy write vault-admin - >/dev/null <<'HCL'
|
|
path "*" {
|
|
capabilities = ["create", "read", "update", "delete", "list", "sudo"]
|
|
}
|
|
HCL
|
|
|
|
vault_exec write auth/oidc/role/default \
|
|
user_claim="sub" \
|
|
allowed_redirect_uris="https://vault.kube.huskypup.net/ui/vault/auth/oidc/oidc/callback,http://localhost:8250/oidc/callback" \
|
|
policies="vault-admin" \
|
|
oidc_scopes="openid,email,profile" \
|
|
token_ttl="1h" >/dev/null
|
|
|
|
echo "==> Done."
|
|
echo "K8s Secret with init creds: ${VAULT_NS}/${SECRET_NAME}"
|
|
echo "IMPORTANT: back these up securely and delete the Secret when you're comfortable:"
|
|
echo " kubectl -n ${VAULT_NS} delete secret ${SECRET_NAME}"
|