#!/bin/bash # scripts/gitlab-bootstrap.sh # GitLab presync bootstrap script - fully automated, no manual interaction required # Auto-creates OAuth provider in Authentik and syncs credentials to GitLab set -euo pipefail echo "=== GitLab Bootstrap - Fully Automated with Authentik Integration ===" echo "Setting up GitLab infrastructure with auto-generated secrets..." # Change to the apps directory for relative paths cd "$(dirname "$0")/../apps" || exit 1 # Ensure namespace exists kubectl get ns gitlab >/dev/null 2>&1 || kubectl create ns gitlab # ============================================================================ # Step 1: Ensure Authentik has GitLab OAuth provider # ============================================================================ echo "" echo "Step 1: Configuring Authentik OAuth provider for GitLab..." # Apply the Authentik blueprint (if not already applied) kubectl apply -f ../infrastructure/authentik/gitlab-blueprint.yaml 2>/dev/null || true # Give Authentik time to process the blueprint (it auto-discovers ConfigMaps with the label) echo "Waiting for Authentik to process GitLab blueprint..." sleep 10 # ============================================================================ # Step 2: Deploy PostgreSQL clusters # ============================================================================ echo "" echo "Step 2: Deploying PostgreSQL clusters..." # Apply CloudNativePG PostgreSQL clusters # NOTE: CNPG auto-generates database passwords in secrets like pg-gitlab-app echo "Applying GitLab PostgreSQL CNPG cluster..." kubectl apply -f gitlab/cnpg-cluster.yaml echo "Applying Praefect PostgreSQL CNPG cluster..." kubectl apply -f gitlab/praefect-cnpg-cluster.yaml # Apply PgBouncer poolers echo "Applying PgBouncer connection poolers..." kubectl apply -f gitlab/pgbouncer-pooler.yaml # Wait for PostgreSQL clusters to be ready echo "Waiting for PostgreSQL clusters to be ready..." for i in {1..60}; do READY=$(kubectl -n gitlab get cluster pg-gitlab -o jsonpath='{.status.instances}' 2>/dev/null || echo "0") if [ "$READY" -ge "1" ]; then echo "GitLab PostgreSQL cluster has $READY instance(s) ready!" break fi sleep 5 done for i in {1..60}; do READY=$(kubectl -n gitlab get cluster pg-praefect -o jsonpath='{.status.instances}' 2>/dev/null || echo "0") if [ "$READY" -ge "1" ]; then echo "Praefect PostgreSQL cluster has $READY instance(s) ready!" break fi sleep 5 done # ============================================================================ # Step 3: Deploy Redis # ============================================================================ echo "" echo "Step 3: Deploying Redis..." # Apply Redis standalone instance and auto-restart automation echo "Applying GitLab Redis standalone instance..." kubectl apply -f gitlab/redis-cluster.yaml echo "Applying GitLab Redis auto-restart automation..." kubectl apply -f gitlab/redis-auto-restart.yaml echo "Applying GitLab PostgreSQL auto-restart automation..." kubectl apply -f gitlab/pg-auto-restart.yaml # Verify auto-restart CronJobs were created echo "Verifying auto-restart CronJobs..." for i in {1..10}; do REDIS_OK=false PG_GITLAB_OK=false PG_PRAEFECT_OK=false if kubectl get cronjob -n gitlab redis-secret-monitor >/dev/null 2>&1; then REDIS_OK=true fi if kubectl get cronjob -n gitlab pg-gitlab-secret-monitor >/dev/null 2>&1; then PG_GITLAB_OK=true fi if kubectl get cronjob -n gitlab pg-praefect-secret-monitor >/dev/null 2>&1; then PG_PRAEFECT_OK=true fi if $REDIS_OK && $PG_GITLAB_OK && $PG_PRAEFECT_OK; then echo "✓ All auto-restart CronJobs are deployed!" break fi echo " waiting for CronJobs to be created... (attempt $i/10)" sleep 1 done # Apply Redis Sentinel HA cluster echo "Applying GitLab Redis Sentinel HA..." kubectl apply -f gitlab/redis-sentinel-ha.yaml # Wait for Redis to be ready echo "Waiting for Redis Sentinel cluster to be ready..." kubectl wait --for=condition=ready pod -n gitlab -l app=redis-gitlab-ha --timeout=120s 2>/dev/null || echo "Redis may still be starting..." # ============================================================================ # Step 4: Set up PgBouncer authentication (SCRAM-SHA-256) # ============================================================================ echo "" echo "Step 4: Setting up PgBouncer authentication with SCRAM-SHA-256..." # Get the primary PostgreSQL pod (read-write) PRIMARY_POD=$(kubectl get pod -n gitlab -l cnpg.io/cluster=pg-gitlab,role=primary -o jsonpath='{.items[0].metadata.name}') if [ -z "$PRIMARY_POD" ]; then echo "❌ ERROR: Could not find primary PostgreSQL pod" exit 1 fi echo "Using primary pod: $PRIMARY_POD" # Create table for password hashes (needed for PgBouncer SCRAM-SHA-256 auth) echo "Creating user passwords table for PgBouncer..." kubectl exec -n gitlab "$PRIMARY_POD" -- psql -U postgres -c "CREATE TABLE IF NOT EXISTS public.user_passwords (usename text primary key, passwd text);" # Create user_search function to return password hashes from the table echo "Creating user_search function for SCRAM-SHA-256 authentication..." kubectl exec -n gitlab "$PRIMARY_POD" -- psql -U postgres -c "DROP FUNCTION IF EXISTS public.user_search(text);" kubectl exec -n gitlab "$PRIMARY_POD" -- psql -U postgres -c "CREATE FUNCTION public.user_search(uname text) RETURNS TABLE(usename text, passwd text) AS \$\$ SELECT usename, passwd FROM public.user_passwords WHERE usename = \$1; \$\$ LANGUAGE sql SECURITY DEFINER;" # Wait for PgBouncer pooler to create its role echo "Waiting for PgBouncer pooler pods to be ready..." for i in {1..30}; do POOLER_READY=$(kubectl get pods -n gitlab -l cnpg.io/poolerName=pgbouncer-gitlab --no-headers 2>/dev/null | grep -c Running || echo "0") if [ "$POOLER_READY" -ge "1" ]; then echo "✅ PgBouncer pooler pods are running" break fi echo " waiting for pooler pods... (attempt $i/30)" sleep 2 done # Grant permissions to PgBouncer auth user (create role if it doesn't exist) echo "Granting permissions to PgBouncer auth user..." kubectl exec -n gitlab "$PRIMARY_POD" -- psql -U postgres -c "DO \$\$ BEGIN IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'cnpg_pooler_pgbouncer') THEN CREATE ROLE cnpg_pooler_pgbouncer LOGIN; END IF; END \$\$;" 2>/dev/null || echo " ↳ Role may already exist" kubectl exec -n gitlab "$PRIMARY_POD" -- psql -U postgres -c "GRANT EXECUTE ON FUNCTION public.user_search(text) TO cnpg_pooler_pgbouncer;" 2>/dev/null || echo " ↳ Permission may already be granted" # Sync password hash from pg_authid to user_passwords table # This ensures PgBouncer can authenticate using SCRAM-SHA-256 echo "Syncing password hash from pg_authid to user_passwords table..." kubectl exec -n gitlab "$PRIMARY_POD" -- psql -U postgres -c "INSERT INTO public.user_passwords (usename, passwd) SELECT rolname, rolpassword FROM pg_authid WHERE rolname = 'app' ON CONFLICT (usename) DO UPDATE SET passwd = EXCLUDED.passwd;" # Verify the password hash was stored correctly echo "Verifying password hash sync..." HASH_COUNT=$(kubectl exec -n gitlab "$PRIMARY_POD" -- psql -U postgres -t -c "SELECT COUNT(*) FROM public.user_passwords WHERE usename = 'app' AND passwd LIKE 'SCRAM-SHA-256%';" | tr -d ' ') if [ "$HASH_COUNT" = "1" ]; then echo "✅ PgBouncer authentication configured with SCRAM-SHA-256" else echo "⚠️ WARNING: Password hash may not be correctly stored" echo " PgBouncer authentication may fail - check user_passwords table" fi # ============================================================================ # Step 5: Configure OAuth/SAML credentials # ============================================================================ echo "" echo "Step 5: Configuring OAuth and SAML credentials..." # Try to sync OAuth credentials from Authentik to Vault echo "Attempting to sync OAuth credentials from Authentik..." if bash ../scripts/sync-gitlab-oauth.sh 2>&1 | grep -q "Successfully stored"; then echo "✓ Successfully synced OAuth credentials from Authentik to Vault" OAUTH_SYNCED=true else echo "⚠ Could not sync OAuth from Authentik (provider may not be ready yet)" echo " GitLab will use placeholder credentials - run './scripts/sync-gitlab-oauth.sh' later to enable SSO" OAUTH_SYNCED=false fi # Apply External Secrets for GitLab OIDC (will sync from Vault if available) echo "Applying GitLab OIDC External Secrets..." kubectl apply -f gitlab/external-secret.yaml # Apply External Secrets for GitLab SAML (optional) echo "Applying GitLab SAML External Secrets..." kubectl apply -f gitlab/external-secret-saml.yaml 2>/dev/null || true # Wait for External Secret to sync (with timeout) echo "Checking if GitLab OIDC External Secret can sync from Vault..." OIDC_SYNCED=false for i in {1..10}; do STATUS=$(kubectl -n gitlab get externalsecret gitlab-oidc -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "False") if [ "$STATUS" = "True" ]; then echo "✓ GitLab OIDC External Secret synced from Vault!" OIDC_SYNCED=true break fi sleep 2 done # Create placeholder OAuth credentials in Vault if they don't exist echo "Checking if OAuth credentials exist in Vault..." VAULT_POD=$(kubectl get pods -n vault -l app.kubernetes.io/name=vault -o jsonpath='{.items[0].metadata.name}') VAULT_TOKEN=$(kubectl get secret -n vault vault-init-keys -o jsonpath='{.data.VAULT_ROOT_TOKEN}' | base64 -d) if ! kubectl exec -n vault "$VAULT_POD" -- env VAULT_TOKEN="$VAULT_TOKEN" vault kv get secret/gitlab-oauth >/dev/null 2>&1; then echo "⚠ Vault doesn't have gitlab-oauth credentials yet. Creating placeholders..." PLACEHOLDER_SECRET=$(openssl rand -hex 32) kubectl exec -n vault "$VAULT_POD" -- env VAULT_TOKEN="$VAULT_TOKEN" vault kv put secret/gitlab-oauth \ client-id="placeholder-gitlab-client-id" \ client-secret="$PLACEHOLDER_SECRET" echo "✓ Placeholder OAuth credentials created in Vault" echo " Update with real credentials: ./scripts/sync-gitlab-oauth.sh" else echo "✓ GitLab OAuth credentials exist in Vault" fi # Wait for external secret to sync the placeholder/real credentials echo "Waiting for GitLab OIDC External Secret to sync..." for i in {1..30}; do STATUS=$(kubectl -n gitlab get externalsecret gitlab-oidc -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "False") if [ "$STATUS" = "True" ]; then echo "✓ GitLab OIDC External Secret synced successfully" break fi sleep 2 done # Check if SAML secret can sync from Vault echo "Checking if GitLab SAML External Secret can sync from Vault..." SAML_SYNCED=false for i in {1..10}; do STATUS=$(kubectl -n gitlab get externalsecret gitlab-saml -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "False") if [ "$STATUS" = "True" ]; then echo "✓ GitLab SAML External Secret synced from Vault!" SAML_SYNCED=true break fi sleep 2 done # Create placeholder SAML credentials in Vault if they don't exist echo "Checking if SAML credentials exist in Vault..." if ! kubectl exec -n vault "$VAULT_POD" -- env VAULT_TOKEN="$VAULT_TOKEN" vault kv get secret/gitlab/saml >/dev/null 2>&1; then echo "⚠ Vault doesn't have gitlab/saml credentials yet. Creating placeholders..." kubectl exec -n vault "$VAULT_POD" -- env VAULT_TOKEN="$VAULT_TOKEN" vault kv put secret/gitlab/saml \ idp_sso_url="https://auth.kube.huskypup.net/application/saml/gitlab/sso/binding/redirect/" \ idp_fingerprint="00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00" echo "✓ Placeholder SAML credentials created in Vault" echo " Update with real credentials: ./scripts/gitlab-saml-bootstrap.sh" else echo "✓ GitLab SAML credentials exist in Vault" fi # Wait for external secret to sync echo "Waiting for GitLab SAML External Secret to sync..." for i in {1..30}; do STATUS=$(kubectl -n gitlab get externalsecret gitlab-saml -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "False") if [ "$STATUS" = "True" ]; then echo "✓ GitLab SAML External Secret synced successfully" break fi sleep 2 done # Force password synchronization to prevent authentication issues echo "=== Synchronizing Database and Redis Passwords ===" # Delete and recreate database secrets to ensure sync echo "Refreshing database secrets..." kubectl delete secret pg-gitlab-app -n gitlab --ignore-not-found=true kubectl delete secret pg-praefect-app -n gitlab --ignore-not-found=true # Wait for ExternalSecrets to recreate secrets echo "Waiting for database secrets to be recreated..." for i in {1..30}; do if kubectl -n gitlab get secret pg-gitlab-app >/dev/null 2>&1 && \ kubectl -n gitlab get secret pg-praefect-app >/dev/null 2>&1; then echo "Database secrets recreated successfully!" break fi echo " waiting for database secrets... (attempt $i/30)" sleep 2 done # Delete and recreate Redis secret to ensure sync echo "Refreshing Redis secrets..." kubectl delete secret redis-gitlab-secret -n gitlab --ignore-not-found=true # Wait for Redis ExternalSecret to recreate secret echo "Waiting for Redis secret to be recreated..." for i in {1..30}; do if kubectl -n gitlab get secret redis-gitlab-secret >/dev/null 2>&1; then echo "Redis secret recreated successfully!" break fi echo " waiting for Redis secret... (attempt $i/30)" sleep 2 done # Trigger PostgreSQL reload to pick up new passwords echo "Triggering PostgreSQL cluster reload..." kubectl annotate cluster pg-gitlab -n gitlab cnpg.io/reload=$(date +%s) --overwrite kubectl annotate cluster pg-praefect -n gitlab cnpg.io/reload=$(date +%s) --overwrite echo "" echo "====================================================================" echo "GitLab Bootstrap Summary - Fully Automated" echo "====================================================================" echo "✓ Authentik GitLab OAuth blueprint applied" echo "✓ PostgreSQL CNPG clusters (GitLab + Praefect) deployed" echo "✓ PgBouncer connection poolers configured with authentication" echo "✓ Redis standalone instance deployed" echo "✓ Redis auto-restart automation (CronJob) deployed" echo "✓ Redis Sentinel HA cluster deployed" if [ "${OAUTH_SYNCED:-false}" = "true" ]; then echo "✓ GitLab OIDC credentials synced from Authentik" else echo "⚠ GitLab OIDC using placeholder (Authentik provider not ready)" fi echo "✓ GitLab SAML secret configured (placeholder)" echo "✓ Database and Redis passwords synchronized" echo "" echo "Password Rotation: Automated via CronJob (hourly checks)" echo "Next password rotation: $(kubectl get externalsecret -n gitlab gitlab-redis-password -o jsonpath='{.status.refreshTime}' 2>/dev/null || echo 'Unknown') + 24h" echo "====================================================================" echo "" echo "🎉 GitLab bootstrap completed - NO MANUAL STEPS REQUIRED!" echo "" echo "GitLab will be accessible at: https://gitlab.kube.huskypup.net" echo "" if [ "${OAUTH_SYNCED:-false}" = "false" ]; then echo "To enable Authentik SSO (optional):" echo " 1. Wait for Authentik to process the GitLab blueprint (~5 min)" echo " 2. Run: ./scripts/sync-gitlab-oauth.sh" echo " 3. Restart GitLab pods to pick up real credentials" echo "" fi echo "For SAML setup (optional): ./scripts/gitlab-saml-bootstrap.sh" echo ""