Homelabv4/scripts/gitlab-bootstrap.sh

#!/bin/bash
# scripts/gitlab-bootstrap.sh
# GitLab presync bootstrap script - fully automated, no manual interaction required
# Auto-creates OAuth provider in Authentik and syncs credentials to GitLab

set -euo pipefail

echo "=== GitLab Bootstrap - Fully Automated with Authentik Integration ==="
echo "Setting up GitLab infrastructure with auto-generated secrets..."

# Change to the apps directory for relative paths
cd "$(dirname "$0")/../apps" || exit 1

# Ensure namespace exists
kubectl get ns gitlab >/dev/null 2>&1 || kubectl create ns gitlab

# ============================================================================
# Step 1: Ensure Authentik has GitLab OAuth provider
# ============================================================================
echo ""
echo "Step 1: Configuring Authentik OAuth provider for GitLab..."

# Apply the Authentik blueprint (if not already applied)
kubectl apply -f ../infrastructure/authentik/gitlab-blueprint.yaml 2>/dev/null || true

# Give Authentik time to process the blueprint (it auto-discovers ConfigMaps with the label)
echo "Waiting for Authentik to process GitLab blueprint..."
sleep 10

# ============================================================================
# Step 2: Deploy PostgreSQL clusters
# ============================================================================
echo ""
echo "Step 2: Deploying PostgreSQL clusters..."

# Apply CloudNativePG PostgreSQL clusters
# NOTE: CNPG auto-generates database passwords in secrets like pg-gitlab-app
echo "Applying GitLab PostgreSQL CNPG cluster..."
kubectl apply -f gitlab/cnpg-cluster.yaml

echo "Applying Praefect PostgreSQL CNPG cluster..."
kubectl apply -f gitlab/praefect-cnpg-cluster.yaml

# Apply PgBouncer poolers
echo "Applying PgBouncer connection poolers..."
kubectl apply -f gitlab/pgbouncer-pooler.yaml

# Wait for PostgreSQL clusters to be ready
echo "Waiting for PostgreSQL clusters to be ready..."
for i in {1..60}; do
  READY=$(kubectl -n gitlab get cluster pg-gitlab -o jsonpath='{.status.instances}' 2>/dev/null || echo "0")
  if [ "$READY" -ge "1" ]; then
    echo "GitLab PostgreSQL cluster has $READY instance(s) ready!"
    break
  fi
  sleep 5
done

for i in {1..60}; do
  READY=$(kubectl -n gitlab get cluster pg-praefect -o jsonpath='{.status.instances}' 2>/dev/null || echo "0")
  if [ "$READY" -ge "1" ]; then
    echo "Praefect PostgreSQL cluster has $READY instance(s) ready!"
    break
  fi
  sleep 5
done

# ============================================================================
# Step 3: Deploy Redis
# ============================================================================
echo ""
echo "Step 3: Deploying Redis..."

# Apply Redis standalone instance and auto-restart automation
echo "Applying GitLab Redis standalone instance..."
kubectl apply -f gitlab/redis-cluster.yaml

echo "Applying GitLab Redis auto-restart automation..."
kubectl apply -f gitlab/redis-auto-restart.yaml

echo "Applying GitLab PostgreSQL auto-restart automation..."
kubectl apply -f gitlab/pg-auto-restart.yaml

# Verify auto-restart CronJobs were created
echo "Verifying auto-restart CronJobs..."
for i in {1..10}; do
  REDIS_OK=false
  PG_GITLAB_OK=false
  PG_PRAEFECT_OK=false

  if kubectl get cronjob -n gitlab redis-secret-monitor >/dev/null 2>&1; then
    REDIS_OK=true
  fi
  if kubectl get cronjob -n gitlab pg-gitlab-secret-monitor >/dev/null 2>&1; then
    PG_GITLAB_OK=true
  fi
  if kubectl get cronjob -n gitlab pg-praefect-secret-monitor >/dev/null 2>&1; then
    PG_PRAEFECT_OK=true
  fi

  if $REDIS_OK && $PG_GITLAB_OK && $PG_PRAEFECT_OK; then
    echo "✓ All auto-restart CronJobs are deployed!"
    break
  fi
  echo "  waiting for CronJobs to be created... (attempt $i/10)"
  sleep 1
done

# Apply Redis Sentinel HA cluster
echo "Applying GitLab Redis Sentinel HA..."
kubectl apply -f gitlab/redis-sentinel-ha.yaml

# Wait for Redis to be ready
echo "Waiting for Redis Sentinel cluster to be ready..."
kubectl wait --for=condition=ready pod -n gitlab -l app=redis-gitlab-ha --timeout=120s 2>/dev/null || echo "Redis may still be starting..."

# ============================================================================
# Step 4: Set up PgBouncer authentication (SCRAM-SHA-256)
# ============================================================================
echo ""
echo "Step 4: Setting up PgBouncer authentication with SCRAM-SHA-256..."

# Get the primary PostgreSQL pod (read-write)
PRIMARY_POD=$(kubectl get pod -n gitlab -l cnpg.io/cluster=pg-gitlab,role=primary -o jsonpath='{.items[0].metadata.name}')

if [ -z "$PRIMARY_POD" ]; then
  echo "❌ ERROR: Could not find primary PostgreSQL pod"
  exit 1
fi

echo "Using primary pod: $PRIMARY_POD"

# Create table for password hashes (needed for PgBouncer SCRAM-SHA-256 auth)
echo "Creating user passwords table for PgBouncer..."
kubectl exec -n gitlab "$PRIMARY_POD" -- psql -U postgres -c "CREATE TABLE IF NOT EXISTS public.user_passwords (usename text primary key, passwd text);"

# Create user_search function to return password hashes from the table
echo "Creating user_search function for SCRAM-SHA-256 authentication..."
kubectl exec -n gitlab "$PRIMARY_POD" -- psql -U postgres -c "DROP FUNCTION IF EXISTS public.user_search(text);"
kubectl exec -n gitlab "$PRIMARY_POD" -- psql -U postgres -c "CREATE FUNCTION public.user_search(uname text) RETURNS TABLE(usename text, passwd text) AS \$\$ SELECT usename, passwd FROM public.user_passwords WHERE usename = \$1; \$\$ LANGUAGE sql SECURITY DEFINER;"

# Wait for PgBouncer pooler to create its role
echo "Waiting for PgBouncer pooler pods to be ready..."
for i in {1..30}; do
  POOLER_READY=$(kubectl get pods -n gitlab -l cnpg.io/poolerName=pgbouncer-gitlab --no-headers 2>/dev/null | grep -c Running || echo "0")
  if [ "$POOLER_READY" -ge "1" ]; then
    echo "✅ PgBouncer pooler pods are running"
    break
  fi
  echo "   waiting for pooler pods... (attempt $i/30)"
  sleep 2
done

# Grant permissions to PgBouncer auth user (create role if it doesn't exist)
echo "Granting permissions to PgBouncer auth user..."
kubectl exec -n gitlab "$PRIMARY_POD" -- psql -U postgres -c "DO \$\$ BEGIN IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'cnpg_pooler_pgbouncer') THEN CREATE ROLE cnpg_pooler_pgbouncer LOGIN; END IF; END \$\$;" 2>/dev/null || echo "   ↳ Role may already exist"
kubectl exec -n gitlab "$PRIMARY_POD" -- psql -U postgres -c "GRANT EXECUTE ON FUNCTION public.user_search(text) TO cnpg_pooler_pgbouncer;" 2>/dev/null || echo "   ↳ Permission may already be granted"

# Sync password hash from pg_authid to user_passwords table
# This ensures PgBouncer can authenticate using SCRAM-SHA-256
echo "Syncing password hash from pg_authid to user_passwords table..."
kubectl exec -n gitlab "$PRIMARY_POD" -- psql -U postgres -c "INSERT INTO public.user_passwords (usename, passwd) SELECT rolname, rolpassword FROM pg_authid WHERE rolname = 'app' ON CONFLICT (usename) DO UPDATE SET passwd = EXCLUDED.passwd;"

# Verify the password hash was stored correctly
echo "Verifying password hash sync..."
HASH_COUNT=$(kubectl exec -n gitlab "$PRIMARY_POD" -- psql -U postgres -t -c "SELECT COUNT(*) FROM public.user_passwords WHERE usename = 'app' AND passwd LIKE 'SCRAM-SHA-256%';" | tr -d ' ')

if [ "$HASH_COUNT" = "1" ]; then
  echo "✅ PgBouncer authentication configured with SCRAM-SHA-256"
else
  echo "⚠️  WARNING: Password hash may not be correctly stored"
  echo "   PgBouncer authentication may fail - check user_passwords table"
fi

# ============================================================================
# Step 5: Configure OAuth/SAML credentials
# ============================================================================
echo ""
echo "Step 5: Configuring OAuth and SAML credentials..."

# Try to sync OAuth credentials from Authentik to Vault
echo "Attempting to sync OAuth credentials from Authentik..."
if bash ../scripts/sync-gitlab-oauth.sh 2>&1 | grep -q "Successfully stored"; then
  echo "✓ Successfully synced OAuth credentials from Authentik to Vault"
  OAUTH_SYNCED=true
else
  echo "⚠ Could not sync OAuth from Authentik (provider may not be ready yet)"
  echo "  GitLab will use placeholder credentials - run './scripts/sync-gitlab-oauth.sh' later to enable SSO"
  OAUTH_SYNCED=false
fi

# Apply External Secrets for GitLab OIDC (will sync from Vault if available)
echo "Applying GitLab OIDC External Secrets..."
kubectl apply -f gitlab/external-secret.yaml

# Apply External Secrets for GitLab SAML (optional)
echo "Applying GitLab SAML External Secrets..."
kubectl apply -f gitlab/external-secret-saml.yaml 2>/dev/null || true

# Wait for External Secret to sync (with timeout)
echo "Checking if GitLab OIDC External Secret can sync from Vault..."
OIDC_SYNCED=false
for i in {1..10}; do
  STATUS=$(kubectl -n gitlab get externalsecret gitlab-oidc -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "False")
  if [ "$STATUS" = "True" ]; then
    echo "✓ GitLab OIDC External Secret synced from Vault!"
    OIDC_SYNCED=true
    break
  fi
  sleep 2
done

# Create placeholder OAuth credentials in Vault if they don't exist
echo "Checking if OAuth credentials exist in Vault..."
VAULT_POD=$(kubectl get pods -n vault -l app.kubernetes.io/name=vault -o jsonpath='{.items[0].metadata.name}')
VAULT_TOKEN=$(kubectl get secret -n vault vault-init-keys -o jsonpath='{.data.VAULT_ROOT_TOKEN}' | base64 -d)

if ! kubectl exec -n vault "$VAULT_POD" -- env VAULT_TOKEN="$VAULT_TOKEN" vault kv get secret/gitlab-oauth >/dev/null 2>&1; then
  echo "⚠ Vault doesn't have gitlab-oauth credentials yet. Creating placeholders..."
  PLACEHOLDER_SECRET=$(openssl rand -hex 32)
  kubectl exec -n vault "$VAULT_POD" -- env VAULT_TOKEN="$VAULT_TOKEN" vault kv put secret/gitlab-oauth \
    client-id="placeholder-gitlab-client-id" \
    client-secret="$PLACEHOLDER_SECRET"
  echo "✓ Placeholder OAuth credentials created in Vault"
  echo "   Update with real credentials: ./scripts/sync-gitlab-oauth.sh"
else
  echo "✓ GitLab OAuth credentials exist in Vault"
fi

# Wait for external secret to sync the placeholder/real credentials
echo "Waiting for GitLab OIDC External Secret to sync..."
for i in {1..30}; do
  STATUS=$(kubectl -n gitlab get externalsecret gitlab-oidc -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "False")
  if [ "$STATUS" = "True" ]; then
    echo "✓ GitLab OIDC External Secret synced successfully"
    break
  fi
  sleep 2
done

# Check if SAML secret can sync from Vault
echo "Checking if GitLab SAML External Secret can sync from Vault..."
SAML_SYNCED=false
for i in {1..10}; do
  STATUS=$(kubectl -n gitlab get externalsecret gitlab-saml -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "False")
  if [ "$STATUS" = "True" ]; then
    echo "✓ GitLab SAML External Secret synced from Vault!"
    SAML_SYNCED=true
    break
  fi
  sleep 2
done

# Create placeholder SAML credentials in Vault if they don't exist
echo "Checking if SAML credentials exist in Vault..."
if ! kubectl exec -n vault "$VAULT_POD" -- env VAULT_TOKEN="$VAULT_TOKEN" vault kv get secret/gitlab/saml >/dev/null 2>&1; then
  echo "⚠ Vault doesn't have gitlab/saml credentials yet. Creating placeholders..."
  kubectl exec -n vault "$VAULT_POD" -- env VAULT_TOKEN="$VAULT_TOKEN" vault kv put secret/gitlab/saml \
    idp_sso_url="https://auth.kube.huskypup.net/application/saml/gitlab/sso/binding/redirect/" \
    idp_fingerprint="00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00"
  echo "✓ Placeholder SAML credentials created in Vault"
  echo "   Update with real credentials: ./scripts/gitlab-saml-bootstrap.sh"
else
  echo "✓ GitLab SAML credentials exist in Vault"
fi

# Wait for external secret to sync
echo "Waiting for GitLab SAML External Secret to sync..."
for i in {1..30}; do
  STATUS=$(kubectl -n gitlab get externalsecret gitlab-saml -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "False")
  if [ "$STATUS" = "True" ]; then
    echo "✓ GitLab SAML External Secret synced successfully"
    break
  fi
  sleep 2
done

# Force password synchronization to prevent authentication issues
echo "=== Synchronizing Database and Redis Passwords ==="

# Delete and recreate database secrets to ensure sync
echo "Refreshing database secrets..."
kubectl delete secret pg-gitlab-app -n gitlab --ignore-not-found=true
kubectl delete secret pg-praefect-app -n gitlab --ignore-not-found=true

# Wait for ExternalSecrets to recreate secrets
echo "Waiting for database secrets to be recreated..."
for i in {1..30}; do
  if kubectl -n gitlab get secret pg-gitlab-app >/dev/null 2>&1 && \
     kubectl -n gitlab get secret pg-praefect-app >/dev/null 2>&1; then
    echo "Database secrets recreated successfully!"
    break
  fi
  echo "  waiting for database secrets... (attempt $i/30)"
  sleep 2
done

# Delete and recreate Redis secret to ensure sync
echo "Refreshing Redis secrets..."
kubectl delete secret redis-gitlab-secret -n gitlab --ignore-not-found=true

# Wait for Redis ExternalSecret to recreate secret
echo "Waiting for Redis secret to be recreated..."
for i in {1..30}; do
  if kubectl -n gitlab get secret redis-gitlab-secret >/dev/null 2>&1; then
    echo "Redis secret recreated successfully!"
    break
  fi
  echo "  waiting for Redis secret... (attempt $i/30)"
  sleep 2
done

# Trigger PostgreSQL reload to pick up new passwords
echo "Triggering PostgreSQL cluster reload..."
kubectl annotate cluster pg-gitlab -n gitlab cnpg.io/reload=$(date +%s) --overwrite
kubectl annotate cluster pg-praefect -n gitlab cnpg.io/reload=$(date +%s) --overwrite

echo ""
echo "===================================================================="
echo "GitLab Bootstrap Summary - Fully Automated"
echo "===================================================================="
echo "✓ Authentik GitLab OAuth blueprint applied"
echo "✓ PostgreSQL CNPG clusters (GitLab + Praefect) deployed"
echo "✓ PgBouncer connection poolers configured with authentication"
echo "✓ Redis standalone instance deployed"
echo "✓ Redis auto-restart automation (CronJob) deployed"
echo "✓ Redis Sentinel HA cluster deployed"

if [ "${OAUTH_SYNCED:-false}" = "true" ]; then
  echo "✓ GitLab OIDC credentials synced from Authentik"
else
  echo "⚠ GitLab OIDC using placeholder (Authentik provider not ready)"
fi

echo "✓ GitLab SAML secret configured (placeholder)"
echo "✓ Database and Redis passwords synchronized"
echo ""
echo "Password Rotation: Automated via CronJob (hourly checks)"
echo "Next password rotation: $(kubectl get externalsecret -n gitlab gitlab-redis-password -o jsonpath='{.status.refreshTime}' 2>/dev/null || echo 'Unknown') + 24h"
echo "===================================================================="
echo ""
echo "🎉 GitLab bootstrap completed - NO MANUAL STEPS REQUIRED!"
echo ""
echo "GitLab will be accessible at: https://gitlab.kube.huskypup.net"
echo ""
if [ "${OAUTH_SYNCED:-false}" = "false" ]; then
  echo "To enable Authentik SSO (optional):"
  echo "  1. Wait for Authentik to process the GitLab blueprint (~5 min)"
  echo "  2. Run: ./scripts/sync-gitlab-oauth.sh"
  echo "  3. Restart GitLab pods to pick up real credentials"
  echo ""
fi
echo "For SAML setup (optional): ./scripts/gitlab-saml-bootstrap.sh"
echo ""