Homelabv4/apps/gitlab/manifests/pg-auto-restart.yaml

---
# ServiceAccount for the CronJob that monitors PostgreSQL secret changes
apiVersion: v1
kind: ServiceAccount
metadata:
  name: pg-restart-sa
  namespace: gitlab
---
# Role to allow patching Deployments, StatefulSets, Clusters and reading Secrets
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: pg-restart-role
  namespace: gitlab
rules:
  - apiGroups: ["apps"]
    resources: ["deployments", "statefulsets"]
    verbs: ["get", "patch"]
  - apiGroups: ["postgresql.cnpg.io"]
    resources: ["clusters"]
    verbs: ["get", "patch"]
  - apiGroups: [""]
    resources: ["secrets"]
    verbs: ["get"]
  - apiGroups: ["apps"]
    resources: ["deployments/status", "statefulsets/status"]
    verbs: ["get"]
  - apiGroups: ["postgresql.cnpg.io"]
    resources: ["clusters/status"]
    verbs: ["get"]
---
# RoleBinding to grant permissions to the ServiceAccount
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: pg-restart-binding
  namespace: gitlab
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: pg-restart-role
subjects:
  - kind: ServiceAccount
    name: pg-restart-sa
    namespace: gitlab
---
# CronJob to monitor pg-gitlab-app secret and trigger restarts on changes
apiVersion: batch/v1
kind: CronJob
metadata:
  name: pg-gitlab-secret-monitor
  namespace: gitlab
spec:
  # Run every 30 minutes to check for secret changes (rotations happen at most daily)
  schedule: "*/30 * * * *"
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 1
  failedJobsHistoryLimit: 1
  jobTemplate:
    spec:
      template:
        metadata:
          labels:
            app: pg-gitlab-secret-monitor
        spec:
          serviceAccountName: pg-restart-sa
          restartPolicy: OnFailure
          containers:
            - name: monitor
              image: docker.io/alpine/k8s:1.32.13
              securityContext:
                runAsUser: 10000
                runAsGroup: 10000
                runAsNonRoot: true
                allowPrivilegeEscalation: false
                readOnlyRootFilesystem: true
                capabilities:
                  drop: ["ALL"]
                seccompProfile:
                  type: RuntimeDefault
              volumeMounts:
                - name: tmp
                  mountPath: /tmp
              command:
                - /bin/bash
                - -c
                - |
                  set -e

                  # Get current secret version
                  SECRET_VERSION=$(kubectl get secret -n gitlab pg-gitlab-app -o jsonpath='{.metadata.resourceVersion}')

                  # Get last known secret version from pgbouncer deployment annotation
                  LAST_VERSION=$(kubectl get deployment -n gitlab pgbouncer-gitlab -o jsonpath='{.spec.template.metadata.annotations.secret-version/pg-password}' 2>/dev/null || echo "")

                  echo "Current secret version: $SECRET_VERSION"
                  echo "Last known version: $LAST_VERSION"

                   # If versions differ, update database password and restart resources
                   if [ "$SECRET_VERSION" != "$LAST_VERSION" ]; then
                     echo "Secret has changed! Updating database password and resources..."

                     # Get the new password from the secret
                     NEW_PASSWORD=$(kubectl get secret -n gitlab pg-gitlab-app -o jsonpath='{.data.password}' | base64 -d)

                     # Update the database user password
                     # Try both pg-gitlab-1 and pg-gitlab-2 in case one is restarting
                     kubectl exec -n gitlab pg-gitlab-1 -c postgres -- psql -U postgres -d gitlabhq_production -c "ALTER USER app PASSWORD '$NEW_PASSWORD';" 2>/dev/null || \
                     kubectl exec -n gitlab pg-gitlab-2 -c postgres -- psql -U postgres -d gitlabhq_production -c "ALTER USER app PASSWORD '$NEW_PASSWORD';" 2>/dev/null || \
                     echo "Database password update failed"

                     # Update password table with new hash for PgBouncer SCRAM auth
                     kubectl exec -n gitlab pg-gitlab-1 -c postgres -- psql -U postgres -d gitlabhq_production -c "INSERT INTO public.user_passwords (usename, passwd) SELECT rolname, rolpassword FROM pg_authid WHERE rolname = 'app' ON CONFLICT (usename) DO UPDATE SET passwd = EXCLUDED.passwd;" 2>/dev/null || \
                     kubectl exec -n gitlab pg-gitlab-2 -c postgres -- psql -U postgres -d gitlabhq_production -c "INSERT INTO public.user_passwords (usename, passwd) SELECT rolname, rolpassword FROM pg_authid WHERE rolname = 'app' ON CONFLICT (usename) DO UPDATE SET passwd = EXCLUDED.passwd;" 2>/dev/null || \
                     echo "Password table update failed, PgBouncer may need manual restart"

                     # Patch pgbouncer deployments to trigger restart
                     kubectl patch deployment -n gitlab pgbouncer-gitlab -p "{\"spec\":{\"template\":{\"metadata\":{\"annotations\":{\"secret-version/pg-password\":\"$SECRET_VERSION\",\"restarted-at\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"}}}}}" 2>/dev/null || echo "Deployment patch failed"

                     echo "Database password updated and resources will restart."
                   else
                     echo "Secret has not changed. No restart needed."
                   fi
          volumes:
            - name: tmp
              emptyDir: {}
---
# CronJob to monitor pg-praefect-app secret and trigger restarts on changes
apiVersion: batch/v1
kind: CronJob
metadata:
  name: pg-praefect-secret-monitor
  namespace: gitlab
spec:
  # Run every 30 minutes to check for secret changes (rotations happen at most daily)
  schedule: "*/30 * * * *"
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 1
  failedJobsHistoryLimit: 1
  jobTemplate:
    spec:
      template:
        metadata:
          labels:
            app: pg-praefect-secret-monitor
        spec:
          serviceAccountName: pg-restart-sa
          restartPolicy: OnFailure
          containers:
            - name: monitor
              image: docker.io/alpine/k8s:1.32.13
              securityContext:
                runAsUser: 10000
                runAsGroup: 10000
                runAsNonRoot: true
                allowPrivilegeEscalation: false
                readOnlyRootFilesystem: true
                capabilities:
                  drop: ["ALL"]
                seccompProfile:
                  type: RuntimeDefault
              volumeMounts:
                - name: tmp
                  mountPath: /tmp
              command:
                - /bin/bash
                - -c
                - |
                  set -e

                  # Get current secret version
                  SECRET_VERSION=$(kubectl get secret -n gitlab pg-praefect-app -o jsonpath='{.metadata.resourceVersion}')

                  # Get last known secret version from gitaly statefulset annotation
                  LAST_VERSION=$(kubectl get statefulset -n gitlab gitlab-gitaly-default -o jsonpath='{.spec.template.metadata.annotations.secret-version/pg-password}' 2>/dev/null || echo "")

                  echo "Current secret version: $SECRET_VERSION"
                  echo "Last known version: $LAST_VERSION"

                   # If versions differ, update database password and restart resources
                   if [ "$SECRET_VERSION" != "$LAST_VERSION" ]; then
                     echo "Secret has changed! Updating database password and resources..."

                     # Get the new password from the secret
                     NEW_PASSWORD=$(kubectl get secret -n gitlab pg-praefect-app -o jsonpath='{.data.password}' | base64 -d)

                     # Update the database user password
                     # Try both pg-praefect-3 and pg-praefect-4
                     kubectl exec -n gitlab pg-praefect-3 -c postgres -- psql -U postgres -d gitlabhq_production -c "ALTER USER app PASSWORD '$NEW_PASSWORD';" 2>/dev/null || \
                     kubectl exec -n gitlab pg-praefect-4 -c postgres -- psql -U postgres -d gitlabhq_production -c "ALTER USER app PASSWORD '$NEW_PASSWORD';" 2>/dev/null || \
                     echo "Database password update failed"

                     # Patch gitaly and praefect statefulsets
                     kubectl patch statefulset -n gitlab gitlab-gitaly-default -p "{\"spec\":{\"template\":{\"metadata\":{\"annotations\":{\"secret-version/pg-password\":\"$SECRET_VERSION\",\"restarted-at\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"}}}}}"
                     kubectl patch statefulset -n gitlab gitlab-praefect -p "{\"spec\":{\"template\":{\"metadata\":{\"annotations\":{\"secret-version/pg-password\":\"$SECRET_VERSION\",\"restarted-at\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"}}}}}"

                     # Patch CNPG cluster
                     kubectl patch cluster -n gitlab pg-praefect -p "{\"metadata\":{\"annotations\":{\"secret-version/pg-password\":\"$SECRET_VERSION\",\"restarted-at\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"}}}" --type merge

                     echo "Database password updated and resources will restart."
                   else
                     echo "Secret has not changed. No restart needed."
                   fi
          volumes:
            - name: tmp
              emptyDir: {}