--- # ServiceAccount for the CronJob that monitors PostgreSQL secret changes apiVersion: v1 kind: ServiceAccount metadata: name: pg-restart-sa namespace: gitlab --- # Role to allow patching Deployments, StatefulSets, Clusters and reading Secrets apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: pg-restart-role namespace: gitlab rules: - apiGroups: ["apps"] resources: ["deployments", "statefulsets"] verbs: ["get", "patch"] - apiGroups: ["postgresql.cnpg.io"] resources: ["clusters"] verbs: ["get", "patch"] - apiGroups: [""] resources: ["secrets"] verbs: ["get"] - apiGroups: ["apps"] resources: ["deployments/status", "statefulsets/status"] verbs: ["get"] - apiGroups: ["postgresql.cnpg.io"] resources: ["clusters/status"] verbs: ["get"] --- # RoleBinding to grant permissions to the ServiceAccount apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: pg-restart-binding namespace: gitlab roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: pg-restart-role subjects: - kind: ServiceAccount name: pg-restart-sa namespace: gitlab --- # CronJob to monitor pg-gitlab-app secret and trigger restarts on changes apiVersion: batch/v1 kind: CronJob metadata: name: pg-gitlab-secret-monitor namespace: gitlab spec: # Run every 30 minutes to check for secret changes (rotations happen at most daily) schedule: "*/30 * * * *" concurrencyPolicy: Forbid successfulJobsHistoryLimit: 1 failedJobsHistoryLimit: 1 jobTemplate: spec: template: metadata: labels: app: pg-gitlab-secret-monitor spec: serviceAccountName: pg-restart-sa restartPolicy: OnFailure containers: - name: monitor image: docker.io/alpine/k8s:1.32.13 securityContext: runAsUser: 10000 runAsGroup: 10000 runAsNonRoot: true allowPrivilegeEscalation: false readOnlyRootFilesystem: true capabilities: drop: ["ALL"] seccompProfile: type: RuntimeDefault volumeMounts: - name: tmp mountPath: /tmp command: - /bin/bash - -c - | set -e # Get current secret version SECRET_VERSION=$(kubectl get secret -n gitlab pg-gitlab-app -o jsonpath='{.metadata.resourceVersion}') # Get last known secret version from pgbouncer deployment annotation LAST_VERSION=$(kubectl get deployment -n gitlab pgbouncer-gitlab -o jsonpath='{.spec.template.metadata.annotations.secret-version/pg-password}' 2>/dev/null || echo "") echo "Current secret version: $SECRET_VERSION" echo "Last known version: $LAST_VERSION" # If versions differ, update database password and restart resources if [ "$SECRET_VERSION" != "$LAST_VERSION" ]; then echo "Secret has changed! Updating database password and resources..." # Get the new password from the secret NEW_PASSWORD=$(kubectl get secret -n gitlab pg-gitlab-app -o jsonpath='{.data.password}' | base64 -d) # Update the database user password # Try both pg-gitlab-1 and pg-gitlab-2 in case one is restarting kubectl exec -n gitlab pg-gitlab-1 -c postgres -- psql -U postgres -d gitlabhq_production -c "ALTER USER app PASSWORD '$NEW_PASSWORD';" 2>/dev/null || \ kubectl exec -n gitlab pg-gitlab-2 -c postgres -- psql -U postgres -d gitlabhq_production -c "ALTER USER app PASSWORD '$NEW_PASSWORD';" 2>/dev/null || \ echo "Database password update failed" # Update password table with new hash for PgBouncer SCRAM auth kubectl exec -n gitlab pg-gitlab-1 -c postgres -- psql -U postgres -d gitlabhq_production -c "INSERT INTO public.user_passwords (usename, passwd) SELECT rolname, rolpassword FROM pg_authid WHERE rolname = 'app' ON CONFLICT (usename) DO UPDATE SET passwd = EXCLUDED.passwd;" 2>/dev/null || \ kubectl exec -n gitlab pg-gitlab-2 -c postgres -- psql -U postgres -d gitlabhq_production -c "INSERT INTO public.user_passwords (usename, passwd) SELECT rolname, rolpassword FROM pg_authid WHERE rolname = 'app' ON CONFLICT (usename) DO UPDATE SET passwd = EXCLUDED.passwd;" 2>/dev/null || \ echo "Password table update failed, PgBouncer may need manual restart" # Patch pgbouncer deployments to trigger restart kubectl patch deployment -n gitlab pgbouncer-gitlab -p "{\"spec\":{\"template\":{\"metadata\":{\"annotations\":{\"secret-version/pg-password\":\"$SECRET_VERSION\",\"restarted-at\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"}}}}}" 2>/dev/null || echo "Deployment patch failed" echo "Database password updated and resources will restart." else echo "Secret has not changed. No restart needed." fi volumes: - name: tmp emptyDir: {} --- # CronJob to monitor pg-praefect-app secret and trigger restarts on changes apiVersion: batch/v1 kind: CronJob metadata: name: pg-praefect-secret-monitor namespace: gitlab spec: # Run every 30 minutes to check for secret changes (rotations happen at most daily) schedule: "*/30 * * * *" concurrencyPolicy: Forbid successfulJobsHistoryLimit: 1 failedJobsHistoryLimit: 1 jobTemplate: spec: template: metadata: labels: app: pg-praefect-secret-monitor spec: serviceAccountName: pg-restart-sa restartPolicy: OnFailure containers: - name: monitor image: docker.io/alpine/k8s:1.32.13 securityContext: runAsUser: 10000 runAsGroup: 10000 runAsNonRoot: true allowPrivilegeEscalation: false readOnlyRootFilesystem: true capabilities: drop: ["ALL"] seccompProfile: type: RuntimeDefault volumeMounts: - name: tmp mountPath: /tmp command: - /bin/bash - -c - | set -e # Get current secret version SECRET_VERSION=$(kubectl get secret -n gitlab pg-praefect-app -o jsonpath='{.metadata.resourceVersion}') # Get last known secret version from gitaly statefulset annotation LAST_VERSION=$(kubectl get statefulset -n gitlab gitlab-gitaly-default -o jsonpath='{.spec.template.metadata.annotations.secret-version/pg-password}' 2>/dev/null || echo "") echo "Current secret version: $SECRET_VERSION" echo "Last known version: $LAST_VERSION" # If versions differ, update database password and restart resources if [ "$SECRET_VERSION" != "$LAST_VERSION" ]; then echo "Secret has changed! Updating database password and resources..." # Get the new password from the secret NEW_PASSWORD=$(kubectl get secret -n gitlab pg-praefect-app -o jsonpath='{.data.password}' | base64 -d) # Update the database user password # Try both pg-praefect-3 and pg-praefect-4 kubectl exec -n gitlab pg-praefect-3 -c postgres -- psql -U postgres -d gitlabhq_production -c "ALTER USER app PASSWORD '$NEW_PASSWORD';" 2>/dev/null || \ kubectl exec -n gitlab pg-praefect-4 -c postgres -- psql -U postgres -d gitlabhq_production -c "ALTER USER app PASSWORD '$NEW_PASSWORD';" 2>/dev/null || \ echo "Database password update failed" # Patch gitaly and praefect statefulsets kubectl patch statefulset -n gitlab gitlab-gitaly-default -p "{\"spec\":{\"template\":{\"metadata\":{\"annotations\":{\"secret-version/pg-password\":\"$SECRET_VERSION\",\"restarted-at\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"}}}}}" kubectl patch statefulset -n gitlab gitlab-praefect -p "{\"spec\":{\"template\":{\"metadata\":{\"annotations\":{\"secret-version/pg-password\":\"$SECRET_VERSION\",\"restarted-at\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"}}}}}" # Patch CNPG cluster kubectl patch cluster -n gitlab pg-praefect -p "{\"metadata\":{\"annotations\":{\"secret-version/pg-password\":\"$SECRET_VERSION\",\"restarted-at\":\"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"}}}" --type merge echo "Database password updated and resources will restart." else echo "Secret has not changed. No restart needed." fi volumes: - name: tmp emptyDir: {}