From 59bff07a5364a9702d96901778b6e07f1fcb2196 Mon Sep 17 00:00:00 2001 From: Scooby Husky Date: Thu, 20 Aug 2026 23:10:48 -0500 Subject: [PATCH] Loosen VPS GitLab webservice/sidekiq liveness probes, cut worker count VPS box is chronically CPU-overcommitted (load avg ~22-26 on 6 cores, confirmed live - unrelated pre-existing services: barman, k3s itself, node/gunicorn apps). Chart-default liveness probes (~200s grace) were SIGTERM-killing both webservice and sidekiq mid-boot (still inside bootsnap's require), every restart resetting progress to zero and never actually finishing. Loosened to a ~7min grace window and dropped webservice to 1 worker process instead of 2, less to fork/boot. Co-Authored-By: Claude Sonnet 5 --- infrastructure/vps-standby/gitlab/values.yaml | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/infrastructure/vps-standby/gitlab/values.yaml b/infrastructure/vps-standby/gitlab/values.yaml index e387622..6b2b0e4 100644 --- a/infrastructure/vps-standby/gitlab/values.yaml +++ b/infrastructure/vps-standby/gitlab/values.yaml @@ -123,7 +123,27 @@ gitlab: webservice: minReplicas: 1 maxReplicas: 1 - workerProcesses: 2 + # 1 worker process, not 2 - single-node VPS, already sharing the box + # with several other unrelated services (barman, k3s itself, node/ + # gunicorn apps) that keep it chronically CPU-overcommitted (load + # average ~22-26 sustained on 6 cores, confirmed live 2026-08-21, + # unrelated to GitLab). Less to fork/boot per pod start. + workerProcesses: 1 + deployment: + # Chart defaults (initialDelaySeconds:20, periodSeconds:60, + # failureThreshold:3 => ~200s grace) aren't enough on this + # contended box - confirmed live 2026-08-21 the webservice + # (puma) container was repeatedly SIGTERM'd by the liveness + # probe mid-`require` (still loading gems via bootsnap), never + # finishing boot, restarting from zero every time. Loosened so + # a slow boot survives instead of getting killed and restarted + # forever. + livenessProbe: + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 30 + successThreshold: 1 + failureThreshold: 15 extraEnvFrom: GITLAB_OIDC_CLIENT_ID: @@ -198,6 +218,17 @@ gitlab: sidekiq: minReplicas: 1 maxReplicas: 1 + # Same probe-loosening as webservice.deployment above, same root + # cause (confirmed live 2026-08-21) - sidekiq's own boot was + # getting SIGTERM'd mid-`require` by the liveness probe on this + # CPU-contended box, every ~90-150s, resetting to zero each time + # and never actually finishing. + livenessProbe: + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 30 + successThreshold: 1 + failureThreshold: 15 resources: requests: cpu: 25m