From 989d42dd505c5cfa08d8ed551fdb6e6b8412fb9b Mon Sep 17 00:00:00 2001 From: Scooby Husky Date: Thu, 20 Aug 2026 22:36:16 -0500 Subject: [PATCH] Fix GitLab OIDC secret FailedMount blocking webservice/sidekiq startup Real bug affecting both home and VPS GitLab, found live diagnosing why VPS's webservice pod was stuck at Init:0/3 for 74+ minutes (surfacing as 404s - nothing was ever actually serving): global.appConfig.omniauth.providers: [{secret: gitlab-oidc-secret, key: provider}] requires that secret to have a 'provider' key (a full YAML omniauth provider config) - gitlab-oidc-secret never has one, only GITLAB_OIDC_CLIENT_ID/SECRET, which get consumed via extraEnvFrom + GITLAB_OMNIBUS_CONFIG's gitlab_rails['omniauth_providers'] instead (the actual, complete mechanism already configuring OIDC - this providers: list was always redundant). Removed it from both values.yaml files. Home's GitLab has run fine so far because Kubernetes doesn't re-validate already-mounted volumes when a referenced Secret's shape changes - webservice/sidekiq there just haven't restarted since gitlab-oidc-secret was last (if ever) in a shape with that key. VPS's identical values shape hit it immediately on first pod creation. This was a live landmine for home too - fixed before it could bite on a future restart. Also: global.kas.enabled: false added to the VPS values (gitlab.kas.enabled alone doesn't stop KAS pods - found live, separate toggle). Co-Authored-By: Claude Sonnet 5 --- apps/gitlab/values.yaml | 22 ++++++++++++++++--- infrastructure/vps-standby/gitlab/values.yaml | 18 ++++++++++++--- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/apps/gitlab/values.yaml b/apps/gitlab/values.yaml index 5b60f60..aaed70f 100644 --- a/apps/gitlab/values.yaml +++ b/apps/gitlab/values.yaml @@ -89,6 +89,25 @@ global: # Application Configuration appConfig: # OmniAuth SSO Configuration + # + # NOTE: no `providers:` list here (was `- secret: gitlab-oidc-secret, + # key: provider`) - found live 2026-08-21 (diagnosing the VPS GitLab + # deployment, which shares this same values shape) that the chart + # requires that secret to actually HAVE a `provider` key (a full + # omniauth provider config block, YAML-encoded) when this list is + # set, and gitlab-oidc-secret never had one - it only has + # GITLAB_OIDC_CLIENT_ID/SECRET, consumed via extraEnvFrom below and + # gitlab.webservice.extraEnv's GITLAB_OMNIBUS_CONFIG instead, which + # is the ACTUAL mechanism configuring the OIDC provider (this + # global.appConfig.omniauth.providers list was always redundant/ + # unused for our setup). Silently didn't matter here at home because + # this Deployment hasn't restarted since gitlab-oidc-secret was last + # in whatever shape included that key (if it ever did) - Kubernetes + # doesn't re-validate already-mounted volumes when a Secret's shape + # changes, only NEW pod creation does, which is exactly why the VPS + # deployment (fresh pods, same values shape) hit it immediately + # ("FailedMount: references non-existent secret key: provider") while + # this one didn't - until its next restart. omniauth: enabled: true allowSingleSignOn: ['openid_connect'] @@ -96,9 +115,6 @@ global: autoLinkUser: ['openid_connect'] syncProfileFromProvider: ['openid_connect'] syncProfileAttributes: ['email', 'name'] - providers: - - secret: gitlab-oidc-secret - key: provider # Settings for Let's Encrypt ACME Issuer - disabled, using cluster-wide cert-manager certmanager-issuer: diff --git a/infrastructure/vps-standby/gitlab/values.yaml b/infrastructure/vps-standby/gitlab/values.yaml index ed8ad0f..e387622 100644 --- a/infrastructure/vps-standby/gitlab/values.yaml +++ b/infrastructure/vps-standby/gitlab/values.yaml @@ -37,6 +37,12 @@ global: secret: pg-gitlab-app key: password + # gitlab.kas.enabled: false alone doesn't stop KAS pods deploying - + # there's a SEPARATE global.kas.enabled toggle (default true) that + # actually controls it, found live 2026-08-21. + kas: + enabled: false + # Single Gitaly node, no Praefect - HA routing is pointless with 1 node gitaly: enabled: true @@ -58,6 +64,15 @@ global: key: password appConfig: + # NOTE: no `providers:` list here - found live 2026-08-21: the chart + # requires gitlab-oidc-secret to have a `provider` key (full YAML + # provider config) when this is set, and it never does - only + # GITLAB_OIDC_CLIENT_ID/SECRET, consumed via extraEnvFrom + + # GITLAB_OMNIBUS_CONFIG below instead (the actual mechanism + # configuring OIDC). Caused "FailedMount: references non-existent + # secret key: provider" blocking webservice/sidekiq/toolbox from ever + # starting - see the matching fix + full explanation in + # apps/gitlab/values.yaml (home). omniauth: enabled: true allowSingleSignOn: ['openid_connect'] @@ -65,9 +80,6 @@ global: autoLinkUser: ['openid_connect'] syncProfileFromProvider: ['openid_connect'] syncProfileAttributes: ['email', 'name'] - providers: - - secret: gitlab-oidc-secret - key: provider # GitLab's pre-upgrade hook checks the deployed version against a # previous-release ConfigMap to enforce supported upgrade paths - found