From 1ab292e1858e3f907a77d3b888ddc17dbe3ffca0 Mon Sep 17 00:00:00 2001 From: Scooby Husky Date: Thu, 20 Aug 2026 23:14:25 -0500 Subject: [PATCH] Fix GitLab Authentik SSO: restore providers list, populate real provider key GITLAB_OMNIBUS_CONFIG (gitlab_rails['omniauth_providers'] = [...]) is an omnibus-Docker-image-only convention. This chart's CNG webservice image never processes it - confirmed via gitlab/charts/gitlab/ templates/_omniauth.tpl, which only reads global.appConfig.omniauth.providers (a list of {secret,key} refs, each pointing at a Secret key holding a whole YAML-encoded provider block loaded via Ruby's YAML.load_file). So GITLAB_OMNIBUS_CONFIG was always a silent no-op on both home and VPS - neither ever actually had SSO configured despite gitlab-oidc-secret existing and looking correct. Yesterday's fix for 'FailedMount: references non-existent secret key: provider' removed the providers: list entirely instead of populating that key - stopped the crash, but also silently deleted the only real OIDC config path on both sites (no error, login page just lost its SSO button). This restores providers: on both, and gitlab-oidc-secret's ExternalSecret template now actually renders a provider key containing a real YAML provider block (with the templated client_id/secret substituted in), matching what _omniauth.tpl expects. Co-Authored-By: Claude Sonnet 5 --- .../manifests/external-secret-oidc.yaml | 38 +++++++++++++++++++ apps/gitlab/values.yaml | 33 ++++++++-------- .../gitlab/manifests/oidc-secret.yaml | 27 +++++++++++++ infrastructure/vps-standby/gitlab/values.yaml | 19 +++++----- 4 files changed, 90 insertions(+), 27 deletions(-) diff --git a/apps/gitlab/manifests/external-secret-oidc.yaml b/apps/gitlab/manifests/external-secret-oidc.yaml index 63d2e00..bb2d21e 100644 --- a/apps/gitlab/manifests/external-secret-oidc.yaml +++ b/apps/gitlab/manifests/external-secret-oidc.yaml @@ -2,6 +2,24 @@ # git-tracked, never rotatable, no record of what it was) - found and # fixed 2026-08-20 alongside infrastructure/authentik/gitlab-blueprint.yaml # (same value, same Vault path - see that file for the full story). +# +# `provider` key added 2026-08-21: found live that GITLAB_OMNIBUS_CONFIG's +# `gitlab_rails['omniauth_providers'] = [...]` (still set in this app's +# values.yaml's extraEnv, for reference/documentation only at this point) +# is an omnibus-image-only convention - the CNG webservice image this +# chart actually runs never processes it, so it was always a no-op and +# GitLab never had SSO configured despite it looking configured. The +# REAL mechanism (gitlab/charts/gitlab/templates/_omniauth.tpl in the +# chart) is global.appConfig.omniauth.providers: a list of +# {secret, key} refs, each pointing at a Secret key whose value is a +# whole YAML-encoded provider block (loaded via Ruby's YAML.load_file +# at container start, baked into gitlab.yml) - not raw client_id/secret +# strings. This is also why the `providers:` list got removed entirely +# a day earlier chasing a "FailedMount: references non-existent secret +# key: provider" error: the fix should have been to populate that key +# correctly (this), not remove the reference to it - doing so silently +# killed SSO on both home and VPS GitLab (no error, the login page just +# had no SSO button). apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: @@ -20,6 +38,26 @@ spec: data: GITLAB_OIDC_CLIENT_ID: "{{ .clientId }}" GITLAB_OIDC_CLIENT_SECRET: "{{ .clientSecret }}" + provider: | + name: openid_connect + label: Authentik + args: + name: openid_connect + scope: + - openid + - profile + - email + response_type: code + issuer: 'https://auth.kube.huskypup.net/application/o/gitlab/' + discovery: true + client_auth_method: query + uid_field: sub + send_scope_to_token_endpoint: true + pkce: true + client_options: + identifier: '{{ .clientId }}' + secret: '{{ .clientSecret }}' + redirect_uri: 'https://gitlab.kube.huskypup.net/users/auth/openid_connect/callback' data: - secretKey: clientId remoteRef: diff --git a/apps/gitlab/values.yaml b/apps/gitlab/values.yaml index aaed70f..7e0aba8 100644 --- a/apps/gitlab/values.yaml +++ b/apps/gitlab/values.yaml @@ -90,24 +90,18 @@ global: appConfig: # OmniAuth SSO Configuration # - # NOTE: no `providers:` list here (was `- secret: gitlab-oidc-secret, - # key: provider`) - found live 2026-08-21 (diagnosing the VPS GitLab - # deployment, which shares this same values shape) that the chart - # requires that secret to actually HAVE a `provider` key (a full - # omniauth provider config block, YAML-encoded) when this list is - # set, and gitlab-oidc-secret never had one - it only has - # GITLAB_OIDC_CLIENT_ID/SECRET, consumed via extraEnvFrom below and - # gitlab.webservice.extraEnv's GITLAB_OMNIBUS_CONFIG instead, which - # is the ACTUAL mechanism configuring the OIDC provider (this - # global.appConfig.omniauth.providers list was always redundant/ - # unused for our setup). Silently didn't matter here at home because - # this Deployment hasn't restarted since gitlab-oidc-secret was last - # in whatever shape included that key (if it ever did) - Kubernetes - # doesn't re-validate already-mounted volumes when a Secret's shape - # changes, only NEW pod creation does, which is exactly why the VPS - # deployment (fresh pods, same values shape) hit it immediately - # ("FailedMount: references non-existent secret key: provider") while - # this one didn't - until its next restart. + # CORRECTED 2026-08-21 (second time - see git history for the wrong + # 2026-08-20 fix that removed this list entirely). This IS the real, + # actually-processed OIDC config mechanism for the CNG webservice + # image (chart's gitlab.appConfig.omniauth.configuration template, + # infra/_omniauth.tpl) - GITLAB_OMNIBUS_CONFIG below is an + # omnibus-image-only convention this image never processes, so it + # was always a documentation-only no-op despite looking functional. + # gitlab-oidc-secret's `provider` key now holds a full YAML-encoded + # provider block (apps/gitlab/manifests/external-secret-oidc.yaml), + # loaded via Ruby's YAML.load_file at container start - not raw + # client_id/secret strings, which is why plain GITLAB_OIDC_CLIENT_ID/ + # SECRET keys alone (yesterday's assumption) were never enough. omniauth: enabled: true allowSingleSignOn: ['openid_connect'] @@ -115,6 +109,9 @@ global: autoLinkUser: ['openid_connect'] syncProfileFromProvider: ['openid_connect'] syncProfileAttributes: ['email', 'name'] + providers: + - secret: gitlab-oidc-secret + key: provider # Settings for Let's Encrypt ACME Issuer - disabled, using cluster-wide cert-manager certmanager-issuer: diff --git a/infrastructure/vps-standby/gitlab/manifests/oidc-secret.yaml b/infrastructure/vps-standby/gitlab/manifests/oidc-secret.yaml index bd6da0d..18b5b42 100644 --- a/infrastructure/vps-standby/gitlab/manifests/oidc-secret.yaml +++ b/infrastructure/vps-standby/gitlab/manifests/oidc-secret.yaml @@ -5,6 +5,13 @@ # (infrastructure/vps-eso/manifests/clustersecretstore.yaml's policy, # applied live via vault CLI, not git-tracked - matches how every other # Vault policy/auth-method in this repo is set up). +# +# `provider` key added 2026-08-21: found live that GITLAB_OMNIBUS_CONFIG +# (this app's values.yaml extraEnv) is an omnibus-image-only convention +# the CNG webservice image never processes - see the matching comment +# in apps/gitlab/manifests/external-secret-oidc.yaml (home) for the +# full story. The real mechanism needs this key to hold a whole +# YAML-encoded provider block, not raw client_id/secret strings. apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: @@ -23,6 +30,26 @@ spec: data: GITLAB_OIDC_CLIENT_ID: "{{ .clientId }}" GITLAB_OIDC_CLIENT_SECRET: "{{ .clientSecret }}" + provider: | + name: openid_connect + label: Authentik + args: + name: openid_connect + scope: + - openid + - profile + - email + response_type: code + issuer: 'https://auth.kube.huskypup.net/application/o/gitlab/' + discovery: true + client_auth_method: query + uid_field: sub + send_scope_to_token_endpoint: true + pkce: true + client_options: + identifier: '{{ .clientId }}' + secret: '{{ .clientSecret }}' + redirect_uri: 'https://gitlab.vps.huskypup.net/users/auth/openid_connect/callback' data: - secretKey: clientId remoteRef: diff --git a/infrastructure/vps-standby/gitlab/values.yaml b/infrastructure/vps-standby/gitlab/values.yaml index 6b2b0e4..03c3a23 100644 --- a/infrastructure/vps-standby/gitlab/values.yaml +++ b/infrastructure/vps-standby/gitlab/values.yaml @@ -64,15 +64,13 @@ global: key: password appConfig: - # NOTE: no `providers:` list here - found live 2026-08-21: the chart - # requires gitlab-oidc-secret to have a `provider` key (full YAML - # provider config) when this is set, and it never does - only - # GITLAB_OIDC_CLIENT_ID/SECRET, consumed via extraEnvFrom + - # GITLAB_OMNIBUS_CONFIG below instead (the actual mechanism - # configuring OIDC). Caused "FailedMount: references non-existent - # secret key: provider" blocking webservice/sidekiq/toolbox from ever - # starting - see the matching fix + full explanation in - # apps/gitlab/values.yaml (home). + # CORRECTED 2026-08-21 (see apps/gitlab/values.yaml (home) for the + # full story) - `providers:` list restored, now pointing at + # gitlab-oidc-secret's `provider` key which holds a real + # YAML-encoded provider block (manifests/oidc-secret.yaml), the + # actual mechanism the CNG webservice image processes. + # GITLAB_OMNIBUS_CONFIG below is an omnibus-image-only convention + # this image never reads - always a no-op here. omniauth: enabled: true allowSingleSignOn: ['openid_connect'] @@ -80,6 +78,9 @@ global: autoLinkUser: ['openid_connect'] syncProfileFromProvider: ['openid_connect'] syncProfileAttributes: ['email', 'name'] + providers: + - secret: gitlab-oidc-secret + key: provider # GitLab's pre-upgrade hook checks the deployed version against a # previous-release ConfigMap to enforce supported upgrade paths - found