Add VPS GitLab mirror-sync CronJob for Homelabv4 repo

GitLab's native pull-mirror feature is Premium/Ultimate-gated on this chart/version - confirmed live: the project API rejects 'mirror' and 'import_url' as unrecognized params on this CE instance entirely. Plain git equivalent instead: CronJob every 15min does 'git clone --mirror' from home's Scooby/Homelabv4 (read-only deploy token) then 'git push --mirror' to the VPS copy (write_repository PAT, since deploy tokens can't push - not a valid scope for them either). Matches the original plan's 'Gitea pull mirror of the GitOps source' intent now that real GitLab replaced Gitea. Both tokens in Vault at secret/vps/gitlab-mirror, pulled via ExternalSecret. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Loosen VPS GitLab sidekiq liveness probe further (~17min grace)
2026-08-21 05:26:49 +00:00 · 2026-08-20 23:55:20 -05:00 · 2026-08-20 23:39:49 -05:00 · 2026-08-20 23:16:56 -05:00 · 2026-08-20 23:14:25 -05:00 · 2026-08-20 23:10:48 -05:00
44 changed files with 2294 additions and 507 deletions
@@ -0,0 +1,69 @@
 # gitlab-oidc-secret was a manually kubectl-created plain Secret (never
 # git-tracked, never rotatable, no record of what it was) - found and
 # fixed 2026-08-20 alongside infrastructure/authentik/gitlab-blueprint.yaml
 # (same value, same Vault path - see that file for the full story).
 #
 # `provider` key added 2026-08-21: found live that GITLAB_OMNIBUS_CONFIG's
 # `gitlab_rails['omniauth_providers'] = [...]` (still set in this app's
 # values.yaml's extraEnv, for reference/documentation only at this point)
 # is an omnibus-image-only convention - the CNG webservice image this
 # chart actually runs never processes it, so it was always a no-op and
 # GitLab never had SSO configured despite it looking configured. The
 # REAL mechanism (gitlab/charts/gitlab/templates/_omniauth.tpl in the
 # chart) is global.appConfig.omniauth.providers: a list of
 # {secret, key} refs, each pointing at a Secret key whose value is a
 # whole YAML-encoded provider block (loaded via Ruby's YAML.load_file
 # at container start, baked into gitlab.yml) - not raw client_id/secret
 # strings. This is also why the `providers:` list got removed entirely
 # a day earlier chasing a "FailedMount: references non-existent secret
 # key: provider" error: the fix should have been to populate that key
 # correctly (this), not remove the reference to it - doing so silently
 # killed SSO on both home and VPS GitLab (no error, the login page just
 # had no SSO button).
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: gitlab-oidc
  namespace: gitlab
 spec:
  refreshInterval: 1h
  secretStoreRef:
    kind: ClusterSecretStore
    name: vault-backend
  target:
    name: gitlab-oidc-secret
    creationPolicy: Owner
    template:
      type: Opaque
      data:
        GITLAB_OIDC_CLIENT_ID: "{{ .clientId }}"
        GITLAB_OIDC_CLIENT_SECRET: "{{ .clientSecret }}"
        provider: |
          name: openid_connect
          label: Authentik
          args:
            name: openid_connect
            scope:
              - openid
              - profile
              - email
            response_type: code
            issuer: 'https://auth.kube.huskypup.net/application/o/gitlab/'
            discovery: true
            client_auth_method: query
            uid_field: sub
            send_scope_to_token_endpoint: true
            pkce: true
            client_options:
              identifier: '{{ .clientId }}'
              secret: '{{ .clientSecret }}'
              redirect_uri: 'https://gitlab.kube.huskypup.net/users/auth/openid_connect/callback'
  data:
    - secretKey: clientId
      remoteRef:
        key: gitlab-oauth
        property: client-id
    - secretKey: clientSecret
      remoteRef:
        key: gitlab-oauth
        property: client-secret
@@ -89,6 +89,19 @@ global:
  # Application Configuration
  appConfig:
    # OmniAuth SSO Configuration
    #
    # CORRECTED 2026-08-21 (second time - see git history for the wrong
    # 2026-08-20 fix that removed this list entirely). This IS the real,
    # actually-processed OIDC config mechanism for the CNG webservice
    # image (chart's gitlab.appConfig.omniauth.configuration template,
    # infra/_omniauth.tpl) - GITLAB_OMNIBUS_CONFIG below is an
    # omnibus-image-only convention this image never processes, so it
    # was always a documentation-only no-op despite looking functional.
    # gitlab-oidc-secret's `provider` key now holds a full YAML-encoded
    # provider block (apps/gitlab/manifests/external-secret-oidc.yaml),
    # loaded via Ruby's YAML.load_file at container start - not raw
    # client_id/secret strings, which is why plain GITLAB_OIDC_CLIENT_ID/
    # SECRET keys alone (yesterday's assumption) were never enough.
    omniauth:
      enabled: true
      allowSingleSignOn: ['openid_connect']
@@ -0,0 +1,22 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: coredns
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "0"  # cluster DNS - as early as anything can depend on it
  finalizers:
    - resources-finalizer.argocd.argoproj.io
 spec:
  project: infrastructure
  source:
    repoURL: https://gitlab.kube.huskypup.net/Scooby/Homelabv4.git
    targetRevision: main
    path: infrastructure/coredns/manifests
  destination:
    server: https://kubernetes.default.svc
    namespace: kube-system
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
@@ -0,0 +1,33 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: vps-gitlab
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "3"  # after vps-cnpg (wave 1) and vps-eso (wave 1)
  finalizers:
    - resources-finalizer.argocd.argoproj.io
 spec:
  project: vps-standby
  sources:
    - repoURL: https://charts.gitlab.io/
      chart: gitlab
      targetRevision: 7.7.0
      helm:
        valueFiles:
          - $values/infrastructure/vps-standby/gitlab/values.yaml
    - repoURL: https://gitlab.kube.huskypup.net/Scooby/Homelabv4.git
      targetRevision: main
      ref: values
    - repoURL: https://gitlab.kube.huskypup.net/Scooby/Homelabv4.git
      targetRevision: main
      path: infrastructure/vps-standby/gitlab/manifests
  destination:
    name: vps-standby
    namespace: gitlab
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
@@ -0,0 +1,22 @@
 apiVersion: argoproj.io/v1alpha1
 kind: Application
 metadata:
  name: vps-eso
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: "1"  # before any Application whose secrets it manages (authentik is wave 3)
  finalizers:
    - resources-finalizer.argocd.argoproj.io
 spec:
  project: vps-standby
  source:
    repoURL: https://gitlab.kube.huskypup.net/Scooby/Homelabv4.git
    targetRevision: main
    path: infrastructure/vps-eso/manifests
  destination:
    name: vps-standby
    namespace: external-secrets
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
@@ -1,49 +1,77 @@
 ---
-apiVersion: v1
+# Was a plain ConfigMap with client_secret hardcoded in plaintext - found
-kind: ConfigMap
+# and fixed 2026-08-20 (user: "make the repo proper and not have
 # sensitive things in the files"). The value itself was already the
 # genuine live secret ArgoCD's own OIDC client uses (apps/argocd/
 # manifests/external-secret.yaml pulls the same value from Vault at
 # secret/argocd-oauth) - Vault was already the source of truth on the
 # APP side, this blueprint (the Authentik PROVIDER side) just had its own
 # separate hardcoded plaintext copy instead of also reading from Vault.
 #
 # Converted ConfigMap -> ExternalSecret (chart supports mounting
 # blueprints from Secrets too - see blueprints.secrets in values.yaml).
 # client_id stays inline - it's not a secret, same convention as every
 # other blueprint here.
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: authentik-blueprints-argocd
  namespace: authentik
-  labels:
+spec:
-    goauthentik.io/blueprint: "true"
+  refreshInterval: 1h
-data:
+  secretStoreRef:
-  argocd.yaml: |-
+    kind: ClusterSecretStore
-    version: 1
+    name: vault-backend
-    metadata:
+  target:
-      name: argocd-oidc
+    name: authentik-blueprints-argocd
-    entries:
+    creationPolicy: Owner
-      - model: authentik_providers_oauth2.oauth2provider
+    template:
-        id: argocd-provider
+      metadata:
-        state: present
+        labels:
-        identifiers:
+          goauthentik.io/blueprint: "true"
-          name: ArgoCD
+      data:
-        attrs:
+        argocd.yaml: |-
-          name: ArgoCD
+          version: 1
-          client_id: 3791058a7de278045e5ec36be29f7a33d635119b8c786f20c86b185139c28841
+          metadata:
-          client_secret: f787416822f5bf4c46fe9170e63d1bf9520bab7703d53341aaaec107c8c4ba253d96c30b70d23a1c0b5a401ae1ae4bfb5db06092d338e29e0b0f04cd29bc0b36
+            name: argocd-oidc
-          authorization_flow: !Find [authentik_flows.flow, [slug, default-provider-authorization-implicit-consent]]
+          entries:
-          authentication_flow: !Find [authentik_flows.flow, [slug, default-authentication-flow]]
+            - model: authentik_providers_oauth2.oauth2provider
-          invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
+              id: argocd-provider
-          redirect_uris:
+              state: present
-            - url: "https://argocd.kube.huskypup.net/auth/callback"
+              identifiers:
-              matching_mode: strict
+                name: ArgoCD
-          property_mappings:
+              attrs:
-            - !Find [authentik_providers_oauth2.scopemapping, [scope_name, openid]]
+                name: ArgoCD
-            - !Find [authentik_providers_oauth2.scopemapping, [scope_name, email]]
+                client_id: 3791058a7de278045e5ec36be29f7a33d635119b8c786f20c86b185139c28841
-            - !Find [authentik_providers_oauth2.scopemapping, [scope_name, profile]]
+                client_secret: "{{ .clientSecret }}"
-          client_type: confidential
+                authorization_flow: !Find [authentik_flows.flow, [slug, default-provider-authorization-implicit-consent]]
-          access_code_validity: "minutes=10"
+                authentication_flow: !Find [authentik_flows.flow, [slug, default-authentication-flow]]
-          access_token_validity: "hours=1"
+                invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
-          refresh_token_validity: "days=30"
+                redirect_uris:
-          signing_key: !Find [authentik_crypto.certificatekeypair, [name, "authentik Internal JWT Certificate"]]
+                  - url: "https://argocd.kube.huskypup.net/auth/callback"
-      
+                    matching_mode: strict
-      - model: authentik_core.application
+                property_mappings:
-        id: argocd-application
+                  - !Find [authentik_providers_oauth2.scopemapping, [scope_name, openid]]
-        state: present
+                  - !Find [authentik_providers_oauth2.scopemapping, [scope_name, email]]
-        identifiers:
+                  - !Find [authentik_providers_oauth2.scopemapping, [scope_name, profile]]
-          slug: argocd
+                client_type: confidential
-        attrs:
+                access_code_validity: "minutes=10"
-          name: ArgoCD
+                access_token_validity: "hours=1"
-          slug: argocd
+                refresh_token_validity: "days=30"
-          policy_engine_mode: any
+                signing_key: !Find [authentik_crypto.certificatekeypair, [name, "authentik Internal JWT Certificate"]]
-          provider: !KeyOf argocd-provider
+
            - model: authentik_core.application
              id: argocd-application
              state: present
              identifiers:
                slug: argocd
              attrs:
                name: ArgoCD
                slug: argocd
                policy_engine_mode: any
                provider: !KeyOf argocd-provider
  data:
    - secretKey: clientSecret
      remoteRef:
        key: argocd-oauth
        property: client-secret
@@ -1,53 +1,82 @@
 ---
-apiVersion: v1
+# Was a plain ConfigMap with client_secret hardcoded in plaintext - found
-kind: ConfigMap
+# and fixed 2026-08-20 (user: "make the repo proper and not have
 # sensitive things in the files"). Unlike argocd/n8n/etc, Vault had NO
 # copy of this at all before this fix - the ONLY place it existed was
 # here and in gitlab-oidc-secret (a manually kubectl-created Secret in
 # the gitlab namespace, apps/gitlab/values.yaml:166 - also fixed
 # alongside this). Created secret/gitlab-oauth in Vault with the existing
 # live value (not rotated - this is GitLab's actual working OIDC
 # credential right now, changing it would break login until both sides
 # are updated in lockstep).
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: authentik-blueprints-gitlab
  namespace: authentik
-  labels:
+spec:
-    goauthentik.io/blueprint: "true"
+  refreshInterval: 1h
-data:
+  secretStoreRef:
-  gitlab.yaml: |-
+    kind: ClusterSecretStore
-    # yaml-language-server: $schema=https://goauthentik.io/blueprints/schema.json
+    name: vault-backend
-    version: 1
+  target:
-    metadata:
+    name: authentik-blueprints-gitlab
-      name: gitlab-oidc
+    creationPolicy: Owner
-    entries:
+    template:
-      # OAuth2/OIDC Provider for GitLab
+      metadata:
-      - model: authentik_providers_oauth2.oauth2provider
+        labels:
-        id: gitlab-provider
+          goauthentik.io/blueprint: "true"
-        state: present
+      data:
-        identifiers:
+        gitlab.yaml: |-
-          name: GitLab
+          # yaml-language-server: $schema=https://goauthentik.io/blueprints/schema.json
-        attrs:
+          version: 1
-          name: GitLab
+          metadata:
-          client_id: 70b9da5c8166cea0ba504a869824020bce6636
+            name: gitlab-oidc
-          client_secret: 2e9c0c5b7a8969eb782d08bf1bf3d50ad96f8a73d9c18d1e4141816aebe14a622e8064bf5a5bce835d80e89cdc5f4cc6cf4a8c8234556647dc8884888866bd7d
+          entries:
-          authorization_flow: !Find [authentik_flows.flow, [slug, default-provider-authorization-implicit-consent]]
+            # OAuth2/OIDC Provider for GitLab
-          authentication_flow: !Find [authentik_flows.flow, [slug, default-authentication-flow]]
+            - model: authentik_providers_oauth2.oauth2provider
-          invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
+              id: gitlab-provider
-          redirect_uris:
+              state: present
-            - url: "https://gitlab.kube.huskypup.net/users/auth/openid_connect/callback"
+              identifiers:
-              matching_mode: strict
+                name: GitLab
-          enabled: true
+              attrs:
-          property_mappings:
+                name: GitLab
-            - !Find [authentik_providers_oauth2.scopemapping, [scope_name, openid]]
+                client_id: 70b9da5c8166cea0ba504a869824020bce6636
-            - !Find [authentik_providers_oauth2.scopemapping, [scope_name, email]]
+                client_secret: "{{ .clientSecret }}"
-            - !Find [authentik_providers_oauth2.scopemapping, [scope_name, profile]]
+                authorization_flow: !Find [authentik_flows.flow, [slug, default-provider-authorization-implicit-consent]]
-          client_type: confidential
+                authentication_flow: !Find [authentik_flows.flow, [slug, default-authentication-flow]]
-          access_code_validity: "minutes=10"
+                invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
-          access_token_validity: "days=7"
+                redirect_uris:
-          refresh_token_validity: "days=30"
+                  - url: "https://gitlab.kube.huskypup.net/users/auth/openid_connect/callback"
-          signing_key: !Find [authentik_crypto.certificatekeypair, [name, default]]
+                    matching_mode: strict
-      
+                  # Real GitLab on the VPS (infrastructure/vps-standby/gitlab/) -
-      # Application for GitLab
+                  # shares this same provider/client_id, added 2026-08-21.
-      - model: authentik_core.application
+                  - url: "https://gitlab.vps.huskypup.net/users/auth/openid_connect/callback"
-        id: gitlab-application
+                    matching_mode: strict
-        state: present
+                enabled: true
-        identifiers:
+                property_mappings:
-          slug: gitlab
+                  - !Find [authentik_providers_oauth2.scopemapping, [scope_name, openid]]
-        attrs:
+                  - !Find [authentik_providers_oauth2.scopemapping, [scope_name, email]]
-          name: GitLab
+                  - !Find [authentik_providers_oauth2.scopemapping, [scope_name, profile]]
-          slug: gitlab
+                client_type: confidential
-          policy_engine_mode: any
+                access_code_validity: "minutes=10"
-          provider: !KeyOf gitlab-provider
+                access_token_validity: "days=7"
                refresh_token_validity: "days=30"
                signing_key: !Find [authentik_crypto.certificatekeypair, [name, default]]
            # Application for GitLab
            - model: authentik_core.application
              id: gitlab-application
              state: present
              identifiers:
                slug: gitlab
              attrs:
                name: GitLab
                slug: gitlab
                policy_engine_mode: any
                provider: !KeyOf gitlab-provider
  data:
    - secretKey: clientSecret
      remoteRef:
        key: gitlab-oauth
        property: client-secret
@@ -1,99 +1,121 @@
 ---
-apiVersion: v1
+# Was a plain ConfigMap with client_secret hardcoded in plaintext - found
-kind: ConfigMap
+# and fixed 2026-08-20. Unlike argocd/nextcloud, Vault had NO copy of
 # this at all - the only places it existed were here AND in
 # infrastructure/grafana/manifests/grafana-oauth-secret.yaml (also fixed
 # alongside this, same Vault path). Created secret/grafana-oauth with the
 # existing live value (not rotated - changing it would break login until
 # both sides are updated together).
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: authentik-blueprints-grafana
  namespace: authentik
-  labels:
+spec:
-    goauthentik.io/blueprint: "true"
+  refreshInterval: 1h
-data:
+  secretStoreRef:
-  grafana.yaml: |-
+    kind: ClusterSecretStore
-    version: 1
+    name: vault-backend
-    metadata:
+  target:
-      name: grafana-oidc
+    name: authentik-blueprints-grafana
-    entries:
+    creationPolicy: Owner
-      # ============================================================================
+    template:
-      # AUTO-CREATE GRAFANA GROUPS
+      metadata:
-      # ============================================================================
+        labels:
-      # These groups control access levels in Grafana via OAuth role mapping:
+          goauthentik.io/blueprint: "true"
-      #   - Grafana Admins  → Admin role (full access)
+      data:
-      #   - Grafana Editors → Editor role (can edit dashboards)
+        grafana.yaml: |-
-      #   - Grafana Viewers → Viewer role (read-only)
+          version: 1
-      #
+          metadata:
-      # AUTOMATIC ADMIN ACCESS:
+            name: grafana-oidc
-      #   - Anyone in "authentik Admins" automatically gets Grafana Admin access
+          entries:
-      #   - No manual configuration needed!
+            # ============================================================================
-      # ============================================================================
+            # AUTO-CREATE GRAFANA GROUPS
-      
+            # ============================================================================
-      - model: authentik_core.group
+            # These groups control access levels in Grafana via OAuth role mapping:
-        id: grafana-admins-group
+            #   - Grafana Admins  → Admin role (full access)
-        state: present
+            #   - Grafana Editors → Editor role (can edit dashboards)
-        identifiers:
+            #   - Grafana Viewers → Viewer role (read-only)
-          name: Grafana Admins
+            #
-        attrs:
+            # AUTOMATIC ADMIN ACCESS:
-          name: Grafana Admins
+            #   - Anyone in "authentik Admins" automatically gets Grafana Admin access
-          is_superuser: false
+            #   - No manual configuration needed!
-          # Note: authentik Admins are automatically granted access via role_attribute_path
+            # ============================================================================
-          # in Grafana configuration (see grafana/values.yaml)
+
-      
+            - model: authentik_core.group
-      - model: authentik_core.group
+              id: grafana-admins-group
-        id: grafana-editors-group
+              state: present
-        state: present
+              identifiers:
-        identifiers:
+                name: Grafana Admins
-          name: Grafana Editors
+              attrs:
-        attrs:
+                name: Grafana Admins
-          name: Grafana Editors
+                is_superuser: false
-          is_superuser: false
+                # Note: authentik Admins are automatically granted access via role_attribute_path
-      
+                # in Grafana configuration (see grafana/values.yaml)
-      - model: authentik_core.group
+
-        id: grafana-viewers-group
+            - model: authentik_core.group
-        state: present
+              id: grafana-editors-group
-        identifiers:
+              state: present
-          name: Grafana Viewers
+              identifiers:
-        attrs:
+                name: Grafana Editors
-          name: Grafana Viewers
+              attrs:
-          is_superuser: false
+                name: Grafana Editors
-      
+                is_superuser: false
-      # ============================================================================
+
-      # GRAFANA OAUTH2 PROVIDER
+            - model: authentik_core.group
-      # ============================================================================
+              id: grafana-viewers-group
-      
+              state: present
-      - model: authentik_providers_oauth2.oauth2provider
+              identifiers:
-        id: grafana-provider
+                name: Grafana Viewers
-        state: present
+              attrs:
-        identifiers:
+                name: Grafana Viewers
-          name: Grafana
+                is_superuser: false
-        attrs:
+
-          name: Grafana
+            # ============================================================================
-          client_id: bd03e9139dd2063c6c44c4d2f65f51d69de3ba0b6d6b1b9b41c255d2376d2dcc
+            # GRAFANA OAUTH2 PROVIDER
-          client_secret: 6b6f2deecfe8fd56cae9c512cc71eedd463d67ff08f24c816b15b15e78ce36bc4f06a2276c5ffe3f67799d935994a32ed32cdc02ec674f279c82d7cfe3ca05d5
+            # ============================================================================
-          authorization_flow: !Find [authentik_flows.flow, [slug, default-provider-authorization-implicit-consent]]
+
-          authentication_flow: !Find [authentik_flows.flow, [slug, default-authentication-flow]]
+            - model: authentik_providers_oauth2.oauth2provider
-          invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
+              id: grafana-provider
-          redirect_uris:
+              state: present
-            - url: "https://grafana.kube.huskypup.net/login/generic_oauth"
+              identifiers:
-              matching_mode: strict
+                name: Grafana
-          property_mappings:
+              attrs:
-            - !Find [authentik_providers_oauth2.scopemapping, [scope_name, openid]]
+                name: Grafana
-            - !Find [authentik_providers_oauth2.scopemapping, [scope_name, email]]
+                client_id: bd03e9139dd2063c6c44c4d2f65f51d69de3ba0b6d6b1b9b41c255d2376d2dcc
-            - !Find [authentik_providers_oauth2.scopemapping, [scope_name, profile]]
+                client_secret: "{{ .clientSecret }}"
-          client_type: confidential
+                authorization_flow: !Find [authentik_flows.flow, [slug, default-provider-authorization-implicit-consent]]
-          access_code_validity: "minutes=10"
+                authentication_flow: !Find [authentik_flows.flow, [slug, default-authentication-flow]]
-          access_token_validity: "hours=1"
+                invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
-          refresh_token_validity: "days=30"
+                redirect_uris:
-          signing_key: !Find [authentik_crypto.certificatekeypair, [name, default]]
+                  - url: "https://grafana.kube.huskypup.net/login/generic_oauth"
-      
+                    matching_mode: strict
-      # ============================================================================
+                property_mappings:
-      # GRAFANA APPLICATION
+                  - !Find [authentik_providers_oauth2.scopemapping, [scope_name, openid]]
-      # ============================================================================
+                  - !Find [authentik_providers_oauth2.scopemapping, [scope_name, email]]
-      
+                  - !Find [authentik_providers_oauth2.scopemapping, [scope_name, profile]]
-      - model: authentik_core.application
+                client_type: confidential
-        id: grafana-application
+                access_code_validity: "minutes=10"
-        state: present
+                access_token_validity: "hours=1"
-        identifiers:
+                refresh_token_validity: "days=30"
-          slug: grafana
+                signing_key: !Find [authentik_crypto.certificatekeypair, [name, default]]
-        attrs:
+
-          name: Grafana
+            # ============================================================================
-          slug: grafana
+            # GRAFANA APPLICATION
-          policy_engine_mode: any
+            # ============================================================================
-          provider: !KeyOf grafana-provider
+
            - model: authentik_core.application
              id: grafana-application
              state: present
              identifiers:
                slug: grafana
              attrs:
                name: Grafana
                slug: grafana
                policy_engine_mode: any
                provider: !KeyOf grafana-provider
  data:
    - secretKey: clientSecret
      remoteRef:
        key: grafana-oauth
        property: client-secret
@@ -1,83 +1,103 @@
 ---
-apiVersion: v1
+# Was a plain ConfigMap with client_secret hardcoded in plaintext - found
-kind: ConfigMap
+# and fixed 2026-08-20. Same story as argocd-blueprint.yaml: the value
 # was already Vault's source of truth on the app side (secret/
 # guacamole-oauth), this blueprint just had its own separate hardcoded
 # plaintext copy.
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: authentik-blueprints-guacamole
  namespace: authentik
-  labels:
+spec:
-    goauthentik.io/blueprint: "true"
+  refreshInterval: 1h
-data:
+  secretStoreRef:
-  guacamole.yaml: |-
+    kind: ClusterSecretStore
-    version: 1
+    name: vault-backend
-    metadata:
+  target:
-      name: guacamole-oidc
+    name: authentik-blueprints-guacamole
-      labels:
+    creationPolicy: Owner
-        blueprints.goauthentik.io/instantiate: "true"
+    template:
-    entries:
+      metadata:
-      # ============================================================================
+        labels:
-      # GUACAMOLE OAUTH2 PROVIDER
+          goauthentik.io/blueprint: "true"
-      # ============================================================================
+      data:
-      # This creates the OAuth2 provider that Guacamole will use for authentication.
+        guacamole.yaml: |-
-      # The client_id and client_secret must match what's configured in Guacamole.
+          version: 1
-      # ============================================================================
+          metadata:
            name: guacamole-oidc
            labels:
              blueprints.goauthentik.io/instantiate: "true"
          entries:
            # ============================================================================
            # GUACAMOLE OAUTH2 PROVIDER
            # ============================================================================
            # This creates the OAuth2 provider that Guacamole will use for authentication.
            # The client_id and client_secret must match what's configured in Guacamole.
            # ============================================================================
-      # ============================================================================
+            # ============================================================================
-      # GROUPS SCOPE MAPPING
+            # GROUPS SCOPE MAPPING
-      # ============================================================================
+            # ============================================================================
-      # Custom scope mapping that includes user's group names in the OIDC token.
+            # Custom scope mapping that includes user's group names in the OIDC token.
-      # ============================================================================
+            # ============================================================================
-      - model: authentik_providers_oauth2.scopemapping
+            - model: authentik_providers_oauth2.scopemapping
-        id: guacamole-groups-scope
+              id: guacamole-groups-scope
-        identifiers:
+              identifiers:
-          managed: goauthentik.io/providers/oauth2/guacamole-groups
+                managed: goauthentik.io/providers/oauth2/guacamole-groups
-        attrs:
+              attrs:
-          name: "Guacamole Groups Scope"
+                name: "Guacamole Groups Scope"
-          scope_name: groups
+                scope_name: groups
-          expression: |
+                expression: |
-            return {
+                  return {
-              "groups": [group.name for group in request.user.ak_groups.all()],
+                    "groups": [group.name for group in request.user.ak_groups.all()],
-            }
+                  }
-      - model: authentik_providers_oauth2.oauth2provider
+            - model: authentik_providers_oauth2.oauth2provider
-        id: guacamole-provider
+              id: guacamole-provider
-        identifiers:
+              identifiers:
-          name: guacamole
+                name: guacamole
-        attrs:
+              attrs:
-          name: Guacamole
+                name: Guacamole
-          client_id: a55cbe3c6d2ae30c05c86385802ffcfd76a972c22c8d039f000281d4a149fd44
+                client_id: a55cbe3c6d2ae30c05c86385802ffcfd76a972c22c8d039f000281d4a149fd44
-          client_secret: 718d4c4816aa68767a6a0da0fa4099d492473c700ba236cbeb64d294762b9548acda7e716ba11b2ad7729d6a1c16c7890b227e3b2dcfc35fd4af1eb0d6662b0f
+                client_secret: "{{ .clientSecret }}"
-          authorization_flow: !Find [authentik_flows.flow, [slug, default-provider-authorization-implicit-consent]]
+                authorization_flow: !Find [authentik_flows.flow, [slug, default-provider-authorization-implicit-consent]]
-          authentication_flow: !Find [authentik_flows.flow, [slug, default-authentication-flow]]
+                authentication_flow: !Find [authentik_flows.flow, [slug, default-authentication-flow]]
-          invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
+                invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
-          redirect_uris:
+                redirect_uris:
-            - url: "https://guacamole.kube.huskypup.net/"
+                  - url: "https://guacamole.kube.huskypup.net/"
-              matching_mode: strict
+                    matching_mode: strict
-            - url: "https://guacamole.kube.huskypup.net/guacamole/"
+                  - url: "https://guacamole.kube.huskypup.net/guacamole/"
-              matching_mode: strict
+                    matching_mode: strict
-          property_mappings:
+                property_mappings:
-            - !Find [authentik_providers_oauth2.scopemapping, [scope_name, openid]]
+                  - !Find [authentik_providers_oauth2.scopemapping, [scope_name, openid]]
-            - !Find [authentik_providers_oauth2.scopemapping, [scope_name, email]]
+                  - !Find [authentik_providers_oauth2.scopemapping, [scope_name, email]]
-            - !Find [authentik_providers_oauth2.scopemapping, [scope_name, profile]]
+                  - !Find [authentik_providers_oauth2.scopemapping, [scope_name, profile]]
-            - !KeyOf guacamole-groups-scope
+                  - !KeyOf guacamole-groups-scope
-          client_type: confidential
+                client_type: confidential
-          access_code_validity: "minutes=10"
+                access_code_validity: "minutes=10"
-          access_token_validity: "hours=1"
+                access_token_validity: "hours=1"
-          refresh_token_validity: "days=30"
+                refresh_token_validity: "days=30"
-          signing_key: !Find [authentik_crypto.certificatekeypair, [name, "Lets Encrypt K8s Cert"]]
+                signing_key: !Find [authentik_crypto.certificatekeypair, [name, "Lets Encrypt K8s Cert"]]
-      # ============================================================================
+            # ============================================================================
-      # GUACAMOLE APPLICATION
+            # GUACAMOLE APPLICATION
-      # ============================================================================
+            # ============================================================================
-      # This creates the application entry that users will see in Authentik.
+            # This creates the application entry that users will see in Authentik.
-      # ============================================================================
+            # ============================================================================
-      - model: authentik_core.application
+            - model: authentik_core.application
-        id: guacamole-application
+              id: guacamole-application
-        identifiers:
+              identifiers:
-          slug: guacamole
+                slug: guacamole
-        attrs:
+              attrs:
-          name: Guacamole
+                name: Guacamole
-          slug: guacamole
+                slug: guacamole
-          policy_engine_mode: any
+                policy_engine_mode: any
-          provider: !KeyOf guacamole-provider
+                provider: !KeyOf guacamole-provider
  data:
    - secretKey: clientSecret
      remoteRef:
        key: guacamole-oauth
        property: client-secret
@@ -0,0 +1,36 @@
 # Multi-site active failover pilot (see
 # /home/scooby/.claude/plans/jiggly-snacking-iverson.md) - closes the gap
 # the user correctly flagged: manually kubectl/ssh-copying secrets between
 # home and the VPS defeats the entire point of having Vault. This is the
 # first piece of a real Vault -> VPS pipeline (infrastructure/vps-eso/ has
 # the other half, ESO running on the VPS itself pulling this back down).
 #
 # pg-authentik-app is CNPG-generated, not Vault-native - each cluster
 # (home and VPS) independently generates its own random password for the
 # `app` role at bootstrap time. Since VPS's Postgres now REPLICATES from
 # home (WAL includes role/password changes), the two clusters' actual live
 # passwords are only in sync because of that replication - but the two
 # clusters' K8S SECRET OBJECTS never resync on their own (confirmed live
 # 2026-08-20: VPS's copy was stale). Pushing home's value into Vault, with
 # VPS's ESO pulling it back down on refreshInterval, makes this self-
 # healing instead of a manual one-time fix that goes stale again on the
 # next password rotation.
 apiVersion: external-secrets.io/v1alpha1
 kind: PushSecret
 metadata:
  name: pg-authentik-app-to-vps
  namespace: authentik
 spec:
  refreshInterval: 5m
  secretStoreRefs:
    - name: vault-backend
      kind: ClusterSecretStore
  selector:
    secret:
      name: pg-authentik-app
  data:
    - match:
        secretKey: password
        remoteRef:
          remoteKey: vps/pg-authentik-app
          property: password
@@ -6,6 +6,18 @@
 # object and won't fight with it. Needs
 # infrastructure/kyverno/policies/ha-failover-nodeport-exception.yaml
 # (disallow-nodeport-services is enforced cluster-wide otherwise).
 #
 # Second port (61432, ClusterIP only - no nodePort needed) added for the
 # CoreDNS rewrite in infrastructure/coredns/manifests/: home's own pods
 # resolving pg-authentik.ha.huskypup.net rewrite to THIS service (not
 # CNPG's own pg-authentik-rw) specifically so they can reach it on the
 # SAME port number the app tier is configured with (61432, matching the
 # external floating hostname's port - see infrastructure/authentik/
 # values.yaml's AUTHENTIK_POSTGRESQL__PORT) - CoreDNS rewrite only
 # changes the resolved NAME, not the port the client then connects to,
 # so the target service has to actually listen on that same port too, or
 # the connection times out even though DNS resolves correctly (confirmed
 # live 2026-08-20).
 apiVersion: v1
 kind: Service
 metadata:
@@ -17,6 +29,10 @@ spec:
    cnpg.io/cluster: pg-authentik
    cnpg.io/instanceRole: primary
  ports:
-    - port: 5432
+    - name: nodeport-5432
      port: 5432
      targetPort: 5432
      nodePort: 32432
    - name: floating-port-61432
      port: 61432
      targetPort: 5432
@@ -38,3 +38,14 @@ spec:
  portLevelMtls:
    "5432":
      mode: PERMISSIVE
    # Multi-site active failover pilot, floating hostname (see
    # infrastructure/authentik/manifests/ha-postgres-nodeport.yaml): the
    # ha-authentik-postgres Service also listens on 61432 (-> targetPort
    # 5432, same pods) so home's own pods can reach it via the
    # pg-authentik.ha.huskypup.net CoreDNS rewrite on the same port the
    # app tier is configured with externally. Needed its own PERMISSIVE
    # entry - confirmed live 2026-08-20 that ambient's port-level mTLS
    # enforcement is keyed on the port actually dialed (61432), not just
    # the pod's real containerPort (5432) it eventually reaches.
    "61432":
      mode: PERMISSIVE
@@ -1,49 +1,68 @@
 ---
-apiVersion: v1
+# Was a plain ConfigMap with client_secret hardcoded in plaintext - found
-kind: ConfigMap
+# and fixed 2026-08-20. Same story as argocd-blueprint.yaml: the value
 # was already Vault's source of truth on the app side (secret/n8n-oauth),
 # this blueprint just had its own separate hardcoded plaintext copy.
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: authentik-blueprints-n8n
  namespace: authentik
-  labels:
+spec:
-    goauthentik.io/blueprint: "true"
+  refreshInterval: 1h
-data:
+  secretStoreRef:
-  n8n.yaml: |-
+    kind: ClusterSecretStore
-    version: 1
+    name: vault-backend
-    metadata:
+  target:
-      name: n8n-oidc
+    name: authentik-blueprints-n8n
-    entries:
+    creationPolicy: Owner
-      - model: authentik_providers_oauth2.oauth2provider
+    template:
-        id: n8n-provider
+      metadata:
-        state: present
+        labels:
-        identifiers:
+          goauthentik.io/blueprint: "true"
-          name: n8n
+      data:
-        attrs:
+        n8n.yaml: |-
-          name: n8n
+          version: 1
-          client_id: e55cbe3c6d2ae30c05c86385802ffcfd76a972c22c8d039f000281d4a149fd44
+          metadata:
-          client_secret: 718d4c4816aa68767a6a0da0fa4099d492473c700ba236cbeb64d294762b9548acda7e716ba11b2ad7729d6a1c16c7890b227e3b2dcfc35fd4af1eb0d6662b0f
+            name: n8n-oidc
-          authorization_flow: !Find [authentik_flows.flow, [slug, default-provider-authorization-implicit-consent]]
+          entries:
-          authentication_flow: !Find [authentik_flows.flow, [slug, default-authentication-flow]]
+            - model: authentik_providers_oauth2.oauth2provider
-          invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
+              id: n8n-provider
-          redirect_uris:
+              state: present
-            - url: "https://n8n.kube.huskypup.net/callback"
+              identifiers:
-              matching_mode: strict
+                name: n8n
-          property_mappings:
+              attrs:
-            - !Find [authentik_providers_oauth2.scopemapping, [scope_name, openid]]
+                name: n8n
-            - !Find [authentik_providers_oauth2.scopemapping, [scope_name, email]]
+                client_id: e55cbe3c6d2ae30c05c86385802ffcfd76a972c22c8d039f000281d4a149fd44
-            - !Find [authentik_providers_oauth2.scopemapping, [scope_name, profile]]
+                client_secret: "{{ .clientSecret }}"
-          client_type: confidential
+                authorization_flow: !Find [authentik_flows.flow, [slug, default-provider-authorization-implicit-consent]]
-          access_code_validity: "minutes=10"
+                authentication_flow: !Find [authentik_flows.flow, [slug, default-authentication-flow]]
-          access_token_validity: "hours=1"
+                invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
-          refresh_token_validity: "days=30"
+                redirect_uris:
-          signing_key: !Find [authentik_crypto.certificatekeypair, [name, "authentik Self-signed Certificate"]]
+                  - url: "https://n8n.kube.huskypup.net/callback"
                    matching_mode: strict
                property_mappings:
                  - !Find [authentik_providers_oauth2.scopemapping, [scope_name, openid]]
                  - !Find [authentik_providers_oauth2.scopemapping, [scope_name, email]]
                  - !Find [authentik_providers_oauth2.scopemapping, [scope_name, profile]]
                client_type: confidential
                access_code_validity: "minutes=10"
                access_token_validity: "hours=1"
                refresh_token_validity: "days=30"
                signing_key: !Find [authentik_crypto.certificatekeypair, [name, "authentik Self-signed Certificate"]]
-      - model: authentik_core.application
+            - model: authentik_core.application
-        id: n8n-application
+              id: n8n-application
-        state: present
+              state: present
-        identifiers:
+              identifiers:
-          slug: n8n
+                slug: n8n
-        attrs:
+              attrs:
-          name: n8n
+                name: n8n
-          slug: n8n
+                slug: n8n
-          policy_engine_mode: any
+                policy_engine_mode: any
-          provider: !KeyOf n8n-provider
+                provider: !KeyOf n8n-provider
  data:
    - secretKey: clientSecret
      remoteRef:
        key: n8n-oauth
        property: client-secret
@@ -1,52 +1,72 @@
---
+---
-apiVersion: v1
+# Was a plain ConfigMap with client_secret hardcoded in plaintext - found
-kind: ConfigMap
+# and fixed 2026-08-20. Same story as argocd-blueprint.yaml: the value
-metadata:
+# was already Vault's source of truth on the app side (secret/
-  name: authentik-blueprints-nextcloud
+# nextcloud-oauth), this blueprint just had its own separate hardcoded
-  namespace: authentik
+# plaintext copy.
-  labels:
+apiVersion: external-secrets.io/v1
-    goauthentik.io/blueprint: "true"
+kind: ExternalSecret
-data:
+metadata:
-   nextcloud.yaml: |-
+  name: authentik-blueprints-nextcloud
-    version: 1
+  namespace: authentik
-    metadata:
+spec:
-      name: nextcloud-oidc
+  refreshInterval: 1h
-    entries:
+  secretStoreRef:
-      - model: authentik_providers_oauth2.oauth2provider
+    kind: ClusterSecretStore
-        id: nextcloud-provider
+    name: vault-backend
-        state: present
+  target:
-        identifiers:
+    name: authentik-blueprints-nextcloud
-          name: Nextcloud
+    creationPolicy: Owner
-        attrs:
+    template:
-          name: Nextcloud
+      metadata:
-          client_id: HmTGnUfhCvjFFbUUAne7JLrucNC2X675JicPbEnQ
+        labels:
-          client_secret: qTUf4NYjzUuAqfNV5D7SFPuuhJJwVltU6U3LAuJVy7dddJWeJSv2B4wKaRxbbHqpIiSfYPDaKujhEysKbOWUnf4n9OHPKJ0bu1I2EzkYtChO1Gq4gUv5EHPTzQlMBPnl
+          goauthentik.io/blueprint: "true"
-          authorization_flow: !Find [authentik_flows.flow, [slug, default-provider-authorization-implicit-consent]]
+      data:
-          authentication_flow: !Find [authentik_flows.flow, [slug, default-authentication-flow]]
+        nextcloud.yaml: |-
-          invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
+          version: 1
-          redirect_uris:
+          metadata:
-            - url: "https://nextcloud.kube.huskypup.net/apps/user_oidc/code"
+            name: nextcloud-oidc
-              matching_mode: strict
+          entries:
-            - url: "http://nextcloud.kube.huskypup.net/apps/user_oidc/code"
+            - model: authentik_providers_oauth2.oauth2provider
-              matching_mode: strict
+              id: nextcloud-provider
-          enabled: true
+              state: present
-          property_mappings:
+              identifiers:
-            - !Find [authentik_providers_oauth2.scopemapping, [scope_name, openid]]
+                name: Nextcloud
-            - !Find [authentik_providers_oauth2.scopemapping, [scope_name, email]]
+              attrs:
-            - !Find [authentik_providers_oauth2.scopemapping, [scope_name, profile]]
+                name: Nextcloud
-          client_type: confidential
+                client_id: HmTGnUfhCvjFFbUUAne7JLrucNC2X675JicPbEnQ
-          access_code_validity: "minutes=10"
+                client_secret: "{{ .clientSecret }}"
-          access_token_validity: "days=7"
+                authorization_flow: !Find [authentik_flows.flow, [slug, default-provider-authorization-implicit-consent]]
-          refresh_token_validity: "days=30"
+                authentication_flow: !Find [authentik_flows.flow, [slug, default-authentication-flow]]
-          signing_key: !Find [authentik_crypto.certificatekeypair, [name, "authentik Internal JWT Certificate"]]
+                invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
-      
+                redirect_uris:
-      - model: authentik_core.application
+                  - url: "https://nextcloud.kube.huskypup.net/apps/user_oidc/code"
-        id: nextcloud-application
+                    matching_mode: strict
-        state: present
+                  - url: "http://nextcloud.kube.huskypup.net/apps/user_oidc/code"
-        identifiers:
+                    matching_mode: strict
-          slug: nextcloud
+                enabled: true
-        attrs:
+                property_mappings:
-          name: Nextcloud
+                  - !Find [authentik_providers_oauth2.scopemapping, [scope_name, openid]]
-          slug: nextcloud
+                  - !Find [authentik_providers_oauth2.scopemapping, [scope_name, email]]
-          policy_engine_mode: any
+                  - !Find [authentik_providers_oauth2.scopemapping, [scope_name, profile]]
-          provider: !KeyOf nextcloud-provider
+                client_type: confidential
                access_code_validity: "minutes=10"
                access_token_validity: "days=7"
                refresh_token_validity: "days=30"
                signing_key: !Find [authentik_crypto.certificatekeypair, [name, "authentik Internal JWT Certificate"]]
            - model: authentik_core.application
              id: nextcloud-application
              state: present
              identifiers:
                slug: nextcloud
              attrs:
                name: Nextcloud
                slug: nextcloud
                policy_engine_mode: any
                provider: !KeyOf nextcloud-provider
  data:
    - secretKey: clientSecret
      remoteRef:
        key: nextcloud-oauth
        property: client-secret
@@ -1,54 +1,73 @@
 ---
-apiVersion: v1
+# Was a plain ConfigMap with client_secret hardcoded in plaintext - found
-kind: ConfigMap
+# and fixed 2026-08-20. Same story as argocd-blueprint.yaml: the value
 # was already Vault's source of truth on the app side (secret/
 # rancher-oauth), this blueprint just had its own separate hardcoded
 # plaintext copy.
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: authentik-blueprints-rancher
  namespace: authentik
-  labels:
+spec:
-    goauthentik.io/blueprint: "true"
+  refreshInterval: 1h
-data:
+  secretStoreRef:
-  rancher.yaml: |-
+    kind: ClusterSecretStore
-    # yaml-language-server: $schema=https://goauthentik.io/blueprints/schema.json
+    name: vault-backend
-    version: 1
+  target:
-    metadata:
+    name: authentik-blueprints-rancher
-      name: rancher-oidc
+    creationPolicy: Owner
-    entries:
+    template:
-      - model: authentik_providers_oauth2.oauth2provider
+      metadata:
-        id: rancher-provider
+        labels:
-        state: present
+          goauthentik.io/blueprint: "true"
-        identifiers:
+      data:
-          name: Rancher
+        rancher.yaml: |-
-        attrs:
+          # yaml-language-server: $schema=https://goauthentik.io/blueprints/schema.json
-          name: Rancher
+          version: 1
-          client_id: 94a6beeaa91d0d0407cf8af807e8208a611cf3f25ae42161e697a8dccd563449
+          metadata:
-          client_secret: 19ebd4545ba74a93c220d4471af634302c5991204cf650cabd522b0864d7617ded2c59b470ab6de7a5801e7de4c84e4b9741b983170815419c9503d55f849794
+            name: rancher-oidc
-          authorization_flow: !Find [authentik_flows.flow, [slug, default-provider-authorization-implicit-consent]]
+          entries:
-          authentication_flow: !Find [authentik_flows.flow, [slug, default-authentication-flow]]
+            - model: authentik_providers_oauth2.oauth2provider
-          invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
+              id: rancher-provider
-          redirect_uris:
+              state: present
-            - url: "https://rancher.kube.huskypup.net/verify-auth"
+              identifiers:
-              matching_mode: strict
+                name: Rancher
-            - url: "https://rancher.kube.huskypup.net"
+              attrs:
-              matching_mode: strict
+                name: Rancher
-          enabled: true
+                client_id: 94a6beeaa91d0d0407cf8af807e8208a611cf3f25ae42161e697a8dccd563449
-          property_mappings:
+                client_secret: "{{ .clientSecret }}"
-            - !Find [authentik_providers_oauth2.scopemapping, [scope_name, openid]]
+                authorization_flow: !Find [authentik_flows.flow, [slug, default-provider-authorization-implicit-consent]]
-            - !Find [authentik_providers_oauth2.scopemapping, [scope_name, email]]
+                authentication_flow: !Find [authentik_flows.flow, [slug, default-authentication-flow]]
-            - !Find [authentik_providers_oauth2.scopemapping, [scope_name, profile]]
+                invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
-          client_type: confidential
+                redirect_uris:
-          access_code_validity: "minutes=10"
+                  - url: "https://rancher.kube.huskypup.net/verify-auth"
-          access_token_validity: "hours=1"
+                    matching_mode: strict
-          refresh_token_validity: "days=30"
+                  - url: "https://rancher.kube.huskypup.net"
-          signing_key: !Find [authentik_crypto.certificatekeypair, [name, "authentik Internal JWT Certificate"]]
+                    matching_mode: strict
-
+                enabled: true
-      - model: authentik_core.application
+                property_mappings:
-        id: rancher-application
+                  - !Find [authentik_providers_oauth2.scopemapping, [scope_name, openid]]
-        state: present
+                  - !Find [authentik_providers_oauth2.scopemapping, [scope_name, email]]
-        identifiers:
+                  - !Find [authentik_providers_oauth2.scopemapping, [scope_name, profile]]
-          slug: rancher
+                client_type: confidential
-        attrs:
+                access_code_validity: "minutes=10"
-          name: Rancher
+                access_token_validity: "hours=1"
-          slug: rancher
+                refresh_token_validity: "days=30"
-          policy_engine_mode: any
+                signing_key: !Find [authentik_crypto.certificatekeypair, [name, "authentik Internal JWT Certificate"]]
          provider: !KeyOf rancher-provider
            - model: authentik_core.application
              id: rancher-application
              state: present
              identifiers:
                slug: rancher
              attrs:
                name: Rancher
                slug: rancher
                policy_engine_mode: any
                provider: !KeyOf rancher-provider
  data:
    - secretKey: clientSecret
      remoteRef:
        key: rancher-oauth
        property: client-secret
@@ -13,20 +13,52 @@ global:
    # First secret: Authentik-specific settings (AUTHENTIK_SECRET_KEY)
    - secretRef:
        name: authentik
-    # Second secret: Database credentials from CNPG
+    # The second secretRef (pg-authentik-app, prefix: AUTHENTIK_POSTGRESQL__)
-    - secretRef:
+    # that used to live here is GONE - found live 2026-08-20 that it was
-        name: pg-authentik-app
+    # actively breaking things, not just redundant. That prefix mechanism
-      prefix: AUTHENTIK_POSTGRESQL__
+    # copies the raw secret's lowercase keys (host, port, dbname, username,
    # password) into lowercase-suffixed env vars (AUTHENTIK_POSTGRESQL__host,
    # etc.) - env var names are case-sensitive to Kubernetes, but authentik's
    # own generic AUTHENTIK_*-prefixed env-var scanner is NOT, and it was
    # picking up the lowercase AUTHENTIK_POSTGRESQL__host=pg-authentik-rw
    # instead of the explicit uppercase AUTHENTIK_POSTGRESQL__HOST override
    # below - silently connecting to the local CNPG service on port 5432
    # the whole time regardless of what HOST/PORT were explicitly set to,
    # surfacing as an unexplained "connection timeout expired" loop (the
    # local hostname resolves fine in-cluster but on the VPS the request
    # apparently doesn't complete - never fully root-caused, just confirmed
    # dropping this duplicate source fixes it). Every key that secretRef
    # used to provide (dbname/username/password) is already explicitly set
    # via valueFrom below, so nothing is lost by removing it.
  env:
    # Configure external URL for proper OIDC discovery responses
    - name: AUTHENTIK_URL
      value: "https://auth.kube.huskypup.net"
-    # Override to use the correct field names from pg-authentik-app
+    # Multi-site active failover pilot (see
    # /home/scooby/.claude/plans/jiggly-snacking-iverson.md): HOST/PORT
    # point at the floating pg-authentik.ha.huskypup.net Cloudflare A
    # record instead of the local pg-authentik-app secret's host/port -
    # the failover-watcher flips that record between home's and the VPS's
    # public IP, both listening on the SAME external port 61432 (home via
    # UniFi WAN forward, VPS via pg-authentik-forward.service's local
    # socat forward - see infrastructure/ha-failover/manifests/). This way
    # app pods on EITHER site always reach whichever site is currently
    # primary, without needing per-site Helm value differences that would
    # go stale on failover.
    #
    # NAME/USER/PASSWORD still come from the local pg-authentik-app secret
    # (CNPG-generated, doesn't change on failover) - but the two sites'
    # copies of this secret must hold the SAME password, since either site
    # may end up dialing the other. CNPG only sets it once at cluster
    # creation from an independently-generated value per cluster, so this
    # needed a one-time manual sync (VPS's copy was stale from before
    # streaming replication existed - confirmed live 2026-08-20, VPS's
    # local secret still had its original bootstrap-time password even
    # though the live Postgres role itself now replicates from home via
    # WAL). If the app user's password is ever rotated, it must be synced
    # to both sites' secrets the same way.
    - name: AUTHENTIK_POSTGRESQL__HOST
-      valueFrom:
+      value: "pg-authentik.ha.huskypup.net"
        secretKeyRef:
          name: pg-authentik-app
          key: host
    - name: AUTHENTIK_POSTGRESQL__NAME
      valueFrom:
        secretKeyRef:
@@ -43,24 +75,28 @@ global:
          name: pg-authentik-app
          key: password
    - name: AUTHENTIK_POSTGRESQL__PORT
-      value: "5432"
+      value: "61432"
-# Blueprints - Mount ConfigMaps for auto-discovery
+# Blueprints - Mount ConfigMaps for auto-discovery (no secrets) or
 # Secrets (client_secret-bearing ones, ExternalSecret-templated from
 # Vault - found and fixed 2026-08-20, see each file's own header for
 # what it used to look like).
 blueprints:
  configMaps:
    - authentik-blueprints-forward-auth
    - authentik-blueprints-prometheus
    - authentik-blueprints-ceph
    - authentik-blueprints-home-assistant
    - authentik-blueprints-percona-everest
    - authentik-blueprints-netbird
  secrets:
    - authentik-blueprints-gitlab
    - authentik-blueprints-nextcloud
    - authentik-blueprints-argocd
    - authentik-blueprints-grafana
    - authentik-blueprints-forward-auth
    - authentik-blueprints-prometheus
    - authentik-blueprints-n8n
    - authentik-blueprints-ceph
    - authentik-blueprints-home-assistant
    - authentik-blueprints-guacamole
    - authentik-blueprints-percona-everest
    - authentik-blueprints-rancher
    - authentik-blueprints-netbird
    - authentik-blueprints-vault
 # Enable Prometheus metrics
@@ -1,51 +1,83 @@
 ---
-apiVersion: v1
+# Was a plain ConfigMap with client_secret hardcoded in plaintext - found
-kind: ConfigMap
+# and fixed 2026-08-20. Unlike argocd/nextcloud, Vault had NO copy of
 # this at all - the only places it existed were here AND in
 # infrastructure/vault/manifests/vault-init-configmap.yaml (embedded in a
 # shell script that runs `vault write auth/oidc/config` once at cluster
 # init - also fixed alongside this, same Vault path). Created
 # secret/vault-oauth with the existing live value (not rotated - this is
 # what Vault's OWN oidc auth method is currently configured with).
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: authentik-blueprints-vault
  namespace: authentik
-  labels:
+spec:
-    goauthentik.io/blueprint: "true"
+  refreshInterval: 1h
-data:
+  secretStoreRef:
-  vault.yaml: |-
+    kind: ClusterSecretStore
-    version: 1
+    name: vault-backend
-    metadata:
+  target:
-      name: vault-oidc
+    name: authentik-blueprints-vault
-    entries:
+    creationPolicy: Owner
-      - model: authentik_providers_oauth2.oauth2provider
+    template:
-        id: vault-provider
+      metadata:
-        state: present
+        labels:
-        identifiers:
+          goauthentik.io/blueprint: "true"
-          name: Vault
+      data:
-        attrs:
+        vault.yaml: |-
-          name: Vault
+          version: 1
-          client_id: 9816a5ae7e7914b5d18f4ab939d011a98f8c8d6b3bb6777c46431afa06ac4a85
+          metadata:
-          client_secret: ed2ba1c6378c7a46341b5162f39a7fab80e37596b01ed387c3719e8e0040344cf1daa307476c2e7a7f75041b3979275b1ebf00bb8bad94c864b4a38ded544f7b
+            name: vault-oidc
-          authorization_flow: !Find [authentik_flows.flow, [slug, default-provider-authorization-implicit-consent]]
+          entries:
-          authentication_flow: !Find [authentik_flows.flow, [slug, default-authentication-flow]]
+            - model: authentik_providers_oauth2.oauth2provider
-          invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
+              id: vault-provider
-          redirect_uris:
+              state: present
-            - url: "https://vault.kube.huskypup.net/ui/vault/auth/oidc/oidc/callback"
+              identifiers:
-              matching_mode: strict
+                name: Vault
-            - url: "http://localhost:8250/oidc/callback"
+              attrs:
-              matching_mode: strict
+                name: Vault
-          property_mappings:
+                client_id: 9816a5ae7e7914b5d18f4ab939d011a98f8c8d6b3bb6777c46431afa06ac4a85
-            - !Find [authentik_providers_oauth2.scopemapping, [scope_name, openid]]
+                client_secret: "{{ .clientSecret }}"
-            - !Find [authentik_providers_oauth2.scopemapping, [scope_name, email]]
+                authorization_flow: !Find [authentik_flows.flow, [slug, default-provider-authorization-implicit-consent]]
-            - !Find [authentik_providers_oauth2.scopemapping, [scope_name, profile]]
+                authentication_flow: !Find [authentik_flows.flow, [slug, default-authentication-flow]]
-          client_type: confidential
+                invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
-          access_code_validity: "minutes=10"
+                redirect_uris:
-          access_token_validity: "hours=1"
+                  - url: "https://vault.kube.huskypup.net/ui/vault/auth/oidc/oidc/callback"
-          refresh_token_validity: "days=30"
+                    matching_mode: strict
-          signing_key: !Find [authentik_crypto.certificatekeypair, [name, "authentik Internal JWT Certificate"]]
+                  # VPS's own standby Vault instance (infrastructure/vps-standby/vault/)
                  # shares this SAME Authentik provider/client_id - found missing
                  # 2026-08-21 while testing VPS vault login ("missing auth_url" -
                  # Vault silently returns no auth_url when the UI's computed
                  # redirect_uri isn't in allowed_redirect_uris; same fix needed on
                  # the VPS Vault's own auth/oidc/role/default, done separately since
                  # it's a fully independent Vault instance/backend).
                  - url: "https://vault.vps.huskypup.net/ui/vault/auth/oidc/oidc/callback"
                    matching_mode: strict
                  - url: "http://localhost:8250/oidc/callback"
                    matching_mode: strict
                property_mappings:
                  - !Find [authentik_providers_oauth2.scopemapping, [scope_name, openid]]
                  - !Find [authentik_providers_oauth2.scopemapping, [scope_name, email]]
                  - !Find [authentik_providers_oauth2.scopemapping, [scope_name, profile]]
                client_type: confidential
                access_code_validity: "minutes=10"
                access_token_validity: "hours=1"
                refresh_token_validity: "days=30"
                signing_key: !Find [authentik_crypto.certificatekeypair, [name, "authentik Internal JWT Certificate"]]
-      - model: authentik_core.application
+            - model: authentik_core.application
-        id: vault-application
+              id: vault-application
-        state: present
+              state: present
-        identifiers:
+              identifiers:
-          slug: vault
+                slug: vault
-        attrs:
+              attrs:
-          name: Vault
+                name: Vault
-          slug: vault
+                slug: vault
-          policy_engine_mode: any
+                policy_engine_mode: any
-          provider: !KeyOf vault-provider
+                provider: !KeyOf vault-provider
  data:
    - secretKey: clientSecret
      remoteRef:
        key: vault-oauth
        property: client-secret
@@ -0,0 +1,76 @@
 # Brings kube-system/coredns's Corefile under GitOps management. It was
 # previously only ever edited live via kubectl (3 existing rewrite rules
 # for auth/rancher/netbird.kube.huskypup.net, none of them git-tracked
 # anywhere in this repo) - every future change meant a manual,
 # classifier-gated live edit to a cluster-wide DNS config instead of a
 # normal git commit ArgoCD applies on its own. No Talos/kubeadm
 # ownerReferences on this ConfigMap or the coredns Deployment (confirmed
 # live 2026-08-20), so it's safe to take over here.
 #
 # No pod restart needed after a sync - the Corefile's own `reload` plugin
 # (already present below) watches the mounted file and hot-reloads within
 # ~30-60s of the ConfigMap volume actually updating on disk (kubelet
 # propagates ConfigMap changes to mounted volumes automatically, typically
 # within ~60-90s).
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: coredns
  namespace: kube-system
 data:
  Corefile: |
    .:53 {
        errors
        health {
            lameduck 5s
        }
        ready
        log . {
            class error
        }
        prometheus :9153
        # Rewrite auth.kube.huskypup.net to ingress gateway (NOT authentik-server directly)
        # This ensures HTTPS works because the ingress gateway has the wildcard TLS cert.
        # The old rewrite to authentik-server caused TLS failures for OIDC token exchange.
        rewrite name auth.kube.huskypup.net istio-ingressgateway.istio-system.svc.cluster.local
        # Rewrite rancher.kube.huskypup.net to internal Istio ingress gateway
        rewrite name rancher.kube.huskypup.net istio-ingressgateway.istio-system.svc.cluster.local
        # Rewrite netbird.kube.huskypup.net to internal Istio ingress gateway
        rewrite name netbird.kube.huskypup.net istio-ingressgateway.istio-system.svc.cluster.local
        # Multi-site active failover pilot (see
        # /home/scooby/.claude/plans/jiggly-snacking-iverson.md): home's
        # own pods resolving the floating pg-authentik.ha.huskypup.net
        # record always go straight to a local service instead of
        # round-tripping out through the UniFi WAN port-forward and back
        # in - this UniFi doesn't support NAT hairpin/loopback for its
        # own forwarded ports (confirmed live 2026-08-20: DNS resolved
        # fine, TCP connect failed). Points at ha-authentik-postgres, NOT
        # CNPG's own pg-authentik-rw directly - rewrite only changes the
        # resolved name, not the port a client then connects to, and the
        # app tier is configured for port 61432 (matching the external
        # floating hostname) - pg-authentik-rw only listens on 5432,
        # ha-authentik-postgres additionally listens on 61432 (see
        # infrastructure/authentik/manifests/ha-postgres-nodeport.yaml)
        # specifically so this rewrite target actually has that port.
        rewrite name pg-authentik.ha.huskypup.net ha-authentik-postgres.authentik.svc.cluster.local
        kubernetes cluster.local in-addr.arpa ip6.arpa {
            pods insecure
            fallthrough in-addr.arpa ip6.arpa
            ttl 30
        }
        forward . 1.1.1.1 8.8.8.8 {
           max_concurrent 1000
        }
        cache 30 {
           disable success cluster.local
           disable denial cluster.local
        }
        loop
        reload
        loadbalance
    }
@@ -1,8 +1,33 @@
 ---
-apiVersion: v1
+# Was a plain Secret with a live UniFi API key committed in plaintext -
-kind: Secret
+# found and fixed 2026-08-20 (same class of issue as the Cloudflare token
 # in infrastructure/cert-manager/manifests/secret-cf-token.yaml, which was
 # already rotated to this pattern - this one was missed at the time).
 # Moved to Vault+ExternalSecret, matching that precedent.
 #
 # The value in Vault right now (secret/unifi-api-key) is still the
 # ORIGINAL key that was exposed in git history - rotating it (Settings ->
 # Admins -> API Keys on the UniFi controller at https://172.28.110.1, no
 # self-service API for this) is a separate, still-pending step. Once
 # rotated:
 #   vault kv put secret/unifi-api-key api-key=<new key>
 # ESO picks it up automatically (refreshInterval below), no restart
 # needed on external-dns's side.
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: external-dns-unifi-secret
  namespace: external-dns
-stringData:
+spec:
-  api-key: 3Qha5hupHsCkkFyQb8z-T2kh-BasIGSH
+  refreshInterval: 1h
  secretStoreRef:
    kind: ClusterSecretStore
    name: vault-backend
  target:
    name: external-dns-unifi-secret
    creationPolicy: Owner
  data:
    - secretKey: api-key
      remoteRef:
        key: unifi-api-key
        property: api-key
@@ -1,12 +1,32 @@
-# infrastructure/grafana/grafana-oauth-secret.yaml
+# Was a plain Secret with the OIDC client_id/client_secret hardcoded in
-# Grafana OAuth secret - hardcoded from blueprint
+# plaintext (the file's own old comment even said "hardcoded from
-
+# blueprint") - found and fixed 2026-08-20 alongside
-apiVersion: v1
+# infrastructure/authentik/grafana-blueprint.yaml (same value, same Vault
-kind: Secret
+# path - see that file for the full story).
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: grafana-authentik-oauth
  namespace: grafana
-type: Opaque
+spec:
-stringData:
+  refreshInterval: 1h
-  GF_AUTH_GENERIC_OAUTH_CLIENT_ID: "bd03e9139dd2063c6c44c4d2f65f51d69de3ba0b6d6b1b9b41c255d2376d2dcc"
+  secretStoreRef:
-  GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: "6b6f2deecfe8fd56cae9c512cc71eedd463d67ff08f24c816b15b15e78ce36bc4f06a2276c5ffe3f67799d935994a32ed32cdc02ec674f279c82d7cfe3ca05d5"
+    kind: ClusterSecretStore
    name: vault-backend
  target:
    name: grafana-authentik-oauth
    creationPolicy: Owner
    template:
      type: Opaque
      data:
        GF_AUTH_GENERIC_OAUTH_CLIENT_ID: "{{ .clientId }}"
        GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: "{{ .clientSecret }}"
  data:
    - secretKey: clientId
      remoteRef:
        key: grafana-oauth
        property: client-id
    - secretKey: clientSecret
      remoteRef:
        key: grafana-oauth
        property: client-secret
@@ -0,0 +1,85 @@
 # Multi-site active failover pilot (see
 # /home/scooby/.claude/plans/jiggly-snacking-iverson.md) - home's half of
 # the failover-watcher. This is deliberately NOT a general Postgres HA
 # controller: its only job is writing a fresh unix timestamp to etcd every
 # 10s at key /ha-failover/home-heartbeat. The VPS's watcher
 # (scripts/ha-failover-watcher.sh, deployed via systemd - see that
 # script's own header) reads this key and decides whether to promote
 # based purely on how stale it is - no custom voting/consensus logic
 # needed here, etcd's own Raft consensus already provides the "majority
 # agrees" guarantee: a write only succeeds if a majority of the 3 etcd
 # members (home/vps/witness) are reachable and agree, and a linearizable
 # read (the JSON gateway's default) only ever returns majority-confirmed
 # state.
 #
 # Talks to etcd via its LOCAL ClusterIP (ha-etcd.ha-failover.svc.cluster.local
 # :2379) - home reads/writes its OWN cluster member directly, no need to
 # round-trip externally for this side.
 #
 # No RBAC/ServiceAccount needed - this pod never touches the K8s API,
 # only etcd's HTTP gateway via curl.
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: ha-heartbeat-script
  namespace: ha-failover
 data:
  heartbeat.sh: |
    #!/bin/sh
    set -eu
    KEY_B64="$(printf '%s' '/ha-failover/home-heartbeat' | base64 | tr -d '\n')"
    while true; do
      NOW="$(date +%s)"
      VAL_B64="$(printf '%s' "$NOW" | base64 | tr -d '\n')"
      if curl -sf --max-time 5 -X POST \
          http://ha-etcd.ha-failover.svc.cluster.local:2379/v3/kv/put \
          -d "{\"key\":\"${KEY_B64}\",\"value\":\"${VAL_B64}\"}" >/dev/null; then
        echo "heartbeat ${NOW} ok"
      else
        echo "heartbeat ${NOW} FAILED (etcd unreachable or no quorum)"
      fi
      sleep 10
    done
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: ha-heartbeat-writer
  namespace: ha-failover
 spec:
  replicas: 1
  selector:
    matchLabels:
      app: ha-heartbeat-writer
  template:
    metadata:
      labels:
        app: ha-heartbeat-writer
    spec:
      securityContext:
        runAsNonRoot: true
        runAsUser: 1000
        seccompProfile:
          type: RuntimeDefault
      containers:
        - name: heartbeat
          image: alpine/k8s:1.32.13
          command: ["/bin/sh", "/scripts/heartbeat.sh"]
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop: ["ALL"]
          resources:
            requests:
              cpu: 5m
              memory: 16Mi
            limits:
              memory: 64Mi
          volumeMounts:
            - name: script
              mountPath: /scripts
      volumes:
        - name: script
          configMap:
            name: ha-heartbeat-script
            defaultMode: 0755
@@ -131,6 +131,7 @@ spec:
        - operation:
            ports:
              - "5432"
              - "61432" # floating-hostname port, see ha-postgres-peerauth.yaml
 ---
 # --- Grafana: Allow ingress + Prometheus datasource queries + scraping ---
@@ -173,8 +173,22 @@ data:
      ttl="24h" >/dev/null
    # --- OIDC Auth (Authentik SSO) ---
-    OIDC_CLIENT_ID="9816a5ae7e7914b5d18f4ab939d011a98f8c8d6b3bb6777c46431afa06ac4a85"
+    # OIDC_CLIENT_ID/SECRET used to be hardcoded here in plaintext - found
-    OIDC_CLIENT_SECRET="ed2ba1c6378c7a46341b5162f39a7fab80e37596b01ed387c3719e8e0040344cf1daa307476c2e7a7f75041b3979275b1ebf00bb8bad94c864b4a38ded544f7b"
+    # and fixed 2026-08-20. Can't source this from Vault itself the way
    # every other app's OIDC secret now does (infrastructure/authentik/
    # vault-blueprint.yaml, secret/vault-oauth) - this job is what GRANTS
    # ESO's own Kubernetes-auth role a few lines above
    # (bound_service_account_names="...,external-secrets"), so ESO can't
    # yet authenticate to Vault at the point this script runs - genuinely
    # circular. Sourced from vault-oidc-bootstrap instead, a plain Secret
    # created once manually (kubectl, not git):
    #   kubectl -n vault create secret generic vault-oidc-bootstrap \
    #     --from-literal=client_id=<same value as secret/vault-oauth> \
    #     --from-literal=client_secret=<same value as secret/vault-oauth>
    # envFrom on the Job (vault-init-job.yaml) injects these as
    # VAULT_OIDC_CLIENT_ID/VAULT_OIDC_CLIENT_SECRET.
    OIDC_CLIENT_ID="${VAULT_OIDC_CLIENT_ID:?VAULT_OIDC_CLIENT_ID not set - see vault-oidc-bootstrap secret}"
    OIDC_CLIENT_SECRET="${VAULT_OIDC_CLIENT_SECRET:?VAULT_OIDC_CLIENT_SECRET not set - see vault-oidc-bootstrap secret}"
    OIDC_DISCOVERY_URL="https://auth.kube.huskypup.net/application/o/vault/"
    echo "==> Configuring OIDC auth (Authentik)..."
@@ -16,6 +16,19 @@ spec:
        - name: vault-init
          image: alpine/k8s:1.32.13
          command: ["/bin/bash", "/scripts/vault-init-and-store.sh"]
          env:
            # See the matching comment in vault-init-configmap.yaml for
            # why this can't come from Vault itself via ESO.
            - name: VAULT_OIDC_CLIENT_ID
              valueFrom:
                secretKeyRef:
                  name: vault-oidc-bootstrap
                  key: client_id
            - name: VAULT_OIDC_CLIENT_SECRET
              valueFrom:
                secretKeyRef:
                  name: vault-oidc-bootstrap
                  key: client_secret
          volumeMounts:
            - name: scripts
              mountPath: /scripts
@@ -0,0 +1,88 @@
 # Multi-site active failover pilot (see
 # /home/scooby/.claude/plans/jiggly-snacking-iverson.md) - closes the gap
 # flagged 2026-08-20: manually kubectl/ssh-copying secrets between home
 # and the VPS defeats the point of having Vault at all. This gives the
 # VPS its own real Vault -> ESO pipeline instead.
 #
 # External Secrets Operator itself is installed directly via helm
 # (out-of-band, like k3s/cert-manager/Netbird - see
 # scripts/vps-bootstrap.sh's existing pattern for why those aren't
 # GitOps-managed either):
 #   helm repo add external-secrets https://charts.external-secrets.io
 #   helm install external-secrets external-secrets/external-secrets \
 #     --namespace external-secrets --create-namespace \
 #     --version 0.20.4 --set installCRDs=true
 # (0.20.4 matches home's version - see argocd-apps/infrastructure/
 # external-secrets.yaml)
 #
 # Auth: AppRole, not Kubernetes auth - home's existing vault-backend
 # ClusterSecretStore (infrastructure/vault/manifests/clustersecretstore.yaml)
 # uses Vault's kubernetes auth method, which verifies a ServiceAccount JWT
 # against THAT cluster's own API server - doesn't work for the VPS, it's a
 # completely separate k3s cluster with no federation to home's API server.
 # AppRole is the standard way to authenticate an external/non-native
 # client to Vault instead.
 #
 # Reachable via https://vault.kube.huskypup.net - a NEW public Cloudflare
 # A record (home was previously deliberately kept off the public
 # internet - this was an explicit, confirmed decision, not a default).
 # Goes through the same Istio ingress gateway that already serves other
 # public *.kube.huskypup.net hosts, so no new UniFi port-forward/NodePort
 # needed, and no Istio PeerAuthentication/AuthorizationPolicy change
 # either - unlike the ha-authentik-postgres NodePort case, this traffic
 # arrives already wrapped in a normal in-mesh call from the ingress
 # gateway's own identity, not raw external TCP straight to a pod.
 #
 # NOTE: the AppRole's issued token/secret_id CANNOT be IP-bound
 # (token_bound_cidrs / secret_id_bound_cidrs) over this path - confirmed
 # live 2026-08-20 that Vault only ever sees the ingress gateway's own pod
 # IP for any request arriving this way, never the VPS's real source IP.
 # Security boundary here is AppRole credential secrecy + the narrow
 # read-only secret/vps/* policy (vps-eso-reader), not network-level
 # restriction - this is the standard/expected shape of AppRole auth for
 # external clients generally, CIDR-binding is normally extra
 # defense-in-depth on top rather than the primary mechanism.
 #
 # One-time manual bootstrap on Vault's side (already done 2026-08-20, not
 # scripted - matches every other Vault policy/auth-method setup in this
 # repo, which are also applied by hand via `vault` CLI, not GitOps):
 #   vault auth enable approle
 #   vault policy write vps-eso-reader - <<'EOF'
 #   path "secret/data/vps/*"     { capabilities = ["read", "list"] }
 #   path "secret/metadata/vps/*" { capabilities = ["read", "list"] }
 #   EOF
 #   vault write auth/approle/role/vps-eso token_policies="vps-eso-reader" \
 #     token_ttl=1h token_max_ttl=4h secret_id_num_uses=0
 #   vault read auth/approle/role/vps-eso/role-id      # -> roleId below
 #   vault write -f auth/approle/role/vps-eso/secret-id  # -> secret_id
 #
 # vault-approle-creds is a plain Secret created manually on the VPS
 # (kubectl, not git - same reasoning as every other VPS secret, and
 # consistent regardless of "how sensitive" a given field is on its own -
 # role_id and secret_id both live in the same Secret, neither in git):
 #   kubectl -n external-secrets create secret generic vault-approle-creds \
 #     --from-literal=role_id=<from role-id above> \
 #     --from-literal=secret_id=<from secret-id above>
 # Rotate the secret_id periodically by writing a new one and patching this
 # Secret.
 apiVersion: external-secrets.io/v1
 kind: ClusterSecretStore
 metadata:
  name: vault-backend
 spec:
  provider:
    vault:
      server: https://vault.kube.huskypup.net
      path: secret # KV v2 mount, same as home's vault-backend
      version: v2
      auth:
        appRole:
          path: approle
          roleRef:
            name: vault-approle-creds
            namespace: external-secrets
            key: role_id
          secretRef:
            name: vault-approle-creds
            namespace: external-secrets
            key: secret_id
@@ -1,6 +1,9 @@
 ---
 # See infrastructure/vps-standby/vault/manifests/ingress.yaml for the
-# vps.huskypup.net subdomain design rationale.
+# vps.huskypup.net subdomain design rationale, and for why this
 # https-redirect Middleware is needed (found live 2026-08-21 diagnosing
 # the exact same "Missing auth_url"-shaped symptom on Vault's OIDC login -
 # applies equally to every VPS-standby app on this Traefik).
 #
 # CAVEAT: Authentik's provider/application configs (redirect_uris, etc.)
 # are replicated byte-for-byte from home and point at *.kube.huskypup.net
@@ -8,6 +11,16 @@
 # won't complete correctly (redirect mismatch) until that's addressed
 # separately. Direct/local Authentik admin login still works fine for
 # browsing and verifying replicated data.
 apiVersion: traefik.io/v1alpha1
 kind: Middleware
 metadata:
  name: https-redirect
  namespace: authentik
 spec:
  redirectScheme:
    scheme: https
    permanent: true
 ---
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -15,6 +28,7 @@ metadata:
  namespace: authentik
  annotations:
    cert-manager.io/cluster-issuer: letsencrypt-production
    traefik.ingress.kubernetes.io/router.middlewares: authentik-https-redirect@kubernetescrd
 spec:
  ingressClassName: traefik
  tls:
@@ -0,0 +1,29 @@
 # Multi-site active failover pilot - VPS's half of the Vault pipeline
 # (infrastructure/authentik/manifests/ha-postgres-app-pushsecret.yaml has
 # home's half, which pushes the authoritative password into Vault at
 # secret/vps/pg-authentik-app). Pulls it back down here.
 #
 # Merge (not Owner/Replace): only overwrites the `password` key, leaving
 # CNPG's own generated host/dbname/username fields on this secret intact
 # - those are correctly LOCAL to each site (this secret's `host` key
 # points at the VPS's own local -rw service, which CNPG itself still
 # needs internally, even though the app tier reads a different value via
 # the pg-authentik.ha.huskypup.net override in values.yaml).
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: pg-authentik-app-password
  namespace: authentik
 spec:
  refreshInterval: 5m
  secretStoreRef:
    name: vault-backend
    kind: ClusterSecretStore
  target:
    name: pg-authentik-app
    creationPolicy: Merge
  data:
    - secretKey: password
      remoteRef:
        key: vps/pg-authentik-app
        property: password
@@ -1,11 +1,16 @@
-# Authentik warm standby on the VPS - Phase 2. The CNPG replica cluster
+# Authentik warm standby on the VPS - Phase 2, later extended into the
-# (manifests/cnpg-cluster.yaml) continuously replays WAL from home in the
+# multi-site active failover pilot (see
-# background. The app itself DOES run continuously (replicas: 1, reachable
+# /home/scooby/.claude/plans/jiggly-snacking-iverson.md). The CNPG cluster
-# at auth.vps.huskypup.net - see manifests/ingress.yaml) so the replicated
+# (manifests/cnpg-cluster.yaml) now uses real streaming replication
-# data is browsable/verifiable at all times, even though the underlying DB
+# (externalClusters connectionParameters + spec.replica.self/primary/
-# is a read-only CNPG replica - writes (new logins, session creation) will
+# source) rather than the original WAL-archive-polling replica.enabled
-# error until a deliberate manual promotion. Promotion runbook: flip the
+# mode - that flag is gone from cnpg-cluster.yaml now, it's mutually
-# CNPG cluster's spec.replica.enabled to false (see cnpg-cluster.yaml).
+# exclusive with the primary/self fields. The app itself DOES run
 # continuously (replicas: 1, reachable at auth.vps.huskypup.net - see
 # manifests/ingress.yaml) so the replicated data is browsable/verifiable
 # at all times, even though the underlying DB is a read-only replica until
 # promoted. Promotion (automatic via the failover-watcher, or manually):
 # patch cnpg-cluster.yaml's spec.replica to self/primary/source: vps.
 #
 # The `authentik:` block below (secret_key/postgresql "env://" indirection)
 # is copied VERBATIM from infrastructure/authentik/values.yaml on purpose -
@@ -29,17 +34,22 @@ global:
  envFrom:
    - secretRef:
        name: authentik
-    - secretRef:
+    # The pg-authentik-app secretRef that used to live here is GONE - see
-        name: pg-authentik-app
+    # the matching comment in infrastructure/authentik/values.yaml (home's
-      prefix: AUTHENTIK_POSTGRESQL__
+    # copy of this same block) for why: its lowercase-suffixed env vars
    # (AUTHENTIK_POSTGRESQL__host etc.) were silently winning over the
    # explicit uppercase HOST/PORT overrides below, connecting straight to
    # the local CNPG service the whole time regardless of what was set.
  env:
    - name: AUTHENTIK_URL
      value: "https://auth.vps.huskypup.net"
    # HOST/PORT point at the floating pg-authentik.ha.huskypup.net record
    # instead of the local secret's host/port - see the matching comment
    # in infrastructure/authentik/values.yaml (home's copy of this same
    # block) for the full explanation, including why pg-authentik-app's
    # password had to be manually synced between the two sites first.
    - name: AUTHENTIK_POSTGRESQL__HOST
-      valueFrom:
+      value: "pg-authentik.ha.huskypup.net"
        secretKeyRef:
          name: pg-authentik-app
          key: host
    - name: AUTHENTIK_POSTGRESQL__NAME
      valueFrom:
        secretKeyRef:
@@ -56,7 +66,7 @@ global:
          name: pg-authentik-app
          key: password
    - name: AUTHENTIK_POSTGRESQL__PORT
-      value: "5432"
+      value: "61432"
 server:
  replicas: 1
@@ -1,6 +1,18 @@
 ---
 # See infrastructure/vps-standby/vault/manifests/ingress.yaml for the
-# vps.huskypup.net subdomain design rationale.
+# vps.huskypup.net subdomain design rationale, and for why this
 # https-redirect Middleware is needed (found live 2026-08-21 diagnosing
 # the exact same symptom on this app's OIDC login).
 apiVersion: traefik.io/v1alpha1
 kind: Middleware
 metadata:
  name: https-redirect
  namespace: gitea
 spec:
  redirectScheme:
    scheme: https
    permanent: true
 ---
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -8,6 +20,7 @@ metadata:
  namespace: gitea
  annotations:
    cert-manager.io/cluster-issuer: letsencrypt-production
    traefik.ingress.kubernetes.io/router.middlewares: gitea-https-redirect@kubernetescrd
 spec:
  ingressClassName: traefik
  tls:
@@ -0,0 +1,54 @@
 ---
 # Real GitLab on the VPS - simplified single-node equivalent of
 # apps/gitlab/manifests/cnpg-cluster.yaml (home's is 3-instance HA with
 # Rook-Ceph; this is 1 instance on local-path, matching the "keep it
 # lighter" pattern already used for the rest of vps-standby).
 apiVersion: postgresql.cnpg.io/v1
 kind: Cluster
 metadata:
  name: pg-gitlab
  namespace: gitlab
 spec:
  imageName: ghcr.io/cloudnative-pg/postgresql:16
  instances: 1
  resources:
    requests:
      memory: "512Mi"
      cpu: "50m"
    limits:
      memory: "2Gi"
      cpu: "500m"
  storage:
    size: 20Gi
    storageClass: local-path
  primaryUpdateStrategy: unsupervised
  bootstrap:
    initdb:
      database: gitlabhq_production
      owner: app
      postInitSQL:
        - CREATE EXTENSION IF NOT EXISTS pg_trgm;
        - CREATE EXTENSION IF NOT EXISTS btree_gist;
  # Same tuning as home's, GitLab needs these regardless of scale
  postgresql:
    parameters:
      max_connections: "200"
      shared_buffers: "512MB"
      effective_cache_size: "1536MB"
      maintenance_work_mem: "128MB"
      checkpoint_completion_target: "0.9"
      wal_buffers: "16MB"
      default_statistics_target: "100"
      random_page_cost: "1.1"
      effective_io_concurrency: "200"
      work_mem: "8MB"
      min_wal_size: "1GB"
      max_wal_size: "4GB"
  monitoring:
    enablePodMonitor: false  # no Prometheus on the VPS cluster
@@ -0,0 +1,42 @@
 ---
 # Real GitLab on the VPS. https-redirect Middleware from the start this
 # time - see infrastructure/vps-standby/vault/manifests/ingress.yaml for
 # why it's needed (found live 2026-08-21: Traefik serves plain HTTP
 # alongside HTTPS unless explicitly redirected, which silently breaks
 # OIDC login since the callback URL computed client-side won't match
 # what's registered in Authentik).
 apiVersion: traefik.io/v1alpha1
 kind: Middleware
 metadata:
  name: https-redirect
  namespace: gitlab
 spec:
  redirectScheme:
    scheme: https
    permanent: true
 ---
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
  name: vps-gitlab
  namespace: gitlab
  annotations:
    cert-manager.io/cluster-issuer: letsencrypt-production
    traefik.ingress.kubernetes.io/router.middlewares: gitlab-https-redirect@kubernetescrd
 spec:
  ingressClassName: traefik
  tls:
    - hosts:
        - gitlab.vps.huskypup.net
      secretName: vps-gitlab-tls
  rules:
    - host: gitlab.vps.huskypup.net
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: vps-gitlab-webservice-default
                port:
                  number: 8181
@@ -0,0 +1,59 @@
 # Periodic git mirror: home's Scooby/Homelabv4 (the actual GitOps
 # source) -> VPS GitLab's copy of the same project. See
 # mirror-sync-secret.yaml for why this is a plain CronJob instead of
 # GitLab's native pull-mirror feature (Premium-gated, confirmed live
 # 2026-08-21). This is the VPS-standby equivalent of the original
 # plan's "Gitea pull mirror of the GitOps source" - same intent,
 # different mechanism now that real GitLab replaced Gitea here.
 apiVersion: batch/v1
 kind: CronJob
 metadata:
  name: gitlab-mirror-sync
  namespace: gitlab
 spec:
  schedule: "*/15 * * * *"
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
  jobTemplate:
    spec:
      backoffLimit: 1
      activeDeadlineSeconds: 600
      template:
        spec:
          restartPolicy: Never
          containers:
            - name: mirror-sync
              image: alpine/git:2.43.0
              command:
                - sh
                - -c
                - |
                  set -eu
                  rm -rf /tmp/mirror.git
                  git clone --mirror "https://${PULL_USERNAME}:${PULL_TOKEN}@gitlab.kube.huskypup.net/Scooby/Homelabv4.git" /tmp/mirror.git
                  cd /tmp/mirror.git
                  git push --mirror "https://root:${PUSH_TOKEN}@gitlab.vps.huskypup.net/Scooby/Homelabv4.git"
                  echo "Mirror sync complete: $(date -u)"
              env:
                - name: PULL_USERNAME
                  valueFrom:
                    secretKeyRef:
                      name: gitlab-mirror-sync-secret
                      key: pull_username
                - name: PULL_TOKEN
                  valueFrom:
                    secretKeyRef:
                      name: gitlab-mirror-sync-secret
                      key: pull_token
                - name: PUSH_TOKEN
                  valueFrom:
                    secretKeyRef:
                      name: gitlab-mirror-sync-secret
                      key: push_token
              resources:
                requests:
                  cpu: 25m
                  memory: 128Mi
                limits:
                  memory: 512Mi
@@ -0,0 +1,39 @@
 # Credentials for mirror-sync-cronjob.yaml. GitLab's native repository
 # mirroring (pull mirror) is a Premium/Ultimate-gated feature as of this
 # chart version - confirmed live 2026-08-21: the API rejects `mirror`
 # and `import_url` as valid params on this CE instance entirely
 # ("allow_merge_on_skipped_pipeline, ... are missing" - i.e. neither
 # field is even recognized). So this CronJob does the plain-git
 # equivalent (clone --mirror / push --mirror) on a schedule instead.
 #
 # - pull_username/pull_token: read-only deploy token on HOME's
 #   Scooby/Homelabv4 project (deploy tokens can't have write_repository
 #   - it's not a valid scope for them, confirmed live).
 # - push_username/push_token: a write_repository-scoped Personal Access
 #   Token for VPS GitLab's root user (deploy tokens can't push either).
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: gitlab-mirror-sync
  namespace: gitlab
 spec:
  refreshInterval: 1h
  secretStoreRef:
    kind: ClusterSecretStore
    name: vault-backend
  target:
    name: gitlab-mirror-sync-secret
    creationPolicy: Owner
  data:
    - secretKey: pull_username
      remoteRef:
        key: vps/gitlab-mirror
        property: username
    - secretKey: pull_token
      remoteRef:
        key: vps/gitlab-mirror
        property: token
    - secretKey: push_token
      remoteRef:
        key: vps/gitlab-mirror
        property: push_token
@@ -0,0 +1,61 @@
 # Shares the same Authentik OAuth2 provider/client_id as home's GitLab
 # (infrastructure/authentik/gitlab-blueprint.yaml has both redirect_uris
 # registered). Same Vault path (secret/gitlab-oauth) home's copy reads
 # from - vps-eso-reader policy widened to allow this one extra path
 # (infrastructure/vps-eso/manifests/clustersecretstore.yaml's policy,
 # applied live via vault CLI, not git-tracked - matches how every other
 # Vault policy/auth-method in this repo is set up).
 #
 # `provider` key added 2026-08-21: found live that GITLAB_OMNIBUS_CONFIG
 # (this app's values.yaml extraEnv) is an omnibus-image-only convention
 # the CNG webservice image never processes - see the matching comment
 # in apps/gitlab/manifests/external-secret-oidc.yaml (home) for the
 # full story. The real mechanism needs this key to hold a whole
 # YAML-encoded provider block, not raw client_id/secret strings.
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: gitlab-oidc
  namespace: gitlab
 spec:
  refreshInterval: 1h
  secretStoreRef:
    kind: ClusterSecretStore
    name: vault-backend
  target:
    name: gitlab-oidc-secret
    creationPolicy: Owner
    template:
      type: Opaque
      data:
        GITLAB_OIDC_CLIENT_ID: "{{ .clientId }}"
        GITLAB_OIDC_CLIENT_SECRET: "{{ .clientSecret }}"
        provider: |
          name: openid_connect
          label: Authentik
          args:
            name: openid_connect
            scope:
              - openid
              - profile
              - email
            response_type: code
            issuer: 'https://auth.kube.huskypup.net/application/o/gitlab/'
            discovery: true
            client_auth_method: query
            uid_field: sub
            send_scope_to_token_endpoint: true
            pkce: true
            client_options:
              identifier: '{{ .clientId }}'
              secret: '{{ .clientSecret }}'
              redirect_uri: 'https://gitlab.vps.huskypup.net/users/auth/openid_connect/callback'
  data:
    - secretKey: clientId
      remoteRef:
        key: gitlab-oauth
        property: client-id
    - secretKey: clientSecret
      remoteRef:
        key: gitlab-oauth
        property: client-secret
@@ -0,0 +1,19 @@
 # Same fix as apps/gitlab/manifests/redis-init-fix.yaml (home) - GitLab
 # chart 7.7.0's initializer ordering bug: 7_prometheus_metrics.rb loads
 # before 7_redis.rb (alphabetical 'p' < 'r'), triggering Redis cache
 # access before Gitlab::Redis::ALL_CLASSES is defined. This ConfigMap
 # provides an early initializer that loads the Redis module first.
 #
 # Mounting this into webservice/sidekiq deployments still has to be a
 # manual kubectl patch after first deploy - chart 7.7.0 doesn't support
 # extraVolumes on those workloads, same limitation home hit, and home's
 # own patch was never git-tracked either (applied live). See this
 # directory's own deploy notes for the exact patch commands.
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: redis-init-fix
  namespace: gitlab
 data:
  0_redis_early.rb: |
    require "gitlab/redis"
@@ -0,0 +1,22 @@
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: redis-gitlab
  namespace: gitlab
 spec:
  refreshInterval: 1h
  secretStoreRef:
    kind: ClusterSecretStore
    name: vault-backend
  target:
    name: redis-gitlab-secret
    creationPolicy: Owner
    template:
      type: Opaque
      data:
        password: "{{ .password }}"
  data:
    - secretKey: password
      remoteRef:
        key: vps/gitlab-redis
        property: password
@@ -0,0 +1,84 @@
 # Real GitLab on the VPS - standalone Redis, same pattern as home's
 # apps/gitlab/manifests/redis-cluster.yaml (despite that file's name,
 # it's already just a single standalone instance - "Sentinel" in the
 # filename is legacy, not what's actually running). local-path storage
 # instead of rook-ceph-block.
 ---
 apiVersion: v1
 kind: Service
 metadata:
  name: redis-gitlab
  namespace: gitlab
 spec:
  ports:
    - port: 6379
      targetPort: 6379
      name: redis
  selector:
    app: redis-gitlab
  type: ClusterIP
 ---
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
  name: redis-gitlab
  namespace: gitlab
 spec:
  serviceName: redis-gitlab
  replicas: 1
  selector:
    matchLabels:
      app: redis-gitlab
  template:
    metadata:
      labels:
        app: redis-gitlab
    spec:
      securityContext:
        fsGroup: 1000
      containers:
        - name: redis
          image: redis:7.0-alpine
          securityContext:
            runAsUser: 999
            runAsGroup: 1000
            runAsNonRoot: true
            allowPrivilegeEscalation: false
            capabilities:
              drop: ["ALL"]
          ports:
            - containerPort: 6379
              name: redis
          command:
            - sh
            - -c
            - redis-server --appendonly yes --requirepass "$REDIS_PASSWORD"
          env:
            - name: REDIS_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: redis-gitlab-secret
                  key: password
          resources:
            requests:
              cpu: 25m
              memory: 128Mi
            limits:
              memory: 512Mi
          volumeMounts:
            - name: data
              mountPath: /data
            - name: tmp
              mountPath: /tmp
      volumes:
        - name: tmp
          emptyDir: {}
  volumeClaimTemplates:
    - metadata:
        name: data
      spec:
        accessModes: ["ReadWriteOnce"]
        storageClassName: local-path
        resources:
          requests:
            storage: 5Gi
@@ -0,0 +1,275 @@
 # Real GitLab on the VPS - simplified single-node equivalent of
 # apps/gitlab/values.yaml (home). Same chart, same version, same
 # Authentik OIDC provider/client (shared, see infrastructure/authentik/
 # gitlab-blueprint.yaml). Deliberately lighter than home's: no Praefect
 # (pointless with 1 Gitaly node), no PgBouncer (direct CNPG connection,
 # low enough traffic not to need pooling), no SAML, no Container
 # Registry, no KAS - all addable later if actually needed, matches the
 # "keep it lighter" decision already made for the rest of vps-standby.
 global:
  hosts:
    domain: vps.huskypup.net
    gitlab:
      name: gitlab.vps.huskypup.net
  # Ingress disabled - own Ingress + Traefik https-redirect Middleware
  # handle routing (infrastructure/vps-standby/gitlab/manifests/ingress.yaml)
  ingress:
    enabled: false
    configureCertmanager: false
  edition: ce
  time_zone: UTC
  email:
    from: 'gitlab@vps.huskypup.net'
    display_name: GitLab (VPS)
    reply_to: 'noreply@vps.huskypup.net'
  # External PostgreSQL via CNPG (infrastructure/vps-standby/gitlab/manifests/cnpg-cluster.yaml)
  psql:
    host: pg-gitlab-rw.gitlab.svc.cluster.local
    port: 5432
    database: gitlabhq_production
    username: app
    password:
      secret: pg-gitlab-app
      key: password
  # gitlab.kas.enabled: false alone doesn't stop KAS pods deploying -
  # there's a SEPARATE global.kas.enabled toggle (default true) that
  # actually controls it, found live 2026-08-21.
  kas:
    enabled: false
  # Single Gitaly node, no Praefect - HA routing is pointless with 1 node
  gitaly:
    enabled: true
    internal:
      names:
        - default
    external: []
  praefect:
    enabled: false
  # Standalone Redis (infrastructure/vps-standby/gitlab/manifests/redis.yaml)
  redis:
    host: redis-gitlab.gitlab.svc.cluster.local
    port: 6379
    auth:
      enabled: true
      secret: redis-gitlab-secret
      key: password
  appConfig:
    # CORRECTED 2026-08-21 (see apps/gitlab/values.yaml (home) for the
    # full story) - `providers:` list restored, now pointing at
    # gitlab-oidc-secret's `provider` key which holds a real
    # YAML-encoded provider block (manifests/oidc-secret.yaml), the
    # actual mechanism the CNG webservice image processes.
    # GITLAB_OMNIBUS_CONFIG below is an omnibus-image-only convention
    # this image never reads - always a no-op here.
    omniauth:
      enabled: true
      allowSingleSignOn: ['openid_connect']
      blockAutoCreatedUsers: false
      autoLinkUser: ['openid_connect']
      syncProfileFromProvider: ['openid_connect']
      syncProfileAttributes: ['email', 'name']
      providers:
        - secret: gitlab-oidc-secret
          key: provider
 # GitLab's pre-upgrade hook checks the deployed version against a
 # previous-release ConfigMap to enforce supported upgrade paths - found
 # live 2026-08-21 that it fails with "unsupported upgrade path" on a
 # genuinely FRESH install too (confirmed: no prior chart-info ConfigMap,
 # no Helm release secret, brand new namespace) - a chart/ArgoCD
 # interaction quirk (ArgoCD doesn't do real `helm install`, so the
 # optional chart-info ConfigMap's absence apparently isn't handled the
 # way the script expects). Not relevant for a fresh install anyway - only
 # matters for real version-to-version upgrades, which is a future concern
 # once this is actually running something to upgrade FROM.
 upgradeCheck:
  enabled: false
 postgresql:
  install: false
 redis:
  install: false
 pgbouncer:
  enabled: false
 # GitLab's own bundled MinIO for object storage (uploads, artifacts) -
 # separate from vps-minio (which backs CNPG WAL archiving for other apps)
 minio:
  persistence:
    storageClass: local-path
    size: 20Gi
  resources:
    requests:
      cpu: 25m
      memory: 256Mi
    limits:
      memory: 1Gi
 registry:
  enabled: false
 gitlab:
  webservice:
    minReplicas: 1
    maxReplicas: 1
    # 1 worker process, not 2 - single-node VPS, already sharing the box
    # with several other unrelated services (barman, k3s itself, node/
    # gunicorn apps) that keep it chronically CPU-overcommitted (load
    # average ~22-26 sustained on 6 cores, confirmed live 2026-08-21,
    # unrelated to GitLab). Less to fork/boot per pod start.
    workerProcesses: 1
    deployment:
      # Recreate, not the chart default RollingUpdate - with only 1
      # replica, RollingUpdate still surges an extra pod during every
      # rollout, and this box can't handle 2 concurrent GitLab Ruby
      # boots (confirmed live 2026-08-21: load average spiked past 39
      # on 6 cores, each pod starving the other's boot until both got
      # liveness-killed - a feedback loop that never converges). No
      # real availability cost since there's only 1 replica anyway.
      strategy:
        type: Recreate
      # Chart defaults (initialDelaySeconds:20, periodSeconds:60,
      # failureThreshold:3 => ~200s grace) aren't enough on this
      # contended box - confirmed live 2026-08-21 the webservice
      # (puma) container was repeatedly SIGTERM'd by the liveness
      # probe mid-`require` (still loading gems via bootsnap), never
      # finishing boot, restarting from zero every time. Loosened so
      # a slow boot survives instead of getting killed and restarted
      # forever.
      livenessProbe:
        initialDelaySeconds: 60
        periodSeconds: 30
        timeoutSeconds: 30
        successThreshold: 1
        failureThreshold: 15
    extraEnvFrom:
      GITLAB_OIDC_CLIENT_ID:
        secretKeyRef:
          name: gitlab-oidc-secret
          key: GITLAB_OIDC_CLIENT_ID
      GITLAB_OIDC_CLIENT_SECRET:
        secretKeyRef:
          name: gitlab-oidc-secret
          key: GITLAB_OIDC_CLIENT_SECRET
    extraEnv:
      GITLAB_OMNIBUS_CONFIG: |
        gitlab_rails['omniauth_enabled'] = true
        gitlab_rails['omniauth_allow_single_sign_on'] = ['openid_connect']
        gitlab_rails['omniauth_block_auto_created_users'] = false
        gitlab_rails['omniauth_auto_link_user'] = ['openid_connect']
        gitlab_rails['omniauth_sync_profile_from_provider'] = ['openid_connect']
        gitlab_rails['omniauth_sync_profile_attributes'] = ['email', 'name']
        gitlab_rails['omniauth_providers'] = [
          {
            'name' => 'openid_connect',
            'label' => 'Authentik',
            'args' => {
              'name' => 'openid_connect',
              'scope' => ['openid', 'profile', 'email'],
              'response_type' => 'code',
              'issuer' => 'https://auth.kube.huskypup.net/application/o/gitlab/',
              'discovery' => true,
              'client_auth_method' => 'query',
              'uid_field' => 'sub',
              'send_scope_to_token_endpoint' => true,
              'pkce' => true,
              'client_options' => {
                'identifier' => ENV['GITLAB_OIDC_CLIENT_ID'],
                'secret' => ENV['GITLAB_OIDC_CLIENT_SECRET'],
                'redirect_uri' => 'https://gitlab.vps.huskypup.net/users/auth/openid_connect/callback'
              }
            }
          }
        ]
    resources:
      requests:
        cpu: 150m
        memory: 1536Mi
      limits:
        memory: 3Gi
  kas:
    enabled: false
  gitaly:
    persistence:
      storageClass: local-path
      size: 30Gi
    resources:
      requests:
        cpu: 50m
        memory: 256Mi
      limits:
        cpu: "1"
        memory: 1Gi
  gitlab-runner:
    install: false
  gitlab-exporter:
    enabled: true
    metrics:
      enabled: false  # no Prometheus on the VPS cluster
  sidekiq:
    minReplicas: 1
    maxReplicas: 1
    # Recreate, not RollingUpdate - see gitlab.webservice.deployment.
    # strategy above for why.
    strategy:
      type: Recreate
    # Same probe-loosening as webservice.deployment above, same root
    # cause (confirmed live 2026-08-21) - sidekiq's own boot was
    # getting SIGTERM'd mid-`require` by the liveness probe on this
    # CPU-contended box, every ~90-150s, resetting to zero each time
    # and never actually finishing.
    # Loosened further 2026-08-21: even with the first pass above, 3
    # consecutive attempts all reached "Booting Sidekiq" (the real
    # engine actually starts, confirmed via "Listening on queues" in
    # logs) but never opened the health-check port (3808) within the
    # ~8min grace window - the box's ambient background load (gitea,
    # restic backup, barman, k3s itself - all unrelated to GitLab, load
    # average sustained 35-55 on 6 cores) is apparently severe enough to
    # starve that specific thread's CPU scheduling even after the main
    # process is otherwise up. Genuine forward progress each time, not
    # a hard hang, so more patience rather than a code fix.
    livenessProbe:
      initialDelaySeconds: 120
      periodSeconds: 30
      timeoutSeconds: 30
      successThreshold: 1
      failureThreshold: 30
    resources:
      requests:
        cpu: 25m
        memory: 384Mi
      limits:
        cpu: 250m
        memory: 1536Mi
 certmanager:
  install: false
  installCRDs: false
 prometheus:
  install: false
 nginx-ingress:
  enabled: false
 gitlab-runner:
  install: false
@@ -1,6 +1,18 @@
 ---
 # See infrastructure/vps-standby/vault/manifests/ingress.yaml for the
-# vps.huskypup.net subdomain design rationale.
+# vps.huskypup.net subdomain design rationale, and for why this
 # https-redirect Middleware is needed (found live 2026-08-21 diagnosing
 # the exact same symptom on this app's OIDC login).
 apiVersion: traefik.io/v1alpha1
 kind: Middleware
 metadata:
  name: https-redirect
  namespace: n8n
 spec:
  redirectScheme:
    scheme: https
    permanent: true
 ---
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -8,6 +20,7 @@ metadata:
  namespace: n8n
  annotations:
    cert-manager.io/cluster-issuer: letsencrypt-production
    traefik.ingress.kubernetes.io/router.middlewares: n8n-https-redirect@kubernetescrd
 spec:
  ingressClassName: traefik
  tls:
@@ -1,6 +1,18 @@
 ---
 # See infrastructure/vps-standby/vault/manifests/ingress.yaml for the
-# vps.huskypup.net subdomain design rationale.
+# vps.huskypup.net subdomain design rationale, and for why this
 # https-redirect Middleware is needed (found live 2026-08-21 diagnosing
 # the exact same symptom on this app's OIDC login).
 apiVersion: traefik.io/v1alpha1
 kind: Middleware
 metadata:
  name: https-redirect
  namespace: nextcloud
 spec:
  redirectScheme:
    scheme: https
    permanent: true
 ---
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -8,6 +20,7 @@ metadata:
  namespace: nextcloud
  annotations:
    cert-manager.io/cluster-issuer: letsencrypt-production
    traefik.ingress.kubernetes.io/router.middlewares: nextcloud-https-redirect@kubernetescrd
 spec:
  ingressClassName: traefik
  tls:
@@ -4,6 +4,29 @@
 # (vault.vps.huskypup.net) that doesn't depend on home being up at all -
 # see the vps.huskypup.net subdomain design note in
 # infrastructure/vps-standby/cert-manager/manifests/cluster-issuer.yaml.
 #
 # https-redirect Middleware: found live 2026-08-21 that this Ingress's
 # tls: block makes HTTPS available but does NOT stop Traefik from ALSO
 # serving plain HTTP on the same host - a browser landing on
 # http://vault.vps.huskypup.net (no redirect) gets a Vault UI that
 # computes its OIDC callback URL using window.location.origin, i.e.
 # http://..., which doesn't match the https:// entry registered in
 # allowed_redirect_uris (auth/oidc/role/default) - Vault silently returns
 # an empty auth_url ("Missing auth_url" in the UI) rather than an
 # obviously-DNS/config-looking error. Applies to every VPS-standby app on
 # this same Traefik, not just Vault (same fix needed and applied in each
 # of infrastructure/vps-standby/{gitea,n8n,nextcloud,authentik}/manifests/
 # ingress.yaml too).
 apiVersion: traefik.io/v1alpha1
 kind: Middleware
 metadata:
  name: https-redirect
  namespace: vault
 spec:
  redirectScheme:
    scheme: https
    permanent: true
 ---
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -11,6 +34,7 @@ metadata:
  namespace: vault
  annotations:
    cert-manager.io/cluster-issuer: letsencrypt-production
    traefik.ingress.kubernetes.io/router.middlewares: vault-https-redirect@kubernetescrd
 spec:
  ingressClassName: traefik
  tls:
@@ -0,0 +1,68 @@
 #!/bin/bash
 # Multi-site active failover pilot (see
 # /home/scooby/.claude/plans/jiggly-snacking-iverson.md) - the deliberate
 # MANUAL counterpart to scripts/ha-failover-watcher.sh's automatic
 # promotion. Run this by hand, on the VPS, once you've confirmed home is
 # genuinely healthy again and want to move primary back - never automated,
 # by design (auto-flipping back immediately on reconnect risks flapping,
 # and a human should confirm home's data/state before handing writes back
 # to it).
 #
 # Order matters:
 #   1. Confirm home's pg-authentik Cluster is healthy and NOT still
 #      thinking it's primary (it shouldn't be, since it was down/
 #      unreachable when the VPS promoted - but check
 #      status.currentPrimary on home before proceeding).
 #   2. Re-point home's Cluster to replicate FROM the vps (it needs to
 #      catch up on everything written to the VPS while it was down)
 #      before flipping primary back - home has to actually BE a caught-up
 #      replica of vps first, or this loses the writes the VPS accepted
 #      during the outage.
 #   3. Only once home shows it's streaming and caught up, flip primary
 #      back to home and update DNS.
 #
 # This script only does step 3 (the fast, symmetric part - same shape as
 # the watcher's own promote() but in reverse). Steps 1-2 are a judgment
 # call requiring you to actually look at both clusters' status first -
 # not scripted here on purpose.
 set -euo pipefail
 echo "This will flip pg-authentik's primary back to home and repoint DNS."
 echo "Before continuing, you MUST have already confirmed:"
 echo "  - home's pg-authentik Cluster is healthy and replicating FROM vps"
 echo "    (kubectl -n authentik get cluster pg-authentik -o jsonpath='{.status}')"
 echo "  - home is caught up (no meaningful replication lag from vps)"
 read -r -p "Confirmed both of the above? [y/N] " ans
 if [ "${ans:-N}" != "y" ] && [ "${ans:-N}" != "Y" ]; then
  echo "Aborted."
  exit 1
 fi
 echo "Flipping home's Cluster CR to primary..."
 kubectl -n authentik patch cluster pg-authentik --type merge \
  -p '{"spec":{"replica":{"self":"home","primary":"home","source":"home"}}}'
 echo "Flipping VPS's Cluster CR back to a replica of home..."
 ssh root@172.93.53.139 "kubectl -n authentik patch cluster pg-authentik --type merge \
  -p '{\"spec\":{\"replica\":{\"self\":\"vps\",\"primary\":\"home\",\"source\":\"home\"}}}'"
 echo "Flipping pg-authentik.ha.huskypup.net back to home's IP..."
 HOME_PUBLIC_IP="$(dig +short home.kube.huskypup.net @1.1.1.1 | tail -1)"
 if [ -z "$HOME_PUBLIC_IP" ]; then
  echo "ERROR: could not resolve home.kube.huskypup.net - fix the DNS record manually"
  exit 1
 fi
 TOKEN="$(kubectl -n cert-manager get secret cloudflare-token-secret -o jsonpath='{.data.cloudflare-token}' | base64 -d)"
 ZONE_ID="$(curl -sf -H "Authorization: Bearer ${TOKEN}" "https://api.cloudflare.com/client/v4/zones?name=huskypup.net" | jq -r '.result[0].id')"
 RECORD_ID="$(curl -sf -H "Authorization: Bearer ${TOKEN}" "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records?name=pg-authentik.ha.huskypup.net&type=A" | jq -r '.result[0].id')"
 curl -sf -X PATCH -H "Authorization: Bearer ${TOKEN}" -H "Content-Type: application/json" \
  -d "{\"type\":\"A\",\"name\":\"pg-authentik.ha.huskypup.net\",\"content\":\"${HOME_PUBLIC_IP}\",\"ttl\":60,\"proxied\":false}" \
  "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records/${RECORD_ID}" >/dev/null
 echo "Clearing the etcd promotion record..."
 curl -sf --max-time 5 -X POST http://172.28.101.41:32379/v3/kv/deleterange \
  -d "{\"key\":\"$(printf '%s' '/ha-failover/promoted-at' | base64)\"}" >/dev/null || true
 curl -sf --max-time 5 -X POST http://172.28.101.41:32379/v3/kv/deleterange \
  -d "{\"key\":\"$(printf '%s' '/ha-failover/promoted-by' | base64)\"}" >/dev/null || true
 echo "Failback complete. home is primary again, pg-authentik.ha.huskypup.net -> ${HOME_PUBLIC_IP}"
@@ -0,0 +1,25 @@
 # Multi-site active failover pilot (see
 # /home/scooby/.claude/plans/jiggly-snacking-iverson.md). Install on the
 # VPS (172.93.53.139) alongside ha-failover-watcher.sh:
 #   scp scripts/ha-failover-watcher.sh root@172.93.53.139:/usr/local/bin/
 #   ssh root@172.93.53.139 chmod +x /usr/local/bin/ha-failover-watcher.sh
 #   mkdir -p /etc/ha-failover on the VPS, put the Cloudflare DNS-edit
 #     token (same one cert-manager's cloudflare-token-secret uses) in
 #     /etc/ha-failover/cloudflare-token, chmod 600, chown root
 #   scp this file to /etc/systemd/system/ha-failover-watcher.service
 #   systemctl daemon-reload && systemctl enable --now ha-failover-watcher
 #
 # Requires: curl, jq, kubectl, base64 - all already present on the VPS
 # from earlier in this session.
 [Unit]
 Description=HA failover watcher - promotes pg-authentik to primary if home's etcd heartbeat goes stale (majority-confirmed via etcd quorum)
 After=network.target k3s.service
 [Service]
 ExecStart=/usr/local/bin/ha-failover-watcher.sh
 Restart=always
 RestartSec=5
 User=root
 [Install]
 WantedBy=multi-user.target
@@ -0,0 +1,154 @@
 #!/bin/bash
 # Multi-site active failover pilot (see
 # /home/scooby/.claude/plans/jiggly-snacking-iverson.md) - the VPS's half
 # of the failover-watcher. Runs as a systemd service on the VPS
 # (172.93.53.139), NOT in k3s - it needs to keep running even if the VPS's
 # own k3s/CNPG cluster is unhealthy, and it's the one thing in this whole
 # pilot that's genuinely new/bespoke rather than reusing an existing
 # operator.
 #
 # Deploy (manual, matches every other VPS systemd unit this session - not
 # git-applied automatically):
 #   scp this file to the VPS as /usr/local/bin/ha-failover-watcher.sh
 #   chmod +x it, then install scripts/ha-failover-watcher.service
 #   (see that file) and `systemctl enable --now ha-failover-watcher`.
 #
 # Logic (deliberately simple - see the plan doc's "what CNPG genuinely
 # does NOT provide" paragraph for why this exists at all):
 #   1. Every 10s, read /ha-failover/home-heartbeat from etcd via THIS
 #      node's own local etcd member (127.0.0.1:2379) using a linearizable
 #      (default, quorum-backed) read - home's heartbeat-writer
 #      (infrastructure/ha-failover/manifests/heartbeat-writer.yaml)
 #      refreshes this key every 10s while home is healthy.
 #   2. If the read itself fails/times out, this VPS can't reach a
 #      majority of the 3-member etcd cluster (needs 2 of 3) - meaning
 #      EITHER home is genuinely down AND the witness is also unreachable
 #      from here, OR this VPS itself is the one that's partitioned.
 #      Either way, we cannot safely tell which, so we do NOT promote -
 #      this is the split-brain-prevention property etcd's own Raft
 #      consensus gives us for free, no custom quorum-counting needed.
 #   3. If the read succeeds and the heartbeat is fresher than
 #      STALE_THRESHOLD seconds, home is confirmed up - no-op.
 #   4. If the read succeeds (so we DO have majority/quorum) and the
 #      heartbeat is older than STALE_THRESHOLD - or missing entirely -
 #      home is confirmed down by majority agreement. Promote, once:
 #        a. Skip if already primary (checked via the Cluster CR itself,
 #           idempotent - safe to run this loop forever).
 #        b. kubectl patch the local pg-authentik Cluster:
 #           spec.replica.{self,primary,source} = vps. No promotionToken -
 #           confirmed live via dry-run that CNPG's admission webhook does
 #           NOT require one (it's optional, used for graceful/planned
 #           switchover to cross-check LSNs - not available for a genuine
 #           unplanned outage since home isn't reachable to generate one).
 #           This means promotion accepts whatever the VPS replica had
 #           already streamed - typically a couple seconds of async lag,
 #           an accepted tradeoff of async cross-WAN replication (there is
 #           no realistic sync-replication option over a home/VPS WAN
 #           link without crippling write latency).
 #        c. Flip the pg-authentik.ha.huskypup.net Cloudflare A record to
 #           this VPS's public IP - both sites listen on the SAME external
 #           port 61432 (home via UniFi WAN forward, VPS via
 #           pg-authentik-forward.service's local socat forward)
 #           specifically so a single floating hostname:port works for
 #           both sites without the app tier needing per-site config.
 #        d. Record the promotion in etcd (/ha-failover/promoted-at,
 #           /ha-failover/promoted-by) - both for the idempotency check
 #           above surviving a script restart, and as an audit trail for
 #           whoever does the (deliberately manual - see the plan doc)
 #           failback later.
 #
 # Failback is NOT automated by this script on purpose - see
 # scripts/ha-failback-authentik.sh, run by a human once home is
 # confirmed healthy again.
 set -u
 ETCD="http://127.0.0.1:2379"
 STALE_THRESHOLD=45  # ~4-5 missed 10s heartbeats before acting - avoids flapping on one blip
 CHECK_INTERVAL=10
 CLOUDFLARE_TOKEN_FILE="/etc/ha-failover/cloudflare-token"
 CF_ZONE_NAME="huskypup.net"
 CF_RECORD_NAME="pg-authentik.ha.huskypup.net"
 VPS_PUBLIC_IP="172.93.53.139"
 LOG_TAG="ha-failover-watcher"
 log() { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) ${LOG_TAG}: $*"; }
 b64() { printf '%s' "$1" | base64 | tr -d '\n'; }
 etcd_get() {
  # $1 = key. Prints the decoded value, or nothing + returns 1 if the
  # read failed (unreachable/no quorum) or the key doesn't exist.
  local key_b64 resp val_b64
  key_b64="$(b64 "$1")"
  resp="$(curl -sf --max-time 5 -X POST "${ETCD}/v3/kv/range" \
    -d "{\"key\":\"${key_b64}\"}")" || return 1
  val_b64="$(echo "$resp" | jq -r '.kvs[0].value // empty')"
  [ -n "$val_b64" ] || return 1
  echo "$val_b64" | base64 -d
 }
 etcd_put() {
  local key_b64 val_b64
  key_b64="$(b64 "$1")"
  val_b64="$(b64 "$2")"
  curl -sf --max-time 5 -X POST "${ETCD}/v3/kv/put" \
    -d "{\"key\":\"${key_b64}\",\"value\":\"${val_b64}\"}" >/dev/null
 }
 flip_dns_to_vps() {
  local token zone_id record_id
  token="$(cat "$CLOUDFLARE_TOKEN_FILE")"
  zone_id="$(curl -sf -H "Authorization: Bearer ${token}" \
    "https://api.cloudflare.com/client/v4/zones?name=${CF_ZONE_NAME}" | jq -r '.result[0].id')"
  if [ -z "$zone_id" ] || [ "$zone_id" = "null" ]; then
    log "ERROR: could not resolve Cloudflare zone id for ${CF_ZONE_NAME}"
    return 1
  fi
  record_id="$(curl -sf -H "Authorization: Bearer ${token}" \
    "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records?name=${CF_RECORD_NAME}&type=A" \
    | jq -r '.result[0].id // empty')"
  if [ -z "$record_id" ]; then
    log "ERROR: no existing A record for ${CF_RECORD_NAME} - refusing to create one blind, fix manually"
    return 1
  fi
  curl -sf -X PATCH -H "Authorization: Bearer ${token}" -H "Content-Type: application/json" \
    -d "{\"type\":\"A\",\"name\":\"${CF_RECORD_NAME}\",\"content\":\"${VPS_PUBLIC_IP}\",\"ttl\":60,\"proxied\":false}" \
    "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records/${record_id}" >/dev/null
 }
 promote() {
  local current_primary
  current_primary="$(kubectl -n authentik get cluster pg-authentik -o jsonpath='{.spec.replica.primary}' 2>/dev/null)"
  if [ "$current_primary" = "vps" ]; then
    return 0  # already promoted, nothing to do
  fi
  log "PROMOTING: home confirmed down by etcd majority (heartbeat stale/missing). Flipping pg-authentik to vps."
  if ! kubectl -n authentik patch cluster pg-authentik --type merge \
      -p '{"spec":{"replica":{"self":"vps","primary":"vps","source":"vps"}}}'; then
    log "ERROR: kubectl patch failed - Cluster CR NOT promoted, will retry next loop"
    return 1
  fi
  if ! flip_dns_to_vps; then
    log "ERROR: Cluster CR promoted but Cloudflare DNS flip failed - fix pg-authentik.ha.huskypup.net manually, it still points at home"
  fi
  etcd_put "/ha-failover/promoted-at" "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
  etcd_put "/ha-failover/promoted-by" "vps"
  log "Promotion complete."
 }
 log "starting, stale threshold=${STALE_THRESHOLD}s check interval=${CHECK_INTERVAL}s"
 while true; do
  hb="$(etcd_get /ha-failover/home-heartbeat)"
  if [ $? -ne 0 ]; then
    log "cannot reach etcd quorum (or no heartbeat key yet) - not acting, will retry"
    sleep "$CHECK_INTERVAL"
    continue
  fi
  now="$(date +%s)"
  age=$((now - hb))
  if [ "$age" -gt "$STALE_THRESHOLD" ]; then
    log "home heartbeat is ${age}s stale (threshold ${STALE_THRESHOLD}s)"
    promote
  fi
  sleep "$CHECK_INTERVAL"
 done