From bce3861083a5f7ea7c8f9fe4d7a433beee8a519a Mon Sep 17 00:00:00 2001 From: Scooby Husky Date: Sat, 22 Aug 2026 14:24:13 -0500 Subject: [PATCH] GitLab cross-site replication Phase 2b: enable Gitaly + workhorse TLS Per the approved plan (Istio federation scoped separately, current NodePort/PeerAuth/cert-manager pattern continues for these 2 remaining links). Real Let's Encrypt certs (not self-signed) via the same DNS-01 ClusterIssuer used everywhere else - both sites already trust Let's Encrypt's public CA, no custom CA distribution needed. - global.gitaly.tls.enabled: adds tls_listen_addr (8076) alongside the existing plaintext listener (8075, stays for in-cluster traffic that doesn't need it). Requires global.praefect.virtualStorages[0].tlsSecretName (chart's own NOTES.txt validation rejects the render without it, confirmed live). - global.workhorse.tls.enabled + global.workhorse.host: Gitaly's [gitlab] url (internal-API callback for gitlab-shell hooks) switches from an unpublishable *.svc.cluster.local name to a real floating hostname (gitlab-internal-api.ha.huskypup.net) that Let's Encrypt CAN certify - confirmed live this replaces the plaintext [[listeners]] block with [listeners.tls] on the SAME port (8181), not a second port like Gitaly. - praefect-ha-configmap.yaml (the manually-patched live config) updated to tls:// + floating hostnames for all 3 home nodes, so home's own Praefect uses the identical addresses it'll hand the VPS as peer addresses once Phase 2b's 4th node lands. - New CoreDNS rewrite for gitlab-internal-api.ha.huskypup.net (same local-rewrite pattern as pg-*/gitaly-* already have). Sequencing note: applying live - Certificates must issue before the TLS-enabling values land, or Gitaly/webservice pods fail to start (missing secret for their init container's cert copy). Co-Authored-By: Claude Sonnet 5 --- .../manifests/gitaly-tls-certificate.yaml | 55 +++++++++++++++++++ apps/gitlab/manifests/ha-gitaly-nodeport.yaml | 42 +++++++------- .../ha-gitlab-internal-api-nodeport.yaml | 15 ++++- .../manifests/praefect-ha-configmap.yaml | 23 +++++--- apps/gitlab/values.yaml | 38 ++++++++++++- .../coredns/manifests/coredns-configmap.yaml | 5 ++ 6 files changed, 145 insertions(+), 33 deletions(-) create mode 100644 apps/gitlab/manifests/gitaly-tls-certificate.yaml diff --git a/apps/gitlab/manifests/gitaly-tls-certificate.yaml b/apps/gitlab/manifests/gitaly-tls-certificate.yaml new file mode 100644 index 0000000..b5f1198 --- /dev/null +++ b/apps/gitlab/manifests/gitaly-tls-certificate.yaml @@ -0,0 +1,55 @@ +# GitLab cross-site replication Phase 2b (see +# /home/scooby/.claude/plans/jiggly-snacking-iverson.md) - Gitaly's gRPC +# has no TLS by default; routing it over the public UniFi WAN forward +# (needed because the direct-Netbird-IP path doesn't accept inbound +# connections to home, confirmed live the same way as the existing etcd +# quorum) would otherwise carry the gitaly_token, gitlab_shell_secret, +# and raw repository contents in plaintext. +# +# Real Let's Encrypt cert via the same DNS-01 ClusterIssuer already +# proven for every other *.huskypup.net cert in this cluster, rather +# than a self-signed cert - sidesteps having to distribute a custom CA +# to both home and the VPS (which already trust Let's Encrypt's public +# CA by default). Covers all 4 Gitaly floating hostnames as SANs - one +# shared cert, referenced by every node's tlsSecretName (or the global +# default), simpler than per-node certs. +# +# secretName matches the chart's own default (confirmed live via +# gitlab/templates/_gitaly.tpl's gitlab.gitaly.tls.secret helper: +# "-gitaly-tls" = "gitlab-gitaly-tls") - global.gitaly.tls. +# enabled: true picks this up with no secretName override needed. +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: gitlab-gitaly-tls + namespace: gitlab +spec: + secretName: gitlab-gitaly-tls + issuerRef: + name: letsencrypt-production + kind: ClusterIssuer + dnsNames: + - gitaly-0.ha.huskypup.net + - gitaly-1.ha.huskypup.net + - gitaly-2.ha.huskypup.net + - gitaly-vps.ha.huskypup.net +--- +# Separate cert/secret for workhorse's internal-API TLS listener (port +# 8181, [gitlab] url every Gitaly node calls for gitlab-shell hooks) - +# different secretName (gitlab-workhorse-tls, chart default per +# gitlab/charts/gitlab/charts/webservice/templates/_helpers.tpl's +# workhorse.tls.secret helper) than Gitaly's own cert, kept separate +# rather than trying to share one Secret object across two different +# chart-expected names. +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: gitlab-workhorse-tls + namespace: gitlab +spec: + secretName: gitlab-workhorse-tls + issuerRef: + name: letsencrypt-production + kind: ClusterIssuer + dnsNames: + - gitlab-internal-api.ha.huskypup.net diff --git a/apps/gitlab/manifests/ha-gitaly-nodeport.yaml b/apps/gitlab/manifests/ha-gitaly-nodeport.yaml index 276a854..bbcb037 100644 --- a/apps/gitlab/manifests/ha-gitaly-nodeport.yaml +++ b/apps/gitlab/manifests/ha-gitaly-nodeport.yaml @@ -1,21 +1,23 @@ --- # GitLab cross-site replication Phase 2b (see # /home/scooby/.claude/plans/jiggly-snacking-iverson.md) - exposes each -# of home's 3 existing Gitaly nodes individually to the VPS over the -# Netbird mesh, so Praefect can register the VPS as a 4th node in the -# SAME virtual storage and inter-Gitaly replication RPCs can reach each -# node directly. Per-pod (not per-StatefulSet), unlike the chart's own -# gitlab-gitaly-default Service - Praefect/Gitaly must address each node -# individually, not load-balance across them. +# of home's 3 existing Gitaly nodes individually to the VPS, so Praefect +# can register the VPS as a 4th node in the SAME virtual storage and +# inter-Gitaly replication RPCs can reach each node directly. Per-pod +# (not per-StatefulSet), unlike the chart's own gitlab-gitaly-default +# Service - Praefect/Gitaly must address each node individually, not +# load-balance across them. # -# Reached over the Netbird mesh (100.108.x.x), NOT the UniFi WAN forward -# used for Postgres - Gitaly's gRPC here is PLAINTEXT (carries the -# gitaly_token, the gitlab_shell_secret, and raw repository contents), -# unlike CNPG's TLS-negotiated stream. Home's nodes are directly -# reachable from Netbird peers on their real node IP (confirmed live for -# the ha-failover etcd precedent - infrastructure/ha-failover/manifests/ -# etcd.yaml), so NodePort binds on that same interface without any -# additional exposure. +# Reached over the same public UniFi WAN forward already used for +# Postgres, NOT the Netbird mesh directly - confirmed live the direct- +# Netbird-IP path doesn't accept inbound connections to home (same gap +# already present for the existing etcd quorum; the earlier assumption +# that NodePort binds on a reachable Netbird interface was wrong). +# Targets Gitaly's TLS port (8076), not the plaintext one (8075) - +# Gitaly's gRPC has no built-in transport security, and this WAN hop +# would otherwise carry the gitaly_token, gitlab_shell_secret, and raw +# repository contents in plaintext. See gitaly-tls-certificate.yaml and +# global.gitaly.tls.enabled in values.yaml. apiVersion: v1 kind: Service metadata: @@ -26,8 +28,8 @@ spec: selector: statefulset.kubernetes.io/pod-name: gitlab-gitaly-default-0 ports: - - port: 8075 - targetPort: 8075 + - port: 8076 + targetPort: 8076 nodePort: 32446 --- apiVersion: v1 @@ -40,8 +42,8 @@ spec: selector: statefulset.kubernetes.io/pod-name: gitlab-gitaly-default-1 ports: - - port: 8075 - targetPort: 8075 + - port: 8076 + targetPort: 8076 nodePort: 32447 --- apiVersion: v1 @@ -54,6 +56,6 @@ spec: selector: statefulset.kubernetes.io/pod-name: gitlab-gitaly-default-2 ports: - - port: 8075 - targetPort: 8075 + - port: 8076 + targetPort: 8076 nodePort: 32448 diff --git a/apps/gitlab/manifests/ha-gitlab-internal-api-nodeport.yaml b/apps/gitlab/manifests/ha-gitlab-internal-api-nodeport.yaml index dad4ba4..657abb2 100644 --- a/apps/gitlab/manifests/ha-gitlab-internal-api-nodeport.yaml +++ b/apps/gitlab/manifests/ha-gitlab-internal-api-nodeport.yaml @@ -4,9 +4,18 @@ # node. Gitaly's gitlab-shell hooks call POST /api/v4/internal/{allowed, # pre_receive,post_receive} against this whenever the VPS node is # primary for a repository (per-repository election can cause this even -# in normal operation) or after a real failover. Carries the shell -# secret in a header - same plaintext-over-Netbird-mesh reasoning as -# ha-gitaly-nodeport.yaml, not the UniFi WAN forward. +# in normal operation) or after a real failover. +# +# Reached over the same public UniFi WAN forward already used for +# Postgres/Gitaly, not the Netbird mesh directly (confirmed live that +# path doesn't accept inbound connections to home). Port stays 8181 +# (unlike Gitaly's separate 8075/8076 split) - global.workhorse.tls. +# enabled in values.yaml replaces the plaintext listener with TLS on +# this SAME port rather than adding a second one. Carries the +# gitlab_shell_secret in a header, so this needed TLS just as much as +# Gitaly's gRPC did - see gitaly-tls-certificate.yaml's second +# Certificate (gitlab-workhorse-tls) and the +# gitlab-internal-api.ha.huskypup.net CoreDNS rewrite. apiVersion: v1 kind: Service metadata: diff --git a/apps/gitlab/manifests/praefect-ha-configmap.yaml b/apps/gitlab/manifests/praefect-ha-configmap.yaml index 4e2d48e..4353ced 100644 --- a/apps/gitlab/manifests/praefect-ha-configmap.yaml +++ b/apps/gitlab/manifests/praefect-ha-configmap.yaml @@ -28,12 +28,17 @@ # jsonpath='{.data}' | kubectl -n gitlab patch cm gitlab-praefect --type # merge -p "{\"data\":$(cat -)}") # -# Content below is intentionally byte-identical to the chart's own -# current rendering for the first pass (confirmed live via `kubectl -n -# gitlab get cm gitlab-praefect -o jsonpath='{.data.config\.toml\.tpl}'`) -# - proves the ignoreDifferences + manual-patch mechanism itself doesn't -# break anything, before a follow-up switches addressing to floating -# hostnames and adds the VPS as a 4th node. +# Content below has since moved past byte-identical: nodes 0-2 now use +# tls:// on port 8076 (matching global.gitaly.tls.enabled in +# values.yaml) and the floating hostnames gitaly-{0,1,2}.ha.huskypup.net +# (CoreDNS-rewritten locally for home's own traffic, same reasoning as +# pg-gitlab.ha.huskypup.net) rather than in-cluster DNS names or the +# plaintext scheme - this is what makes the SAME addresses usable once +# the VPS joins as a 4th node, without home's own Praefect needing a +# different config than what it hands the VPS as peer addresses. Praefect +# trusts the cert via the pod's normal system CA bundle - it's a real +# Let's Encrypt cert (gitaly-tls-certificate.yaml), not self-signed, so +# no extra CA trust config is needed here. apiVersion: v1 kind: ConfigMap metadata: @@ -62,15 +67,15 @@ data: name = 'default' [[virtual_storage.node]] storage = 'gitlab-gitaly-default-0' - address = 'tcp://gitlab-gitaly-default-0.gitlab-gitaly-default.gitlab.svc:8075' + address = 'tls://gitaly-0.ha.huskypup.net:8076' token = {% file.Read "/etc/gitlab-secrets/praefect/gitaly_token" | strings.TrimSpace | data.ToJSON %} [[virtual_storage.node]] storage = 'gitlab-gitaly-default-1' - address = 'tcp://gitlab-gitaly-default-1.gitlab-gitaly-default.gitlab.svc:8075' + address = 'tls://gitaly-1.ha.huskypup.net:8076' token = {% file.Read "/etc/gitlab-secrets/praefect/gitaly_token" | strings.TrimSpace | data.ToJSON %} [[virtual_storage.node]] storage = 'gitlab-gitaly-default-2' - address = 'tcp://gitlab-gitaly-default-2.gitlab-gitaly-default.gitlab.svc:8075' + address = 'tls://gitaly-2.ha.huskypup.net:8076' token = {% file.Read "/etc/gitlab-secrets/praefect/gitaly_token" | strings.TrimSpace | data.ToJSON %} diff --git a/apps/gitlab/values.yaml b/apps/gitlab/values.yaml index 75846d9..a76c3bc 100644 --- a/apps/gitlab/values.yaml +++ b/apps/gitlab/values.yaml @@ -54,7 +54,37 @@ global: internal: names: [] # No internal Gitaly, using Praefect external: [] # Praefect configured below - + # GitLab cross-site replication (see + # /home/scooby/.claude/plans/jiggly-snacking-iverson.md) - Gitaly's + # gRPC has no TLS by default. The VPS's 4th node has to be reached + # over the same public UniFi WAN forward already used for Postgres + # (confirmed live the direct-Netbird-IP path doesn't accept inbound + # connections to home, same as the existing etcd quorum) - without + # this, that WAN hop would carry the gitaly_token, + # gitlab_shell_secret, and raw repository contents in plaintext. + # Chart-native support (tls_listen_addr alongside the existing + # plaintext listener, both stay active) - cert is a real Let's + # Encrypt cert via the same DNS-01 ClusterIssuer already used + # everywhere else (gitaly-tls-certificate.yaml), not a self-signed + # one, so no custom CA needs distributing to either site. + tls: + enabled: true + + # Gitaly's [gitlab] url (the internal-API callback every Gitaly node + # uses for gitlab-shell hooks) - a real, TLS-supported override, + # confirmed live via gitlab/charts/gitlab/templates/_workhorse.tpl. + # Uses the SAME floating-hostname + CoreDNS-local-rewrite pattern + # already proven for Postgres and Gitaly's own node addresses: home's + # 3 Gitaly nodes resolve this locally (no WAN round-trip), the VPS's + # 4th node resolves it via the UniFi WAN forward - one consistent + # https:// URL for every node regardless of site, rather than trying + # to TLS-certify an unpublishable *.svc.cluster.local name (which + # Let's Encrypt fundamentally can't issue for). + workhorse: + host: gitlab-internal-api.ha.huskypup.net + tls: + enabled: true + # Praefect configuration praefect: enabled: true @@ -68,6 +98,12 @@ global: - name: default gitalyReplicas: 3 # Production HA maxUnavailable: 1 + # Required once global.gitaly.tls.enabled: true - the chart's own + # NOTES.txt validation rejects the render without it ("not + # specified ('default')... not supported", confirmed live). + # Matches gitlab.gitaly.tls.secret's default name/keys + # (gitaly-tls-certificate.yaml issues into this exact secret). + tlsSecretName: gitlab-gitaly-tls # Praefect PostgreSQL configuration psql: diff --git a/infrastructure/coredns/manifests/coredns-configmap.yaml b/infrastructure/coredns/manifests/coredns-configmap.yaml index 86e44ad..3079eac 100644 --- a/infrastructure/coredns/manifests/coredns-configmap.yaml +++ b/infrastructure/coredns/manifests/coredns-configmap.yaml @@ -77,6 +77,11 @@ data: rewrite name gitaly-1.ha.huskypup.net ha-gitaly-1.gitlab.svc.cluster.local rewrite name gitaly-2.ha.huskypup.net ha-gitaly-2.gitlab.svc.cluster.local + # global.workhorse.host override (apps/gitlab/values.yaml) - every + # Gitaly node's [gitlab] url, home's own included, same local- + # rewrite reasoning as the entries above. + rewrite name gitlab-internal-api.ha.huskypup.net ha-gitlab-internal-api.gitlab.svc.cluster.local + kubernetes cluster.local in-addr.arpa ip6.arpa { pods insecure fallthrough in-addr.arpa ip6.arpa