From 04fa80ece3d0649b8b575670c6d6a1efa230717c Mon Sep 17 00:00:00 2001 From: Scooby Husky Date: Sat, 22 Aug 2026 12:47:03 -0500 Subject: [PATCH] GitLab cross-site replication Phase 1a: Postgres network plumbing Per the approved plan (jiggly-snacking-iverson.md, 'GitLab True Cross-Site Replication' section) - replacing the git-mirror CronJob with real CNPG streaming replication for pg-gitlab and pg-praefect, reusing the exact pattern already proven for pg-authentik. This commit is network plumbing only, no DB replication yet: - Repoint both CNPG clusters' barmanObjectStore backup target from in-cluster gitlab-minio-svc to VPS MinIO (also becomes the initial seed source for the VPS's replicas in Phase 1c). - New ha-gitlab-postgres/ha-praefect-postgres NodePort Services (bypass CNPG's own -rw Services) on both home and VPS. - New PeerAuthentication with portLevelMtls PERMISSIVE on the replication ports - ambient mesh STRICT default would otherwise reset the VPS/witness's connection before the Postgres TLS handshake starts. - Extended allow-gitlab-access AuthorizationPolicy with a port-scoped rule for the same traffic (no source.namespaces rule can match external, non-mesh peers). - CoreDNS rewrites for pg-gitlab.ha.huskypup.net / pg-praefect.ha.huskypup.net so home's own pods reach the floating hostname locally instead of round-tripping through the WAN forward (no NAT hairpin support). Still needed before Phase 1b/1c: UniFi WAN port-forwards for external ports 61442/61443 -> talos-cp-01:32442/32443, source-restricted to the VPS + witness public IPs (manual, same as the existing 61432 rule). Co-Authored-By: Claude Sonnet 5 --- apps/gitlab/manifests/cnpg-cluster.yaml | 18 ++++-- apps/gitlab/manifests/ha-peerauth.yaml | 51 +++++++++++++++++ .../manifests/ha-postgres-nodeport.yaml | 56 +++++++++++++++++++ .../manifests/praefect-cnpg-cluster.yaml | 43 ++++++++++++++ .../coredns/manifests/coredns-configmap.yaml | 9 +++ .../apps/authorization-policies-apps.yaml | 17 ++++++ .../manifests/ha-postgres-nodeport.yaml | 47 ++++++++++++++++ 7 files changed, 236 insertions(+), 5 deletions(-) create mode 100644 apps/gitlab/manifests/ha-peerauth.yaml create mode 100644 apps/gitlab/manifests/ha-postgres-nodeport.yaml create mode 100644 infrastructure/vps-standby/gitlab/manifests/ha-postgres-nodeport.yaml diff --git a/apps/gitlab/manifests/cnpg-cluster.yaml b/apps/gitlab/manifests/cnpg-cluster.yaml index f003dd2..f7e034c 100644 --- a/apps/gitlab/manifests/cnpg-cluster.yaml +++ b/apps/gitlab/manifests/cnpg-cluster.yaml @@ -53,17 +53,25 @@ spec: max_parallel_workers_per_gather: "2" max_parallel_workers: "4" - # Backup configuration to MinIO + # Backup configuration - repointed to VPS MinIO 2026-08-22 (GitLab + # cross-site replication plan, see + # /home/scooby/.claude/plans/jiggly-snacking-iverson.md's "GitLab True + # Cross-Site Replication" section). Was gitlab-minio-svc (in-cluster, + # same Ceph the backup is meant to protect against). Also doubles as + # the initial-seed source for the VPS's own pg-gitlab replica + # (bootstrap.recovery, Phase 1c) - same barmanObjectStore block serves + # both purposes, matching the Authentik precedent + # (infrastructure/authentik/manifests/cnpg-cluster.yaml). backup: barmanObjectStore: - destinationPath: s3://gitlab-backups/pg-gitlab - endpointURL: http://gitlab-minio-svc.gitlab.svc.cluster.local:9000 + destinationPath: s3://cnpg-backups/pg-gitlab + endpointURL: http://100.108.113.41:9000 s3Credentials: accessKeyId: - name: gitlab-minio-secret + name: vps-minio-secret key: accesskey secretAccessKey: - name: gitlab-minio-secret + name: vps-minio-secret key: secretkey wal: compression: gzip diff --git a/apps/gitlab/manifests/ha-peerauth.yaml b/apps/gitlab/manifests/ha-peerauth.yaml new file mode 100644 index 0000000..ace7673 --- /dev/null +++ b/apps/gitlab/manifests/ha-peerauth.yaml @@ -0,0 +1,51 @@ +# GitLab cross-site replication (see +# /home/scooby/.claude/plans/jiggly-snacking-iverson.md) - direct copy of +# infrastructure/authentik/manifests/ha-postgres-peerauth.yaml's proven +# fix. The gitlab namespace carries the same ambient-mesh STRICT default +# as authentik; without this, ztunnel resets the VPS/witness's connection +# (no mesh identity - they're not in this cluster) before the Postgres +# TLS handshake can even start, surfacing as "server closed the +# connection unexpectedly" on the replica side despite the NodePort/ +# firewall path being entirely correct. +# +# Scoped to just each CNPG primary pod's ports via portLevelMtls, not the +# whole namespace - GitLab's own in-mesh east-west traffic (webservice +# <-> gitaly <-> redis <-> sidekiq <-> praefect) stays STRICT. +apiVersion: security.istio.io/v1 +kind: PeerAuthentication +metadata: + name: allow-ha-gitlab-postgres-replication + namespace: gitlab +spec: + selector: + matchLabels: + cnpg.io/cluster: pg-gitlab + mtls: + mode: STRICT + portLevelMtls: + "5432": + mode: PERMISSIVE + # Floating-hostname port (see ha-postgres-nodeport.yaml) - needs its + # own entry, ambient's port-level mTLS enforcement is keyed on the + # port actually dialed (61442), not just the pod's real containerPort + # (5432) it eventually reaches (confirmed live for the authentik + # precedent, same mechanism here). + "61442": + mode: PERMISSIVE +--- +apiVersion: security.istio.io/v1 +kind: PeerAuthentication +metadata: + name: allow-ha-praefect-postgres-replication + namespace: gitlab +spec: + selector: + matchLabels: + cnpg.io/cluster: pg-praefect + mtls: + mode: STRICT + portLevelMtls: + "5432": + mode: PERMISSIVE + "61443": + mode: PERMISSIVE diff --git a/apps/gitlab/manifests/ha-postgres-nodeport.yaml b/apps/gitlab/manifests/ha-postgres-nodeport.yaml new file mode 100644 index 0000000..bf95f41 --- /dev/null +++ b/apps/gitlab/manifests/ha-postgres-nodeport.yaml @@ -0,0 +1,56 @@ +--- +# GitLab cross-site replication (see +# /home/scooby/.claude/plans/jiggly-snacking-iverson.md's "GitLab True +# Cross-Site Replication" section) - direct copy of the proven pattern +# from infrastructure/authentik/manifests/ha-postgres-nodeport.yaml. +# Exposes pg-gitlab's and pg-praefect's current primaries to the +# VPS/witness over the Netbird mesh for streaming replication - same +# selectors CNPG's own -rw ClusterIP Services use, just NodePort instead, +# since CNPG doesn't manage these objects and won't fight with them. +# Needs infrastructure/kyverno/policies/ha-failover-nodeport-exception.yaml +# (name pattern ha-* already matches, no change needed there). +# +# Second port on each (61442/61443, ClusterIP only) for the CoreDNS +# rewrite in infrastructure/coredns/manifests/: home's own pods resolving +# pg-gitlab.ha.huskypup.net / pg-praefect.ha.huskypup.net rewrite to +# THESE services specifically so they can reach them on the SAME port +# number the app tier will eventually be configured with (Phase 3) - +# rewrite only changes the resolved NAME, not the port a client then +# connects to (confirmed live for the authentik precedent). +apiVersion: v1 +kind: Service +metadata: + name: ha-gitlab-postgres + namespace: gitlab +spec: + type: NodePort + selector: + cnpg.io/cluster: pg-gitlab + cnpg.io/instanceRole: primary + ports: + - name: nodeport-5432 + port: 5432 + targetPort: 5432 + nodePort: 32442 + - name: floating-port-61442 + port: 61442 + targetPort: 5432 +--- +apiVersion: v1 +kind: Service +metadata: + name: ha-praefect-postgres + namespace: gitlab +spec: + type: NodePort + selector: + cnpg.io/cluster: pg-praefect + cnpg.io/instanceRole: primary + ports: + - name: nodeport-5432 + port: 5432 + targetPort: 5432 + nodePort: 32443 + - name: floating-port-61443 + port: 61443 + targetPort: 5432 diff --git a/apps/gitlab/manifests/praefect-cnpg-cluster.yaml b/apps/gitlab/manifests/praefect-cnpg-cluster.yaml index 7c8f9dd..a84baf4 100644 --- a/apps/gitlab/manifests/praefect-cnpg-cluster.yaml +++ b/apps/gitlab/manifests/praefect-cnpg-cluster.yaml @@ -26,10 +26,53 @@ spec: primaryUpdateStrategy: unsupervised + # Explicit params added 2026-08-22 (GitLab cross-site replication plan) + # - this cluster ran on bare Postgres defaults before. A VPS replica + # can't replay WAL past a checkpoint recorded with higher settings than + # its own ("recovery aborted because of insufficient parameter + # settings", confirmed live for pg-authentik) - giving both sides an + # identical explicit block up front means nobody can tune one side + # later and silently break replication. + postgresql: + parameters: + max_connections: "200" + shared_buffers: "512MB" + effective_cache_size: "1536MB" + maintenance_work_mem: "128MB" + checkpoint_completion_target: "0.9" + wal_buffers: "16MB" + default_statistics_target: "100" + random_page_cost: "1.1" + effective_io_concurrency: "200" + work_mem: "4MB" + min_wal_size: "1GB" + max_wal_size: "4GB" + max_worker_processes: "4" + bootstrap: initdb: database: praefect_production owner: app + # Backup to VPS MinIO, also the initial-seed source for the VPS's own + # pg-praefect replica (bootstrap.recovery, Phase 1c) - this cluster had + # no backup block at all before. See the matching comment on + # cnpg-cluster.yaml (pg-gitlab) in this same directory. + backup: + barmanObjectStore: + destinationPath: s3://cnpg-backups/pg-praefect + endpointURL: http://100.108.113.41:9000 + s3Credentials: + accessKeyId: + name: vps-minio-secret + key: accesskey + secretAccessKey: + name: vps-minio-secret + key: secretkey + wal: + compression: gzip + maxParallel: 2 + retentionPolicy: "30d" + monitoring: enablePodMonitor: true diff --git a/infrastructure/coredns/manifests/coredns-configmap.yaml b/infrastructure/coredns/manifests/coredns-configmap.yaml index 186f0e7..5a6e327 100644 --- a/infrastructure/coredns/manifests/coredns-configmap.yaml +++ b/infrastructure/coredns/manifests/coredns-configmap.yaml @@ -58,6 +58,15 @@ data: # specifically so this rewrite target actually has that port. rewrite name pg-authentik.ha.huskypup.net ha-authentik-postgres.authentik.svc.cluster.local + # GitLab cross-site replication (see + # /home/scooby/.claude/plans/jiggly-snacking-iverson.md) - same + # reasoning as pg-authentik.ha.huskypup.net above: home's own pods + # go straight to the local NodePort Service instead of round- + # tripping out through the UniFi WAN forward and back in (no NAT + # hairpin support, confirmed live for the authentik precedent). + rewrite name pg-gitlab.ha.huskypup.net ha-gitlab-postgres.gitlab.svc.cluster.local + rewrite name pg-praefect.ha.huskypup.net ha-praefect-postgres.gitlab.svc.cluster.local + kubernetes cluster.local in-addr.arpa ip6.arpa { pods insecure fallthrough in-addr.arpa ip6.arpa diff --git a/infrastructure/istio/manifests/apps/authorization-policies-apps.yaml b/infrastructure/istio/manifests/apps/authorization-policies-apps.yaml index 293b2ca..34adfff 100644 --- a/infrastructure/istio/manifests/apps/authorization-policies-apps.yaml +++ b/infrastructure/istio/manifests/apps/authorization-policies-apps.yaml @@ -83,6 +83,23 @@ spec: - source: ipBlocks: - "10.244.0.0/16" + # GitLab cross-site replication (see + # /home/scooby/.claude/plans/jiggly-snacking-iverson.md): CNPG + # streaming replication from the VPS/witness, arriving via the + # ha-gitlab-postgres/ha-praefect-postgres NodePorts - no mesh + # identity at all, so no source.namespaces rule above can match them. + # Scoped by destination port instead, matching the port-scoped + # PERMISSIVE PeerAuthentication in ha-peerauth.yaml (same root cause, + # same fix, one layer up - mTLS being allowed through isn't enough by + # itself, this ALLOW policy independently denies anything not + # matching one of the rules above). Same pattern already proven for + # Authentik in infrastructure/istio/manifests/mesh/authorization-policies.yaml. + - to: + - operation: + ports: + - "5432" + - "61442" # pg-gitlab floating-hostname port, see ha-postgres-nodeport.yaml + - "61443" # pg-praefect floating-hostname port, see ha-postgres-nodeport.yaml --- # --- n8n: Allow ingress + Prometheus + intra-namespace --- diff --git a/infrastructure/vps-standby/gitlab/manifests/ha-postgres-nodeport.yaml b/infrastructure/vps-standby/gitlab/manifests/ha-postgres-nodeport.yaml new file mode 100644 index 0000000..9256a4e --- /dev/null +++ b/infrastructure/vps-standby/gitlab/manifests/ha-postgres-nodeport.yaml @@ -0,0 +1,47 @@ +--- +# Exposes the VPS's pg-gitlab/pg-praefect primaries (replicas today) back +# to home over the Netbird mesh, for GitLab cross-site replication (see +# /home/scooby/.claude/plans/jiggly-snacking-iverson.md) - needed for +# failback (home rejoining as a replica once it's back). Same pattern as +# infrastructure/vps-standby/authentik/manifests/ha-postgres-nodeport.yaml. +# +# No UniFi port-forward or Kyverno PolicyException needed here - the +# VPS's k3s has no NodePort restriction, and home reaching OUT to the VPS +# already works today via the netbird-egress DaemonSet's route (confirmed +# live all session for MinIO/CrowdSec/pg-authentik) - it's only the +# reverse direction (external peers reaching INTO home) that needs the +# UniFi workaround (see apps/gitlab/manifests/ha-postgres-nodeport.yaml). +# +# Different nodePort numbers than home's side (32442/32443) - not +# required since these are different machines, but matches the +# authentik precedent's convention of keeping each site's numbers +# distinct to avoid confusion. +apiVersion: v1 +kind: Service +metadata: + name: ha-gitlab-postgres + namespace: gitlab +spec: + type: NodePort + selector: + cnpg.io/cluster: pg-gitlab + cnpg.io/instanceRole: primary + ports: + - port: 5432 + targetPort: 5432 + nodePort: 32444 +--- +apiVersion: v1 +kind: Service +metadata: + name: ha-praefect-postgres + namespace: gitlab +spec: + type: NodePort + selector: + cnpg.io/cluster: pg-praefect + cnpg.io/instanceRole: primary + ports: + - port: 5432 + targetPort: 5432 + nodePort: 32445