GitLab cross-site replication Phase 1c (stage 2): live streaming replication

Adds externalClusters connectionParameters + replica.self/primary/source
to all four Cluster CRs (home pg-gitlab/pg-praefect, VPS
pg-gitlab/pg-praefect), turning last commit's recovery-only replicas
into real CNPG streaming replicas - exact pattern already proven for
pg-authentik. TLS certs copied cross-cluster manually (kubectl, not
git, same as every other cross-cluster secret in this plan).

Also adds pg-gitlab-app/pg-praefect-app password sync (PushSecret home
-> Vault -> ExternalSecret VPS, Merge policy) - CNPG-generated
passwords stay in sync via WAL replication but the K8s Secret OBJECTS
never resync on their own, same gap already closed for pg-authentik-app.

VPS's standalone clusters were already destroyed and recreated as
recovery-mode replicas in the prior commit (with the user's explicit
help running the classifier-blocked kubectl delete) - this commit turns
on the actual streaming replication on top of that.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

apps/gitlab/manifests/cnpg-cluster.yaml

@@ -80,3 +80,58 @@ spec:
 
   monitoring:
     enablePodMonitor: true
+
+  # GitLab cross-site replication Phase 1c stage 2 (see
+  # /home/scooby/.claude/plans/jiggly-snacking-iverson.md) - streaming
+  # replication to/from the VPS's pg-gitlab cluster, alongside (not
+  # replacing) the barmanObjectStore backup above. Exact pattern proven
+  # for pg-authentik (infrastructure/authentik/manifests/cnpg-cluster.yaml).
+  #
+  # Auth: reuses CNPG's own generated streaming_replica client cert,
+  # copied here manually as pg-gitlab-vps-{ca,replication} (kubectl, not
+  # git). sslmode: verify-ca (not verify-full) - the VPS's server cert is
+  # issued for its in-cluster DNS names, not the NodePort IP dialed here.
+  #
+  # Reachable via 100.108.113.41 (VPS's Netbird IP) - home reaching OUT
+  # to the VPS already works via the netbird-egress DaemonSet's route, no
+  # UniFi port-forward needed for this direction (only the reverse,
+  # VPS->home, needed that - see ha-postgres-nodeport.yaml).
+  externalClusters:
+    - name: home
+      connectionParameters:
+        host: pg-gitlab-rw
+        port: "5432"
+        dbname: gitlabhq_production
+        user: streaming_replica
+        sslmode: verify-ca
+      sslCert:
+        name: pg-gitlab-replication
+        key: tls.crt
+      sslKey:
+        name: pg-gitlab-replication
+        key: tls.key
+      sslRootCert:
+        name: pg-gitlab-ca
+        key: ca.crt
+    - name: vps
+      connectionParameters:
+        host: 100.108.113.41
+        port: "32444"
+        dbname: gitlabhq_production
+        user: streaming_replica
+        sslmode: verify-ca
+      sslCert:
+        name: pg-gitlab-vps-replication
+        key: tls.crt
+      sslKey:
+        name: pg-gitlab-vps-replication
+        key: tls.key
+      sslRootCert:
+        name: pg-gitlab-vps-ca
+        key: ca.crt
+
+  # Distributed topology: both sides agree home is primary today.
+  replica:
+    self: home
+    primary: home
+    source: home  # required unconditionally by the admission webhook even though home isn't replicating from anyone (self == primary)

apps/gitlab/manifests/ha-postgres-app-pushsecret.yaml

@@ -0,0 +1,50 @@
+# GitLab cross-site replication (see
+# /home/scooby/.claude/plans/jiggly-snacking-iverson.md) - same pattern
+# and same reasoning as
+# infrastructure/authentik/manifests/ha-postgres-app-pushsecret.yaml:
+# pg-gitlab-app/pg-praefect-app are CNPG-generated, not Vault-native -
+# each cluster independently generated its own random `app` role
+# password at bootstrap time. Streaming replication keeps the two
+# clusters' actual live passwords in sync (WAL includes role/password
+# changes), but the two clusters' K8s Secret OBJECTS never resync on
+# their own - this closes that gap, self-healing on refreshInterval
+# instead of a one-time fix that goes stale on the next rotation.
+apiVersion: external-secrets.io/v1alpha1
+kind: PushSecret
+metadata:
+  name: pg-gitlab-app-to-vps
+  namespace: gitlab
+spec:
+  refreshInterval: 5m
+  secretStoreRefs:
+    - name: vault-backend
+      kind: ClusterSecretStore
+  selector:
+    secret:
+      name: pg-gitlab-app
+  data:
+    - match:
+        secretKey: password
+        remoteRef:
+          remoteKey: vps/pg-gitlab-app
+          property: password
+---
+apiVersion: external-secrets.io/v1alpha1
+kind: PushSecret
+metadata:
+  name: pg-praefect-app-to-vps
+  namespace: gitlab
+spec:
+  refreshInterval: 5m
+  secretStoreRefs:
+    - name: vault-backend
+      kind: ClusterSecretStore
+  selector:
+    secret:
+      name: pg-praefect-app
+  data:
+    - match:
+        secretKey: password
+        remoteRef:
+          remoteKey: vps/pg-praefect-app
+          property: password

apps/gitlab/manifests/praefect-cnpg-cluster.yaml

@@ -76,3 +76,44 @@ spec:
 
   monitoring:
     enablePodMonitor: true
+
+  # GitLab cross-site replication Phase 1c stage 2 - same pattern as
+  # cnpg-cluster.yaml (pg-gitlab) in this same directory.
+  externalClusters:
+    - name: home
+      connectionParameters:
+        host: pg-praefect-rw
+        port: "5432"
+        dbname: praefect_production
+        user: streaming_replica
+        sslmode: verify-ca
+      sslCert:
+        name: pg-praefect-replication
+        key: tls.crt
+      sslKey:
+        name: pg-praefect-replication
+        key: tls.key
+      sslRootCert:
+        name: pg-praefect-ca
+        key: ca.crt
+    - name: vps
+      connectionParameters:
+        host: 100.108.113.41
+        port: "32445"
+        dbname: praefect_production
+        user: streaming_replica
+        sslmode: verify-ca
+      sslCert:
+        name: pg-praefect-vps-replication
+        key: tls.crt
+      sslKey:
+        name: pg-praefect-vps-replication
+        key: tls.key
+      sslRootCert:
+        name: pg-praefect-vps-ca
+        key: ca.crt
+
+  replica:
+    self: home
+    primary: home
+    source: home

infrastructure/vps-standby/gitlab/manifests/cnpg-cluster.yaml

@@ -65,6 +65,22 @@ spec:
     recovery:
       source: home
 
+  # Stage 2 (see /home/scooby/.claude/plans/jiggly-snacking-iverson.md):
+  # connectionParameters added alongside the existing barmanObjectStore
+  # on the SAME "home" entry - the recovery-mode initial seed stays
+  # exactly as-is, ongoing replication now streams instead of relying on
+  # WAL-archive polling (the source of the crash-looping/probe-tuning
+  # fights documented for the pg-authentik precedent). Also needs a
+  # self-referencing "vps" entry - replica.self must match an
+  # externalClusters name too, even though it's this very cluster
+  # (confirmed live: "External cluster vps not found" until one was
+  # added).
+  #
+  # Reachable via home.kube.huskypup.net:61442 - a UniFi WAN port-forward
+  # to talos-cp-01's real LAN IP, source-IP-restricted to the VPS/witness
+  # public IPs (see apps/gitlab/manifests/ha-postgres-nodeport.yaml). NOT
+  # the Netbird mesh directly - UniFi has no route to that CIDR
+  # (confirmed live for the pg-authentik precedent).
   externalClusters:
     - name: home
       barmanObjectStore:
@@ -84,6 +100,45 @@ spec:
           secretAccessKey:
             name: vps-minio-secret
             key: secretkey
+      connectionParameters:
+        host: home.kube.huskypup.net
+        port: "61442"
+        dbname: gitlabhq_production
+        user: streaming_replica
+        sslmode: verify-ca
+      sslCert:
+        name: pg-gitlab-home-replication
+        key: tls.crt
+      sslKey:
+        name: pg-gitlab-home-replication
+        key: tls.key
+      sslRootCert:
+        name: pg-gitlab-home-ca
+        key: ca.crt
+    - name: vps
+      connectionParameters:
+        host: pg-gitlab-rw
+        port: "5432"
+        dbname: gitlabhq_production
+        user: streaming_replica
+        sslmode: verify-ca
+      sslCert:
+        name: pg-gitlab-replication
+        key: tls.crt
+      sslKey:
+        name: pg-gitlab-replication
+        key: tls.key
+      sslRootCert:
+        name: pg-gitlab-ca
+        key: ca.crt
+
+  # Distributed topology: both sides agree home is primary today. No
+  # replica.enabled - mutually exclusive with primary/self (confirmed
+  # live for the pg-authentik precedent).
+  replica:
+    self: vps
+    primary: home
+    source: home
 
   monitoring:
     enablePodMonitor: false  # no Prometheus on the VPS cluster

infrastructure/vps-standby/gitlab/manifests/ha-postgres-app-externalsecret.yaml

@@ -0,0 +1,45 @@
+# GitLab cross-site replication - VPS's half of the Vault pipeline
+# (apps/gitlab/manifests/ha-postgres-app-pushsecret.yaml has home's half).
+# Pulls the authoritative app-role passwords back down here.
+#
+# Merge (not Owner/Replace): only overwrites the `password` key, leaving
+# CNPG's own generated host/dbname/username fields intact - those are
+# correctly LOCAL to each site (this secret's `host` key points at the
+# VPS's own local -rw service, which CNPG itself still needs internally).
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: pg-gitlab-app-password
+  namespace: gitlab
+spec:
+  refreshInterval: 5m
+  secretStoreRef:
+    name: vault-backend
+    kind: ClusterSecretStore
+  target:
+    name: pg-gitlab-app
+    creationPolicy: Merge
+  data:
+    - secretKey: password
+      remoteRef:
+        key: vps/pg-gitlab-app
+        property: password
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: pg-praefect-app-password
+  namespace: gitlab
+spec:
+  refreshInterval: 5m
+  secretStoreRef:
+    name: vault-backend
+    kind: ClusterSecretStore
+  target:
+    name: pg-praefect-app
+    creationPolicy: Merge
+  data:
+    - secretKey: password
+      remoteRef:
+        key: vps/pg-praefect-app
+        property: password

infrastructure/vps-standby/gitlab/manifests/praefect-cnpg-cluster.yaml

@@ -52,6 +52,8 @@ spec:
     recovery:
       source: home
 
+  # Stage 2 - same pattern as cnpg-cluster.yaml (pg-gitlab) in this same
+  # directory.
   externalClusters:
     - name: home
       barmanObjectStore:
@@ -65,6 +67,42 @@ spec:
           secretAccessKey:
             name: vps-minio-secret
             key: secretkey
+      connectionParameters:
+        host: home.kube.huskypup.net
+        port: "61443"
+        dbname: praefect_production
+        user: streaming_replica
+        sslmode: verify-ca
+      sslCert:
+        name: pg-praefect-home-replication
+        key: tls.crt
+      sslKey:
+        name: pg-praefect-home-replication
+        key: tls.key
+      sslRootCert:
+        name: pg-praefect-home-ca
+        key: ca.crt
+    - name: vps
+      connectionParameters:
+        host: pg-praefect-rw
+        port: "5432"
+        dbname: praefect_production
+        user: streaming_replica
+        sslmode: verify-ca
+      sslCert:
+        name: pg-praefect-replication
+        key: tls.crt
+      sslKey:
+        name: pg-praefect-replication
+        key: tls.key
+      sslRootCert:
+        name: pg-praefect-ca
+        key: ca.crt
+
+  replica:
+    self: vps
+    primary: home
+    source: home
 
   monitoring:
     enablePodMonitor: false  # no Prometheus on the VPS cluster