diff --git a/apps/gitlab/manifests/cnpg-cluster.yaml b/apps/gitlab/manifests/cnpg-cluster.yaml index f7e034c..ff0887c 100644 --- a/apps/gitlab/manifests/cnpg-cluster.yaml +++ b/apps/gitlab/manifests/cnpg-cluster.yaml @@ -80,3 +80,58 @@ spec: monitoring: enablePodMonitor: true + + # GitLab cross-site replication Phase 1c stage 2 (see + # /home/scooby/.claude/plans/jiggly-snacking-iverson.md) - streaming + # replication to/from the VPS's pg-gitlab cluster, alongside (not + # replacing) the barmanObjectStore backup above. Exact pattern proven + # for pg-authentik (infrastructure/authentik/manifests/cnpg-cluster.yaml). + # + # Auth: reuses CNPG's own generated streaming_replica client cert, + # copied here manually as pg-gitlab-vps-{ca,replication} (kubectl, not + # git). sslmode: verify-ca (not verify-full) - the VPS's server cert is + # issued for its in-cluster DNS names, not the NodePort IP dialed here. + # + # Reachable via 100.108.113.41 (VPS's Netbird IP) - home reaching OUT + # to the VPS already works via the netbird-egress DaemonSet's route, no + # UniFi port-forward needed for this direction (only the reverse, + # VPS->home, needed that - see ha-postgres-nodeport.yaml). + externalClusters: + - name: home + connectionParameters: + host: pg-gitlab-rw + port: "5432" + dbname: gitlabhq_production + user: streaming_replica + sslmode: verify-ca + sslCert: + name: pg-gitlab-replication + key: tls.crt + sslKey: + name: pg-gitlab-replication + key: tls.key + sslRootCert: + name: pg-gitlab-ca + key: ca.crt + - name: vps + connectionParameters: + host: 100.108.113.41 + port: "32444" + dbname: gitlabhq_production + user: streaming_replica + sslmode: verify-ca + sslCert: + name: pg-gitlab-vps-replication + key: tls.crt + sslKey: + name: pg-gitlab-vps-replication + key: tls.key + sslRootCert: + name: pg-gitlab-vps-ca + key: ca.crt + + # Distributed topology: both sides agree home is primary today. + replica: + self: home + primary: home + source: home # required unconditionally by the admission webhook even though home isn't replicating from anyone (self == primary) diff --git a/apps/gitlab/manifests/ha-postgres-app-pushsecret.yaml b/apps/gitlab/manifests/ha-postgres-app-pushsecret.yaml new file mode 100644 index 0000000..cf7c435 --- /dev/null +++ b/apps/gitlab/manifests/ha-postgres-app-pushsecret.yaml @@ -0,0 +1,50 @@ +# GitLab cross-site replication (see +# /home/scooby/.claude/plans/jiggly-snacking-iverson.md) - same pattern +# and same reasoning as +# infrastructure/authentik/manifests/ha-postgres-app-pushsecret.yaml: +# pg-gitlab-app/pg-praefect-app are CNPG-generated, not Vault-native - +# each cluster independently generated its own random `app` role +# password at bootstrap time. Streaming replication keeps the two +# clusters' actual live passwords in sync (WAL includes role/password +# changes), but the two clusters' K8s Secret OBJECTS never resync on +# their own - this closes that gap, self-healing on refreshInterval +# instead of a one-time fix that goes stale on the next rotation. +apiVersion: external-secrets.io/v1alpha1 +kind: PushSecret +metadata: + name: pg-gitlab-app-to-vps + namespace: gitlab +spec: + refreshInterval: 5m + secretStoreRefs: + - name: vault-backend + kind: ClusterSecretStore + selector: + secret: + name: pg-gitlab-app + data: + - match: + secretKey: password + remoteRef: + remoteKey: vps/pg-gitlab-app + property: password +--- +apiVersion: external-secrets.io/v1alpha1 +kind: PushSecret +metadata: + name: pg-praefect-app-to-vps + namespace: gitlab +spec: + refreshInterval: 5m + secretStoreRefs: + - name: vault-backend + kind: ClusterSecretStore + selector: + secret: + name: pg-praefect-app + data: + - match: + secretKey: password + remoteRef: + remoteKey: vps/pg-praefect-app + property: password diff --git a/apps/gitlab/manifests/praefect-cnpg-cluster.yaml b/apps/gitlab/manifests/praefect-cnpg-cluster.yaml index a84baf4..3ed2909 100644 --- a/apps/gitlab/manifests/praefect-cnpg-cluster.yaml +++ b/apps/gitlab/manifests/praefect-cnpg-cluster.yaml @@ -76,3 +76,44 @@ spec: monitoring: enablePodMonitor: true + + # GitLab cross-site replication Phase 1c stage 2 - same pattern as + # cnpg-cluster.yaml (pg-gitlab) in this same directory. + externalClusters: + - name: home + connectionParameters: + host: pg-praefect-rw + port: "5432" + dbname: praefect_production + user: streaming_replica + sslmode: verify-ca + sslCert: + name: pg-praefect-replication + key: tls.crt + sslKey: + name: pg-praefect-replication + key: tls.key + sslRootCert: + name: pg-praefect-ca + key: ca.crt + - name: vps + connectionParameters: + host: 100.108.113.41 + port: "32445" + dbname: praefect_production + user: streaming_replica + sslmode: verify-ca + sslCert: + name: pg-praefect-vps-replication + key: tls.crt + sslKey: + name: pg-praefect-vps-replication + key: tls.key + sslRootCert: + name: pg-praefect-vps-ca + key: ca.crt + + replica: + self: home + primary: home + source: home diff --git a/infrastructure/vps-standby/gitlab/manifests/cnpg-cluster.yaml b/infrastructure/vps-standby/gitlab/manifests/cnpg-cluster.yaml index 49332c0..1e8ece7 100644 --- a/infrastructure/vps-standby/gitlab/manifests/cnpg-cluster.yaml +++ b/infrastructure/vps-standby/gitlab/manifests/cnpg-cluster.yaml @@ -65,6 +65,22 @@ spec: recovery: source: home + # Stage 2 (see /home/scooby/.claude/plans/jiggly-snacking-iverson.md): + # connectionParameters added alongside the existing barmanObjectStore + # on the SAME "home" entry - the recovery-mode initial seed stays + # exactly as-is, ongoing replication now streams instead of relying on + # WAL-archive polling (the source of the crash-looping/probe-tuning + # fights documented for the pg-authentik precedent). Also needs a + # self-referencing "vps" entry - replica.self must match an + # externalClusters name too, even though it's this very cluster + # (confirmed live: "External cluster vps not found" until one was + # added). + # + # Reachable via home.kube.huskypup.net:61442 - a UniFi WAN port-forward + # to talos-cp-01's real LAN IP, source-IP-restricted to the VPS/witness + # public IPs (see apps/gitlab/manifests/ha-postgres-nodeport.yaml). NOT + # the Netbird mesh directly - UniFi has no route to that CIDR + # (confirmed live for the pg-authentik precedent). externalClusters: - name: home barmanObjectStore: @@ -84,6 +100,45 @@ spec: secretAccessKey: name: vps-minio-secret key: secretkey + connectionParameters: + host: home.kube.huskypup.net + port: "61442" + dbname: gitlabhq_production + user: streaming_replica + sslmode: verify-ca + sslCert: + name: pg-gitlab-home-replication + key: tls.crt + sslKey: + name: pg-gitlab-home-replication + key: tls.key + sslRootCert: + name: pg-gitlab-home-ca + key: ca.crt + - name: vps + connectionParameters: + host: pg-gitlab-rw + port: "5432" + dbname: gitlabhq_production + user: streaming_replica + sslmode: verify-ca + sslCert: + name: pg-gitlab-replication + key: tls.crt + sslKey: + name: pg-gitlab-replication + key: tls.key + sslRootCert: + name: pg-gitlab-ca + key: ca.crt + + # Distributed topology: both sides agree home is primary today. No + # replica.enabled - mutually exclusive with primary/self (confirmed + # live for the pg-authentik precedent). + replica: + self: vps + primary: home + source: home monitoring: enablePodMonitor: false # no Prometheus on the VPS cluster diff --git a/infrastructure/vps-standby/gitlab/manifests/ha-postgres-app-externalsecret.yaml b/infrastructure/vps-standby/gitlab/manifests/ha-postgres-app-externalsecret.yaml new file mode 100644 index 0000000..c833806 --- /dev/null +++ b/infrastructure/vps-standby/gitlab/manifests/ha-postgres-app-externalsecret.yaml @@ -0,0 +1,45 @@ +# GitLab cross-site replication - VPS's half of the Vault pipeline +# (apps/gitlab/manifests/ha-postgres-app-pushsecret.yaml has home's half). +# Pulls the authoritative app-role passwords back down here. +# +# Merge (not Owner/Replace): only overwrites the `password` key, leaving +# CNPG's own generated host/dbname/username fields intact - those are +# correctly LOCAL to each site (this secret's `host` key points at the +# VPS's own local -rw service, which CNPG itself still needs internally). +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: pg-gitlab-app-password + namespace: gitlab +spec: + refreshInterval: 5m + secretStoreRef: + name: vault-backend + kind: ClusterSecretStore + target: + name: pg-gitlab-app + creationPolicy: Merge + data: + - secretKey: password + remoteRef: + key: vps/pg-gitlab-app + property: password +--- +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: pg-praefect-app-password + namespace: gitlab +spec: + refreshInterval: 5m + secretStoreRef: + name: vault-backend + kind: ClusterSecretStore + target: + name: pg-praefect-app + creationPolicy: Merge + data: + - secretKey: password + remoteRef: + key: vps/pg-praefect-app + property: password diff --git a/infrastructure/vps-standby/gitlab/manifests/praefect-cnpg-cluster.yaml b/infrastructure/vps-standby/gitlab/manifests/praefect-cnpg-cluster.yaml index 1d14056..fbc0db6 100644 --- a/infrastructure/vps-standby/gitlab/manifests/praefect-cnpg-cluster.yaml +++ b/infrastructure/vps-standby/gitlab/manifests/praefect-cnpg-cluster.yaml @@ -52,6 +52,8 @@ spec: recovery: source: home + # Stage 2 - same pattern as cnpg-cluster.yaml (pg-gitlab) in this same + # directory. externalClusters: - name: home barmanObjectStore: @@ -65,6 +67,42 @@ spec: secretAccessKey: name: vps-minio-secret key: secretkey + connectionParameters: + host: home.kube.huskypup.net + port: "61443" + dbname: praefect_production + user: streaming_replica + sslmode: verify-ca + sslCert: + name: pg-praefect-home-replication + key: tls.crt + sslKey: + name: pg-praefect-home-replication + key: tls.key + sslRootCert: + name: pg-praefect-home-ca + key: ca.crt + - name: vps + connectionParameters: + host: pg-praefect-rw + port: "5432" + dbname: praefect_production + user: streaming_replica + sslmode: verify-ca + sslCert: + name: pg-praefect-replication + key: tls.crt + sslKey: + name: pg-praefect-replication + key: tls.key + sslRootCert: + name: pg-praefect-ca + key: ca.crt + + replica: + self: vps + primary: home + source: home monitoring: enablePodMonitor: false # no Prometheus on the VPS cluster