From 5f52f2b36b743e38dba929a6a4e0191b21e2980c Mon Sep 17 00:00:00 2001 From: Scooby Husky Date: Tue, 18 Aug 2026 21:12:45 -0500 Subject: [PATCH] Multi-site active failover pilot: home etcd member + Postgres NodePort exposure Part of the Authentik HA pilot (see plan doc). Home's etcd quorum member (StatefulSet, pinned to talos-cp-01 for a stable advertised address) plus a NodePort exposing pg-authentik's current primary - both reachable from the VPS/witness over the already-authenticated Netbird mesh (confirmed live: home nodes are directly reachable from Netbird peers on their real node IP, via the netbird-egress DaemonSet's route). Deliberately NOT going through UniFi/public-internet exposure - this stays entirely inside the private mesh, a materially safer path than the WAN port- forward originally considered. Needs a scoped Kyverno PolicyException (ha-failover-nodeport-exception.yaml) since disallow-nodeport-services is enforced cluster-wide - narrowly scoped to Services named ha-*, matching the existing netbird-egress- exemption.yaml precedent for exceptions. --- argocd-apps/infrastructure/ha-failover.yaml | 24 ++++ .../manifests/ha-postgres-nodeport.yaml | 22 ++++ .../ha-failover/manifests/etcd.yaml | 106 ++++++++++++++++++ .../ha-failover-nodeport-exception.yaml | 31 +++++ 4 files changed, 183 insertions(+) create mode 100644 argocd-apps/infrastructure/ha-failover.yaml create mode 100644 infrastructure/authentik/manifests/ha-postgres-nodeport.yaml create mode 100644 infrastructure/ha-failover/manifests/etcd.yaml create mode 100644 infrastructure/kyverno/policies/ha-failover-nodeport-exception.yaml diff --git a/argocd-apps/infrastructure/ha-failover.yaml b/argocd-apps/infrastructure/ha-failover.yaml new file mode 100644 index 0000000..49440a6 --- /dev/null +++ b/argocd-apps/infrastructure/ha-failover.yaml @@ -0,0 +1,24 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: ha-failover + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "23" # after CNPG/storage (10-14), alongside other security/HA infra + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + project: infrastructure + source: + repoURL: https://gitlab.kube.huskypup.net/Scooby/Homelabv4.git + targetRevision: main + path: infrastructure/ha-failover/manifests + destination: + server: https://kubernetes.default.svc + namespace: ha-failover + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/infrastructure/authentik/manifests/ha-postgres-nodeport.yaml b/infrastructure/authentik/manifests/ha-postgres-nodeport.yaml new file mode 100644 index 0000000..781ba7a --- /dev/null +++ b/infrastructure/authentik/manifests/ha-postgres-nodeport.yaml @@ -0,0 +1,22 @@ +--- +# Exposes pg-authentik's current primary to the VPS/witness over the +# Netbird mesh for streaming replication (see the "Multi-Site Active +# Failover" plan) - same selector CNPG's own pg-authentik-rw ClusterIP +# Service uses, just NodePort instead, since CNPG doesn't manage this +# object and won't fight with it. Needs +# infrastructure/kyverno/policies/ha-failover-nodeport-exception.yaml +# (disallow-nodeport-services is enforced cluster-wide otherwise). +apiVersion: v1 +kind: Service +metadata: + name: ha-authentik-postgres + namespace: authentik +spec: + type: NodePort + selector: + cnpg.io/cluster: pg-authentik + cnpg.io/instanceRole: primary + ports: + - port: 5432 + targetPort: 5432 + nodePort: 32432 diff --git a/infrastructure/ha-failover/manifests/etcd.yaml b/infrastructure/ha-failover/manifests/etcd.yaml new file mode 100644 index 0000000..759ab89 --- /dev/null +++ b/infrastructure/ha-failover/manifests/etcd.yaml @@ -0,0 +1,106 @@ +--- +# Home's member of the 3-way etcd quorum used by the failover-watcher +# (see /home/scooby/.claude/plans/jiggly-snacking-iverson.md, "Multi-Site +# Active Failover" section) to safely decide when to trigger a CNPG +# distributed-topology promotion. VPS and witness run this same etcd +# version as plain systemd services (simpler - no k3s/Kyverno involved +# there); home has to run it as a real K8s workload since Talos doesn't +# allow bare-metal process installs. +# +# Pinned to a specific node (talos-cp-01) via nodeSelector so its +# advertised peer/client URLs (which must be stable, not "whichever node +# it landed on today") stay correct. Reachable from the VPS/witness via +# that node's real IP - confirmed live 2026-08-19 that home nodes are +# directly reachable from Netbird mesh peers on their node IP (the +# netbird-egress DaemonSet's route) - and exposed via a NodePort Service, +# which needs infrastructure/kyverno/policies/ha-failover-nodeport-exception.yaml +# (disallow-nodeport-services is enforced cluster-wide otherwise). +apiVersion: v1 +kind: Namespace +metadata: + name: ha-failover +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: ha-etcd + namespace: ha-failover +spec: + serviceName: ha-etcd + replicas: 1 + selector: + matchLabels: + app: ha-etcd + template: + metadata: + labels: + app: ha-etcd + spec: + nodeSelector: + kubernetes.io/hostname: talos-cp-01 + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: etcd + image: gcr.io/etcd-development/etcd:v3.5.17 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + command: + - etcd + - --name=home + - --data-dir=/var/lib/etcd/data + - --listen-client-urls=http://0.0.0.0:2379 + - --advertise-client-urls=http://100.108.42.109:32379 + - --listen-peer-urls=http://0.0.0.0:2380 + - --initial-advertise-peer-urls=http://100.108.42.109:32380 + - --initial-cluster=home=http://100.108.42.109:32380,vps=http://100.108.113.41:2380,witness=http://100.108.130.74:2380 + - --initial-cluster-state=new + - --initial-cluster-token=ha-failover-quorum + ports: + - containerPort: 2379 + name: client + - containerPort: 2380 + name: peer + volumeMounts: + - name: data + mountPath: /var/lib/etcd + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + memory: 256Mi + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: rook-ceph-block + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: ha-etcd + namespace: ha-failover +spec: + type: NodePort + selector: + app: ha-etcd + ports: + - name: client + port: 2379 + targetPort: 2379 + nodePort: 32379 + - name: peer + port: 2380 + targetPort: 2380 + nodePort: 32380 diff --git a/infrastructure/kyverno/policies/ha-failover-nodeport-exception.yaml b/infrastructure/kyverno/policies/ha-failover-nodeport-exception.yaml new file mode 100644 index 0000000..e17fe32 --- /dev/null +++ b/infrastructure/kyverno/policies/ha-failover-nodeport-exception.yaml @@ -0,0 +1,31 @@ +# Scoped exception to the disallow-nodeport-services policy, narrowly for +# the multi-site active-failover pilot's cross-site services (etcd quorum, +# Postgres streaming replication) - not a namespace-wide exclusion. +# +# Why NodePort is genuinely needed here: home nodes are directly reachable +# from Netbird mesh peers (VPS, witness) on their real node IP - confirmed +# live 2026-08-19 (ping succeeded from the VPS to a node's InternalIP, +# which is itself a Netbird-mesh address via the netbird-egress DaemonSet's +# route). A NodePort Service binds on that same real interface on every +# node, giving the VPS/witness a way to reach it directly over the already- +# authenticated Netbird mesh - no public internet exposure, no UniFi +# port-forward, no new WAN-facing attack surface. LoadBalancer (MetalLB) +# only gets a LAN-side VIP, which isn't reachable from Netbird peers at all +# without the same underlying NodePort-style exposure anyway. +apiVersion: kyverno.io/v2 +kind: PolicyException +metadata: + name: ha-failover-nodeport-exemption + namespace: kyverno +spec: + exceptions: + - policyName: disallow-nodeport-services + ruleNames: + - disallow-nodeport + match: + any: + - resources: + kinds: + - Service + names: + - ha-*