From 2643d5e0bfe87c1f8e89e33c5f206dfe98ee1f5b Mon Sep 17 00:00:00 2001 From: Scooby Husky Date: Sat, 14 Mar 2026 14:53:31 -0500 Subject: [PATCH] Fix TeslaMate DB corruption and Rook-Ceph reconcile hang - Scale CNPG pg-teslamate to 1 instance: on-disk storage corruption (corrupted sequences, system catalog indexes) means replicas cloned via pg_basebackup inherit corruption and can't open any database. Primary works from cached catalogs. Sequences already rebuilt live. - Remove talos-cp-02 from Rook-Ceph nodes list: ceph-volume inventory hangs on BIOS partition via nsenter lvs in host namespace. Talos read-only /etc prevents LVM filter fix. OSD 1 keeps running from its existing deployment. Co-Authored-By: Claude Opus 4.6 --- apps/teslamate/manifests/cnpg-cluster.yaml | 2 +- infrastructure/rook-ceph/cluster-values.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/teslamate/manifests/cnpg-cluster.yaml b/apps/teslamate/manifests/cnpg-cluster.yaml index 84e46a6..ed2b626 100644 --- a/apps/teslamate/manifests/cnpg-cluster.yaml +++ b/apps/teslamate/manifests/cnpg-cluster.yaml @@ -5,7 +5,7 @@ metadata: namespace: teslamate spec: imageName: ghcr.io/cloudnative-pg/postgresql:16 - instances: 2 + instances: 1 # Resource limits to prevent OOM resources: diff --git a/infrastructure/rook-ceph/cluster-values.yaml b/infrastructure/rook-ceph/cluster-values.yaml index 3bfcdd0..5719563 100644 --- a/infrastructure/rook-ceph/cluster-values.yaml +++ b/infrastructure/rook-ceph/cluster-values.yaml @@ -120,6 +120,9 @@ cephClusterSpec: memory: "50Mi" # Storage configuration - explicit nodes with passthrough NVMe + # NOTE: talos-cp-02 excluded from nodes list because ceph-volume inventory + # hangs on its BIOS partition (lvs kernel I/O hang via nsenter on Talos). + # OSD 1 (nvme1n1) continues running from its existing deployment. storage: useAllNodes: false useAllDevices: false @@ -127,9 +130,6 @@ cephClusterSpec: - name: "talos-cp-01" devices: - name: "nvme0n1" - - name: "talos-cp-02" - devices: - - name: "nvme1n1" - name: "talos-cp-03" devices: - name: "nvme0n1"