From 511ba55b3067247178cc1047435c42e58e734a05 Mon Sep 17 00:00:00 2001
From: Scooby Husky <garrettstone499@gmail.com>
Date: Tue, 10 Mar 2026 22:06:34 -0500
Subject: [PATCH] Remove dedicated DB devices from Rook-Ceph OSD configuration

Remove metadataDevice (sdb) from all 4 nodes in cluster-values.yaml
since the SATA SSDs are slower than the NVMe data drives. Add temporary
Kyverno mutate policy to protect existing OSD deployments when the
operator is re-enabled - replaces expand-bluefs with a no-op and adds
conditional block.db symlink creation based on BlueStore label detection.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .../fix-rook-osd-metadata-device.yaml         | 63 +++++++++++++++++++
 infrastructure/rook-ceph/cluster-values.yaml  |  9 ---
 2 files changed, 63 insertions(+), 9 deletions(-)
 create mode 100644 infrastructure/kyverno/policies/fix-rook-osd-metadata-device.yaml

diff --git a/infrastructure/kyverno/policies/fix-rook-osd-metadata-device.yaml b/infrastructure/kyverno/policies/fix-rook-osd-metadata-device.yaml
new file mode 100644
index 0000000..3526bf4
--- /dev/null
+++ b/infrastructure/kyverno/policies/fix-rook-osd-metadata-device.yaml
@@ -0,0 +1,63 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: fix-rook-osd-metadata-device
+  annotations:
+    policies.kyverno.io/title: Fix Rook OSD Metadata Device
+    policies.kyverno.io/category: Rook-Ceph
+    policies.kyverno.io/severity: high
+    policies.kyverno.io/description: >-
+      Temporary policy to fix rook-ceph OSD deployments that use metadataDevice.
+      Replaces expand-bluefs with a no-op and adds conditional block.db symlink
+      creation. Remove this policy after all OSDs are recreated without DB devices.
+spec:
+  background: false
+  rules:
+    - name: fix-osd-init-containers
+      match:
+        any:
+          - resources:
+              kinds:
+                - Deployment
+              namespaces:
+                - rook-ceph
+              selector:
+                matchLabels:
+                  app: rook-ceph-osd
+      mutate:
+        patchStrategicMerge:
+          spec:
+            template:
+              spec:
+                initContainers:
+                  - name: expand-bluefs
+                    command: ["/bin/true"]
+                    args: []
+                  - name: create-block-db-symlink
+                    image: quay.io/ceph/ceph:v19.2.0
+                    command: ["/bin/bash", "-c"]
+                    args:
+                      - |
+                        set -e
+                        # Only create block.db symlink if /dev/sdb has a valid BlueStore label
+                        if ! ceph-bluestore-tool show-label --dev /dev/sdb 2>/dev/null; then
+                          echo "No BlueStore label on /dev/sdb, skipping block.db symlink"
+                          exit 0
+                        fi
+                        echo "BlueStore label found on /dev/sdb"
+                        # Find OSD data directory under rook data path
+                        for d in /var/lib/rook/rook-ceph/*/; do
+                          if [ -L "$d/block" ]; then
+                            ln -sf /dev/sdb "$d/block.db"
+                            echo "Created block.db -> /dev/sdb in $d"
+                            break
+                          fi
+                        done
+                    securityContext:
+                      privileged: true
+                      runAsUser: 0
+                    volumeMounts:
+                      - name: rook-data
+                        mountPath: /var/lib/rook
+                      - name: devices
+                        mountPath: /dev
diff --git a/infrastructure/rook-ceph/cluster-values.yaml b/infrastructure/rook-ceph/cluster-values.yaml
index 5759437..28c6caa 100644
--- a/infrastructure/rook-ceph/cluster-values.yaml
+++ b/infrastructure/rook-ceph/cluster-values.yaml
@@ -120,7 +120,6 @@ cephClusterSpec:
         memory: "50Mi"
 
   # Storage configuration - explicit nodes with passthrough NVMe
-  # sdb = dedicated 60GB WAL/DB device per node
   storage:
     useAllNodes: false
     useAllDevices: false
@@ -128,23 +127,15 @@ cephClusterSpec:
       - name: "talos-cp-01"
         devices:
           - name: "nvme0n1"
-        config:
-          metadataDevice: "sdb"
       - name: "talos-cp-02"
         devices:
           - name: "nvme0n1"
-        config:
-          metadataDevice: "sdb"
       - name: "talos-cp-03"
         devices:
           - name: "nvme0n1"
-        config:
-          metadataDevice: "sdb"
       - name: "talos-cp-04"
         devices:
           - name: "nvme0n1"
-        config:
-          metadataDevice: "sdb"
     config:
       osdsPerDevice: "1"