diff --git a/infrastructure/kyverno/policies/fix-rook-osd-metadata-device.yaml b/infrastructure/kyverno/policies/fix-rook-osd-metadata-device.yaml new file mode 100644 index 0000000..3526bf4 --- /dev/null +++ b/infrastructure/kyverno/policies/fix-rook-osd-metadata-device.yaml @@ -0,0 +1,63 @@ +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: fix-rook-osd-metadata-device + annotations: + policies.kyverno.io/title: Fix Rook OSD Metadata Device + policies.kyverno.io/category: Rook-Ceph + policies.kyverno.io/severity: high + policies.kyverno.io/description: >- + Temporary policy to fix rook-ceph OSD deployments that use metadataDevice. + Replaces expand-bluefs with a no-op and adds conditional block.db symlink + creation. Remove this policy after all OSDs are recreated without DB devices. +spec: + background: false + rules: + - name: fix-osd-init-containers + match: + any: + - resources: + kinds: + - Deployment + namespaces: + - rook-ceph + selector: + matchLabels: + app: rook-ceph-osd + mutate: + patchStrategicMerge: + spec: + template: + spec: + initContainers: + - name: expand-bluefs + command: ["/bin/true"] + args: [] + - name: create-block-db-symlink + image: quay.io/ceph/ceph:v19.2.0 + command: ["/bin/bash", "-c"] + args: + - | + set -e + # Only create block.db symlink if /dev/sdb has a valid BlueStore label + if ! ceph-bluestore-tool show-label --dev /dev/sdb 2>/dev/null; then + echo "No BlueStore label on /dev/sdb, skipping block.db symlink" + exit 0 + fi + echo "BlueStore label found on /dev/sdb" + # Find OSD data directory under rook data path + for d in /var/lib/rook/rook-ceph/*/; do + if [ -L "$d/block" ]; then + ln -sf /dev/sdb "$d/block.db" + echo "Created block.db -> /dev/sdb in $d" + break + fi + done + securityContext: + privileged: true + runAsUser: 0 + volumeMounts: + - name: rook-data + mountPath: /var/lib/rook + - name: devices + mountPath: /dev diff --git a/infrastructure/rook-ceph/cluster-values.yaml b/infrastructure/rook-ceph/cluster-values.yaml index 5759437..28c6caa 100644 --- a/infrastructure/rook-ceph/cluster-values.yaml +++ b/infrastructure/rook-ceph/cluster-values.yaml @@ -120,7 +120,6 @@ cephClusterSpec: memory: "50Mi" # Storage configuration - explicit nodes with passthrough NVMe - # sdb = dedicated 60GB WAL/DB device per node storage: useAllNodes: false useAllDevices: false @@ -128,23 +127,15 @@ cephClusterSpec: - name: "talos-cp-01" devices: - name: "nvme0n1" - config: - metadataDevice: "sdb" - name: "talos-cp-02" devices: - name: "nvme0n1" - config: - metadataDevice: "sdb" - name: "talos-cp-03" devices: - name: "nvme0n1" - config: - metadataDevice: "sdb" - name: "talos-cp-04" devices: - name: "nvme0n1" - config: - metadataDevice: "sdb" config: osdsPerDevice: "1"