mirror of
https://gitlab.kube.huskypup.net/Scooby/Homelabv4.git
synced 2026-08-20 23:16:49 +00:00
Initial commit
This commit is contained in:
Scooby Husky
Executable
+231
@@ -0,0 +1,231 @@
|
||||
#!/bin/bash
|
||||
# Rook-Ceph Preparation Script
|
||||
# Wipes NVMe drives on all nodes, creates namespace with proper PSS labels,
|
||||
# cleans up stale resources, and verifies node availability
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
NS="rook-ceph"
|
||||
WIPE_IMAGE="quay.io/ceph/ceph:v19.2.0"
|
||||
|
||||
# Node to NVMe device mapping
|
||||
declare -A NODE_NVME=(
|
||||
["talos-cp-01"]="nvme0n1"
|
||||
["talos-cp-02"]="nvme1n1"
|
||||
["talos-cp-03"]="nvme0n1"
|
||||
["talos-cp-04"]="nvme0n1"
|
||||
)
|
||||
|
||||
echo "=== Rook-Ceph Preparation ==="
|
||||
|
||||
# Create namespace if it doesn't exist
|
||||
if ! kubectl get ns "${NS}" >/dev/null 2>&1; then
|
||||
echo "Creating namespace ${NS}..."
|
||||
kubectl create ns "${NS}"
|
||||
fi
|
||||
|
||||
# Apply privileged Pod Security Standards (required for Ceph)
|
||||
echo "Applying privileged Pod Security Standards..."
|
||||
kubectl label namespace "${NS}" pod-security.kubernetes.io/enforce=privileged --overwrite
|
||||
kubectl label namespace "${NS}" pod-security.kubernetes.io/audit=privileged --overwrite
|
||||
kubectl label namespace "${NS}" pod-security.kubernetes.io/warn=privileged --overwrite
|
||||
|
||||
# Clean up any stale OSD deployments
|
||||
echo "Cleaning up stale OSD resources..."
|
||||
for osd_deploy in $(kubectl -n "${NS}" get deployments -o name 2>/dev/null | grep rook-ceph-osd- || true); do
|
||||
echo " Deleting ${osd_deploy}..."
|
||||
kubectl -n "${NS}" delete "${osd_deploy}" --ignore-not-found=true
|
||||
done
|
||||
|
||||
# Clean up stale OSD prepare pods
|
||||
kubectl -n "${NS}" delete pods -l app=rook-ceph-osd-prepare --ignore-not-found=true 2>/dev/null || true
|
||||
|
||||
# Clean up any stale finalizers from previous failed deployments
|
||||
echo "Checking for stale CephCluster..."
|
||||
if kubectl -n "${NS}" get cephcluster rook-ceph >/dev/null 2>&1; then
|
||||
PHASE=$(kubectl -n "${NS}" get cephcluster rook-ceph -o jsonpath='{.status.phase}' 2>/dev/null || echo "Unknown")
|
||||
if [ "$PHASE" = "Deleting" ]; then
|
||||
echo "Found stale CephCluster in Deleting state, cleaning up finalizers..."
|
||||
kubectl -n "${NS}" patch cephcluster rook-ceph --type merge -p '{"metadata":{"finalizers":[]}}' 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
|
||||
# Function to wipe NVMe on a specific node
|
||||
wipe_nvme() {
|
||||
local node=$1
|
||||
local device=$2
|
||||
local pod_name="wipe-nvme-${node}"
|
||||
|
||||
echo "Wiping /dev/${device} on ${node}..."
|
||||
|
||||
# Delete any existing wipe pod
|
||||
kubectl -n "${NS}" delete pod "${pod_name}" --ignore-not-found=true 2>/dev/null || true
|
||||
|
||||
# Create wipe pod
|
||||
cat <<EOF | kubectl apply -f -
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: ${pod_name}
|
||||
namespace: ${NS}
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
nodeName: ${node}
|
||||
tolerations:
|
||||
- operator: Exists
|
||||
containers:
|
||||
- name: wipe
|
||||
image: ${WIPE_IMAGE}
|
||||
securityContext:
|
||||
privileged: true
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- |
|
||||
set -ex
|
||||
DEVICE="/dev/${device}"
|
||||
|
||||
# Skip if device doesn't exist
|
||||
if [ ! -b "\${DEVICE}" ]; then
|
||||
echo "Device \${DEVICE} not found, skipping"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "=== Wiping \${DEVICE} on ${node} ==="
|
||||
|
||||
# Remove any Ceph LVM volumes
|
||||
for vg in \$(vgs --noheadings -o vg_name 2>/dev/null | grep -i ceph || true); do
|
||||
echo "Removing VG: \${vg}"
|
||||
vgremove -ff "\${vg}" 2>/dev/null || true
|
||||
done
|
||||
|
||||
# Remove PV if exists
|
||||
pvremove -ff "\${DEVICE}" 2>/dev/null || true
|
||||
|
||||
# Remove device mapper entries
|
||||
dmsetup remove_all -f 2>/dev/null || true
|
||||
|
||||
# Wipe filesystem signatures
|
||||
wipefs -af "\${DEVICE}"
|
||||
|
||||
# Zap GPT/MBR
|
||||
sgdisk --zap-all "\${DEVICE}"
|
||||
|
||||
# Zero first 100MB (clears any remaining metadata)
|
||||
dd if=/dev/zero of="\${DEVICE}" bs=1M count=100 conv=fsync
|
||||
|
||||
# Zero last 100MB (clears backup GPT)
|
||||
SECTORS=\$(blockdev --getsz "\${DEVICE}")
|
||||
dd if=/dev/zero of="\${DEVICE}" bs=1M count=100 seek=\$((SECTORS/2048 - 100)) conv=fsync
|
||||
|
||||
echo "=== Wipe complete for \${DEVICE} ==="
|
||||
lsblk -f "\${DEVICE}"
|
||||
blkid "\${DEVICE}" || echo "No signatures (clean)"
|
||||
volumeMounts:
|
||||
- name: dev
|
||||
mountPath: /dev
|
||||
volumes:
|
||||
- name: dev
|
||||
hostPath:
|
||||
path: /dev
|
||||
EOF
|
||||
|
||||
# Wait for pod to complete
|
||||
echo " Waiting for wipe to complete on ${node}..."
|
||||
if ! kubectl -n "${NS}" wait --for=condition=Ready pod/"${pod_name}" --timeout=30s 2>/dev/null; then
|
||||
# Pod might have completed already
|
||||
true
|
||||
fi
|
||||
|
||||
# Wait for completion (up to 2 minutes)
|
||||
local timeout=120
|
||||
local elapsed=0
|
||||
while [ $elapsed -lt $timeout ]; do
|
||||
local phase=$(kubectl -n "${NS}" get pod "${pod_name}" -o jsonpath='{.status.phase}' 2>/dev/null || echo "Unknown")
|
||||
if [ "$phase" = "Succeeded" ]; then
|
||||
echo " ✓ Wipe completed on ${node}"
|
||||
kubectl -n "${NS}" logs "${pod_name}" 2>/dev/null | tail -5
|
||||
kubectl -n "${NS}" delete pod "${pod_name}" --ignore-not-found=true
|
||||
return 0
|
||||
elif [ "$phase" = "Failed" ]; then
|
||||
echo " ✗ Wipe failed on ${node}"
|
||||
kubectl -n "${NS}" logs "${pod_name}" 2>/dev/null | tail -20
|
||||
kubectl -n "${NS}" delete pod "${pod_name}" --ignore-not-found=true
|
||||
return 1
|
||||
fi
|
||||
sleep 5
|
||||
elapsed=$((elapsed + 5))
|
||||
done
|
||||
|
||||
echo " ✗ Wipe timed out on ${node}"
|
||||
kubectl -n "${NS}" delete pod "${pod_name}" --ignore-not-found=true
|
||||
return 1
|
||||
}
|
||||
|
||||
# Verify nodes are available
|
||||
echo "Verifying node availability..."
|
||||
NODE_COUNT=$(kubectl get nodes --no-headers 2>/dev/null | wc -l)
|
||||
if [ "${NODE_COUNT}" -lt 1 ]; then
|
||||
echo "ERROR: No nodes available in cluster"
|
||||
exit 1
|
||||
fi
|
||||
echo "Found ${NODE_COUNT} node(s) available"
|
||||
|
||||
# Check for Ready nodes
|
||||
READY_NODES=$(kubectl get nodes --no-headers 2>/dev/null | grep -c " Ready" || echo "0")
|
||||
if [ "${READY_NODES}" -lt 1 ]; then
|
||||
echo "ERROR: No Ready nodes available in cluster"
|
||||
exit 1
|
||||
fi
|
||||
echo "Found ${READY_NODES} Ready node(s)"
|
||||
|
||||
# Check if Ceph OSDs already exist (skip wipe if so, unless forced)
|
||||
FORCE_WIPE="${FORCE_WIPE:-false}"
|
||||
if [ "$FORCE_WIPE" != "true" ]; then
|
||||
OSD_COUNT=$(kubectl -n "${NS}" get pods -l app=rook-ceph-osd --no-headers 2>/dev/null | grep -c Running || echo "0")
|
||||
if [ "$OSD_COUNT" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "WARNING: Found ${OSD_COUNT} running OSD(s). Skipping disk wipe to protect existing data."
|
||||
echo "To force wipe, run: FORCE_WIPE=true ./scripts/rook-ceph-prepare.sh"
|
||||
echo ""
|
||||
echo "=== Rook-Ceph preparation complete (no wipe) ==="
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# Wipe NVMe drives on all configured nodes
|
||||
echo ""
|
||||
echo "=== Wiping NVMe drives ==="
|
||||
WIPE_FAILED=0
|
||||
for node in "${!NODE_NVME[@]}"; do
|
||||
device="${NODE_NVME[$node]}"
|
||||
|
||||
# Check if node exists in cluster
|
||||
if ! kubectl get node "${node}" >/dev/null 2>&1; then
|
||||
echo "Node ${node} not found in cluster, skipping..."
|
||||
continue
|
||||
fi
|
||||
|
||||
if ! wipe_nvme "${node}" "${device}"; then
|
||||
echo "WARNING: Failed to wipe ${device} on ${node}"
|
||||
WIPE_FAILED=1
|
||||
fi
|
||||
done
|
||||
|
||||
if [ $WIPE_FAILED -eq 1 ]; then
|
||||
echo ""
|
||||
echo "WARNING: Some wipe operations failed. Check logs above."
|
||||
echo "Continuing with Ceph setup anyway..."
|
||||
fi
|
||||
|
||||
# List nodes for informational purposes
|
||||
echo ""
|
||||
echo "Cluster nodes:"
|
||||
kubectl get nodes -o wide --no-headers 2>/dev/null | while read -r line; do
|
||||
echo " $line"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=== Rook-Ceph preparation complete ==="
|
||||
echo "NVMe drives have been wiped and namespace is ready."
|
||||
echo "Run 'helmfile apply' to deploy Ceph."
|
||||
Reference in New Issue
Block a user