mirror of
https://gitlab.kube.huskypup.net/Scooby/Homelabv4.git
synced 2026-08-20 23:16:49 +00:00
Root cause of tonight's earlier CrowdSec/CNPG-backup workarounds: no node in the cluster had any route into the Netbird mesh CIDR (100.108.0.0/16) for pod-originated traffic. The per-namespace netbird 'router' pods are inbound-only infrastructure (external peers reaching K8s services); their own architecture has no reverse path. Fix, in two parts: 1. infrastructure/netbird/manifests/egress-daemonset.yaml - one netbird client per node, hostNetwork so its wt0 interface lives in the node's real network namespace, plus a sidecar that adds a host route sending 100.108.0.0/16 out via it. hostNetwork requires a scoped Kyverno PolicyException (infrastructure/kyverno/policies/netbird-egress-exception.yaml) to the disallow-host-namespaces STIG policy - narrowly for this one DaemonSet by name, not a namespace-wide exclusion. 2. Discovered the route alone wasn't enough for k3s NodePort traffic (vps-minio:30900): Netbird manages its own nftables ACLs independent of iptables/Kyverno, and its forward chain (netbird-rt-fwd) only permits *established* connections through a peer acting as a router - never new ones, by design, unless a Netbird 'Network Route' policy is explicitly configured (it isn't, for this VPS). Locally-terminated connections (tinyproxy) go through a separate, already-permissive ACL chain, which is why the CrowdSec proxy fix from earlier tonight worked. Replicated that working pattern for MinIO: minio-forward.service on the VPS host (systemd, socat) forwards 100.108.113.41:9000 -> MinIO's ClusterIP, avoiding the NodePort path entirely. Re-enabled everything that was disabled/suspended earlier tonight because of this gap, pointed at the new endpoint: - CrowdSec CAPI/console-enroll (removed DISABLE_ONLINE_API, restored the VPS proxy env vars) - n8n/nextcloud/authentik CNPG backup.barmanObjectStore - vault-raft-snapshot CronJob (unsuspended) - nextcloud PVC content sync CronJob endpoint vps-minio.netbird.internal is retired everywhere - it was never actually resolvable (Netbird has no DNS configured) even before today's routing fix. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
137 lines
4.9 KiB
YAML
137 lines
4.9 KiB
YAML
---
|
|
# Syncs Nextcloud's file PVC content (nextcloud-nextcloud, 200Gi) to the VPS
|
|
# MinIO backup receiver via restic. The one piece of the Nextcloud backup story
|
|
# no operator covers - actual file blobs, not database rows (DB is handled by
|
|
# CNPG's barmanObjectStore in cnpg-cluster.yaml).
|
|
#
|
|
# Caveat: this backs up the live-mounted volume with no application-level
|
|
# quiesce (no Nextcloud maintenance-mode pause around the snapshot). Acceptable
|
|
# for a no-real-data test environment; for anything with real user data, pair
|
|
# this with `occ maintenance:mode --on` before / `--off` after.
|
|
#
|
|
# Uses podAffinity to land on the same node as a running Nextcloud pod, since
|
|
# the PVC is RWO (Ceph RBD) - RWO allows multiple pods to mount it concurrently
|
|
# only when co-located on the same node. Verify the label selector below
|
|
# matches the actual Nextcloud chart's pod labels before relying on this.
|
|
apiVersion: generators.external-secrets.io/v1alpha1
|
|
kind: Password
|
|
metadata:
|
|
name: nextcloud-restic-password
|
|
namespace: nextcloud
|
|
spec:
|
|
length: 48
|
|
digits: 5
|
|
symbols: 5
|
|
symbolCharacters: "-_$@"
|
|
noUpper: false
|
|
allowRepeat: true
|
|
---
|
|
apiVersion: external-secrets.io/v1
|
|
kind: ExternalSecret
|
|
metadata:
|
|
name: nextcloud-restic-password
|
|
namespace: nextcloud
|
|
spec:
|
|
refreshInterval: "0" # generate once - changing this after the repo is initialized would lock you out of existing backups
|
|
target:
|
|
name: nextcloud-restic-password
|
|
creationPolicy: Owner
|
|
data: []
|
|
dataFrom:
|
|
- sourceRef:
|
|
generatorRef:
|
|
apiVersion: generators.external-secrets.io/v1alpha1
|
|
kind: Password
|
|
name: nextcloud-restic-password
|
|
---
|
|
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: nextcloud-pvc-sync
|
|
namespace: nextcloud
|
|
spec:
|
|
schedule: "0 2 * * *" # daily at 02:00 - bulky, don't run it as often as the DB backups
|
|
concurrencyPolicy: Forbid
|
|
successfulJobsHistoryLimit: 3
|
|
failedJobsHistoryLimit: 3
|
|
jobTemplate:
|
|
spec:
|
|
backoffLimit: 2
|
|
template:
|
|
spec:
|
|
restartPolicy: Never
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 65534
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
affinity:
|
|
podAffinity:
|
|
requiredDuringSchedulingIgnoredDuringExecution:
|
|
- labelSelector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: nextcloud
|
|
topologyKey: kubernetes.io/hostname
|
|
containers:
|
|
- name: restic-backup
|
|
image: restic/restic:0.16.4
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop: ["ALL"]
|
|
command:
|
|
- /bin/sh
|
|
- -c
|
|
- |
|
|
set -eu
|
|
export RESTIC_REPOSITORY="s3:http://${VPS_MINIO_ENDPOINT}/nextcloud-files/restic-repo"
|
|
export RESTIC_PASSWORD_FILE=/restic-secret/password
|
|
export AWS_ACCESS_KEY_ID="${MINIO_ACCESS_KEY}"
|
|
export AWS_SECRET_ACCESS_KEY="${MINIO_SECRET_KEY}"
|
|
|
|
restic snapshots >/dev/null 2>&1 || restic init
|
|
|
|
echo "==> Backing up /data..."
|
|
restic backup /data --tag nextcloud --host nextcloud-k8s
|
|
|
|
echo "==> Pruning: keep 7 daily, 4 weekly, 6 monthly..."
|
|
restic forget --tag nextcloud --host nextcloud-k8s \
|
|
--keep-daily 7 --keep-weekly 4 --keep-monthly 6 --prune
|
|
|
|
echo "==> Done."
|
|
env:
|
|
# Locally-terminated socat forward on the VPS host to
|
|
# MinIO's ClusterIP, not the NodePort - see
|
|
# infrastructure/vault/manifests/raft-snapshot-cronjob.yaml
|
|
# for why (Netbird has no DNS, and blocks new forwarded
|
|
# connections through a peer by default).
|
|
- name: VPS_MINIO_ENDPOINT
|
|
value: "100.108.113.41:9000"
|
|
- name: MINIO_ACCESS_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: vps-minio-secret
|
|
key: accesskey
|
|
- name: MINIO_SECRET_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: vps-minio-secret
|
|
key: secretkey
|
|
volumeMounts:
|
|
- name: nextcloud-data
|
|
mountPath: /data
|
|
readOnly: true
|
|
- name: restic-secret
|
|
mountPath: /restic-secret
|
|
readOnly: true
|
|
volumes:
|
|
- name: nextcloud-data
|
|
persistentVolumeClaim:
|
|
claimName: nextcloud-nextcloud
|
|
- name: restic-secret
|
|
secret:
|
|
secretName: nextcloud-restic-password
|
|
items:
|
|
- key: password
|
|
path: password
|