mirror of
https://gitlab.kube.huskypup.net/Scooby/Homelabv4.git
synced 2026-08-23 12:56:46 +00:00
- infrastructure/vps-standby/gitlab/manifests/ha-postgres-app-externalsecret.yaml: creationPolicy Merge -> Owner (already applied live, committing to match). Merge assumed CNPG creates a base pg-gitlab-app/pg-praefect-app secret to merge a password into - wrong for these VPS Clusters since Phase 1c's replica-cluster recreate; CNPG doesn't create a local owner secret for a replicated role. Left pg-gitlab-app/pg-praefect-app missing entirely, crash-looping vps-gitlab-webservice on ActiveRecord::DatabaseConnectionError for ~39h. - infrastructure/vps-standby/nextcloud/manifests/pvc-restore-cronjob.yaml: exclude config.php from the restic restore. The job's own comment assumed nextcloud runs at replicas: 0 on the VPS; live confirms replicas: 1 (no such key was ever actually set) - restic couldn't overwrite the live pod's config.php (permission denied), failing the whole job every run for 2+ days even though everything else restored fine. Excluding it is correct regardless of permissions - a live pod's own config.php shouldn't be overwritten by a background restore job. Also fixed live (not git-tracked, config.php is PVC-persisted runtime state, not sourced from git): - home + VPS nextcloud config.php dbpassword: out of sync with CNPG's actual current pg-nextcloud-app password (baked in once at install, never re-synced). VPS's case was two-layered - its own pg-nextcloud-app secret also didn't match the real Postgres role password, since VPS's pg-nextcloud is a read-only streaming replica (spec.replica.enabled) and the authoritative password lives on home. - grafana: broke a RollingUpdate deadlock (single-replica Deployment + RWO Ceph volume - new pod couldn't start while the old pod still held the only-one-node-at-a-time attachment, and the rollout wouldn't scale down the old pod until the new one was ready). - suspended the stale gitlab-mirror-sync CronJob on the VPS (spamming auth failures every ~15min since its stored token predates the Postgres-replication cutover of VPS GitLab's DB) rather than deleting it, per the plan's own note to defer that until Phase 2b is verified.
114 lines
4.8 KiB
YAML
114 lines
4.8 KiB
YAML
---
|
|
# Restores the latest restic snapshot from home's nextcloud-pvc-sync
|
|
# CronJob (apps/nextcloud/manifests/pvc-sync-cronjob.yaml, which backs up
|
|
# to VPS MinIO's nextcloud-files/restic-repo daily at 02:00) into this
|
|
# standby's own Nextcloud PVC. Runs daily at 04:00 - enough margin after
|
|
# home's job to be sure that day's snapshot has landed.
|
|
#
|
|
# `restic restore latest --target /` restores into /data/... because
|
|
# home's backup stored an absolute /data path (`restic backup /data ...`)
|
|
# - mounting the destination PVC at /data here mirrors that exactly.
|
|
#
|
|
# nextcloud-restic-password must be the SAME password used to init the
|
|
# repo at home (it's the decryption key for the whole restic repository,
|
|
# not something that can differ per-consumer) - copied here manually
|
|
# (kubectl, not git):
|
|
# kubectl -n nextcloud create secret generic nextcloud-restic-password \
|
|
# --from-literal=password=<home's nextcloud-restic-password>
|
|
#
|
|
# CORRECTED 2026-08-22: the "replicas: 0" assumption below was wrong -
|
|
# confirmed live vps-nextcloud actually runs continuously at replicas: 1
|
|
# (no replicas key was ever set in values.yaml; the chart's own default
|
|
# applies). This job had been silently failing every run for 2+ days as
|
|
# a result: restic couldn't overwrite the live pod's config.php (mode
|
|
# 0640, owned by www-data/33; this job runs as UID 65534 with only
|
|
# group 33 via fsGroup, so it can read but not write that one file) -
|
|
# "ignoring error for /data/config/config.php: permission denied",
|
|
# Fatal: 1 error, even though the other 31000+ files/15GiB restored
|
|
# fine every time. Fixed by excluding config.php from the restore
|
|
# below rather than chasing permissions - overwriting a live, running
|
|
# app's own config.php from a background restore job is the wrong
|
|
# move regardless (it holds this site's actual current db credentials,
|
|
# etc.), not just a permissions bug to route around.
|
|
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: nextcloud-pvc-restore
|
|
namespace: nextcloud
|
|
spec:
|
|
schedule: "0 4 * * *"
|
|
concurrencyPolicy: Forbid
|
|
successfulJobsHistoryLimit: 3
|
|
failedJobsHistoryLimit: 3
|
|
jobTemplate:
|
|
spec:
|
|
backoffLimit: 2
|
|
template:
|
|
spec:
|
|
restartPolicy: Never
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 65534
|
|
fsGroup: 33 # matches the real nextcloud Deployment's fsGroup, same reasoning as home's pvc-sync-cronjob
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
containers:
|
|
- name: restic-restore
|
|
image: restic/restic:0.16.4
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop: ["ALL"]
|
|
command:
|
|
- /bin/sh
|
|
- -c
|
|
- |
|
|
set -eu
|
|
export RESTIC_REPOSITORY="s3:http://vps-minio.minio.svc.cluster.local:9000/nextcloud-files/restic-repo"
|
|
export RESTIC_PASSWORD_FILE=/restic-secret/password
|
|
export AWS_ACCESS_KEY_ID="${MINIO_ACCESS_KEY}"
|
|
export AWS_SECRET_ACCESS_KEY="${MINIO_SECRET_KEY}"
|
|
export RESTIC_CACHE_DIR=/tmp/restic-cache
|
|
|
|
if ! restic snapshots >/dev/null 2>&1; then
|
|
echo "No repo/snapshots reachable yet - nothing to restore."
|
|
exit 0
|
|
fi
|
|
|
|
echo "==> Restoring latest snapshot into /data..."
|
|
restic restore latest --tag nextcloud --host nextcloud-k8s --target / --exclude /data/config/config.php
|
|
|
|
echo "==> Done."
|
|
env:
|
|
- name: MINIO_ACCESS_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: vps-minio-secret
|
|
key: accesskey
|
|
- name: MINIO_SECRET_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: vps-minio-secret
|
|
key: secretkey
|
|
volumeMounts:
|
|
- name: nextcloud-data
|
|
mountPath: /data
|
|
- name: restic-secret
|
|
mountPath: /restic-secret
|
|
readOnly: true
|
|
volumes:
|
|
- name: nextcloud-data
|
|
persistentVolumeClaim:
|
|
# Chart-generated PVC name, follows the Helm release name
|
|
# (Application metadata.name: vps-nextcloud) - same
|
|
# release-name-based naming gotcha hit with vault-0/
|
|
# vps-vault-0 and gitea-http/vps-gitea-http. Verify against
|
|
# `kubectl -n nextcloud get pvc` after first deploy.
|
|
claimName: vps-nextcloud-nextcloud
|
|
- name: restic-secret
|
|
secret:
|
|
secretName: nextcloud-restic-password
|
|
items:
|
|
- key: password
|
|
path: password
|