Homelabv4/infrastructure/prometheus/manifests/talos-metrics-proxy.yaml

# Talos Metrics Proxy
# Workaround for HTTP 415 errors when scraping Talos apid metrics
# This deploys an nginx proxy that strips problematic HTTP headers

apiVersion: v1
kind: ConfigMap
metadata:
  name: talos-metrics-proxy-config
  namespace: kube-system
data:
  nginx.conf: |
    pid /tmp/nginx.pid;

    events {
        worker_connections 1024;
    }

    http {
        proxy_temp_path /tmp/proxy_temp;
        client_body_temp_path /tmp/client_temp;
        fastcgi_temp_path /tmp/fastcgi_temp;
        uwsgi_temp_path /tmp/uwsgi_temp;
        scgi_temp_path /tmp/scgi_temp;
        # Upstream Talos nodes
        upstream talos01 {
            server 172.28.101.21:50000;
        }
        upstream talos02 {
            server 172.28.101.22:50000;
        }
        upstream talos03 {
            server 172.28.101.23:50000;
        }
        upstream talos04 {
            server 172.28.101.24:50000;
        }
        
        # Proxy for talos01
        server {
            listen 9100;
            server_name talos01-metrics;
            
            location /metrics {
                proxy_pass https://talos01/metrics;
                proxy_ssl_certificate /certs/client.crt;
                proxy_ssl_certificate_key /certs/client.key;
                proxy_ssl_trusted_certificate /certs/ca.crt;
                proxy_ssl_verify on;
                proxy_http_version 1.1;
                proxy_set_header Accept "text/plain;version=0.0.4";
                proxy_set_header Connection "";
            }
        }
        
        # Proxy for talos02
        server {
            listen 9101;
            server_name talos02-metrics;
            
            location /metrics {
                proxy_pass https://talos02/metrics;
                proxy_ssl_certificate /certs/client.crt;
                proxy_ssl_certificate_key /certs/client.key;
                proxy_ssl_trusted_certificate /certs/ca.crt;
                proxy_ssl_verify on;
                proxy_http_version 1.1;
                proxy_set_header Accept "text/plain;version=0.0.4";
                proxy_set_header Connection "";
            }
        }
        
        # Proxy for talos03
        server {
            listen 9102;
            server_name talos03-metrics;
            
            location /metrics {
                proxy_pass https://talos03/metrics;
                proxy_ssl_certificate /certs/client.crt;
                proxy_ssl_certificate_key /certs/client.key;
                proxy_ssl_trusted_certificate /certs/ca.crt;
                proxy_ssl_verify on;
                proxy_http_version 1.1;
                proxy_set_header Accept "text/plain;version=0.0.4";
                proxy_set_header Connection "";
            }
        }
        
        # Proxy for talos04
        server {
            listen 9103;
            server_name talos04-metrics;
            
            location /metrics {
                proxy_pass https://talos04/metrics;
                proxy_ssl_certificate /certs/client.crt;
                proxy_ssl_certificate_key /certs/client.key;
                proxy_ssl_trusted_certificate /certs/ca.crt;
                proxy_ssl_verify on;
                proxy_http_version 1.1;
                proxy_set_header Accept "text/plain;version=0.0.4";
                proxy_set_header Connection "";
            }
        }
    }
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: talos-metrics-proxy
  namespace: kube-system
  labels:
    app: talos-metrics-proxy
spec:
  replicas: 1
  selector:
    matchLabels:
      app: talos-metrics-proxy
  template:
    metadata:
      labels:
        app: talos-metrics-proxy
    spec:
      automountServiceAccountToken: false
      containers:
      - name: nginx
        image: nginx:1.27-alpine
        securityContext:
          runAsUser: 10101
          runAsGroup: 10101
          runAsNonRoot: true
          allowPrivilegeEscalation: false
          readOnlyRootFilesystem: true
          capabilities:
            drop:
              - ALL
          seccompProfile:
            type: RuntimeDefault
        ports:
        - containerPort: 9100
          name: talos01
        - containerPort: 9101
          name: talos02
        - containerPort: 9102
          name: talos03
        - containerPort: 9103
          name: talos04
        volumeMounts:
        - name: nginx-config
          mountPath: /etc/nginx/nginx.conf
          subPath: nginx.conf
        - name: talos-certs
          mountPath: /certs
          readOnly: true
        - name: tmp
          mountPath: /tmp
        - name: cache
          mountPath: /var/cache/nginx
        resources:
          requests:
            memory: "64Mi"
            cpu: "10m"
          limits:
            memory: "128Mi"
      volumes:
      - name: nginx-config
        configMap:
          name: talos-metrics-proxy-config
      - name: talos-certs
        secret:
          secretName: talos-client-cert
      - name: tmp
        emptyDir: {}
      - name: cache
        emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
  name: talos-metrics-proxy
  namespace: kube-system
  labels:
    app: talos-metrics-proxy
spec:
  type: ClusterIP
  ports:
  - port: 9100
    targetPort: 9100
    protocol: TCP
    name: talos01
  - port: 9101
    targetPort: 9101
    protocol: TCP
    name: talos02
  - port: 9102
    targetPort: 9102
    protocol: TCP
    name: talos03
  - port: 9103
    targetPort: 9103
    protocol: TCP
    name: talos04
  selector:
    app: talos-metrics-proxy
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: talos-metrics
  namespace: kube-system
  labels:
    app: talos-metrics-proxy
spec:
  endpoints:
  - port: talos01
    path: /metrics
    interval: 30s
  - port: talos02
    path: /metrics
    interval: 30s
  - port: talos03
    path: /metrics
    interval: 30s
  - port: talos04
    path: /metrics
    interval: 30s
  selector:
    matchLabels:
      app: talos-metrics-proxy
  namespaceSelector:
    matchNames:
    - kube-system