Initial commit

infrastructure/prometheus/manifests/envoy-route.yaml

@@ -0,0 +1,20 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: prometheus
+  namespace: prometheus
+spec:
+  parentRefs:
+    - name: edge
+      namespace: gateway
+      sectionName: https
+  hostnames:
+    - prometheus.kube.huskypup.net
+  rules:
+    - matches:
+        - path:
+            type: PathPrefix
+            value: /
+      backendRefs:
+        - name: kube-prometheus-stack-prometheus
+          port: 9090

infrastructure/prometheus/manifests/istio-virtualservice.yaml

@@ -0,0 +1,16 @@
+apiVersion: networking.istio.io/v1beta1
+kind: VirtualService
+metadata:
+  name: prometheus
+  namespace: prometheus
+spec:
+  hosts:
+    - prometheus.kube.huskypup.net
+  gateways:
+    - istio-system/edge
+  http:
+    - route:
+        - destination:
+            host: kube-prometheus-stack-prometheus.prometheus.svc.cluster.local
+            port:
+              number: 9090

infrastructure/prometheus/manifests/talos-client-cert-secret.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: talos-client-cert
+  namespace: prometheus
+type: Opaque
+data:
+  ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJQekNCOHFBREFnRUNBaEVBelVmaHJ2MW5uS21DcHVLTW90MUhOakFGQmdNclpYQXdFREVPTUF3R0ExVUUKQ2hNRmRHRnNiM013SGhjTk1qVXhNVE13TVRrME56UXlXaGNOTXpVeE1USTRNVGswTnpReVdqQVFNUTR3REFZRApWUVFLRXdWMFlXeHZjekFxTUFVR0F5dGxjQU1oQUpWWThDUDAyN0ZQWnJONzJPMlZkaGRCYjY3Z0plaUFINloxClNlUDAvWFlBbzJFd1h6QU9CZ05WSFE4QkFmOEVCQU1DQW9Rd0hRWURWUjBsQkJZd0ZBWUlLd1lCQlFVSEF3RUcKQ0NzR0FRVUZCd01DTUE4R0ExVWRFd0VCL3dRRk1BTUJBZjh3SFFZRFZSME9CQllFRk1zbmNVTVo4ZGZhVlR3Ugo3RVVZNEl1TjZCZEJNQVVHQXl0bGNBTkJBTjlkK2laekxVdjJXQXNxNWdPbDh2bmhzSHhRMXY3SU1zOGtEZFNHClNUTVZSa1hvTzNKZmdqNUZ4QUMrN2JmV3hGZytzdW9rU3dWdWVBZ1VJTzFrUXdFPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
+  client.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJLRENCMjZBREFnRUNBaEJEZTFWQ2QvZVkyQkcyenpWRWdSdkVNQVVHQXl0bGNEQVFNUTR3REFZRFZRUUsKRXdWMFlXeHZjekFlRncweU5URXhNekF4T1RRM05ESmFGdzB5TmpFeE16QXhPVFEzTkRKYU1CTXhFVEFQQmdOVgpCQW9UQ0c5ek9tRmtiV2x1TUNvd0JRWURLMlZ3QXlFQWdkMUgyQi9wS0ZoaCtsdExwbTY5YkxCUEZVVU5PTE5KCjlFQWFxbUVmRjkyalNEQkdNQTRHQTFVZER3RUIvd1FFQXdJSGdEQVRCZ05WSFNVRUREQUtCZ2dyQmdFRkJRY0QKQWpBZkJnTlZIU01FR0RBV2dCVExKM0ZER2ZIWDJsVThFZXhGR09DTGplZ1hRVEFGQmdNclpYQURRUUNHU1RPUwpHckxpbDVycWcxUDd2NkVkY0dpVXNEZ0x1WVUxcGF5STUzM1NUVTNDVTNnTCtiNkZDY3FWUSt6aTllejM1bWZZClkyWlBlenBXRlVKdUEva0sKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
+  client.key: LS0tLS1CRUdJTiBFRDI1NTE5IFBSSVZBVEUgS0VZLS0tLS0KTUM0Q0FRQXdCUVlESzJWd0JDSUVJTWhDTXVWU0gzcG9XZlR3em14MmxjSm43bjE0ZmtLb0cxQStjejVOSWlLRwotLS0tLUVORCBFRDI1NTE5IFBSSVZBVEUgS0VZLS0tLS0K

infrastructure/prometheus/manifests/talos-metrics-proxy.yaml

@@ -0,0 +1,232 @@
+# Talos Metrics Proxy
+# Workaround for HTTP 415 errors when scraping Talos apid metrics
+# This deploys an nginx proxy that strips problematic HTTP headers
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: talos-metrics-proxy-config
+  namespace: kube-system
+data:
+  nginx.conf: |
+    pid /tmp/nginx.pid;
+
+    events {
+        worker_connections 1024;
+    }
+
+    http {
+        proxy_temp_path /tmp/proxy_temp;
+        client_body_temp_path /tmp/client_temp;
+        fastcgi_temp_path /tmp/fastcgi_temp;
+        uwsgi_temp_path /tmp/uwsgi_temp;
+        scgi_temp_path /tmp/scgi_temp;
+        # Upstream Talos nodes
+        upstream talos01 {
+            server 172.28.101.21:50000;
+        }
+        upstream talos02 {
+            server 172.28.101.22:50000;
+        }
+        upstream talos03 {
+            server 172.28.101.23:50000;
+        }
+        upstream talos04 {
+            server 172.28.101.24:50000;
+        }
+        
+        # Proxy for talos01
+        server {
+            listen 9100;
+            server_name talos01-metrics;
+            
+            location /metrics {
+                proxy_pass https://talos01/metrics;
+                proxy_ssl_certificate /certs/client.crt;
+                proxy_ssl_certificate_key /certs/client.key;
+                proxy_ssl_trusted_certificate /certs/ca.crt;
+                proxy_ssl_verify on;
+                proxy_http_version 1.1;
+                proxy_set_header Accept "text/plain;version=0.0.4";
+                proxy_set_header Connection "";
+            }
+        }
+        
+        # Proxy for talos02
+        server {
+            listen 9101;
+            server_name talos02-metrics;
+            
+            location /metrics {
+                proxy_pass https://talos02/metrics;
+                proxy_ssl_certificate /certs/client.crt;
+                proxy_ssl_certificate_key /certs/client.key;
+                proxy_ssl_trusted_certificate /certs/ca.crt;
+                proxy_ssl_verify on;
+                proxy_http_version 1.1;
+                proxy_set_header Accept "text/plain;version=0.0.4";
+                proxy_set_header Connection "";
+            }
+        }
+        
+        # Proxy for talos03
+        server {
+            listen 9102;
+            server_name talos03-metrics;
+            
+            location /metrics {
+                proxy_pass https://talos03/metrics;
+                proxy_ssl_certificate /certs/client.crt;
+                proxy_ssl_certificate_key /certs/client.key;
+                proxy_ssl_trusted_certificate /certs/ca.crt;
+                proxy_ssl_verify on;
+                proxy_http_version 1.1;
+                proxy_set_header Accept "text/plain;version=0.0.4";
+                proxy_set_header Connection "";
+            }
+        }
+        
+        # Proxy for talos04
+        server {
+            listen 9103;
+            server_name talos04-metrics;
+            
+            location /metrics {
+                proxy_pass https://talos04/metrics;
+                proxy_ssl_certificate /certs/client.crt;
+                proxy_ssl_certificate_key /certs/client.key;
+                proxy_ssl_trusted_certificate /certs/ca.crt;
+                proxy_ssl_verify on;
+                proxy_http_version 1.1;
+                proxy_set_header Accept "text/plain;version=0.0.4";
+                proxy_set_header Connection "";
+            }
+        }
+    }
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: talos-metrics-proxy
+  namespace: kube-system
+  labels:
+    app: talos-metrics-proxy
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: talos-metrics-proxy
+  template:
+    metadata:
+      labels:
+        app: talos-metrics-proxy
+    spec:
+      automountServiceAccountToken: false
+      containers:
+      - name: nginx
+        image: nginx:1.27-alpine
+        securityContext:
+          runAsUser: 10101
+          runAsGroup: 10101
+          runAsNonRoot: true
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          capabilities:
+            drop:
+              - ALL
+          seccompProfile:
+            type: RuntimeDefault
+        ports:
+        - containerPort: 9100
+          name: talos01
+        - containerPort: 9101
+          name: talos02
+        - containerPort: 9102
+          name: talos03
+        - containerPort: 9103
+          name: talos04
+        volumeMounts:
+        - name: nginx-config
+          mountPath: /etc/nginx/nginx.conf
+          subPath: nginx.conf
+        - name: talos-certs
+          mountPath: /certs
+          readOnly: true
+        - name: tmp
+          mountPath: /tmp
+        - name: cache
+          mountPath: /var/cache/nginx
+        resources:
+          requests:
+            memory: "64Mi"
+            cpu: "10m"
+          limits:
+            memory: "128Mi"
+      volumes:
+      - name: nginx-config
+        configMap:
+          name: talos-metrics-proxy-config
+      - name: talos-certs
+        secret:
+          secretName: talos-client-cert
+      - name: tmp
+        emptyDir: {}
+      - name: cache
+        emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: talos-metrics-proxy
+  namespace: kube-system
+  labels:
+    app: talos-metrics-proxy
+spec:
+  type: ClusterIP
+  ports:
+  - port: 9100
+    targetPort: 9100
+    protocol: TCP
+    name: talos01
+  - port: 9101
+    targetPort: 9101
+    protocol: TCP
+    name: talos02
+  - port: 9102
+    targetPort: 9102
+    protocol: TCP
+    name: talos03
+  - port: 9103
+    targetPort: 9103
+    protocol: TCP
+    name: talos04
+  selector:
+    app: talos-metrics-proxy
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: talos-metrics
+  namespace: kube-system
+  labels:
+    app: talos-metrics-proxy
+spec:
+  endpoints:
+  - port: talos01
+    path: /metrics
+    interval: 30s
+  - port: talos02
+    path: /metrics
+    interval: 30s
+  - port: talos03
+    path: /metrics
+    interval: 30s
+  - port: talos04
+    path: /metrics
+    interval: 30s
+  selector:
+    matchLabels:
+      app: talos-metrics-proxy
+  namespaceSelector:
+    matchNames:
+    - kube-system

infrastructure/prometheus/values.yaml

@@ -0,0 +1,87 @@
+# values/prometheus.values.yaml
+
+# Prometheus Operator configuration
+prometheus:
+  prometheusSpec:
+    # High availability with 2 replicas
+    replicas: 2
+    
+    # Reduce retention to lower I/O on shared NVMe storage
+    retention: 7d
+
+    storageSpec:
+      volumeClaimTemplate:
+        spec:
+          storageClassName: rook-ceph-block
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: 20Gi
+
+    # Enable ServiceMonitor discovery across all namespaces
+    # NOTE: empty {} is falsy in Go templates, so the chart falls back to
+    # requiring release: <release-name> label. Setting NilUsesHelmValues=false
+    # ensures the empty selector matches ALL monitors.
+    serviceMonitorSelector: {}
+    serviceMonitorSelectorNilUsesHelmValues: false
+    serviceMonitorNamespaceSelector: {}
+    podMonitorSelector: {}
+    podMonitorSelectorNilUsesHelmValues: false
+    podMonitorNamespaceSelector: {}
+
+    # Mount Talos client certificates
+    secrets:
+      - talos-client-cert
+
+    # Additional scrape configurations for Talos
+    # NOTE: Talos apid endpoint at port 50000 returns HTTP 415 errors with Prometheus v2.x
+    # This is a known compatibility issue between Talos v1.10.x and modern Prometheus
+    # The issue is that Prometheus sends Accept headers for content negotiation that
+    # Talos apid doesn't understand. Workarounds require a reverse proxy or using kubelet.
+    # For now, Talos metrics are disabled. Node metrics are still available via kubelet.
+    additionalScrapeConfigs: []
+    
+    # DISABLED - Talos apid scraping (returns 415 Unsupported Media Type)
+    # - job_name: 'talos-apid'
+    #   honor_timestamps: true
+    #   scrape_interval: 30s
+    #   scrape_timeout: 10s
+    #   metrics_path: /metrics
+    #   scheme: https
+    #   scrape_protocols: []
+    #   enable_http2: false
+    #   enable_compression: false
+    #   tls_config:
+    #     ca_file: /etc/prometheus/secrets/talos-client-cert/ca.crt
+    #     cert_file: /etc/prometheus/secrets/talos-client-cert/client.crt
+    #     key_file: /etc/prometheus/secrets/talos-client-cert/client.key
+    #     insecure_skip_verify: false
+    #   kubernetes_sd_configs:
+    #     - role: node
+    #   relabel_configs:
+    #     - source_labels: [__address__]
+    #       regex: '(.*):.*'
+    #       replacement: '${1}:50000'
+    #       target_label: __address__
+    #     - source_labels: [__meta_kubernetes_node_name]
+    #       target_label: node
+
+  # Ingress disabled - Istio VirtualService handles routing
+  ingress:
+    enabled: false
+
+alertmanager:
+  enabled: true
+
+# Disable node-exporter due to PodSecurity restrictions
+prometheus-node-exporter:
+  enabled: false
+
+# Enable kube-state-metrics for Kubernetes object metrics
+kube-state-metrics:
+  enabled: true
+
+# Grafana (included in kube-prometheus-stack)
+grafana:
+  enabled: false  # We deploy Grafana separately