monitoring: remove prometheus stack, add alloy

services/monitoring/README.md

@@ -1,10 +1,5 @@
 # monitoring
  
-## prometheus-stack
-
-- <https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack>
-- <https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/values.yaml>
-  
 ## alloy
 
 **promtail Encrypt secrets:**

services/monitoring/deploy/alloy/global/secrets.yaml

@@ -3,20 +3,20 @@ kind: Secret
 metadata:
     name: alloy-global
 stringData:
-    loki_url: ENC[AES256_GCM,data:xuMvbrKhvwvJK38ES59SuD/X5CywK3qlnDfSsYmr0W3TLv0wB08wT9kCd57pctlvAqL5RNYTaTIgJ1w=,iv:YqPv1DBbfmVC8P1KgEJPiOtYzx+OcfySIxPjqMjCnhA=,tag:aw7iW8gljmVhPvkKSKh5mw==,type:str]
+    loki_url: ENC[AES256_GCM,data:2DeZZ+7deDbojQ/+YNgT8KwDmjFFyAqMcTX8l6itlF15hilXKgYP1p/SNoyw1JGJ7x5VZqUl0VIVhVY=,iv:SlFiLvEeIqj+zQ/8Jspc9FL/ElneIlWBzGsM+6mGZLc=,tag:9tIl6fi+xfmF+TJpMXgbKg==,type:str]
-    prometheus_url: ENC[AES256_GCM,data:MqNjAcGzi87gC2A2+G6xcn7FroYUczfkNHj0eCXZVi46AJH+dKjFU/pVwtMIrz1Aikz4BhX0ZviijrvYlw==,iv:IydlpoIOq4bipZV/xihvpmYO/sBerDX2z+8LGl3Gu6k=,tag:dJv6nUncoJGcuoR8eQFitg==,type:str]
+    prometheus_url: ENC[AES256_GCM,data:k8Z3lvZRxOmlw6Wx4aLAgV93hPcEawtoprWU2Kn6j/WIe/YDHM9iuB+pRy7ght4I6jxyNFLotN0BaHE1yg==,iv:EEnRoS287e9EV8+25NtlEO31zN0eyfRBOUe9GVBah18=,tag:2K/MnM1QlWPbceQBqCwylw==,type:str]
 sops:
     age:
         - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBpemRxQTNtUTQ2a1dUcnpu
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBJYmx1dGJTSGZvbVdpV1di
-            VjNlN29OeGRrdDN1bm8yWC9hYllBNVNjSVJvCnR5RUU2Z1B0RWFxREFoUlVIVStM
+            Z3dKNUJPVGNkK1hBSnVoRXZrb0R4TmxnUjM4CmpyUXpGN2lneDViN3VPNC9Fa3d6
-            c1AvbUlXa0dOTDF3VkhJRGdGME1oS1UKLS0tIDdncERRSDFKRnBlMW5kMkQ1VFhs
+            RURxZTFuV0o1SzF5ZEhzSVZyYkY5alkKLS0tIEF1T0tINnRUa0tDOGlaSVRJajF5
-            bXJYbmZyUXJ4NU1YOEhpdmhTT2I3S2MK+Zx6ntTO08AxawYo+jKOPTfpMVXdsvlC
+            bTFTd0c4Z3p1aUNWWTZ0ZFVXQVdrMjAKku2YU048qf4E9vEPO19HNfAg78E3fSFQ
-            5hNPs6bqxei+Fa6q87LZrFENwdxjeti6QuKoi45acoOQ6zI2TM9yNw==
+            WBiUgiqtqG/T4cI5cf4CUWoUogtABawBG4z5G5+ud7sfVMzfqW+W1g==
             -----END AGE ENCRYPTED FILE-----
           recipient: age1f9e4pvp5y8gzuk8mz2s5xm85dd7znxhk56tcpuxqwn78qfjwja0qekwlju
     encrypted_regex: ^(data|stringData)$
-    lastmodified: "2026-08-30T10:21:08Z"
+    lastmodified: "2026-08-30T22:14:34Z"
-    mac: ENC[AES256_GCM,data:FZkuO1JP6Ds3XUAWUa0tpb2gVcnFC9Ki3owd+f7On+Nu0EvVRqVBzrSLBXxPVOQ69jj9HDiQVFmYq2fsRd5aAItCA+Jry/2M7m6j6BfyzPMmEChemTYSJQKvQy7l9r4FYwXqvBJ8TmPQv3wc+VPrhiYegWusF/YzcZzFvvUwVOE=,iv:gQUZEbK5l/3Gc6Xcm3RAF5kMaV14dYH/Og+SKrdhUtw=,tag:YVE3seBOcDl31huyDog+FA==,type:str]
+    mac: ENC[AES256_GCM,data:XQ2zRn6zGBONfBiyaip3DpwzIfDBN1mPI65fVfH3c7tsEeY1mV/ZRPD+5iJsiwxlnBa7Rq0iJNOr2f84JE+CiuqYz8UKuQMS02gsvqXuBSgUn5vDRcX/aBV3+1GSOzv3ilmwMkJI/jxfyyXL4YsxKl0fsFEOJRsa3VCCVF0OvrM=,iv:enk8h+TfYBIOFln3SlOw30arg2T/kSU3IMwusrm68Qc=,tag:yp9eQc56Ht6g3x3QnMqMZg==,type:str]
     version: 3.13.1

services/monitoring/deploy/alloy/kube-scraper/helm-values.yaml

@@ -147,15 +147,182 @@ alloy:
           }
         }
 
-      //  stage.labels {
+        discovery.kubernetes "k8s_nodes" {
-      //    values = {
+            role = "node"
-      //      kubernetes_cluster_events = "job",
+        }
-      //    }
+
-      //  }
+
+        discovery.relabel "cadvisor" {
+          targets = discovery.kubernetes.k8s_nodes.targets
+
+          rule {
+            replacement  = "/metrics/cadvisor"
+            target_label = "__metrics_path__"
+          }
+          rule {
+            target_label = "instance"
+            replacement  = env("NODE_IP") + ":10250"
+          }
+
+          rule {
+            target_label = "node"
+            replacement  = env("HOSTNAME")
+          }
+
+          rule {
+            target_label = "service"
+            replacement  = "kube-scrapper"
+          }
+
+          rule {
+            target_label = "cluster"
+            replacement  = env("CLUSTER_NAME")
+          }
+
+          rule {
+            target_label = "job"
+            replacement  = "kubelet"
+          }
+        }
+      
+        prometheus.remote_write "metrics_service" {
+        
+          endpoint {
+            url = env("PROMETHEUS_URL")
+          }
+        
+        }
+        prometheus.scrape "cadvisor" {
+          job_name = "integrations/kubernetes/cadvisor"
+          targets  = discovery.relabel.cadvisor.output
+          scheme   = "https"
+
+          tls_config {
+            server_name          = "kubernetes"
+            ca_file              = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+            insecure_skip_verify = true
+          }
+
+          bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+          forward_to        = [prometheus.remote_write.metrics_service.receiver]
+        }
+
+        
+        discovery.kubernetes "k8s_services" {
+          role = "service"
+        }
+
+        discovery.relabel "ksm" {
+          targets = discovery.kubernetes.k8s_services.targets
+
+          rule {
+            source_labels = ["__meta_kubernetes_service_name"]
+            regex         = "kube-state-metrics"
+            action        = "keep"
+          }
+        }
+
+        prometheus.scrape "kube_state_metrics" {
+          job_name = "integrations/kubernetes/kube-state-metrics"
+          targets  = discovery.relabel.ksm.output
+
+          scheme = "http"
+
+          forward_to = [prometheus.remote_write.metrics_service.receiver]
+        }
+
+        // ============================================
+        // CoreDNS discovery
+        // ============================================
+        discovery.kubernetes "coredns_pod" {
+          role = "pod"
+        }
+
+        discovery.relabel "coredns" {
+          targets = discovery.kubernetes.pod.targets
+
+          rule {
+            source_labels = ["__meta_kubernetes_pod_label_k8s_app"]
+            regex         = "kube-dns"
+            action        = "keep"
+          }
+          rule {
+            target_label = "node"
+            replacement  = env("HOSTNAME")
+          }
+
+          rule {
+            target_label = "cluster"
+            replacement  = env("CLUSTER_NAME")
+          }
+        }
+
+        // ============================================
+        // CoreDNS scrape
+        // ============================================
+        prometheus.scrape "coredns" {
+          job_name = "integrations/kubernetes/coredns-pods"
+          targets  = discovery.relabel.coredns.output
+
+          scheme = "http"
+          metrics_path = "/metrics"
+
+          forward_to = [prometheus.remote_write.metrics_service.receiver]
+        }
+
+        // ============================================
+        // API Server discovery
+        // ============================================
+        discovery.kubernetes "apiserver_services" {
+          role = "service"
+        }
+
+        discovery.relabel "apiserver" {
+          targets = discovery.kubernetes.apiserver_services.targets
+
+          rule {
+            source_labels = ["__meta_kubernetes_service_name"]
+            regex         = "kubernetes"
+            action        = "keep"
+          }
+            rule {
+            target_label = "node"
+            replacement  = env("HOSTNAME")
+          }
+
+          rule {
+            target_label = "cluster"
+            replacement  = env("CLUSTER_NAME")
+          }
+        }
+
+        // ============================================
+        // API Server scrape
+        // ============================================
+        prometheus.scrape "apiserver" {
+          job_name = "integrations/kubernetes/apiserver"
+          targets  = discovery.relabel.apiserver.output
+
+          scheme = "https"
+          metrics_path = "/metrics"
+
+          tls_config {
+            server_name          = "kubernetes"
+            ca_file              = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+            insecure_skip_verify = true
+          }
+
+          bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+
+          forward_to = [prometheus.remote_write.metrics_service.receiver]
         }
 
  
   extraEnv:
+    - name: NODE_IP
+      valueFrom:
+        fieldRef:
+          fieldPath: status.hostIP
     - name: NODE_NAME
       valueFrom:
         fieldRef:
@@ -169,3 +336,8 @@ alloy:
         secretKeyRef:
           name: alloy-global
           key: loki_url
+    - name: PROMETHEUS_URL
+      valueFrom:
+        secretKeyRef:
+          name: alloy-global
+          key: prometheus_url

services/monitoring/deploy/prometheus/helm-release.yaml

@@ -1,19 +1,15 @@
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
-  name: kube-prometheus-stack
+  name: kube-state-metrics
 spec:
-  releaseName: prometheus
+  releaseName: kube-state-metrics
   interval: 1m
   chart:
     spec:
-      chart: kube-prometheus-stack
+      chart: kube-state-metrics
-      version: 86.x.x
+      version: 8.x.x
       sourceRef:
         kind: HelmRepository
         name: prometheus
       interval: 40h
-  valuesFrom:
-    - kind: Secret
-      name: prometheus-stack-helm-values
-      valuesKey: values.yaml

services/monitoring/deploy/prometheus/helm-values.yaml

@@ -1,141 +0,0 @@
-
-defaultRules:
-  create: false
-
-alertmanager:
-  enabled: false
-
-grafana:
-  enabled: false
-
-prometheus:
-  enabled: false
-
-kubeStateMetrics:
-  enabled: true
-
-nodeExporter:
-  enabled: false
-
-prometheus-node-exporter:
-  prometheus:
-    monitor:
-      enabled: false
-
-coreDns: 
-  enabled: true
-  serviceMonitor:
-    relabelings:
-      - targetLabel: cluster
-        replacement: casa
- 
-kubeApiServer: 
-  enabled: true
-  serviceMonitor:
-    relabelings:
-      - targetLabel: cluster
-        replacement: casa
- 
-kubeControllerManager:
-  enabled: false
-
-kubeEtcd:
-  enabled: false
-
-kubeProxy:
-  enabled: false
-
-kubeScheduler:
-  enabled: false
-
-kube-state-metrics:
-  podLabels:
-    role: worker-node
-  nodeSelector:
-    role: worker-node
-  prometheus:
-    monitor:
-        relabelings:
-        - targetLabel: cluster
-          replacement: casa
-
-prometheusOperator:
-  kubeletService:
-    enabled: true
-  enabled: true
-  createCustomResource: true
-  tls:
-    enabled: false
-  admissionWebhooks:
-    enabled: false
-  cleanupCustomResource: false
-  serviceMonitor:
-    selfMonitor: false
- 
-
-kubelet:
-  enabled: true
-  namespace: kube-system
-  serviceMonitor:
-    interval: 60s #WARN: Error on ingesting out-of-order samples. https://github.com/prometheus-community/helm-charts/issues/5483
-    enabled: true 
-    ## Enable scraping /metrics from kubelet's service
-    kubelet: true
-
-    probesMetricRelabelings:
-      - targetLabel: cluster
-        replacement: casa
-      - sourceLabels: [__name__, image]
-        separator: ;
-        regex: container_([a-z_]+);
-        replacement: $1
-        action: drop
-      - sourceLabels: [__name__]
-        separator: ;
-        regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s)
-        replacement: $1
-        action: drop
-
-
-#     # RelabelConfigs to apply to samples before scraping
-#     # ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api-reference/api.md#relabelconfig
-#     #
-#     # metrics_path is required to match upstream rules and charts
-    cAdvisorRelabelings:
-      - targetLabel: cluster
-        replacement: casa    
-      - action: replace
-        sourceLabels: [__metrics_path__]
-        targetLabel: metrics_path
-      - sourceLabels: [__meta_kubernetes_pod_node_name]
-        separator: ;
-        regex: ^(.*)$
-        targetLabel: nodename
-        replacement: $1
-        action: replace
-
-#     # RelabelConfigs to apply to samples before scraping
-#     # ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api-reference/api.md#relabelconfig
-#     #
-    probesRelabelings:
-
-      - targetLabel: cluster
-        replacement: casa
-      - action: replace
-        sourceLabels: [__metrics_path__]
-        targetLabel: metrics_path
-
-      - sourceLabels: [__meta_kubernetes_pod_node_name]
-        separator: ;
-        regex: ^(.*)$
-        targetLabel: nodename
-        replacement: $1
-        action: replace
-
-    resourceRelabelings:
-      - targetLabel: cluster
-        replacement: casa
-      - action: replace
-        sourceLabels: [__metrics_path__]
-        targetLabel: metrics_path
-    

services/monitoring/deploy/prometheus/prometheus-agent.yaml

@@ -1,76 +0,0 @@
-
-apiVersion: monitoring.coreos.com/v1alpha1
-kind: PrometheusAgent
-metadata:
-  name: prometheus-agent
-spec:
-  podMonitorNamespaceSelector:
-    matchLabels:
-      prometheus-monitoring: enabled
-
-  podMonitorSelector:
-    matchLabels:
-      release: prometheus
-
-  serviceMonitorNamespaceSelector:
-    matchLabels:
-      prometheus-monitoring: enabled
-
-  serviceMonitorSelector:
-    matchLabels:
-      release: prometheus
-
-  replicas: 1
-  remoteWrite:
-    - url: https://prometheus.monitoring.limbosolutions.com/api/v1/write
-  scrapeInterval: 60s
-  resources:
-    requests:
-      cpu: 100m
-      memory: 128Mi
-    limits:
-      cpu: 200m
-      memory: 512Mi # minimum stable
-  serviceAccountName: prometheus-agent
-  nodeSelector:
-    role: worker-node
----
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: prometheus-agent
-rules:
-  - apiGroups: [""]
-    resources: ["nodes", "nodes/metrics", "nodes/proxy", "services", "endpoints", "pods"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["monitoring.coreos.com"]
-    resources: ["servicemonitors", "podmonitors"]
-    verbs: ["get", "list", "watch"]
-  - nonResourceURLs:
-      - /metrics
-      - /metrics/cadvisor
-      - /metrics/probes
-    verbs: ["get"]
----
-
-
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: prometheus-agent-monitoring
-roleRef:
-  kind: ClusterRole
-  name: prometheus-agent
-  apiGroup: rbac.authorization.k8s.io
-subjects:
-  - kind: ServiceAccount
-    name: prometheus-agent
-    namespace: monitoring
-
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: prometheus-agent
-  namespace: monitoring