monitoring: remove prometheus stack, add alloy

2026-08-30 22:15:39 +00:00
parent 4913882963
commit 115a600055
6 changed files with 192 additions and 246 deletions
@@ -147,15 +147,182 @@ alloy:
          }
        }

-      //  stage.labels {
-      //    values = {
-      //      kubernetes_cluster_events = "job",
-      //    }
-      //  }
-      }
+        discovery.kubernetes "k8s_nodes" {
+            role = "node"
+        }
+
+
+        discovery.relabel "cadvisor" {
+          targets = discovery.kubernetes.k8s_nodes.targets
+
+          rule {
+            replacement  = "/metrics/cadvisor"
+            target_label = "__metrics_path__"
+          }
+          rule {
+            target_label = "instance"
+            replacement  = env("NODE_IP") + ":10250"
+          }
+
+          rule {
+            target_label = "node"
+            replacement  = env("HOSTNAME")
+          }
+
+          rule {
+            target_label = "service"
+            replacement  = "kube-scrapper"
+          }
+
+          rule {
+            target_label = "cluster"
+            replacement  = env("CLUSTER_NAME")
+          }
+
+          rule {
+            target_label = "job"
+            replacement  = "kubelet"
+          }
+        }
+      
+        prometheus.remote_write "metrics_service" {
+        
+          endpoint {
+            url = env("PROMETHEUS_URL")
+          }
+        
+        }
+        prometheus.scrape "cadvisor" {
+          job_name = "integrations/kubernetes/cadvisor"
+          targets  = discovery.relabel.cadvisor.output
+          scheme   = "https"
+
+          tls_config {
+            server_name          = "kubernetes"
+            ca_file              = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+            insecure_skip_verify = true
+          }
+
+          bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+          forward_to        = [prometheus.remote_write.metrics_service.receiver]
+        }
+
+        
+        discovery.kubernetes "k8s_services" {
+          role = "service"
+        }
+
+        discovery.relabel "ksm" {
+          targets = discovery.kubernetes.k8s_services.targets
+
+          rule {
+            source_labels = ["__meta_kubernetes_service_name"]
+            regex         = "kube-state-metrics"
+            action        = "keep"
+          }
+        }
+
+        prometheus.scrape "kube_state_metrics" {
+          job_name = "integrations/kubernetes/kube-state-metrics"
+          targets  = discovery.relabel.ksm.output
+
+          scheme = "http"
+
+          forward_to = [prometheus.remote_write.metrics_service.receiver]
+        }
+
+        // ============================================
+        // CoreDNS discovery
+        // ============================================
+        discovery.kubernetes "coredns_pod" {
+          role = "pod"
+        }
+
+        discovery.relabel "coredns" {
+          targets = discovery.kubernetes.pod.targets
+
+          rule {
+            source_labels = ["__meta_kubernetes_pod_label_k8s_app"]
+            regex         = "kube-dns"
+            action        = "keep"
+          }
+          rule {
+            target_label = "node"
+            replacement  = env("HOSTNAME")
+          }
+
+          rule {
+            target_label = "cluster"
+            replacement  = env("CLUSTER_NAME")
+          }
+        }
+
+        // ============================================
+        // CoreDNS scrape
+        // ============================================
+        prometheus.scrape "coredns" {
+          job_name = "integrations/kubernetes/coredns-pods"
+          targets  = discovery.relabel.coredns.output
+
+          scheme = "http"
+          metrics_path = "/metrics"
+
+          forward_to = [prometheus.remote_write.metrics_service.receiver]
+        }
+
+        // ============================================
+        // API Server discovery
+        // ============================================
+        discovery.kubernetes "apiserver_services" {
+          role = "service"
+        }
+
+        discovery.relabel "apiserver" {
+          targets = discovery.kubernetes.apiserver_services.targets
+
+          rule {
+            source_labels = ["__meta_kubernetes_service_name"]
+            regex         = "kubernetes"
+            action        = "keep"
+          }
+            rule {
+            target_label = "node"
+            replacement  = env("HOSTNAME")
+          }
+
+          rule {
+            target_label = "cluster"
+            replacement  = env("CLUSTER_NAME")
+          }
+        }
+
+        // ============================================
+        // API Server scrape
+        // ============================================
+        prometheus.scrape "apiserver" {
+          job_name = "integrations/kubernetes/apiserver"
+          targets  = discovery.relabel.apiserver.output
+
+          scheme = "https"
+          metrics_path = "/metrics"
+
+          tls_config {
+            server_name          = "kubernetes"
+            ca_file              = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+            insecure_skip_verify = true
+          }
+
+          bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+
+          forward_to = [prometheus.remote_write.metrics_service.receiver]
+        }

 
  extraEnv:
+    - name: NODE_IP
+      valueFrom:
+        fieldRef:
+          fieldPath: status.hostIP
    - name: NODE_NAME
      valueFrom:
        fieldRef:
@@ -169,3 +336,8 @@ alloy:
        secretKeyRef:
          name: alloy-global
          key: loki_url
+    - name: PROMETHEUS_URL
+      valueFrom:
+        secretKeyRef:
+          name: alloy-global
+          key: prometheus_url