alloy; revision

services/monitoring/deploy/alloy/kube-scraper/helm-values.yaml

@@ -8,26 +8,83 @@ alloy:
   configMap:
     content: |
       // ============================================
-      // LOKI WRITE TARGET
+      // Discovery: Kubernetes - Nodes
+      // used by:
+      // - prometheus - Kube_node_metrics - cAdvisor 
       // ============================================
+      
+      discovery.kubernetes "kube_nodes" {
+        role = "node"
+      }
+
+      // ============================================
+      // Discovery: Kubernetes - pods
+      // used by:
+      // - loki kubernetes pod logs
+      // ============================================
+      
+      discovery.kubernetes "kube_pods" {
+        role = "pod"
+      }
+
+      // ============================================
+      // Discovery: Kubernetes - pods
+      // used by:
+      // - loki kubernetes pod logs
+      // ============================================
+      
+
+      // ============================================
+      // Discovery: Kubernetes - pods
+      // used by:
+      // - loki kubernetes kube state metrics
+      // ============================================
+      discovery.kubernetes "kube_services" {
+        role = "service"
+      }
+
+      // ============================================
+      // Logs: Loki config
+      // ============================================
+
       loki.write "loki_srv" {
         endpoint {
           url = env("LOKI_URL")
         }
       }
 
-      // ============================================
+      loki.process "loki_srv" {
-      // POD LOGS: DISCOVERY + RELABEL
+        stage.static_labels {
-      // ============================================
+          values = {
-
+            cluster = env("CLUSTER_NAME"),
-      // Discover Kubernetes pods
+            zone    = env("ZONE"),
-      discovery.kubernetes "pod_logs" {
+          }
-        role = "pod"
+        }
+        forward_to = [loki.write.loki_srv.receiver]
       }
 
-      discovery.relabel "pod_logs" {
-        targets = discovery.kubernetes.pod_logs.targets
 
+      // ============================================
+      // Loki.source  kube_pod_logs
+      // Requirements: 
+      //  - discovery.relabel: kube_pod_logs
+      //  - discovery.process: loki_srv
+      // ============================================
+
+      loki.source.kubernetes "kube_pod_logs" {
+        targets    = discovery.relabel.kube_pod_logs.output
+        forward_to = [loki.process.loki_srv.receiver]
+      }
+
+
+      // ============================================
+      // Discovery:relabel: kube_pod_logs
+      // Requirements: 
+      //  - discovery.kubernetes.role.pod: kube_pods
+      // ============================================
+    
+      discovery.relabel "kube_pod_logs" {
+        targets = discovery.kubernetes.kube_pods.targets
         rule {
           source_labels = ["__meta_kubernetes_namespace"]
           target_label  = "namespace"
@@ -98,75 +155,62 @@ alloy:
 
 
       // ============================================
-      // POD LOGS: SOURCE → PROCESS → WRITE
+      // Prometheus: prometheus_srv
       // ============================================
 
-      // Tail logs from Kubernetes pods
-      loki.source.kubernetes "pod_logs" {
-        // targets    = discovery.kubernetes.pod_logs.targets
-        targets    = discovery.relabel.pod_logs.output
-        forward_to = [loki.process.pod_logs.receiver]
-      }
-
-      // Add static labels and forward logs to Loki
-      loki.process "pod_logs" {
-
-        stage.static_labels {
-          values = {
-            cluster = env("CLUSTER_NAME"),
-            zone    = env("ZONE"),
-          }
-        }
-
-        forward_to = [loki.write.loki_srv.receiver]
-      }
-
-      // ============================================
-      // KUBERNETES EVENTS
-      // ============================================
-
-      // Tail Kubernetes events and convert them to log entries
-      loki.source.kubernetes_events "cluster_events" {
-        job_name   = "integrations/kubernetes/eventhandler"
-        log_format = "logfmt"
-        forward_to = [
-          loki.process.cluster_events.receiver,
-        ]
-      }
-
-      // Add static labels and forward events to Loki
-      loki.process "cluster_events" {
-        forward_to = [loki.write.loki_srv.receiver]
-
-        stage.static_labels {
-          values = {
-            cluster = env("CLUSTER_NAME"),
-            host    = "env("CLUSTER_NAME")",
-            job     = "cluster_events",
-            zone    = env("ZONE"),
-          }
-        }
-
-      }
-
-
-      // Metrics
-
       prometheus.remote_write "prometheus_srv" {
+  
         endpoint {
           url = env("PROMETHEUS_URL")
         }
+
+        // global labels
+        external_labels = {
+          cluster = env("CLUSTER_NAME"),
+          zone    = env("ZONE"),
+        }
+
       }
 
-      // discovery kubernetes
+      // ===============================================================================
+      // prometheus / Metrics - Scrape kube_state_metrics
+      // Requirements: 
+      //  - discovery.relabel - kube_node_metrics (cAdviser)
+      // ===============================================================================
+      prometheus.scrape "kube_state_metrics" {
+        job_name = "integrations/kubernetes/kube-state-metrics"
+        targets  = discovery.relabel.kube_state_metrics.output
 
-      discovery.kubernetes "k8s_nodes" {
+        scheme = "http"
-          role = "node"
+
+        forward_to = [prometheus.remote_write.prometheus_srv.receiver]
       }
 
+      discovery.relabel "kube_state_metrics" {
+        targets = discovery.kubernetes.kube_services.targets
+
+        rule {
+          source_labels = ["__meta_kubernetes_service_name"]
+          regex         = "kube-state-metrics"
+          action        = "keep"
+        }
+
+        rule {
+            source_labels = ["__meta_kubernetes_node_name"]
+            target_label  = "node"
+        }
+      }
+
+
+      // ===============================================================================
+      // prometheus / Metrics - Scrape Kubernetes nodes / cAdvisor
+      // Requirements: 
+      //  - discovery.relabel - kube_node_metrics (cAdviser)
+      // ===============================================================================
+
       prometheus.scrape "cadvisor" {
         job_name = "integrations/kubernetes/cadvisor"
-        targets  = discovery.relabel.cadvisor.output
+        targets  = discovery.relabel.kube_node_metrics.output
         scheme   = "https"
 
         tls_config {
@@ -178,129 +222,36 @@ alloy:
         bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
         forward_to        = [prometheus.remote_write.prometheus_srv.receiver]
       }
+  
 
- 
+
-      discovery.relabel "cadvisor" {
+
-        targets = discovery.kubernetes.k8s_nodes.targets
+      // ===========================================================
+      // Discovery:relabel: Kube_node_metrics (cAdviser)
+      // Requirements: 
+      //  - discovery.kubernetes.role.node: kube_nodes
+      // ===========================================================
+
+       discovery.relabel "kube_node_metrics" {
+        targets = discovery.kubernetes.kube_nodes.targets
+        rule {
+          action        = "replace"
+          source_labels = ["__meta_kubernetes_node_name"]
+          regex         = "(.+)"
+          target_label  = "node"
+        }
 
         rule {
+          source_labels = ["node"]
+          target_label = "host"
+        }
+
+       rule {
           replacement  = "/metrics/cadvisor"
           target_label = "__metrics_path__"
         }
-        
-        rule {
-          target_label = "instance"
-          replacement  = env("NODE_IP") + ":10250"
-        }
 
-        rule {
+      }   
-          source_labels = ["exported_node"]
-          target_label  = "node"
-        }
-
-        rule {
-          target_label = "cluster"
-          replacement  = env("CLUSTER_NAME")
-        }
-
-      }
-
-
-      discovery.kubernetes "k8s_services" {
-        role = "service"
-      }
-
-      discovery.relabel "kube_state_metrics" {
-        targets = discovery.kubernetes.k8s_services.targets
-
-        rule {
-          source_labels = ["__meta_kubernetes_service_name"]
-          regex         = "kube-state-metrics"
-          action        = "keep"
-        }
-
-        rule {
-          source_labels = ["exported_node"]
-          target_label  = "node"
-        }
-
-        rule {
-          target_label = "cluster"
-          replacement  = env("CLUSTER_NAME")
-        }
-      }
-
-      prometheus.scrape "kube_state_metrics" {
-        job_name = "integrations/kubernetes/kube-state-metrics"
-        targets  = discovery.relabel.kube_state_metrics.output
-
-        scheme = "http"
-
-        forward_to = [prometheus.remote_write.prometheus_srv.receiver]
-      }
-
-      // ============================================
-      // API Server discovery
-      // ============================================
-
-      discovery.relabel "apiserver" {
-        targets = discovery.kubernetes.k8s_services.targets
-
-        rule {
-          source_labels = ["__meta_kubernetes_service_name"]
-          regex         = "kubernetes"
-          action        = "keep"
-        }
-        rule {
-          target_label = "cluster"
-          replacement  = env("CLUSTER_NAME")
-        }
-      }
-
-      prometheus.scrape "apiserver" {
-        job_name = "integrations/kubernetes/apiserver"
-        targets  = discovery.relabel.apiserver.output
-
-        scheme = "https"
-        metrics_path = "/metrics"
-
-        tls_config {
-          server_name          = "kubernetes"
-          ca_file              = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
-          insecure_skip_verify = true
-        }
-
-        bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
-
-        forward_to = [prometheus.remote_write.prometheus_srv.receiver]
-      }
-
-      // ============================================
-      // CoreDNS discovery
-      // ============================================
-      discovery.relabel "coredns" {
-        targets = discovery.kubernetes.k8s_services.targets
-
-        rule {
-          source_labels = ["__meta_kubernetes_pod_label_k8s_app"]
-          regex         = "kube-dns"
-          action        = "keep"
-        }
-        rule {
-          target_label = "cluster"
-          replacement  = env("CLUSTER_NAME")
-        }
-      }
-
-      prometheus.scrape "coredns" {
-        job_name = "integrations/kubernetes/coredns-pods"
-        targets  = discovery.relabel.coredns.output
-
-        scheme = "http"
-        metrics_path = "/metrics"
-
-        forward_to = [prometheus.remote_write.prometheus_srv.receiver]
-      }
 
  
   extraEnv:
@@ -308,10 +259,7 @@ alloy:
       valueFrom:
         fieldRef:
           fieldPath: status.hostIP
-    - name: NODE_NAME
+ 
-      valueFrom:
-        fieldRef:
-          fieldPath: spec.nodeName
     - name: CLUSTER_NAME
       value: casa
     - name: ZONE

services/monitoring/deploy/alloy/node-scraper/helm-values.yaml

@@ -58,6 +58,7 @@ alloy:
   # https://grafana.com/docs/alloy/latest/collect/logs-in-kubernetes/
   configMap:
     content: |
+    
       // ============================================
       // LOKI WRITE TARGET
       // ============================================
@@ -123,9 +124,30 @@ alloy:
 
         forward_to = [loki.write.loki_srv.receiver]
       }
+
+
       // ============================================
-      // prometheus unix exporter
+      // Metrics: Prometheus connection config
       // ============================================
+
+      prometheus.remote_write "metrics_service" {
+        endpoint {
+          url = env("PROMETHEUS_URL")
+        }
+        
+        // global labels
+        external_labels = {
+          cluster = env("CLUSTER_NAME"),
+          zone    = env("ZONE"),
+          node    = env("HOSTNAME")
+        }
+      }
+
+
+      // ============================================
+      // Metrics: Prometheus Unix Exporter
+      // ============================================
+
       prometheus.exporter.unix "node_exporter" {
         rootfs_path = "/host"
         disable_collectors = ["ipvs", "btrfs", "infiniband", "xfs", "zfs"]
@@ -153,78 +175,18 @@ alloy:
           device_exclude = "^(veth.*|cali.*|[a-f0-9]{15})$"
         }
       }
-    
+
       prometheus.scrape "node_exporter" {
         targets         = prometheus.exporter.unix.node_exporter.targets
         scrape_interval = "30s"
 
         forward_to = [prometheus.relabel.node_exporter.receiver]
       }
-
+  
       prometheus.relabel "node_exporter" {
-
-        rule {
-          target_label = "instance"
-          replacement  = env("NODE_IP") + ":9100"
-        }
-
-        rule {
-          target_label = "nodename"
-          replacement  = env("HOSTNAME")
-        }
-
-        rule {
-          target_label = "node"
-          replacement  = env("HOSTNAME")
-        }
-
-        rule {
-          target_label = "kubernetes_node"
-          replacement  = env("HOSTNAME")
-        }
-
-        rule {
-          target_label = "cluster"
-          replacement  = env("CLUSTER_NAME")
-        }
-
-        rule {
-          target_label = "job"
-          replacement  = env("JOB")
-        }
-
-        rule {
-
-          target_label = "container"
-          replacement  = env("CONTAINER")
-        }
-        rule {
-
-          target_label = "NS"
-          replacement  = env("NS")
-        }
-
-        rule {
-          target_label = "pod"
-          replacement  = env("POD")
-        }
-
-        rule {
-          target_label = "service"
-          replacement  = env("SERVICE")
-        }
-
         forward_to = [prometheus.remote_write.metrics_service.receiver]
-
       }
 
-      prometheus.remote_write "metrics_service" {
-
-        endpoint {
-          url = env("PROMETHEUS_URL")
-        }
-
-      }
 
   extraEnv:
     - name: NODE_IP