deleted: services/monitoring/deploy/alloy/alloy-loki-examples copy.config

deleted:    services/monitoring/deploy/alloy/alloy-prometheus-examples.config
modified:   services/monitoring/deploy/alloy/kube-scraper/helm-values.yaml

services/monitoring/deploy/alloy/alloy-loki-examples copy.config

@@ -1,144 +0,0 @@
-      // ============================================
-      // LOKI WRITE TARGET
-      // ============================================
-      loki.write "loki_srv" {
-        endpoint {
-          url = env("LOKI_URL")
-        }
-      }
-
-      // ============================================
-      // POD LOGS: DISCOVERY + RELABEL
-      // ============================================
-
-      // Discover Kubernetes pods
-      discovery.kubernetes "pod_logs" {
-        role = "pod"
-      }
-
-      discovery.relabel "pod_logs" {
-        targets = discovery.kubernetes.pod_logs.targets
-
-        rule {
-          source_labels = ["__meta_kubernetes_namespace"]
-          target_label  = "namespace"
-        }
-
-        rule {
-          source_labels = ["__meta_kubernetes_pod_name"]
-          regex        = "^(.+?)-[0-9a-zA-Z]{4,16}(?:-[0-9a-zA-Z]{4,16})?$"
-          replacement  = "$1"
-          target_label = "pod"
-        }
-
-        rule {
-          source_labels = ["__meta_kubernetes_pod_container_name"]
-          target_label = "container"
-        }
-
-        rule {
-          source_labels = ["__meta_kubernetes_node_name"]
-          target_label  = "node_name"
-        }
-
-        rule {
-          source_labels = [
-            "__meta_kubernetes_pod_controller_name",
-          ]
-          regex        = "([0-9a-z-.]+?)(-[0-9a-f]{8,10})?"
-          target_label = "controller_name"
-        }
-
-        rule {
-          source_labels = [
-            "__meta_kubernetes_pod_label_app_kubernetes_io_name",
-            "__meta_kubernetes_pod_label_app",
-            "controller_name",
-            "__meta_kubernetes_pod_name",
-          ]
-          regex        = "^;*([^;]+)(;.*)?$"
-          target_label = "app"
-        }
-
-        rule {
-          source_labels = [
-            "__meta_kubernetes_pod_label_app_kubernetes_io_component",
-            "__meta_kubernetes_pod_label_component",
-          ]
-          regex        = "^;*([^;]+)(;.*)?$"
-          target_label = "component"
-        }
-
-        rule {
-          source_labels = ["namespace", "app"]
-          separator     = "/"
-          target_label  = "job"
-        }
-
-        rule {
-          source_labels = ["job", "container"]
-          separator     = ":"
-          target_label  = "instance"
-        }
-        rule {
-          source_labels = ["__meta_kubernetes_pod_node_name"]
-          separator     = ":"
-          target_label  = "host"
-        }
-      }
-
-
-      // ============================================
-      // POD LOGS: SOURCE → PROCESS → WRITE
-      // ============================================
-
-      // Tail logs from Kubernetes pods
-      loki.source.kubernetes "pod_logs" {
-        // targets    = discovery.kubernetes.pod_logs.targets
-        targets    = discovery.relabel.pod_logs.output
-        forward_to = [loki.process.pod_logs.receiver]
-      }
-
-      // Add static labels and forward logs to Loki
-      loki.process "pod_logs" {
-
-        stage.static_labels {
-          values = {
-            cluster = env("CLUSTER_NAME"),
-            zone    = env("ZONE"),
-          }
-        }
-
-        forward_to = [loki.write.loki_srv.receiver]
-      }
-
-      // ============================================
-      // KUBERNETES EVENTS
-      // ============================================
-
-      // Tail Kubernetes events and convert them to log entries
-      loki.source.kubernetes_events "cluster_events" {
-        job_name   = "integrations/kubernetes/eventhandler"
-        log_format = "logfmt"
-        forward_to = [
-          loki.process.cluster_events.receiver,
-        ]
-      }
-
-      // Add static labels and forward events to Loki
-      loki.process "cluster_events" {
-        forward_to = [loki.write.loki_srv.receiver]
-
-        stage.static_labels {
-          values = {
-            cluster = env("CLUSTER_NAME"),
-            host    = "env("CLUSTER_NAME")",
-            job     = "cluster_events",
-            zone    = env("ZONE"),
-          }
-        }
-
-      }
-
-
-

services/monitoring/deploy/alloy/alloy-prometheus-examples.config

@@ -1,146 +0,0 @@
-      // ============================================
-      // Metrics: Discovery Kubernetes Services
-      // ============================================
-      
-      // discovery.kubernetes "k8s_services" {
-      //   role = "service"
-      // }
-
-      // ============================================
-      // Metrics: Kube State Metrics (k8s_services)
-      // ============================================
-
-      // prometheus.scrape "kube_state_metrics" {
-      //   job_name = "integrations/kubernetes/kube-state-metrics"
-      //   targets  = discovery.relabel.kube_state_metrics.output
-      // 
-      //   scheme = "http"
-      // 
-      //   forward_to = [prometheus.remote_write.prometheus_srv.receiver]
-      //  }
-      
-      // ============================================
-      // API Server discovery
-      // ============================================
-
-      // discovery.relabel "apiserver" {
-      //   targets = discovery.kubernetes.k8s_services.targets
-
-      //   rule {
-      //     source_labels = ["__meta_kubernetes_service_name"]
-      //     regex         = "kubernetes"
-      //     action        = "keep"
-      //   }
-      //   rule {
-      //     target_label = "cluster"
-      //     replacement  = env("CLUSTER_NAME")
-      //   }
-      // }
-
-      // prometheus.scrape "apiserver" {
-      //   job_name = "integrations/kubernetes/apiserver"
-      //   targets  = discovery.relabel.apiserver.output
-
-      //   scheme = "https"
-      //   metrics_path = "/metrics"
-
-      //   tls_config {
-      //     server_name          = "kubernetes"
-      //     ca_file              = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
-      //     insecure_skip_verify = true
-      //   }
-
-      //   bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
-
-      //   forward_to = [prometheus.remote_write.prometheus_srv.receiver]
-      // }
-
-      // ============================================
-      // CoreDNS discovery
-      // ============================================
-      // discovery.relabel "coredns" {
-      //   targets = discovery.kubernetes.k8s_services.targets
-      // 
-      //  rule {
-      //    source_labels = ["__meta_kubernetes_pod_label_k8s_app"]
-      //    regex         = "kube-dns"
-      //    action        = "keep"
-      //  }
-      //  rule {
-      //     target_label = "cluster"
-      //    replacement  = env("CLUSTER_NAME")
-      //   }
-      // }
-
-      // prometheus.scrape "coredns" {
-      //   job_name = "integrations/kubernetes/coredns-pods"
-      //   targets  = discovery.relabel.coredns.output
-
-      //   scheme = "http"
-      //   metrics_path = "/metrics"
-      // 
-      //   forward_to = [prometheus.remote_write.prometheus_srv.receiver]
-      // }
-
-
-      // ============================================
-      // Prometheus: prometheus_srv
-      // ============================================
-
-      prometheus.remote_write "prometheus_srv" {
-  
-        endpoint {
-          url = env("PROMETHEUS_URL")
-        }
-
-        // global labels
-        external_labels = {
-          cluster = env("CLUSTER_NAME"),
-          zone    = env("ZONE"),
-        }
-
-      }
-
-      // ===============================================================================
-      // prometheus / Metrics - Node exporter
-      // Requirements: 
-      //  - discovery.relabel - node_exporter
-      // ===============================================================================
-      prometheus.scrape "node_exporter" {
-        job_name   = "integrations/kubernetes/node-exporter"
-        targets    = discovery.relabel.node_exporter.output
-        scheme     = "http"
-        forward_to = [prometheus.remote_write.prometheus_srv.receiver]
-      }
-
-
-      discovery.relabel "node_exporter" {
-        targets = discovery.kubernetes.kube_pods.targets
-
-        rule {
-          source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
-          regex         = "node-exporter|prometheus-node-exporter"
-          action        = "keep"
-        }
-
-        rule {
-          source_labels = ["__meta_kubernetes_pod_container_port_number"]
-          regex         = "9100"
-          action        = "keep"
-        }
-
-        rule {
-          source_labels = ["__meta_kubernetes_pod_node_name"]
-          target_label  = "node"
-        }
-
-        rule {
-          source_labels = ["__meta_kubernetes_pod_node_name"]
-          target_label  = "nodename"
-        }
-
-        rule {
-          source_labels = ["__meta_kubernetes_pod_node_name"]
-          target_label  = "instance"
-        }
-      }

services/monitoring/deploy/alloy/kube-scraper/helm-values.yaml

@@ -27,6 +27,14 @@ alloy:
         role = "pod"
       }
 
+      // ============================================
+      // Discovery: Kubernetes - services
+      // used by:
+      // - loki kubernetes kube state metrics
+      // ============================================
+      discovery.kubernetes "kube_services" {
+        role = "service"
+      }
       
       // ============================================
       // Discovery: Kubernetes - endpoints
@@ -34,7 +42,7 @@ alloy:
       // - loki kubernetes kube state metrics
       // ============================================
       discovery.kubernetes "kube_endpoints" {
-        role = "endpoint"
+        role = "endpoints"
       }
 
       // ============================================
@@ -44,6 +52,10 @@ alloy:
         endpoint {
           url = env("PROMETHEUS_URL")
         }
+        external_labels = {
+          cluster = env("CLUSTER_NAME"),
+          zone    = env("ZONE"),
+        }
       }
 
       // ============================================
@@ -181,20 +193,14 @@ alloy:
         }
 
         rule {
-            source_labels = ["__meta_kubernetes_endpoint_node_name"]
+          target_label = "cluster"
-            target_label  = "node"
+          replacement  = env("CLUSTER_NAME")
-        }
-        
-        rule {
-          source_labels = ["__meta_kubernetes_endpoint_node_name"]
-          target_label = "host"
         }
 
         rule {
-          source_labels = ["__meta_kubernetes_endpoint_node_name"]
+          target_label = "zone"
-          target_label = "nodename"
+          replacement  = env("ZONE")
         }
-
       }
 
 
@@ -245,8 +251,110 @@ alloy:
           replacement  = "/metrics/cadvisor"
           target_label = "__metrics_path__"
         }
+
+        rule {
+          target_label = "cluster"
+          replacement  = env("CLUSTER_NAME")
+        }
+
+        rule {
+          target_label = "zone"
+          replacement  = env("ZONE")
+        }
       }
- 
+
+      // ============================================
+      // API Server discovery
+      // ============================================
+
+      discovery.relabel "apiserver" {
+        targets = discovery.kubernetes.kube_services.targets
+
+        rule {
+          source_labels = ["__meta_kubernetes_service_name"]
+          regex         = "kubernetes"
+          action        = "keep"
+        }
+     
+        rule {
+          target_label = "cluster"
+          replacement  = env("CLUSTER_NAME")
+        }
+
+        rule {
+          target_label = "zone"
+          replacement  = env("ZONE")
+        }
+      }
+
+      prometheus.scrape "apiserver" {
+        job_name = "integrations/kubernetes/apiserver"
+        targets  = discovery.relabel.apiserver.output
+
+        scheme = "https"
+        metrics_path = "/metrics"
+
+        tls_config {
+          server_name          = "kubernetes"
+          ca_file              = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+          insecure_skip_verify = true
+        }
+
+        bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+
+        forward_to = [prometheus.remote_write.prometheus_srv.receiver]
+      }
+      
+      // ============================================
+      // CoreDNS discovery (ENDPOINTS)
+      // ============================================
+
+      discovery.relabel "coredns" {
+        targets = discovery.kubernetes.kube_endpoints.targets
+
+        rule {
+          source_labels = ["__meta_kubernetes_service_name"]
+          regex         = "(kube-dns|coredns)"
+          action        = "keep"
+        }
+
+        rule {
+          source_labels = ["__meta_kubernetes_endpoint_node_name"]
+          target_label  = "node"
+        }
+
+        rule {
+          action        = "replace"
+          target_label  = "__address__"
+          replacement   = "$__meta_kubernetes_endpoint_address:9153"
+        }
+
+        rule {
+          target_label = "cluster"
+          replacement  = env("CLUSTER_NAME")
+        }
+
+        rule {
+          target_label = "zone"
+          replacement  = env("ZONE")
+        }
+
+      }
+
+      // ============================================
+      // CoreDNS scrape
+      // ============================================
+      prometheus.scrape "coredns" {
+        job_name = "integrations/kubernetes/coredns"
+        targets  = discovery.relabel.coredns.output
+
+        scheme = "http"
+        metrics_path = "/metrics"
+
+        forward_to = [prometheus.remote_write.prometheus_srv.receiver]
+      }
+
+
   extraEnv:
 
     - name: CLUSTER_NAME