From 629bde526b13303949344e75aaafbe4512de9c7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Fernandes?= Date: Sat, 5 Sep 2026 14:28:42 +0000 Subject: [PATCH] alloy; revision --- .../alloy/kube-scraper/helm-values.yaml | 320 ++++++++---------- .../alloy/node-scraper/helm-values.yaml | 88 ++--- 2 files changed, 159 insertions(+), 249 deletions(-) diff --git a/services/monitoring/deploy/alloy/kube-scraper/helm-values.yaml b/services/monitoring/deploy/alloy/kube-scraper/helm-values.yaml index d5b2dd4..e190f39 100644 --- a/services/monitoring/deploy/alloy/kube-scraper/helm-values.yaml +++ b/services/monitoring/deploy/alloy/kube-scraper/helm-values.yaml @@ -8,26 +8,83 @@ alloy: configMap: content: | // ============================================ - // LOKI WRITE TARGET + // Discovery: Kubernetes - Nodes + // used by: + // - prometheus - Kube_node_metrics - cAdvisor // ============================================ + + discovery.kubernetes "kube_nodes" { + role = "node" + } + + // ============================================ + // Discovery: Kubernetes - pods + // used by: + // - loki kubernetes pod logs + // ============================================ + + discovery.kubernetes "kube_pods" { + role = "pod" + } + + // ============================================ + // Discovery: Kubernetes - pods + // used by: + // - loki kubernetes pod logs + // ============================================ + + + // ============================================ + // Discovery: Kubernetes - pods + // used by: + // - loki kubernetes kube state metrics + // ============================================ + discovery.kubernetes "kube_services" { + role = "service" + } + + // ============================================ + // Logs: Loki config + // ============================================ + loki.write "loki_srv" { endpoint { url = env("LOKI_URL") } } - // ============================================ - // POD LOGS: DISCOVERY + RELABEL - // ============================================ - - // Discover Kubernetes pods - discovery.kubernetes "pod_logs" { - role = "pod" + loki.process "loki_srv" { + stage.static_labels { + values = { + cluster = env("CLUSTER_NAME"), + zone = env("ZONE"), + } + } + forward_to = [loki.write.loki_srv.receiver] } - discovery.relabel "pod_logs" { - targets = discovery.kubernetes.pod_logs.targets + // ============================================ + // Loki.source kube_pod_logs + // Requirements: + // - discovery.relabel: kube_pod_logs + // - discovery.process: loki_srv + // ============================================ + + loki.source.kubernetes "kube_pod_logs" { + targets = discovery.relabel.kube_pod_logs.output + forward_to = [loki.process.loki_srv.receiver] + } + + + // ============================================ + // Discovery:relabel: kube_pod_logs + // Requirements: + // - discovery.kubernetes.role.pod: kube_pods + // ============================================ + + discovery.relabel "kube_pod_logs" { + targets = discovery.kubernetes.kube_pods.targets rule { source_labels = ["__meta_kubernetes_namespace"] target_label = "namespace" @@ -98,75 +155,62 @@ alloy: // ============================================ - // POD LOGS: SOURCE → PROCESS → WRITE + // Prometheus: prometheus_srv // ============================================ - // Tail logs from Kubernetes pods - loki.source.kubernetes "pod_logs" { - // targets = discovery.kubernetes.pod_logs.targets - targets = discovery.relabel.pod_logs.output - forward_to = [loki.process.pod_logs.receiver] - } - - // Add static labels and forward logs to Loki - loki.process "pod_logs" { - - stage.static_labels { - values = { - cluster = env("CLUSTER_NAME"), - zone = env("ZONE"), - } - } - - forward_to = [loki.write.loki_srv.receiver] - } - - // ============================================ - // KUBERNETES EVENTS - // ============================================ - - // Tail Kubernetes events and convert them to log entries - loki.source.kubernetes_events "cluster_events" { - job_name = "integrations/kubernetes/eventhandler" - log_format = "logfmt" - forward_to = [ - loki.process.cluster_events.receiver, - ] - } - - // Add static labels and forward events to Loki - loki.process "cluster_events" { - forward_to = [loki.write.loki_srv.receiver] - - stage.static_labels { - values = { - cluster = env("CLUSTER_NAME"), - host = "env("CLUSTER_NAME")", - job = "cluster_events", - zone = env("ZONE"), - } - } - - } - - - // Metrics - prometheus.remote_write "prometheus_srv" { + endpoint { url = env("PROMETHEUS_URL") } + + // global labels + external_labels = { + cluster = env("CLUSTER_NAME"), + zone = env("ZONE"), + } + } - // discovery kubernetes + // =============================================================================== + // prometheus / Metrics - Scrape kube_state_metrics + // Requirements: + // - discovery.relabel - kube_node_metrics (cAdviser) + // =============================================================================== + prometheus.scrape "kube_state_metrics" { + job_name = "integrations/kubernetes/kube-state-metrics" + targets = discovery.relabel.kube_state_metrics.output - discovery.kubernetes "k8s_nodes" { - role = "node" + scheme = "http" + + forward_to = [prometheus.remote_write.prometheus_srv.receiver] } + discovery.relabel "kube_state_metrics" { + targets = discovery.kubernetes.kube_services.targets + + rule { + source_labels = ["__meta_kubernetes_service_name"] + regex = "kube-state-metrics" + action = "keep" + } + + rule { + source_labels = ["__meta_kubernetes_node_name"] + target_label = "node" + } + } + + + // =============================================================================== + // prometheus / Metrics - Scrape Kubernetes nodes / cAdvisor + // Requirements: + // - discovery.relabel - kube_node_metrics (cAdviser) + // =============================================================================== + prometheus.scrape "cadvisor" { job_name = "integrations/kubernetes/cadvisor" - targets = discovery.relabel.cadvisor.output + targets = discovery.relabel.kube_node_metrics.output scheme = "https" tls_config { @@ -178,129 +222,36 @@ alloy: bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" forward_to = [prometheus.remote_write.prometheus_srv.receiver] } + - - discovery.relabel "cadvisor" { - targets = discovery.kubernetes.k8s_nodes.targets + + + // =========================================================== + // Discovery:relabel: Kube_node_metrics (cAdviser) + // Requirements: + // - discovery.kubernetes.role.node: kube_nodes + // =========================================================== + + discovery.relabel "kube_node_metrics" { + targets = discovery.kubernetes.kube_nodes.targets + rule { + action = "replace" + source_labels = ["__meta_kubernetes_node_name"] + regex = "(.+)" + target_label = "node" + } rule { + source_labels = ["node"] + target_label = "host" + } + + rule { replacement = "/metrics/cadvisor" target_label = "__metrics_path__" } - - rule { - target_label = "instance" - replacement = env("NODE_IP") + ":10250" - } - rule { - source_labels = ["exported_node"] - target_label = "node" - } - - rule { - target_label = "cluster" - replacement = env("CLUSTER_NAME") - } - - } - - - discovery.kubernetes "k8s_services" { - role = "service" - } - - discovery.relabel "kube_state_metrics" { - targets = discovery.kubernetes.k8s_services.targets - - rule { - source_labels = ["__meta_kubernetes_service_name"] - regex = "kube-state-metrics" - action = "keep" - } - - rule { - source_labels = ["exported_node"] - target_label = "node" - } - - rule { - target_label = "cluster" - replacement = env("CLUSTER_NAME") - } - } - - prometheus.scrape "kube_state_metrics" { - job_name = "integrations/kubernetes/kube-state-metrics" - targets = discovery.relabel.kube_state_metrics.output - - scheme = "http" - - forward_to = [prometheus.remote_write.prometheus_srv.receiver] - } - - // ============================================ - // API Server discovery - // ============================================ - - discovery.relabel "apiserver" { - targets = discovery.kubernetes.k8s_services.targets - - rule { - source_labels = ["__meta_kubernetes_service_name"] - regex = "kubernetes" - action = "keep" - } - rule { - target_label = "cluster" - replacement = env("CLUSTER_NAME") - } - } - - prometheus.scrape "apiserver" { - job_name = "integrations/kubernetes/apiserver" - targets = discovery.relabel.apiserver.output - - scheme = "https" - metrics_path = "/metrics" - - tls_config { - server_name = "kubernetes" - ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - insecure_skip_verify = true - } - - bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" - - forward_to = [prometheus.remote_write.prometheus_srv.receiver] - } - - // ============================================ - // CoreDNS discovery - // ============================================ - discovery.relabel "coredns" { - targets = discovery.kubernetes.k8s_services.targets - - rule { - source_labels = ["__meta_kubernetes_pod_label_k8s_app"] - regex = "kube-dns" - action = "keep" - } - rule { - target_label = "cluster" - replacement = env("CLUSTER_NAME") - } - } - - prometheus.scrape "coredns" { - job_name = "integrations/kubernetes/coredns-pods" - targets = discovery.relabel.coredns.output - - scheme = "http" - metrics_path = "/metrics" - - forward_to = [prometheus.remote_write.prometheus_srv.receiver] - } + } extraEnv: @@ -308,10 +259,7 @@ alloy: valueFrom: fieldRef: fieldPath: status.hostIP - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName + - name: CLUSTER_NAME value: casa - name: ZONE diff --git a/services/monitoring/deploy/alloy/node-scraper/helm-values.yaml b/services/monitoring/deploy/alloy/node-scraper/helm-values.yaml index f3a07d4..86b9e2d 100644 --- a/services/monitoring/deploy/alloy/node-scraper/helm-values.yaml +++ b/services/monitoring/deploy/alloy/node-scraper/helm-values.yaml @@ -58,6 +58,7 @@ alloy: # https://grafana.com/docs/alloy/latest/collect/logs-in-kubernetes/ configMap: content: | + // ============================================ // LOKI WRITE TARGET // ============================================ @@ -123,9 +124,30 @@ alloy: forward_to = [loki.write.loki_srv.receiver] } + + // ============================================ - // prometheus unix exporter + // Metrics: Prometheus connection config // ============================================ + + prometheus.remote_write "metrics_service" { + endpoint { + url = env("PROMETHEUS_URL") + } + + // global labels + external_labels = { + cluster = env("CLUSTER_NAME"), + zone = env("ZONE"), + node = env("HOSTNAME") + } + } + + + // ============================================ + // Metrics: Prometheus Unix Exporter + // ============================================ + prometheus.exporter.unix "node_exporter" { rootfs_path = "/host" disable_collectors = ["ipvs", "btrfs", "infiniband", "xfs", "zfs"] @@ -153,78 +175,18 @@ alloy: device_exclude = "^(veth.*|cali.*|[a-f0-9]{15})$" } } - + prometheus.scrape "node_exporter" { targets = prometheus.exporter.unix.node_exporter.targets scrape_interval = "30s" forward_to = [prometheus.relabel.node_exporter.receiver] } - + prometheus.relabel "node_exporter" { - - rule { - target_label = "instance" - replacement = env("NODE_IP") + ":9100" - } - - rule { - target_label = "nodename" - replacement = env("HOSTNAME") - } - - rule { - target_label = "node" - replacement = env("HOSTNAME") - } - - rule { - target_label = "kubernetes_node" - replacement = env("HOSTNAME") - } - - rule { - target_label = "cluster" - replacement = env("CLUSTER_NAME") - } - - rule { - target_label = "job" - replacement = env("JOB") - } - - rule { - - target_label = "container" - replacement = env("CONTAINER") - } - rule { - - target_label = "NS" - replacement = env("NS") - } - - rule { - target_label = "pod" - replacement = env("POD") - } - - rule { - target_label = "service" - replacement = env("SERVICE") - } - forward_to = [prometheus.remote_write.metrics_service.receiver] - } - prometheus.remote_write "metrics_service" { - - endpoint { - url = env("PROMETHEUS_URL") - } - - } extraEnv: - name: NODE_IP