From 097bbad83fd114f765c928fe4db43d099f519a95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Fernandes?= Date: Sat, 12 Sep 2026 12:01:15 +0000 Subject: [PATCH] alloy: configMapGenerator setup config.alloy --- services/monitoring/README.md | 8 +- .../deploy/alloy/kube-scraper/config.alloy | 435 ++++++++++++++++++ .../alloy/kube-scraper/helm-values.yaml | 349 +------------- .../alloy/kube-scraper/kustomization.yaml | 6 + .../deploy/alloy/node-scraper/config.alloy | 145 ++++++ .../alloy/node-scraper/helm-values.yaml | 151 +----- .../alloy/node-scraper/kustomization.yaml | 6 + 7 files changed, 600 insertions(+), 500 deletions(-) create mode 100644 services/monitoring/deploy/alloy/kube-scraper/config.alloy create mode 100644 services/monitoring/deploy/alloy/node-scraper/config.alloy diff --git a/services/monitoring/README.md b/services/monitoring/README.md index 832345a..4748941 100644 --- a/services/monitoring/README.md +++ b/services/monitoring/README.md @@ -2,16 +2,12 @@ ## alloy -**promtail Encrypt secrets:** +**Encrypt global secrets:** ``` bash sops -e deploy/alloy/global/secrets.dec.yaml > deploy/alloy/global/secrets.yaml ``` -**Check agent:** - -``` bash -http://localhost:9090/agent -``` +[Official Helm chart default values.yaml](https://github.com/grafana/alloy/blob/main/operations/helm/charts/alloy/values.yaml) ## prometheus - kube-state-metrics diff --git a/services/monitoring/deploy/alloy/kube-scraper/config.alloy b/services/monitoring/deploy/alloy/kube-scraper/config.alloy new file mode 100644 index 0000000..26c2d5c --- /dev/null +++ b/services/monitoring/deploy/alloy/kube-scraper/config.alloy @@ -0,0 +1,435 @@ +logging { + level = "warn" +} + +// ============================================ +// Discovery: Kubernetes - Nodes +// used by: +// - prometheus - Kube_node_metrics - cAdvisor +// ============================================ + +discovery.kubernetes "kube_nodes" { + role = "node" +} + +// ============================================ +// Discovery: Kubernetes - pods +// ============================================ + +discovery.kubernetes "kube_pods" { + role = "pod" +} + +// ============================================ +// Discovery: Kubernetes - services +// used by: +// - loki kubernetes kube state metrics +// ============================================ +discovery.kubernetes "kube_services" { + role = "service" +} + +// ============================================ +// Discovery: Kubernetes - endpoints +// used by: +// - loki kubernetes kube state metrics +// ============================================ +discovery.kubernetes "kube_endpoints" { + role = "endpoints" +} + + +// ============================================ +// Logs: Loki config +// ============================================ + +loki.write "loki_srv" { + endpoint { + url = sys.env("LOKI_URL") + } +} + +loki.process "loki_srv" { + stage.static_labels { + values = { + cluster = sys.env("CLUSTER_NAME"), + zone = sys.env("ZONE"), + job = "integrations/kubernetes/pod-logs", + } + } + forward_to = [loki.write.loki_srv.receiver] +} + + +// ============================================ +// Loki.source kube_pod_logs +// Requirements: +// - discovery.relabel: kube_pod_logs +// - discovery.process: loki_srv +// ============================================ + +loki.source.kubernetes "kube_pod_logs" { + targets = discovery.relabel.kube_pod_logs.output + forward_to = [loki.process.loki_srv.receiver] +} + + +// ============================================ +// Discovery:relabel: kube_pod_logs +// Requirements: +// - discovery.kubernetes.role.pod: kube_pods +// ============================================ + +discovery.relabel "kube_pod_logs" { + targets = discovery.kubernetes.kube_pods.targets + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + + rule { + source_labels = ["__meta_kubernetes_pod_name"] + regex = "^(.+?)-[0-9a-zA-Z]{4,16}(?:-[0-9a-zA-Z]{4,16})?$" + replacement = "$1" + target_label = "pod" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + + rule { + source_labels = ["__meta_kubernetes_node_name"] + target_label = "node_name" + } + + rule { + source_labels = [ + "__meta_kubernetes_pod_controller_name", + ] + regex = "([0-9a-z-.]+?)(-[0-9a-f]{8,10})?" + target_label = "controller" + } + + rule { + source_labels = [ + "__meta_kubernetes_pod_label_app_kubernetes_io_name", + "__meta_kubernetes_pod_label_app", + "controller_name", + "__meta_kubernetes_pod_name", + ] + regex = "^;*([^;]+)(;.*)?$" + target_label = "app" + } + + rule { + source_labels = ["__meta_kubernetes_pod_node_name"] + separator = ":" + target_label = "host" + } +} + + +// ============================================ +// Metrics: Prometheus Remote Write +// ============================================ +prometheus.remote_write "prometheus_srv" { + endpoint { + url = sys.env("PROMETHEUS_URL") + } + external_labels = { + cluster = sys.env("CLUSTER_NAME"), + zone = sys.env("ZONE"), + } +} + +discovery.relabel "kubelet" { + targets = discovery.kubernetes.kube_nodes.targets + + rule { + regex = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$" + action = "drop" + } + + rule { + regex = "container_fs_.*|container_network_.*|container_hugetlb_.*|container_blkio_.*|container_tasks_state_.*" + action = "drop" + } + + + // duplicate metrics from cAdviser + rule { + regex = "container_cpu_usage_seconds_total|container_memory_working_set_bytes|container_memory_rss|container_memory_cache" + action = "drop" + } + + rule { + source_labels = ["__meta_kubernetes_node_address_InternalIP"] + target_label = "__address__" + replacement = "$1:10250" + } + + rule { + action = "replace" + source_labels = ["__meta_kubernetes_node_name"] + regex = "(.+)" + target_label = "node" + } + + rule { + source_labels = ["node"] + target_label = "host" + } + + + rule { + target_label = "__metrics_path__" + replacement = "/metrics" + } + + rule { + target_label = "cluster" + replacement = sys.env("CLUSTER_NAME") + } + + rule { + target_label = "zone" + replacement = sys.env("ZONE") + } +} + + +prometheus.scrape "kubelet" { + job_name = "integrations/kubernetes/kubelet" + targets = discovery.relabel.kubelet.output + + scheme = "https" + metrics_path = "/metrics" + + tls_config { + server_name = "kubernetes" + ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + insecure_skip_verify = true + } + + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + forward_to = [prometheus.remote_write.prometheus_srv.receiver] +} + +// =============================================================================== +// prometheus / Metrics - Scrape kube_state_metrics +// Requirements: +// - discovery.relabel - kube_state_metrics +// =============================================================================== +prometheus.scrape "kube_state_metrics" { + job_name = "integrations/kubernetes/kube-state-metrics" + targets = discovery.relabel.kube_state_metrics.output + + scheme = "http" + + forward_to = [prometheus.remote_write.prometheus_srv.receiver] +} + +discovery.relabel "kube_state_metrics" { + targets = discovery.kubernetes.kube_endpoints.targets + + rule { + source_labels = ["__meta_kubernetes_service_name"] + regex = "kube-state-metrics" + action = "keep" + } + + rule { + regex = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$" + action = "drop" + } + + rule { + target_label = "cluster" + replacement = sys.env("CLUSTER_NAME") + } + + rule { + target_label = "zone" + replacement = sys.env("ZONE") + } +} + + +// =============================================================================== +// Scrape Kubernetes pods containers metrics / cAdvisor +// Requirements: +// - discovery.relabel - kube_node_metrics +// =============================================================================== + +prometheus.scrape "cadvisor" { + job_name = "integrations/kubernetes/cadvisor" + targets = discovery.relabel.kube_node_metrics.output + scheme = "https" + + tls_config { + server_name = "kubernetes" + ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + insecure_skip_verify = true + } + + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + forward_to = [prometheus.remote_write.prometheus_srv.receiver] +} + + + +// =========================================================== +// Discovery:relabel: kube_node_metrics (cAdviser) +// Requirements: +// - discovery.kubernetes.role.node: kube_nodes +// =========================================================== + + discovery.relabel "kube_node_metrics" { + targets = discovery.kubernetes.kube_nodes.targets + + rule { + regex = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$" + action = "drop" + } + + rule { + action = "replace" + source_labels = ["__meta_kubernetes_node_name"] + regex = "(.+)" + target_label = "node" + } + + rule { + source_labels = ["node"] + target_label = "host" + } + + rule { + replacement = "/metrics/cadvisor" + target_label = "__metrics_path__" + } + + rule { + target_label = "cluster" + replacement = sys.env("CLUSTER_NAME") + } + + rule { + target_label = "zone" + replacement = sys.env("ZONE") + } +} + +// ============================================ +// API Server discovery +// ============================================ + +discovery.relabel "apiserver" { + targets = discovery.kubernetes.kube_services.targets + + rule { + source_labels = ["__meta_kubernetes_service_name"] + regex = "kubernetes" + action = "keep" + } + + rule { + regex = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$" + action = "drop" + } + + rule { + regex = "apiserver_flowcontrol_.*|apiserver_envelope_.*" + action = "drop" + } + + rule { + target_label = "cluster" + replacement = sys.env("CLUSTER_NAME") + } + + rule { + target_label = "zone" + replacement = sys.env("ZONE") + } +} + +prometheus.scrape "apiserver" { + job_name = "integrations/kubernetes/apiserver" + targets = discovery.relabel.apiserver.output + + scheme = "https" + metrics_path = "/metrics" + + tls_config { + server_name = "kubernetes" + ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + insecure_skip_verify = true + } + + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + forward_to = [prometheus.remote_write.prometheus_srv.receiver] +} + +// ============================================ +// CoreDNS discovery (ENDPOINTS) +// ============================================ + +discovery.relabel "coredns" { + targets = discovery.kubernetes.kube_endpoints.targets + + rule { + source_labels = ["__meta_kubernetes_service_name"] + regex = "(kube-dns|coredns)" + action = "keep" + } + + rule { + regex = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$" + action = "drop" + } + + rule { + regex = "coredns_cache_.*|coredns_forward_.*" + action = "drop" + } + + rule { + source_labels = ["__meta_kubernetes_endpoint_node_name"] + target_label = "node" + } + + rule { + action = "replace" + target_label = "__address__" + replacement = "$__meta_kubernetes_endpoint_address:9153" + } + + rule { + target_label = "cluster" + replacement = sys.env("CLUSTER_NAME") + } + + rule { + target_label = "zone" + replacement = sys.env("ZONE") + } + +} + +// ============================================ +// CoreDNS scrape +// ============================================ +prometheus.scrape "coredns" { + job_name = "integrations/kubernetes/coredns" + targets = discovery.relabel.coredns.output + + scheme = "http" + metrics_path = "/metrics" + + forward_to = [prometheus.remote_write.prometheus_srv.receiver] +} \ No newline at end of file diff --git a/services/monitoring/deploy/alloy/kube-scraper/helm-values.yaml b/services/monitoring/deploy/alloy/kube-scraper/helm-values.yaml index d39f5ee..75c21f6 100644 --- a/services/monitoring/deploy/alloy/kube-scraper/helm-values.yaml +++ b/services/monitoring/deploy/alloy/kube-scraper/helm-values.yaml @@ -6,352 +6,9 @@ alloy: # https://grafana.com/docs/alloy/latest/collect/logs-in-kubernetes/ configMap: - content: | - logging { - level = "warn" - } - - // ============================================ - // Discovery: Kubernetes - Nodes - // used by: - // - prometheus - Kube_node_metrics - cAdvisor - // ============================================ - - discovery.kubernetes "kube_nodes" { - role = "node" - } - - // ============================================ - // Discovery: Kubernetes - pods - // used by: - // - loki kubernetes pod logs - // ============================================ - - discovery.kubernetes "kube_pods" { - role = "pod" - } - - // ============================================ - // Discovery: Kubernetes - services - // used by: - // - loki kubernetes kube state metrics - // ============================================ - discovery.kubernetes "kube_services" { - role = "service" - } - - // ============================================ - // Discovery: Kubernetes - endpoints - // used by: - // - loki kubernetes kube state metrics - // ============================================ - discovery.kubernetes "kube_endpoints" { - role = "endpoints" - } - - // ============================================ - // Metrics: Prometheus Remote Write - // ============================================ - prometheus.remote_write "prometheus_srv" { - endpoint { - url = sys.env("PROMETHEUS_URL") - } - external_labels = { - cluster = sys.env("CLUSTER_NAME"), - zone = sys.env("ZONE"), - } - } - - discovery.relabel "kubelet" { - targets = discovery.kubernetes.kube_nodes.targets - - rule { - regex = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$" - action = "drop" - } - - rule { - regex = "container_fs_.*|container_network_.*|container_hugetlb_.*|container_blkio_.*|container_tasks_state_.*" - action = "drop" - } - - - // duplicate metrics from cAdviser - rule { - regex = "container_cpu_usage_seconds_total|container_memory_working_set_bytes|container_memory_rss|container_memory_cache" - action = "drop" - } - - rule { - source_labels = ["__meta_kubernetes_node_address_InternalIP"] - target_label = "__address__" - replacement = "$1:10250" - } - - rule { - action = "replace" - source_labels = ["__meta_kubernetes_node_name"] - regex = "(.+)" - target_label = "node" - } - - rule { - source_labels = ["node"] - target_label = "host" - } - - - rule { - target_label = "__metrics_path__" - replacement = "/metrics" - } - - rule { - target_label = "cluster" - replacement = sys.env("CLUSTER_NAME") - } - - rule { - target_label = "zone" - replacement = sys.env("ZONE") - } - } - - - prometheus.scrape "kubelet" { - job_name = "integrations/kubernetes/kubelet" - targets = discovery.relabel.kubelet.output - - scheme = "https" - metrics_path = "/metrics" - - tls_config { - server_name = "kubernetes" - ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - insecure_skip_verify = true - } - - bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" - - forward_to = [prometheus.remote_write.prometheus_srv.receiver] - } - - // =============================================================================== - // prometheus / Metrics - Scrape kube_state_metrics - // Requirements: - // - discovery.relabel - kube_state_metrics - // =============================================================================== - prometheus.scrape "kube_state_metrics" { - job_name = "integrations/kubernetes/kube-state-metrics" - targets = discovery.relabel.kube_state_metrics.output - - scheme = "http" - - forward_to = [prometheus.remote_write.prometheus_srv.receiver] - } - - discovery.relabel "kube_state_metrics" { - targets = discovery.kubernetes.kube_endpoints.targets - - rule { - source_labels = ["__meta_kubernetes_service_name"] - regex = "kube-state-metrics" - action = "keep" - } - - rule { - regex = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$" - action = "drop" - } - - rule { - target_label = "cluster" - replacement = sys.env("CLUSTER_NAME") - } - - rule { - target_label = "zone" - replacement = sys.env("ZONE") - } - } - - - // =============================================================================== - // Scrape Kubernetes pods containers metrics / cAdvisor - // Requirements: - // - discovery.relabel - kube_node_metrics - // =============================================================================== - - prometheus.scrape "cadvisor" { - job_name = "integrations/kubernetes/cadvisor" - targets = discovery.relabel.kube_node_metrics.output - scheme = "https" - - tls_config { - server_name = "kubernetes" - ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - insecure_skip_verify = true - } - - bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" - forward_to = [prometheus.remote_write.prometheus_srv.receiver] - } - - - - // =========================================================== - // Discovery:relabel: kube_node_metrics (cAdviser) - // Requirements: - // - discovery.kubernetes.role.node: kube_nodes - // =========================================================== - - discovery.relabel "kube_node_metrics" { - targets = discovery.kubernetes.kube_nodes.targets - - rule { - regex = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$" - action = "drop" - } - - rule { - action = "replace" - source_labels = ["__meta_kubernetes_node_name"] - regex = "(.+)" - target_label = "node" - } - - rule { - source_labels = ["node"] - target_label = "host" - } - - rule { - replacement = "/metrics/cadvisor" - target_label = "__metrics_path__" - } - - rule { - target_label = "cluster" - replacement = sys.env("CLUSTER_NAME") - } - - rule { - target_label = "zone" - replacement = sys.env("ZONE") - } - } - - // ============================================ - // API Server discovery - // ============================================ - - discovery.relabel "apiserver" { - targets = discovery.kubernetes.kube_services.targets - - rule { - source_labels = ["__meta_kubernetes_service_name"] - regex = "kubernetes" - action = "keep" - } - - rule { - regex = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$" - action = "drop" - } - - rule { - regex = "apiserver_flowcontrol_.*|apiserver_envelope_.*" - action = "drop" - } - - rule { - target_label = "cluster" - replacement = sys.env("CLUSTER_NAME") - } - - rule { - target_label = "zone" - replacement = sys.env("ZONE") - } - } - - prometheus.scrape "apiserver" { - job_name = "integrations/kubernetes/apiserver" - targets = discovery.relabel.apiserver.output - - scheme = "https" - metrics_path = "/metrics" - - tls_config { - server_name = "kubernetes" - ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - insecure_skip_verify = true - } - - bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" - - forward_to = [prometheus.remote_write.prometheus_srv.receiver] - } - - // ============================================ - // CoreDNS discovery (ENDPOINTS) - // ============================================ - - discovery.relabel "coredns" { - targets = discovery.kubernetes.kube_endpoints.targets - - rule { - source_labels = ["__meta_kubernetes_service_name"] - regex = "(kube-dns|coredns)" - action = "keep" - } - - rule { - regex = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$" - action = "drop" - } - - rule { - regex = "coredns_cache_.*|coredns_forward_.*" - action = "drop" - } - - rule { - source_labels = ["__meta_kubernetes_endpoint_node_name"] - target_label = "node" - } - - rule { - action = "replace" - target_label = "__address__" - replacement = "$__meta_kubernetes_endpoint_address:9153" - } - - rule { - target_label = "cluster" - replacement = sys.env("CLUSTER_NAME") - } - - rule { - target_label = "zone" - replacement = sys.env("ZONE") - } - - } - - // ============================================ - // CoreDNS scrape - // ============================================ - prometheus.scrape "coredns" { - job_name = "integrations/kubernetes/coredns" - targets = discovery.relabel.coredns.output - - scheme = "http" - metrics_path = "/metrics" - - forward_to = [prometheus.remote_write.prometheus_srv.receiver] - } - + create: false + name: alloy-kube-scraper + key: config.alloy extraEnv: diff --git a/services/monitoring/deploy/alloy/kube-scraper/kustomization.yaml b/services/monitoring/deploy/alloy/kube-scraper/kustomization.yaml index 5298a73..94bec25 100644 --- a/services/monitoring/deploy/alloy/kube-scraper/kustomization.yaml +++ b/services/monitoring/deploy/alloy/kube-scraper/kustomization.yaml @@ -3,6 +3,12 @@ kind: Kustomization namespace: monitoring resources: - helm-release.yaml + +configMapGenerator: + - name: alloy-kube-scraper + files: + - config.alloy=config.alloy + secretGenerator: - name: alloy-kube-scraper-helm-values files: diff --git a/services/monitoring/deploy/alloy/node-scraper/config.alloy b/services/monitoring/deploy/alloy/node-scraper/config.alloy new file mode 100644 index 0000000..52b93c2 --- /dev/null +++ b/services/monitoring/deploy/alloy/node-scraper/config.alloy @@ -0,0 +1,145 @@ +logging { + level = "warn" +} + +// ============================================ +// LOKI WRITE TARGET +// ============================================ +loki.write "loki_srv" { + endpoint { + url = sys.env("LOKI_URL") + } +} + +// ============================================ +// HOST JOURNALD SOURCE +// ============================================ + +// Read journald logs from the host +loki.source.journal "journal" { + max_age = "24h" + relabel_rules = discovery.relabel.host_journal.rules + forward_to = [loki.process.host_journal.receiver] +} + +// ============================================ +// HOST JOURNALD RELABELING +// ============================================ + +// Convert journald metadata into Loki labels +discovery.relabel "host_journal" { + targets = [] + + // Systemd unit name + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "systemd_unit" + } + + // Transport type (stdout, syslog, kernel, audit, etc.) + rule { + source_labels = ["__journal__transport"] + target_label = "journal_transport" + } + + // Priority (info, warning, error) + rule { + source_labels = ["__journal_priority_keyword"] + target_label = "level" + } +} + +// ============================================ +// HOST JOURNALD PROCESSING +// ============================================ + +// Add static labels and forward journald logs to Loki +loki.process "host_journal" { + stage.static_labels { + values = { + cluster = sys.env("CLUSTER_NAME"), + host = sys.env("HOSTNAME"), + zone = sys.env("ZONE"), + job = "integrations/journald", + } + } + + forward_to = [loki.write.loki_srv.receiver] +} + + + + +// ============================================ +// Metrics: Prometheus connection config +// ============================================ + +prometheus.remote_write "metrics_service" { + endpoint { + url = sys.env("PROMETHEUS_URL") + } + + // global labels + external_labels = { + cluster = sys.env("CLUSTER_NAME"), + zone = sys.env("ZONE"), + } +} + + +// ============================================ +// Metrics: Prometheus Unix Exporter +// ============================================ + +prometheus.exporter.unix "node_exporter" { + rootfs_path = "/host" + disable_collectors = ["ipvs", "btrfs", "infiniband", "xfs", "zfs"] + enable_collectors = [ + "cpu", + "meminfo", + "vmstat", + "loadavg", + "filesystem", + "netdev", + "uname", + ] + + filesystem { + mount_points_exclude = "^(/host|/)(etc/resolv.conf|etc/hosts|etc/hostname|etc/machine-id|dataDisks/nvme01-ephemeral/.+|proc|sys|run/containerd/.+|run/k3s/containerd/.+|var/lib/docker/.+|var/lib/kubelet/.+)($|/)" + fs_types_exclude = "^(tmpfs|shm|autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs|erofs)$" + mount_timeout = "5s" + } + + netclass { + ignored_devices = "^(veth.*|cali.*|[a-f0-9]{15})$" + } + + netdev { + device_exclude = "^(veth.*|cali.*|[a-f0-9]{15})$" + } +} + +prometheus.scrape "node_exporter" { + targets = prometheus.exporter.unix.node_exporter.targets + scrape_interval = "30s" + + forward_to = [prometheus.relabel.node_exporter.receiver] +} + +prometheus.relabel "node_exporter" { + forward_to = [prometheus.remote_write.metrics_service.receiver] + + rule { + action = "replace" + replacement = sys.env("HOSTNAME") + target_label = "nodename" + } + + rule { + action = "replace" + replacement = sys.env("HOSTNAME") + target_label = "node" + } +} + + diff --git a/services/monitoring/deploy/alloy/node-scraper/helm-values.yaml b/services/monitoring/deploy/alloy/node-scraper/helm-values.yaml index 77c0f24..e13c9ca 100644 --- a/services/monitoring/deploy/alloy/node-scraper/helm-values.yaml +++ b/services/monitoring/deploy/alloy/node-scraper/helm-values.yaml @@ -57,154 +57,9 @@ alloy: readOnly: true # https://grafana.com/docs/alloy/latest/collect/logs-in-kubernetes/ configMap: - content: | - logging { - level = "warn" - } - - // ============================================ - // LOKI WRITE TARGET - // ============================================ - loki.write "loki_srv" { - endpoint { - url = sys.env("LOKI_URL") - } - } - - // ============================================ - // HOST JOURNALD SOURCE - // ============================================ - - // Read journald logs from the host - loki.source.journal "journal" { - max_age = "24h" - relabel_rules = discovery.relabel.host_journal.rules - forward_to = [loki.process.host_journal.receiver] - } - - // ============================================ - // HOST JOURNALD RELABELING - // ============================================ - - // Convert journald metadata into Loki labels - discovery.relabel "host_journal" { - targets = [] - - // Systemd unit name - rule { - source_labels = ["__journal__systemd_unit"] - target_label = "systemd_unit" - } - - // Transport type (stdout, syslog, kernel, audit, etc.) - rule { - source_labels = ["__journal__transport"] - target_label = "journal_transport" - } - - // Priority (info, warning, error) - rule { - source_labels = ["__journal_priority_keyword"] - target_label = "level" - } - } - - // ============================================ - // HOST JOURNALD PROCESSING - // ============================================ - - // Add static labels and forward journald logs to Loki - loki.process "host_journal" { - stage.static_labels { - values = { - cluster = sys.env("CLUSTER_NAME"), - host = sys.env("HOSTNAME"), - zone = sys.env("ZONE"), - job = "journald", - service_name = "journald", - } - } - - forward_to = [loki.write.loki_srv.receiver] - } - - - - - // ============================================ - // Metrics: Prometheus connection config - // ============================================ - - prometheus.remote_write "metrics_service" { - endpoint { - url = sys.env("PROMETHEUS_URL") - } - - // global labels - external_labels = { - cluster = sys.env("CLUSTER_NAME"), - zone = sys.env("ZONE"), - } - } - - - // ============================================ - // Metrics: Prometheus Unix Exporter - // ============================================ - - prometheus.exporter.unix "node_exporter" { - rootfs_path = "/host" - disable_collectors = ["ipvs", "btrfs", "infiniband", "xfs", "zfs"] - enable_collectors = [ - "cpu", - "meminfo", - "vmstat", - "loadavg", - "filesystem", - "netdev", - "uname", - ] - - filesystem { - mount_points_exclude = "^(/host|/)(etc/resolv.conf|etc/hosts|etc/hostname|etc/machine-id|dataDisks/nvme01-ephemeral/.+|proc|sys|run/containerd/.+|run/k3s/containerd/.+|var/lib/docker/.+|var/lib/kubelet/.+)($|/)" - fs_types_exclude = "^(tmpfs|shm|autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs|erofs)$" - mount_timeout = "5s" - } - - netclass { - ignored_devices = "^(veth.*|cali.*|[a-f0-9]{15})$" - } - - netdev { - device_exclude = "^(veth.*|cali.*|[a-f0-9]{15})$" - } - } - - prometheus.scrape "node_exporter" { - targets = prometheus.exporter.unix.node_exporter.targets - scrape_interval = "30s" - - forward_to = [prometheus.relabel.node_exporter.receiver] - } - - prometheus.relabel "node_exporter" { - forward_to = [prometheus.remote_write.metrics_service.receiver] - - rule { - action = "replace" - replacement = sys.env("HOSTNAME") - target_label = "nodename" - } - - rule { - action = "replace" - replacement = sys.env("HOSTNAME") - target_label = "node" - } - } - - - + create: false + name: alloy-node-scraper + key: config.alloy extraEnv: - name: NODE_IP valueFrom: diff --git a/services/monitoring/deploy/alloy/node-scraper/kustomization.yaml b/services/monitoring/deploy/alloy/node-scraper/kustomization.yaml index f445d22..5c1a09b 100644 --- a/services/monitoring/deploy/alloy/node-scraper/kustomization.yaml +++ b/services/monitoring/deploy/alloy/node-scraper/kustomization.yaml @@ -3,6 +3,12 @@ kind: Kustomization namespace: monitoring resources: - helm-release.yaml + +configMapGenerator: + - name: alloy-node-scraper + files: + - config.alloy=config.alloy + secretGenerator: - name: alloy-node-scraper-helm-values files: