diff --git a/services/monitoring/deploy/alloy/node-scraper/helm-values.yaml b/services/monitoring/deploy/alloy/node-scraper/helm-values.yaml index 5fa1a9b..208ab1e 100644 --- a/services/monitoring/deploy/alloy/node-scraper/helm-values.yaml +++ b/services/monitoring/deploy/alloy/node-scraper/helm-values.yaml @@ -10,7 +10,19 @@ controller: - name: etc-machine-id hostPath: path: /etc/machine-id - + - name: rootfs + hostPath: + path: / + - name: proc + hostPath: + path: /proc + - name: sys + hostPath: + path: /sys + - name: dev + hostPath: + path: /dev + tolerations: - key: "node-role.kubernetes.io/control-plane" operator: "Exists" @@ -31,7 +43,18 @@ alloy: - name: etc-machine-id mountPath: /etc/machine-id readOnly: true - + - name: rootfs + mountPath: /host + readOnly: true + - name: proc + mountPath: /host/proc + readOnly: true + - name: sys + mountPath: /host/sys + readOnly: true + - name: dev + mountPath: /host/dev + readOnly: true # https://grafana.com/docs/alloy/latest/collect/logs-in-kubernetes/ configMap: content: | @@ -100,6 +123,108 @@ alloy: forward_to = [loki.write.loki_srv.receiver] } + // ============================================ + // prometheus unix exporter + // ============================================ + prometheus.exporter.unix "node_exporter" { + rootfs_path = "/host" + disable_collectors = ["ipvs", "btrfs", "infiniband", "xfs", "zfs"] + enable_collectors = [ + "cpu", + "meminfo", + "vmstat", + "loadavg", + "filesystem", + "netdev", + "uname", + ] + + filesystem { + mount_points_exclude = "^/(dev|proc|sys|run/containerd/.+|var/lib/docker/.+|var/lib/kubelet/.+)($|/)" + fs_types_exclude = "^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs|erofs)$" + mount_timeout = "5s" + } + + netclass { + ignored_devices = "^(veth.*|cali.*|[a-f0-9]{15})$" + } + + netdev { + device_exclude = "^(veth.*|cali.*|[a-f0-9]{15})$" + } + } + + prometheus.scrape "node_exporter" { + targets = prometheus.exporter.unix.node_exporter.targets + scrape_interval = "30s" + + forward_to = [prometheus.relabel.node_exporter.receiver] + } + + prometheus.relabel "node_exporter" { + + rule { + target_label = "instance" + replacement = env("NODE_IP") + ":9100" + } + + rule { + target_label = "nodename" + replacement = env("HOSTNAME") + } + + rule { + target_label = "node" + replacement = env("HOSTNAME") + } + + rule { + target_label = "kubernetes_node" + replacement = env("HOSTNAME") + } + + rule { + target_label = "cluster" + replacement = env("CLUSTER_NAME") + } + + rule { + target_label = "job" + replacement = env("JOB") + } + + rule { + + target_label = "container" + replacement = env("CONTAINER") + } + rule { + + target_label = "NS" + replacement = env("NS") + } + + rule { + target_label = "pod" + replacement = env("POD") + } + + rule { + target_label = "service" + replacement = env("SERVICE") + } + + forward_to = [prometheus.remote_write.metrics_service.receiver] + + } + + prometheus.remote_write "metrics_service" { + + endpoint { + url = "https://prometheus.monitoring.limbosolutions.com/api/v1/write" + } + + } extraEnv: - name: NODE_NAME @@ -108,6 +233,16 @@ alloy: fieldPath: spec.nodeName - name: CLUSTER_NAME value: casa + - name: JOB + value: node-exporter + - name: CONTAINER + value: node-exporter + - name: NS + value: monitoring + - name: POD + value: alloy-node-scraper + - name: SERVICE + value: alloy-node-scraper - name: ZONE value: casa-vlan - name: LOKI_URL diff --git a/services/monitoring/deploy/prometheus/helm-values.yaml b/services/monitoring/deploy/prometheus/helm-values.yaml index ed8868a..c22480e 100644 --- a/services/monitoring/deploy/prometheus/helm-values.yaml +++ b/services/monitoring/deploy/prometheus/helm-values.yaml @@ -15,21 +15,12 @@ kubeStateMetrics: enabled: true nodeExporter: - enabled: true + enabled: false prometheus-node-exporter: prometheus: monitor: - enabled: true - relabelings: - # https://github.com/dotdc/grafana-dashboards-kubernetes - - action: replace - sourceLabels: [__meta_kubernetes_pod_node_name] - targetLabel: nodename - - targetLabel: cluster - replacement: casa - - + enabled: false coreDns: enabled: true