monitoring: add Grafana alloy and Prometheus kube-state-metrics
This commit is contained in:
Márcio Fernandes
@@ -0,0 +1,145 @@
|
||||
logging {
|
||||
level = "warn"
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// LOKI WRITE TARGET
|
||||
// ============================================
|
||||
loki.write "loki_srv" {
|
||||
endpoint {
|
||||
url = sys.env("LOKI_URL")
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// HOST JOURNALD SOURCE
|
||||
// ============================================
|
||||
|
||||
// Read journald logs from the host
|
||||
loki.source.journal "journal" {
|
||||
max_age = "24h"
|
||||
relabel_rules = discovery.relabel.host_journal.rules
|
||||
forward_to = [loki.process.host_journal.receiver]
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// HOST JOURNALD RELABELING
|
||||
// ============================================
|
||||
|
||||
// Convert journald metadata into Loki labels
|
||||
discovery.relabel "host_journal" {
|
||||
targets = []
|
||||
|
||||
// Systemd unit name
|
||||
rule {
|
||||
source_labels = ["__journal__systemd_unit"]
|
||||
target_label = "systemd_unit"
|
||||
}
|
||||
|
||||
// Transport type (stdout, syslog, kernel, audit, etc.)
|
||||
rule {
|
||||
source_labels = ["__journal__transport"]
|
||||
target_label = "journal_transport"
|
||||
}
|
||||
|
||||
// Priority (info, warning, error)
|
||||
rule {
|
||||
source_labels = ["__journal_priority_keyword"]
|
||||
target_label = "level"
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// HOST JOURNALD PROCESSING
|
||||
// ============================================
|
||||
|
||||
// Add static labels and forward journald logs to Loki
|
||||
loki.process "host_journal" {
|
||||
stage.static_labels {
|
||||
values = {
|
||||
cluster = sys.env("CLUSTER_NAME"),
|
||||
host = sys.env("HOSTNAME"),
|
||||
zone = sys.env("ZONE"),
|
||||
job = "integrations/journald",
|
||||
}
|
||||
}
|
||||
|
||||
forward_to = [loki.write.loki_srv.receiver]
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
// ============================================
|
||||
// Metrics: Prometheus connection config
|
||||
// ============================================
|
||||
|
||||
prometheus.remote_write "metrics_service" {
|
||||
endpoint {
|
||||
url = sys.env("PROMETHEUS_URL")
|
||||
}
|
||||
|
||||
// global labels
|
||||
external_labels = {
|
||||
cluster = sys.env("CLUSTER_NAME"),
|
||||
zone = sys.env("ZONE"),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// ============================================
|
||||
// Metrics: Prometheus Unix Exporter
|
||||
// ============================================
|
||||
|
||||
prometheus.exporter.unix "node_exporter" {
|
||||
rootfs_path = "/host"
|
||||
disable_collectors = ["ipvs", "btrfs", "infiniband", "xfs", "zfs"]
|
||||
enable_collectors = [
|
||||
"cpu",
|
||||
"meminfo",
|
||||
"vmstat",
|
||||
"loadavg",
|
||||
"filesystem",
|
||||
"netdev",
|
||||
"uname",
|
||||
]
|
||||
|
||||
filesystem {
|
||||
mount_points_exclude = "^(/host|/)(etc/resolv.conf|etc/hosts|etc/hostname|etc/machine-id|dataDisks/nvme01-ephemeral/.+|proc|sys|run/containerd/.+|run/k3s/containerd/.+|var/lib/docker/.+|var/lib/kubelet/.+)($|/)"
|
||||
fs_types_exclude = "^(tmpfs|shm|autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs|erofs)$"
|
||||
mount_timeout = "5s"
|
||||
}
|
||||
|
||||
netclass {
|
||||
ignored_devices = "^(veth.*|cali.*|[a-f0-9]{15})$"
|
||||
}
|
||||
|
||||
netdev {
|
||||
device_exclude = "^(veth.*|cali.*|[a-f0-9]{15})$"
|
||||
}
|
||||
}
|
||||
|
||||
prometheus.scrape "node_exporter" {
|
||||
targets = prometheus.exporter.unix.node_exporter.targets
|
||||
scrape_interval = "30s"
|
||||
|
||||
forward_to = [prometheus.relabel.node_exporter.receiver]
|
||||
}
|
||||
|
||||
prometheus.relabel "node_exporter" {
|
||||
forward_to = [prometheus.remote_write.metrics_service.receiver]
|
||||
|
||||
rule {
|
||||
action = "replace"
|
||||
replacement = sys.env("HOSTNAME")
|
||||
target_label = "nodename"
|
||||
}
|
||||
|
||||
rule {
|
||||
action = "replace"
|
||||
replacement = sys.env("HOSTNAME")
|
||||
target_label = "node"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
apiVersion: helm.toolkit.fluxcd.io/v2
|
||||
kind: HelmRelease
|
||||
metadata:
|
||||
name: alloy-node-scraper
|
||||
spec:
|
||||
releaseName: alloy-node-scraper
|
||||
interval: 1m
|
||||
chart:
|
||||
spec:
|
||||
chart: alloy
|
||||
version: 1.x.x
|
||||
sourceRef:
|
||||
kind: HelmRepository
|
||||
name: grafana
|
||||
interval: 40h
|
||||
valuesFrom:
|
||||
- kind: ConfigMap
|
||||
name: alloy-node-scraper-helm-values
|
||||
valuesKey: values.yaml
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
controller:
|
||||
nameOverride: "alloy-node-scrapper"
|
||||
volumes:
|
||||
extra:
|
||||
# requires host journal configuration storage = volatile
|
||||
- name: run-log-journal
|
||||
hostPath:
|
||||
path: /run/log/journal
|
||||
# Log scrapers (e.g., Alloy/Loki) read this ID from the journal metadata and use it to separate log streams per machine.
|
||||
- name: etc-machine-id
|
||||
hostPath:
|
||||
path: /etc/machine-id
|
||||
- name: rootfs
|
||||
hostPath:
|
||||
path: /
|
||||
- name: proc
|
||||
hostPath:
|
||||
path: /proc
|
||||
- name: sys
|
||||
hostPath:
|
||||
path: /sys
|
||||
- name: dev
|
||||
hostPath:
|
||||
path: /dev
|
||||
|
||||
tolerations:
|
||||
- key: "node-role.kubernetes.io/control-plane"
|
||||
operator: "Exists"
|
||||
effect: "NoSchedule"
|
||||
- key: "infra.limbosolutions.com/dedicated"
|
||||
operator: "Exists"
|
||||
effect: "NoSchedule"
|
||||
|
||||
alloy:
|
||||
mounts:
|
||||
varlog: true # Mounts /var/log from the host (persistent journal)
|
||||
extra:
|
||||
# requires host journal configuration storage = volatile
|
||||
- name: run-log-journal
|
||||
mountPath: /run/log/journal
|
||||
readOnly: true
|
||||
# required
|
||||
- name: etc-machine-id
|
||||
mountPath: /etc/machine-id
|
||||
readOnly: true
|
||||
- name: rootfs
|
||||
mountPath: /host
|
||||
readOnly: true
|
||||
- name: proc
|
||||
mountPath: /host/proc
|
||||
readOnly: true
|
||||
- name: sys
|
||||
mountPath: /host/sys
|
||||
readOnly: true
|
||||
- name: dev
|
||||
mountPath: /host/dev
|
||||
readOnly: true
|
||||
# https://grafana.com/docs/alloy/latest/collect/logs-in-kubernetes/
|
||||
configMap:
|
||||
create: false
|
||||
name: alloy-node-scraper
|
||||
key: config.alloy
|
||||
extraEnv:
|
||||
- name: NODE_IP
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: status.hostIP
|
||||
- name: CLUSTER_NAME
|
||||
value: casa
|
||||
- name: JOB
|
||||
value: node-exporter
|
||||
- name: CONTAINER
|
||||
value: node-exporter
|
||||
- name: NS
|
||||
value: monitoring
|
||||
- name: POD
|
||||
value: alloy-node-scraper
|
||||
- name: SERVICE
|
||||
value: alloy-node-scraper
|
||||
- name: ZONE
|
||||
value: casa-vlan
|
||||
- name: LOKI_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: alloy-global
|
||||
key: loki_url
|
||||
- name: PROMETHEUS_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: alloy-global
|
||||
key: prometheus_url
|
||||
@@ -0,0 +1,14 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- helm-release.yaml
|
||||
|
||||
configMapGenerator:
|
||||
- name: alloy-node-scraper
|
||||
files:
|
||||
- config.alloy=config.alloy
|
||||
- name: alloy-node-scraper-helm-values
|
||||
files:
|
||||
- values.yaml=helm-values.yaml
|
||||
generatorOptions:
|
||||
disableNameSuffixHash: true
|
||||
Reference in New Issue
Block a user