This commit is contained in:
Márcio Fernandes
2026-09-12 14:11:02 +00:00
parent 097bbad83f
commit 28022eb5f3
21 changed files with 6 additions and 875 deletions
@@ -3,6 +3,5 @@ kind: Kustomization
namespace: monitoring
resources:
- secrets.yaml
- grafana-helm-repo.yaml
generatorOptions:
disableNameSuffixHash: true
@@ -1,8 +0,0 @@
apiVersion: source.toolkit.fluxcd.io/v1
kind: HelmRepository
metadata:
name: grafana
spec:
interval: 24h
url: https://grafana.github.io/helm-charts
@@ -1,435 +0,0 @@
logging {
level = "warn"
}
// ============================================
// Discovery: Kubernetes - Nodes
// used by:
// - prometheus - Kube_node_metrics - cAdvisor
// ============================================
discovery.kubernetes "kube_nodes" {
role = "node"
}
// ============================================
// Discovery: Kubernetes - pods
// ============================================
discovery.kubernetes "kube_pods" {
role = "pod"
}
// ============================================
// Discovery: Kubernetes - services
// used by:
// - loki kubernetes kube state metrics
// ============================================
discovery.kubernetes "kube_services" {
role = "service"
}
// ============================================
// Discovery: Kubernetes - endpoints
// used by:
// - loki kubernetes kube state metrics
// ============================================
discovery.kubernetes "kube_endpoints" {
role = "endpoints"
}
// ============================================
// Logs: Loki config
// ============================================
loki.write "loki_srv" {
endpoint {
url = sys.env("LOKI_URL")
}
}
loki.process "loki_srv" {
stage.static_labels {
values = {
cluster = sys.env("CLUSTER_NAME"),
zone = sys.env("ZONE"),
job = "integrations/kubernetes/pod-logs",
}
}
forward_to = [loki.write.loki_srv.receiver]
}
// ============================================
// Loki.source kube_pod_logs
// Requirements:
// - discovery.relabel: kube_pod_logs
// - discovery.process: loki_srv
// ============================================
loki.source.kubernetes "kube_pod_logs" {
targets = discovery.relabel.kube_pod_logs.output
forward_to = [loki.process.loki_srv.receiver]
}
// ============================================
// Discovery:relabel: kube_pod_logs
// Requirements:
// - discovery.kubernetes.role.pod: kube_pods
// ============================================
discovery.relabel "kube_pod_logs" {
targets = discovery.kubernetes.kube_pods.targets
rule {
source_labels = ["__meta_kubernetes_namespace"]
target_label = "namespace"
}
rule {
source_labels = ["__meta_kubernetes_pod_name"]
regex = "^(.+?)-[0-9a-zA-Z]{4,16}(?:-[0-9a-zA-Z]{4,16})?$"
replacement = "$1"
target_label = "pod"
}
rule {
source_labels = ["__meta_kubernetes_pod_container_name"]
target_label = "container"
}
rule {
source_labels = ["__meta_kubernetes_node_name"]
target_label = "node_name"
}
rule {
source_labels = [
"__meta_kubernetes_pod_controller_name",
]
regex = "([0-9a-z-.]+?)(-[0-9a-f]{8,10})?"
target_label = "controller"
}
rule {
source_labels = [
"__meta_kubernetes_pod_label_app_kubernetes_io_name",
"__meta_kubernetes_pod_label_app",
"controller_name",
"__meta_kubernetes_pod_name",
]
regex = "^;*([^;]+)(;.*)?$"
target_label = "app"
}
rule {
source_labels = ["__meta_kubernetes_pod_node_name"]
separator = ":"
target_label = "host"
}
}
// ============================================
// Metrics: Prometheus Remote Write
// ============================================
prometheus.remote_write "prometheus_srv" {
endpoint {
url = sys.env("PROMETHEUS_URL")
}
external_labels = {
cluster = sys.env("CLUSTER_NAME"),
zone = sys.env("ZONE"),
}
}
discovery.relabel "kubelet" {
targets = discovery.kubernetes.kube_nodes.targets
rule {
regex = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$"
action = "drop"
}
rule {
regex = "container_fs_.*|container_network_.*|container_hugetlb_.*|container_blkio_.*|container_tasks_state_.*"
action = "drop"
}
// duplicate metrics from cAdviser
rule {
regex = "container_cpu_usage_seconds_total|container_memory_working_set_bytes|container_memory_rss|container_memory_cache"
action = "drop"
}
rule {
source_labels = ["__meta_kubernetes_node_address_InternalIP"]
target_label = "__address__"
replacement = "$1:10250"
}
rule {
action = "replace"
source_labels = ["__meta_kubernetes_node_name"]
regex = "(.+)"
target_label = "node"
}
rule {
source_labels = ["node"]
target_label = "host"
}
rule {
target_label = "__metrics_path__"
replacement = "/metrics"
}
rule {
target_label = "cluster"
replacement = sys.env("CLUSTER_NAME")
}
rule {
target_label = "zone"
replacement = sys.env("ZONE")
}
}
prometheus.scrape "kubelet" {
job_name = "integrations/kubernetes/kubelet"
targets = discovery.relabel.kubelet.output
scheme = "https"
metrics_path = "/metrics"
tls_config {
server_name = "kubernetes"
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
insecure_skip_verify = true
}
bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
forward_to = [prometheus.remote_write.prometheus_srv.receiver]
}
// ===============================================================================
// prometheus / Metrics - Scrape kube_state_metrics
// Requirements:
// - discovery.relabel - kube_state_metrics
// ===============================================================================
prometheus.scrape "kube_state_metrics" {
job_name = "integrations/kubernetes/kube-state-metrics"
targets = discovery.relabel.kube_state_metrics.output
scheme = "http"
forward_to = [prometheus.remote_write.prometheus_srv.receiver]
}
discovery.relabel "kube_state_metrics" {
targets = discovery.kubernetes.kube_endpoints.targets
rule {
source_labels = ["__meta_kubernetes_service_name"]
regex = "kube-state-metrics"
action = "keep"
}
rule {
regex = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$"
action = "drop"
}
rule {
target_label = "cluster"
replacement = sys.env("CLUSTER_NAME")
}
rule {
target_label = "zone"
replacement = sys.env("ZONE")
}
}
// ===============================================================================
// Scrape Kubernetes pods containers metrics / cAdvisor
// Requirements:
// - discovery.relabel - kube_node_metrics
// ===============================================================================
prometheus.scrape "cadvisor" {
job_name = "integrations/kubernetes/cadvisor"
targets = discovery.relabel.kube_node_metrics.output
scheme = "https"
tls_config {
server_name = "kubernetes"
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
insecure_skip_verify = true
}
bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
forward_to = [prometheus.remote_write.prometheus_srv.receiver]
}
// ===========================================================
// Discovery:relabel: kube_node_metrics (cAdviser)
// Requirements:
// - discovery.kubernetes.role.node: kube_nodes
// ===========================================================
discovery.relabel "kube_node_metrics" {
targets = discovery.kubernetes.kube_nodes.targets
rule {
regex = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$"
action = "drop"
}
rule {
action = "replace"
source_labels = ["__meta_kubernetes_node_name"]
regex = "(.+)"
target_label = "node"
}
rule {
source_labels = ["node"]
target_label = "host"
}
rule {
replacement = "/metrics/cadvisor"
target_label = "__metrics_path__"
}
rule {
target_label = "cluster"
replacement = sys.env("CLUSTER_NAME")
}
rule {
target_label = "zone"
replacement = sys.env("ZONE")
}
}
// ============================================
// API Server discovery
// ============================================
discovery.relabel "apiserver" {
targets = discovery.kubernetes.kube_services.targets
rule {
source_labels = ["__meta_kubernetes_service_name"]
regex = "kubernetes"
action = "keep"
}
rule {
regex = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$"
action = "drop"
}
rule {
regex = "apiserver_flowcontrol_.*|apiserver_envelope_.*"
action = "drop"
}
rule {
target_label = "cluster"
replacement = sys.env("CLUSTER_NAME")
}
rule {
target_label = "zone"
replacement = sys.env("ZONE")
}
}
prometheus.scrape "apiserver" {
job_name = "integrations/kubernetes/apiserver"
targets = discovery.relabel.apiserver.output
scheme = "https"
metrics_path = "/metrics"
tls_config {
server_name = "kubernetes"
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
insecure_skip_verify = true
}
bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
forward_to = [prometheus.remote_write.prometheus_srv.receiver]
}
// ============================================
// CoreDNS discovery (ENDPOINTS)
// ============================================
discovery.relabel "coredns" {
targets = discovery.kubernetes.kube_endpoints.targets
rule {
source_labels = ["__meta_kubernetes_service_name"]
regex = "(kube-dns|coredns)"
action = "keep"
}
rule {
regex = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$"
action = "drop"
}
rule {
regex = "coredns_cache_.*|coredns_forward_.*"
action = "drop"
}
rule {
source_labels = ["__meta_kubernetes_endpoint_node_name"]
target_label = "node"
}
rule {
action = "replace"
target_label = "__address__"
replacement = "$__meta_kubernetes_endpoint_address:9153"
}
rule {
target_label = "cluster"
replacement = sys.env("CLUSTER_NAME")
}
rule {
target_label = "zone"
replacement = sys.env("ZONE")
}
}
// ============================================
// CoreDNS scrape
// ============================================
prometheus.scrape "coredns" {
job_name = "integrations/kubernetes/coredns"
targets = discovery.relabel.coredns.output
scheme = "http"
metrics_path = "/metrics"
forward_to = [prometheus.remote_write.prometheus_srv.receiver]
}
@@ -1,24 +0,0 @@
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: alloy-kube-scraper
spec:
releaseName: alloy-kube-scraper
interval: 1m
chart:
spec:
chart: alloy
version: 1.x.x
sourceRef:
kind: HelmRepository
name: grafana
interval: 40h
valuesFrom:
- kind: Secret
name: alloy-kube-scraper-helm-values
valuesKey: values.yaml
@@ -1,30 +0,0 @@
controller:
type: 'deployment'
nodeSelector:
kubernetes.io/hostname: casa-minion-01
alloy:
# https://grafana.com/docs/alloy/latest/collect/logs-in-kubernetes/
configMap:
create: false
name: alloy-kube-scraper
key: config.alloy
extraEnv:
- name: CLUSTER_NAME
value: casa
- name: ZONE
value: casa-vlan
- name: LOKI_URL
valueFrom:
secretKeyRef:
name: alloy-global
key: loki_url
- name: PROMETHEUS_URL
valueFrom:
secretKeyRef:
name: alloy-global
key: prometheus_url
@@ -1,17 +0,0 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: monitoring
resources:
- helm-release.yaml
configMapGenerator:
- name: alloy-kube-scraper
files:
- config.alloy=config.alloy
secretGenerator:
- name: alloy-kube-scraper-helm-values
files:
- values.yaml=helm-values.yaml
generatorOptions:
disableNameSuffixHash: true
@@ -1,9 +0,0 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: monitoring
resources:
- global
- node-scraper
- kube-scraper
generatorOptions:
disableNameSuffixHash: true
@@ -1,145 +0,0 @@
logging {
level = "warn"
}
// ============================================
// LOKI WRITE TARGET
// ============================================
loki.write "loki_srv" {
endpoint {
url = sys.env("LOKI_URL")
}
}
// ============================================
// HOST JOURNALD SOURCE
// ============================================
// Read journald logs from the host
loki.source.journal "journal" {
max_age = "24h"
relabel_rules = discovery.relabel.host_journal.rules
forward_to = [loki.process.host_journal.receiver]
}
// ============================================
// HOST JOURNALD RELABELING
// ============================================
// Convert journald metadata into Loki labels
discovery.relabel "host_journal" {
targets = []
// Systemd unit name
rule {
source_labels = ["__journal__systemd_unit"]
target_label = "systemd_unit"
}
// Transport type (stdout, syslog, kernel, audit, etc.)
rule {
source_labels = ["__journal__transport"]
target_label = "journal_transport"
}
// Priority (info, warning, error)
rule {
source_labels = ["__journal_priority_keyword"]
target_label = "level"
}
}
// ============================================
// HOST JOURNALD PROCESSING
// ============================================
// Add static labels and forward journald logs to Loki
loki.process "host_journal" {
stage.static_labels {
values = {
cluster = sys.env("CLUSTER_NAME"),
host = sys.env("HOSTNAME"),
zone = sys.env("ZONE"),
job = "integrations/journald",
}
}
forward_to = [loki.write.loki_srv.receiver]
}
// ============================================
// Metrics: Prometheus connection config
// ============================================
prometheus.remote_write "metrics_service" {
endpoint {
url = sys.env("PROMETHEUS_URL")
}
// global labels
external_labels = {
cluster = sys.env("CLUSTER_NAME"),
zone = sys.env("ZONE"),
}
}
// ============================================
// Metrics: Prometheus Unix Exporter
// ============================================
prometheus.exporter.unix "node_exporter" {
rootfs_path = "/host"
disable_collectors = ["ipvs", "btrfs", "infiniband", "xfs", "zfs"]
enable_collectors = [
"cpu",
"meminfo",
"vmstat",
"loadavg",
"filesystem",
"netdev",
"uname",
]
filesystem {
mount_points_exclude = "^(/host|/)(etc/resolv.conf|etc/hosts|etc/hostname|etc/machine-id|dataDisks/nvme01-ephemeral/.+|proc|sys|run/containerd/.+|run/k3s/containerd/.+|var/lib/docker/.+|var/lib/kubelet/.+)($|/)"
fs_types_exclude = "^(tmpfs|shm|autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs|erofs)$"
mount_timeout = "5s"
}
netclass {
ignored_devices = "^(veth.*|cali.*|[a-f0-9]{15})$"
}
netdev {
device_exclude = "^(veth.*|cali.*|[a-f0-9]{15})$"
}
}
prometheus.scrape "node_exporter" {
targets = prometheus.exporter.unix.node_exporter.targets
scrape_interval = "30s"
forward_to = [prometheus.relabel.node_exporter.receiver]
}
prometheus.relabel "node_exporter" {
forward_to = [prometheus.remote_write.metrics_service.receiver]
rule {
action = "replace"
replacement = sys.env("HOSTNAME")
target_label = "nodename"
}
rule {
action = "replace"
replacement = sys.env("HOSTNAME")
target_label = "node"
}
}
@@ -1,24 +0,0 @@
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: alloy-node-scraper
spec:
releaseName: alloy-node-scraper
interval: 1m
chart:
spec:
chart: alloy
version: 1.x.x
sourceRef:
kind: HelmRepository
name: grafana
interval: 40h
valuesFrom:
- kind: Secret
name: alloy-node-scraper-helm-values
valuesKey: values.yaml
@@ -1,91 +0,0 @@
controller:
nameOverride: "alloy-node-scrapper"
volumes:
extra:
# requires host journal configuration storage = volatile
- name: run-log-journal
hostPath:
path: /run/log/journal
# Log scrapers (e.g., Alloy/Loki) read this ID from the journal metadata and use it to separate log streams per machine.
- name: etc-machine-id
hostPath:
path: /etc/machine-id
- name: rootfs
hostPath:
path: /
- name: proc
hostPath:
path: /proc
- name: sys
hostPath:
path: /sys
- name: dev
hostPath:
path: /dev
tolerations:
- key: "node-role.kubernetes.io/control-plane"
operator: "Exists"
effect: "NoSchedule"
- key: "infra.limbosolutions.com/dedicated"
operator: "Exists"
effect: "NoSchedule"
alloy:
mounts:
varlog: true # Mounts /var/log from the host (persistent journal)
extra:
# requires host journal configuration storage = volatile
- name: run-log-journal
mountPath: /run/log/journal
readOnly: true
# required
- name: etc-machine-id
mountPath: /etc/machine-id
readOnly: true
- name: rootfs
mountPath: /host
readOnly: true
- name: proc
mountPath: /host/proc
readOnly: true
- name: sys
mountPath: /host/sys
readOnly: true
- name: dev
mountPath: /host/dev
readOnly: true
# https://grafana.com/docs/alloy/latest/collect/logs-in-kubernetes/
configMap:
create: false
name: alloy-node-scraper
key: config.alloy
extraEnv:
- name: NODE_IP
valueFrom:
fieldRef:
fieldPath: status.hostIP
- name: CLUSTER_NAME
value: casa
- name: JOB
value: node-exporter
- name: CONTAINER
value: node-exporter
- name: NS
value: monitoring
- name: POD
value: alloy-node-scraper
- name: SERVICE
value: alloy-node-scraper
- name: ZONE
value: casa-vlan
- name: LOKI_URL
valueFrom:
secretKeyRef:
name: alloy-global
key: loki_url
- name: PROMETHEUS_URL
valueFrom:
secretKeyRef:
name: alloy-global
key: prometheus_url
@@ -1,17 +0,0 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: monitoring
resources:
- helm-release.yaml
configMapGenerator:
- name: alloy-node-scraper
files:
- config.alloy=config.alloy
secretGenerator:
- name: alloy-node-scraper-helm-values
files:
- values.yaml=helm-values.yaml
generatorOptions:
disableNameSuffixHash: true
@@ -1,2 +0,0 @@
**
!.gitignore
@@ -1,16 +0,0 @@
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: alloy
spec:
interval: 1m
sourceRef:
kind: GitRepository
name: casa
namespace: casa-limbosolutions-com
path: services/monitoring/deploy/alloy
prune: true
decryption:
provider: sops
secretRef:
name: flux-sops-age
@@ -1,16 +0,0 @@
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: kube-state-metrics
spec:
interval: 1m
sourceRef:
kind: GitRepository
name: casa
namespace: casa-limbosolutions-com
path: services/monitoring/deploy/prometheus
prune: true
decryption:
provider: sops
secretRef:
name: flux-sops-age
@@ -1,12 +0,0 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: monitoring
resources:
- flux-kustomization-alloy.yaml
- flux-kustomization-prometheus.yaml
secretGenerator:
- name: flux-sops-age
files:
- "age.agekey=./.env.d/age.agekey"
generatorOptions:
disableNameSuffixHash: true
@@ -1,15 +0,0 @@
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: kube-state-metrics
spec:
releaseName: kube-state-metrics
interval: 1m
chart:
spec:
chart: kube-state-metrics
version: 8.x.x
sourceRef:
kind: HelmRepository
name: prometheus
interval: 40h
@@ -1,7 +0,0 @@
apiVersion: source.toolkit.fluxcd.io/v1
kind: HelmRepository
metadata:
name: prometheus
spec:
interval: 40h
url: https://prometheus-community.github.io/helm-charts
@@ -1,6 +0,0 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: monitoring
resources:
- helm-repo.yaml
- helm-release.yaml