alloy: configMapGenerator setup config.alloy
This commit is contained in:
Márcio Fernandes
@@ -0,0 +1,435 @@
|
||||
logging {
|
||||
level = "warn"
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Discovery: Kubernetes - Nodes
|
||||
// used by:
|
||||
// - prometheus - Kube_node_metrics - cAdvisor
|
||||
// ============================================
|
||||
|
||||
discovery.kubernetes "kube_nodes" {
|
||||
role = "node"
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Discovery: Kubernetes - pods
|
||||
// ============================================
|
||||
|
||||
discovery.kubernetes "kube_pods" {
|
||||
role = "pod"
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Discovery: Kubernetes - services
|
||||
// used by:
|
||||
// - loki kubernetes kube state metrics
|
||||
// ============================================
|
||||
discovery.kubernetes "kube_services" {
|
||||
role = "service"
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Discovery: Kubernetes - endpoints
|
||||
// used by:
|
||||
// - loki kubernetes kube state metrics
|
||||
// ============================================
|
||||
discovery.kubernetes "kube_endpoints" {
|
||||
role = "endpoints"
|
||||
}
|
||||
|
||||
|
||||
// ============================================
|
||||
// Logs: Loki config
|
||||
// ============================================
|
||||
|
||||
loki.write "loki_srv" {
|
||||
endpoint {
|
||||
url = sys.env("LOKI_URL")
|
||||
}
|
||||
}
|
||||
|
||||
loki.process "loki_srv" {
|
||||
stage.static_labels {
|
||||
values = {
|
||||
cluster = sys.env("CLUSTER_NAME"),
|
||||
zone = sys.env("ZONE"),
|
||||
job = "integrations/kubernetes/pod-logs",
|
||||
}
|
||||
}
|
||||
forward_to = [loki.write.loki_srv.receiver]
|
||||
}
|
||||
|
||||
|
||||
// ============================================
|
||||
// Loki.source kube_pod_logs
|
||||
// Requirements:
|
||||
// - discovery.relabel: kube_pod_logs
|
||||
// - discovery.process: loki_srv
|
||||
// ============================================
|
||||
|
||||
loki.source.kubernetes "kube_pod_logs" {
|
||||
targets = discovery.relabel.kube_pod_logs.output
|
||||
forward_to = [loki.process.loki_srv.receiver]
|
||||
}
|
||||
|
||||
|
||||
// ============================================
|
||||
// Discovery:relabel: kube_pod_logs
|
||||
// Requirements:
|
||||
// - discovery.kubernetes.role.pod: kube_pods
|
||||
// ============================================
|
||||
|
||||
discovery.relabel "kube_pod_logs" {
|
||||
targets = discovery.kubernetes.kube_pods.targets
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_namespace"]
|
||||
target_label = "namespace"
|
||||
}
|
||||
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_name"]
|
||||
regex = "^(.+?)-[0-9a-zA-Z]{4,16}(?:-[0-9a-zA-Z]{4,16})?$"
|
||||
replacement = "$1"
|
||||
target_label = "pod"
|
||||
}
|
||||
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_container_name"]
|
||||
target_label = "container"
|
||||
}
|
||||
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_node_name"]
|
||||
target_label = "node_name"
|
||||
}
|
||||
|
||||
rule {
|
||||
source_labels = [
|
||||
"__meta_kubernetes_pod_controller_name",
|
||||
]
|
||||
regex = "([0-9a-z-.]+?)(-[0-9a-f]{8,10})?"
|
||||
target_label = "controller"
|
||||
}
|
||||
|
||||
rule {
|
||||
source_labels = [
|
||||
"__meta_kubernetes_pod_label_app_kubernetes_io_name",
|
||||
"__meta_kubernetes_pod_label_app",
|
||||
"controller_name",
|
||||
"__meta_kubernetes_pod_name",
|
||||
]
|
||||
regex = "^;*([^;]+)(;.*)?$"
|
||||
target_label = "app"
|
||||
}
|
||||
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_pod_node_name"]
|
||||
separator = ":"
|
||||
target_label = "host"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// ============================================
|
||||
// Metrics: Prometheus Remote Write
|
||||
// ============================================
|
||||
prometheus.remote_write "prometheus_srv" {
|
||||
endpoint {
|
||||
url = sys.env("PROMETHEUS_URL")
|
||||
}
|
||||
external_labels = {
|
||||
cluster = sys.env("CLUSTER_NAME"),
|
||||
zone = sys.env("ZONE"),
|
||||
}
|
||||
}
|
||||
|
||||
discovery.relabel "kubelet" {
|
||||
targets = discovery.kubernetes.kube_nodes.targets
|
||||
|
||||
rule {
|
||||
regex = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$"
|
||||
action = "drop"
|
||||
}
|
||||
|
||||
rule {
|
||||
regex = "container_fs_.*|container_network_.*|container_hugetlb_.*|container_blkio_.*|container_tasks_state_.*"
|
||||
action = "drop"
|
||||
}
|
||||
|
||||
|
||||
// duplicate metrics from cAdviser
|
||||
rule {
|
||||
regex = "container_cpu_usage_seconds_total|container_memory_working_set_bytes|container_memory_rss|container_memory_cache"
|
||||
action = "drop"
|
||||
}
|
||||
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_node_address_InternalIP"]
|
||||
target_label = "__address__"
|
||||
replacement = "$1:10250"
|
||||
}
|
||||
|
||||
rule {
|
||||
action = "replace"
|
||||
source_labels = ["__meta_kubernetes_node_name"]
|
||||
regex = "(.+)"
|
||||
target_label = "node"
|
||||
}
|
||||
|
||||
rule {
|
||||
source_labels = ["node"]
|
||||
target_label = "host"
|
||||
}
|
||||
|
||||
|
||||
rule {
|
||||
target_label = "__metrics_path__"
|
||||
replacement = "/metrics"
|
||||
}
|
||||
|
||||
rule {
|
||||
target_label = "cluster"
|
||||
replacement = sys.env("CLUSTER_NAME")
|
||||
}
|
||||
|
||||
rule {
|
||||
target_label = "zone"
|
||||
replacement = sys.env("ZONE")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
prometheus.scrape "kubelet" {
|
||||
job_name = "integrations/kubernetes/kubelet"
|
||||
targets = discovery.relabel.kubelet.output
|
||||
|
||||
scheme = "https"
|
||||
metrics_path = "/metrics"
|
||||
|
||||
tls_config {
|
||||
server_name = "kubernetes"
|
||||
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
|
||||
insecure_skip_verify = true
|
||||
}
|
||||
|
||||
bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
|
||||
|
||||
forward_to = [prometheus.remote_write.prometheus_srv.receiver]
|
||||
}
|
||||
|
||||
// ===============================================================================
|
||||
// prometheus / Metrics - Scrape kube_state_metrics
|
||||
// Requirements:
|
||||
// - discovery.relabel - kube_state_metrics
|
||||
// ===============================================================================
|
||||
prometheus.scrape "kube_state_metrics" {
|
||||
job_name = "integrations/kubernetes/kube-state-metrics"
|
||||
targets = discovery.relabel.kube_state_metrics.output
|
||||
|
||||
scheme = "http"
|
||||
|
||||
forward_to = [prometheus.remote_write.prometheus_srv.receiver]
|
||||
}
|
||||
|
||||
discovery.relabel "kube_state_metrics" {
|
||||
targets = discovery.kubernetes.kube_endpoints.targets
|
||||
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_service_name"]
|
||||
regex = "kube-state-metrics"
|
||||
action = "keep"
|
||||
}
|
||||
|
||||
rule {
|
||||
regex = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$"
|
||||
action = "drop"
|
||||
}
|
||||
|
||||
rule {
|
||||
target_label = "cluster"
|
||||
replacement = sys.env("CLUSTER_NAME")
|
||||
}
|
||||
|
||||
rule {
|
||||
target_label = "zone"
|
||||
replacement = sys.env("ZONE")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// ===============================================================================
|
||||
// Scrape Kubernetes pods containers metrics / cAdvisor
|
||||
// Requirements:
|
||||
// - discovery.relabel - kube_node_metrics
|
||||
// ===============================================================================
|
||||
|
||||
prometheus.scrape "cadvisor" {
|
||||
job_name = "integrations/kubernetes/cadvisor"
|
||||
targets = discovery.relabel.kube_node_metrics.output
|
||||
scheme = "https"
|
||||
|
||||
tls_config {
|
||||
server_name = "kubernetes"
|
||||
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
|
||||
insecure_skip_verify = true
|
||||
}
|
||||
|
||||
bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
|
||||
forward_to = [prometheus.remote_write.prometheus_srv.receiver]
|
||||
}
|
||||
|
||||
|
||||
|
||||
// ===========================================================
|
||||
// Discovery:relabel: kube_node_metrics (cAdviser)
|
||||
// Requirements:
|
||||
// - discovery.kubernetes.role.node: kube_nodes
|
||||
// ===========================================================
|
||||
|
||||
discovery.relabel "kube_node_metrics" {
|
||||
targets = discovery.kubernetes.kube_nodes.targets
|
||||
|
||||
rule {
|
||||
regex = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$"
|
||||
action = "drop"
|
||||
}
|
||||
|
||||
rule {
|
||||
action = "replace"
|
||||
source_labels = ["__meta_kubernetes_node_name"]
|
||||
regex = "(.+)"
|
||||
target_label = "node"
|
||||
}
|
||||
|
||||
rule {
|
||||
source_labels = ["node"]
|
||||
target_label = "host"
|
||||
}
|
||||
|
||||
rule {
|
||||
replacement = "/metrics/cadvisor"
|
||||
target_label = "__metrics_path__"
|
||||
}
|
||||
|
||||
rule {
|
||||
target_label = "cluster"
|
||||
replacement = sys.env("CLUSTER_NAME")
|
||||
}
|
||||
|
||||
rule {
|
||||
target_label = "zone"
|
||||
replacement = sys.env("ZONE")
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// API Server discovery
|
||||
// ============================================
|
||||
|
||||
discovery.relabel "apiserver" {
|
||||
targets = discovery.kubernetes.kube_services.targets
|
||||
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_service_name"]
|
||||
regex = "kubernetes"
|
||||
action = "keep"
|
||||
}
|
||||
|
||||
rule {
|
||||
regex = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$"
|
||||
action = "drop"
|
||||
}
|
||||
|
||||
rule {
|
||||
regex = "apiserver_flowcontrol_.*|apiserver_envelope_.*"
|
||||
action = "drop"
|
||||
}
|
||||
|
||||
rule {
|
||||
target_label = "cluster"
|
||||
replacement = sys.env("CLUSTER_NAME")
|
||||
}
|
||||
|
||||
rule {
|
||||
target_label = "zone"
|
||||
replacement = sys.env("ZONE")
|
||||
}
|
||||
}
|
||||
|
||||
prometheus.scrape "apiserver" {
|
||||
job_name = "integrations/kubernetes/apiserver"
|
||||
targets = discovery.relabel.apiserver.output
|
||||
|
||||
scheme = "https"
|
||||
metrics_path = "/metrics"
|
||||
|
||||
tls_config {
|
||||
server_name = "kubernetes"
|
||||
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
|
||||
insecure_skip_verify = true
|
||||
}
|
||||
|
||||
bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
|
||||
|
||||
forward_to = [prometheus.remote_write.prometheus_srv.receiver]
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// CoreDNS discovery (ENDPOINTS)
|
||||
// ============================================
|
||||
|
||||
discovery.relabel "coredns" {
|
||||
targets = discovery.kubernetes.kube_endpoints.targets
|
||||
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_service_name"]
|
||||
regex = "(kube-dns|coredns)"
|
||||
action = "keep"
|
||||
}
|
||||
|
||||
rule {
|
||||
regex = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$"
|
||||
action = "drop"
|
||||
}
|
||||
|
||||
rule {
|
||||
regex = "coredns_cache_.*|coredns_forward_.*"
|
||||
action = "drop"
|
||||
}
|
||||
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_endpoint_node_name"]
|
||||
target_label = "node"
|
||||
}
|
||||
|
||||
rule {
|
||||
action = "replace"
|
||||
target_label = "__address__"
|
||||
replacement = "$__meta_kubernetes_endpoint_address:9153"
|
||||
}
|
||||
|
||||
rule {
|
||||
target_label = "cluster"
|
||||
replacement = sys.env("CLUSTER_NAME")
|
||||
}
|
||||
|
||||
rule {
|
||||
target_label = "zone"
|
||||
replacement = sys.env("ZONE")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// CoreDNS scrape
|
||||
// ============================================
|
||||
prometheus.scrape "coredns" {
|
||||
job_name = "integrations/kubernetes/coredns"
|
||||
targets = discovery.relabel.coredns.output
|
||||
|
||||
scheme = "http"
|
||||
metrics_path = "/metrics"
|
||||
|
||||
forward_to = [prometheus.remote_write.prometheus_srv.receiver]
|
||||
}
|
||||
@@ -6,352 +6,9 @@ alloy:
|
||||
|
||||
# https://grafana.com/docs/alloy/latest/collect/logs-in-kubernetes/
|
||||
configMap:
|
||||
content: |
|
||||
logging {
|
||||
level = "warn"
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Discovery: Kubernetes - Nodes
|
||||
// used by:
|
||||
// - prometheus - Kube_node_metrics - cAdvisor
|
||||
// ============================================
|
||||
|
||||
discovery.kubernetes "kube_nodes" {
|
||||
role = "node"
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Discovery: Kubernetes - pods
|
||||
// used by:
|
||||
// - loki kubernetes pod logs
|
||||
// ============================================
|
||||
|
||||
discovery.kubernetes "kube_pods" {
|
||||
role = "pod"
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Discovery: Kubernetes - services
|
||||
// used by:
|
||||
// - loki kubernetes kube state metrics
|
||||
// ============================================
|
||||
discovery.kubernetes "kube_services" {
|
||||
role = "service"
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Discovery: Kubernetes - endpoints
|
||||
// used by:
|
||||
// - loki kubernetes kube state metrics
|
||||
// ============================================
|
||||
discovery.kubernetes "kube_endpoints" {
|
||||
role = "endpoints"
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Metrics: Prometheus Remote Write
|
||||
// ============================================
|
||||
prometheus.remote_write "prometheus_srv" {
|
||||
endpoint {
|
||||
url = sys.env("PROMETHEUS_URL")
|
||||
}
|
||||
external_labels = {
|
||||
cluster = sys.env("CLUSTER_NAME"),
|
||||
zone = sys.env("ZONE"),
|
||||
}
|
||||
}
|
||||
|
||||
discovery.relabel "kubelet" {
|
||||
targets = discovery.kubernetes.kube_nodes.targets
|
||||
|
||||
rule {
|
||||
regex = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$"
|
||||
action = "drop"
|
||||
}
|
||||
|
||||
rule {
|
||||
regex = "container_fs_.*|container_network_.*|container_hugetlb_.*|container_blkio_.*|container_tasks_state_.*"
|
||||
action = "drop"
|
||||
}
|
||||
|
||||
|
||||
// duplicate metrics from cAdviser
|
||||
rule {
|
||||
regex = "container_cpu_usage_seconds_total|container_memory_working_set_bytes|container_memory_rss|container_memory_cache"
|
||||
action = "drop"
|
||||
}
|
||||
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_node_address_InternalIP"]
|
||||
target_label = "__address__"
|
||||
replacement = "$1:10250"
|
||||
}
|
||||
|
||||
rule {
|
||||
action = "replace"
|
||||
source_labels = ["__meta_kubernetes_node_name"]
|
||||
regex = "(.+)"
|
||||
target_label = "node"
|
||||
}
|
||||
|
||||
rule {
|
||||
source_labels = ["node"]
|
||||
target_label = "host"
|
||||
}
|
||||
|
||||
|
||||
rule {
|
||||
target_label = "__metrics_path__"
|
||||
replacement = "/metrics"
|
||||
}
|
||||
|
||||
rule {
|
||||
target_label = "cluster"
|
||||
replacement = sys.env("CLUSTER_NAME")
|
||||
}
|
||||
|
||||
rule {
|
||||
target_label = "zone"
|
||||
replacement = sys.env("ZONE")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
prometheus.scrape "kubelet" {
|
||||
job_name = "integrations/kubernetes/kubelet"
|
||||
targets = discovery.relabel.kubelet.output
|
||||
|
||||
scheme = "https"
|
||||
metrics_path = "/metrics"
|
||||
|
||||
tls_config {
|
||||
server_name = "kubernetes"
|
||||
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
|
||||
insecure_skip_verify = true
|
||||
}
|
||||
|
||||
bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
|
||||
|
||||
forward_to = [prometheus.remote_write.prometheus_srv.receiver]
|
||||
}
|
||||
|
||||
// ===============================================================================
|
||||
// prometheus / Metrics - Scrape kube_state_metrics
|
||||
// Requirements:
|
||||
// - discovery.relabel - kube_state_metrics
|
||||
// ===============================================================================
|
||||
prometheus.scrape "kube_state_metrics" {
|
||||
job_name = "integrations/kubernetes/kube-state-metrics"
|
||||
targets = discovery.relabel.kube_state_metrics.output
|
||||
|
||||
scheme = "http"
|
||||
|
||||
forward_to = [prometheus.remote_write.prometheus_srv.receiver]
|
||||
}
|
||||
|
||||
discovery.relabel "kube_state_metrics" {
|
||||
targets = discovery.kubernetes.kube_endpoints.targets
|
||||
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_service_name"]
|
||||
regex = "kube-state-metrics"
|
||||
action = "keep"
|
||||
}
|
||||
|
||||
rule {
|
||||
regex = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$"
|
||||
action = "drop"
|
||||
}
|
||||
|
||||
rule {
|
||||
target_label = "cluster"
|
||||
replacement = sys.env("CLUSTER_NAME")
|
||||
}
|
||||
|
||||
rule {
|
||||
target_label = "zone"
|
||||
replacement = sys.env("ZONE")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// ===============================================================================
|
||||
// Scrape Kubernetes pods containers metrics / cAdvisor
|
||||
// Requirements:
|
||||
// - discovery.relabel - kube_node_metrics
|
||||
// ===============================================================================
|
||||
|
||||
prometheus.scrape "cadvisor" {
|
||||
job_name = "integrations/kubernetes/cadvisor"
|
||||
targets = discovery.relabel.kube_node_metrics.output
|
||||
scheme = "https"
|
||||
|
||||
tls_config {
|
||||
server_name = "kubernetes"
|
||||
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
|
||||
insecure_skip_verify = true
|
||||
}
|
||||
|
||||
bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
|
||||
forward_to = [prometheus.remote_write.prometheus_srv.receiver]
|
||||
}
|
||||
|
||||
|
||||
|
||||
// ===========================================================
|
||||
// Discovery:relabel: kube_node_metrics (cAdviser)
|
||||
// Requirements:
|
||||
// - discovery.kubernetes.role.node: kube_nodes
|
||||
// ===========================================================
|
||||
|
||||
discovery.relabel "kube_node_metrics" {
|
||||
targets = discovery.kubernetes.kube_nodes.targets
|
||||
|
||||
rule {
|
||||
regex = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$"
|
||||
action = "drop"
|
||||
}
|
||||
|
||||
rule {
|
||||
action = "replace"
|
||||
source_labels = ["__meta_kubernetes_node_name"]
|
||||
regex = "(.+)"
|
||||
target_label = "node"
|
||||
}
|
||||
|
||||
rule {
|
||||
source_labels = ["node"]
|
||||
target_label = "host"
|
||||
}
|
||||
|
||||
rule {
|
||||
replacement = "/metrics/cadvisor"
|
||||
target_label = "__metrics_path__"
|
||||
}
|
||||
|
||||
rule {
|
||||
target_label = "cluster"
|
||||
replacement = sys.env("CLUSTER_NAME")
|
||||
}
|
||||
|
||||
rule {
|
||||
target_label = "zone"
|
||||
replacement = sys.env("ZONE")
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// API Server discovery
|
||||
// ============================================
|
||||
|
||||
discovery.relabel "apiserver" {
|
||||
targets = discovery.kubernetes.kube_services.targets
|
||||
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_service_name"]
|
||||
regex = "kubernetes"
|
||||
action = "keep"
|
||||
}
|
||||
|
||||
rule {
|
||||
regex = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$"
|
||||
action = "drop"
|
||||
}
|
||||
|
||||
rule {
|
||||
regex = "apiserver_flowcontrol_.*|apiserver_envelope_.*"
|
||||
action = "drop"
|
||||
}
|
||||
|
||||
rule {
|
||||
target_label = "cluster"
|
||||
replacement = sys.env("CLUSTER_NAME")
|
||||
}
|
||||
|
||||
rule {
|
||||
target_label = "zone"
|
||||
replacement = sys.env("ZONE")
|
||||
}
|
||||
}
|
||||
|
||||
prometheus.scrape "apiserver" {
|
||||
job_name = "integrations/kubernetes/apiserver"
|
||||
targets = discovery.relabel.apiserver.output
|
||||
|
||||
scheme = "https"
|
||||
metrics_path = "/metrics"
|
||||
|
||||
tls_config {
|
||||
server_name = "kubernetes"
|
||||
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
|
||||
insecure_skip_verify = true
|
||||
}
|
||||
|
||||
bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
|
||||
|
||||
forward_to = [prometheus.remote_write.prometheus_srv.receiver]
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// CoreDNS discovery (ENDPOINTS)
|
||||
// ============================================
|
||||
|
||||
discovery.relabel "coredns" {
|
||||
targets = discovery.kubernetes.kube_endpoints.targets
|
||||
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_service_name"]
|
||||
regex = "(kube-dns|coredns)"
|
||||
action = "keep"
|
||||
}
|
||||
|
||||
rule {
|
||||
regex = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$"
|
||||
action = "drop"
|
||||
}
|
||||
|
||||
rule {
|
||||
regex = "coredns_cache_.*|coredns_forward_.*"
|
||||
action = "drop"
|
||||
}
|
||||
|
||||
rule {
|
||||
source_labels = ["__meta_kubernetes_endpoint_node_name"]
|
||||
target_label = "node"
|
||||
}
|
||||
|
||||
rule {
|
||||
action = "replace"
|
||||
target_label = "__address__"
|
||||
replacement = "$__meta_kubernetes_endpoint_address:9153"
|
||||
}
|
||||
|
||||
rule {
|
||||
target_label = "cluster"
|
||||
replacement = sys.env("CLUSTER_NAME")
|
||||
}
|
||||
|
||||
rule {
|
||||
target_label = "zone"
|
||||
replacement = sys.env("ZONE")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// CoreDNS scrape
|
||||
// ============================================
|
||||
prometheus.scrape "coredns" {
|
||||
job_name = "integrations/kubernetes/coredns"
|
||||
targets = discovery.relabel.coredns.output
|
||||
|
||||
scheme = "http"
|
||||
metrics_path = "/metrics"
|
||||
|
||||
forward_to = [prometheus.remote_write.prometheus_srv.receiver]
|
||||
}
|
||||
|
||||
create: false
|
||||
name: alloy-kube-scraper
|
||||
key: config.alloy
|
||||
|
||||
extraEnv:
|
||||
|
||||
|
||||
@@ -3,6 +3,12 @@ kind: Kustomization
|
||||
namespace: monitoring
|
||||
resources:
|
||||
- helm-release.yaml
|
||||
|
||||
configMapGenerator:
|
||||
- name: alloy-kube-scraper
|
||||
files:
|
||||
- config.alloy=config.alloy
|
||||
|
||||
secretGenerator:
|
||||
- name: alloy-kube-scraper-helm-values
|
||||
files:
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
logging {
|
||||
level = "warn"
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// LOKI WRITE TARGET
|
||||
// ============================================
|
||||
loki.write "loki_srv" {
|
||||
endpoint {
|
||||
url = sys.env("LOKI_URL")
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// HOST JOURNALD SOURCE
|
||||
// ============================================
|
||||
|
||||
// Read journald logs from the host
|
||||
loki.source.journal "journal" {
|
||||
max_age = "24h"
|
||||
relabel_rules = discovery.relabel.host_journal.rules
|
||||
forward_to = [loki.process.host_journal.receiver]
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// HOST JOURNALD RELABELING
|
||||
// ============================================
|
||||
|
||||
// Convert journald metadata into Loki labels
|
||||
discovery.relabel "host_journal" {
|
||||
targets = []
|
||||
|
||||
// Systemd unit name
|
||||
rule {
|
||||
source_labels = ["__journal__systemd_unit"]
|
||||
target_label = "systemd_unit"
|
||||
}
|
||||
|
||||
// Transport type (stdout, syslog, kernel, audit, etc.)
|
||||
rule {
|
||||
source_labels = ["__journal__transport"]
|
||||
target_label = "journal_transport"
|
||||
}
|
||||
|
||||
// Priority (info, warning, error)
|
||||
rule {
|
||||
source_labels = ["__journal_priority_keyword"]
|
||||
target_label = "level"
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// HOST JOURNALD PROCESSING
|
||||
// ============================================
|
||||
|
||||
// Add static labels and forward journald logs to Loki
|
||||
loki.process "host_journal" {
|
||||
stage.static_labels {
|
||||
values = {
|
||||
cluster = sys.env("CLUSTER_NAME"),
|
||||
host = sys.env("HOSTNAME"),
|
||||
zone = sys.env("ZONE"),
|
||||
job = "integrations/journald",
|
||||
}
|
||||
}
|
||||
|
||||
forward_to = [loki.write.loki_srv.receiver]
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
// ============================================
|
||||
// Metrics: Prometheus connection config
|
||||
// ============================================
|
||||
|
||||
prometheus.remote_write "metrics_service" {
|
||||
endpoint {
|
||||
url = sys.env("PROMETHEUS_URL")
|
||||
}
|
||||
|
||||
// global labels
|
||||
external_labels = {
|
||||
cluster = sys.env("CLUSTER_NAME"),
|
||||
zone = sys.env("ZONE"),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// ============================================
|
||||
// Metrics: Prometheus Unix Exporter
|
||||
// ============================================
|
||||
|
||||
prometheus.exporter.unix "node_exporter" {
|
||||
rootfs_path = "/host"
|
||||
disable_collectors = ["ipvs", "btrfs", "infiniband", "xfs", "zfs"]
|
||||
enable_collectors = [
|
||||
"cpu",
|
||||
"meminfo",
|
||||
"vmstat",
|
||||
"loadavg",
|
||||
"filesystem",
|
||||
"netdev",
|
||||
"uname",
|
||||
]
|
||||
|
||||
filesystem {
|
||||
mount_points_exclude = "^(/host|/)(etc/resolv.conf|etc/hosts|etc/hostname|etc/machine-id|dataDisks/nvme01-ephemeral/.+|proc|sys|run/containerd/.+|run/k3s/containerd/.+|var/lib/docker/.+|var/lib/kubelet/.+)($|/)"
|
||||
fs_types_exclude = "^(tmpfs|shm|autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs|erofs)$"
|
||||
mount_timeout = "5s"
|
||||
}
|
||||
|
||||
netclass {
|
||||
ignored_devices = "^(veth.*|cali.*|[a-f0-9]{15})$"
|
||||
}
|
||||
|
||||
netdev {
|
||||
device_exclude = "^(veth.*|cali.*|[a-f0-9]{15})$"
|
||||
}
|
||||
}
|
||||
|
||||
prometheus.scrape "node_exporter" {
|
||||
targets = prometheus.exporter.unix.node_exporter.targets
|
||||
scrape_interval = "30s"
|
||||
|
||||
forward_to = [prometheus.relabel.node_exporter.receiver]
|
||||
}
|
||||
|
||||
prometheus.relabel "node_exporter" {
|
||||
forward_to = [prometheus.remote_write.metrics_service.receiver]
|
||||
|
||||
rule {
|
||||
action = "replace"
|
||||
replacement = sys.env("HOSTNAME")
|
||||
target_label = "nodename"
|
||||
}
|
||||
|
||||
rule {
|
||||
action = "replace"
|
||||
replacement = sys.env("HOSTNAME")
|
||||
target_label = "node"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -57,154 +57,9 @@ alloy:
|
||||
readOnly: true
|
||||
# https://grafana.com/docs/alloy/latest/collect/logs-in-kubernetes/
|
||||
configMap:
|
||||
content: |
|
||||
logging {
|
||||
level = "warn"
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// LOKI WRITE TARGET
|
||||
// ============================================
|
||||
loki.write "loki_srv" {
|
||||
endpoint {
|
||||
url = sys.env("LOKI_URL")
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// HOST JOURNALD SOURCE
|
||||
// ============================================
|
||||
|
||||
// Read journald logs from the host
|
||||
loki.source.journal "journal" {
|
||||
max_age = "24h"
|
||||
relabel_rules = discovery.relabel.host_journal.rules
|
||||
forward_to = [loki.process.host_journal.receiver]
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// HOST JOURNALD RELABELING
|
||||
// ============================================
|
||||
|
||||
// Convert journald metadata into Loki labels
|
||||
discovery.relabel "host_journal" {
|
||||
targets = []
|
||||
|
||||
// Systemd unit name
|
||||
rule {
|
||||
source_labels = ["__journal__systemd_unit"]
|
||||
target_label = "systemd_unit"
|
||||
}
|
||||
|
||||
// Transport type (stdout, syslog, kernel, audit, etc.)
|
||||
rule {
|
||||
source_labels = ["__journal__transport"]
|
||||
target_label = "journal_transport"
|
||||
}
|
||||
|
||||
// Priority (info, warning, error)
|
||||
rule {
|
||||
source_labels = ["__journal_priority_keyword"]
|
||||
target_label = "level"
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// HOST JOURNALD PROCESSING
|
||||
// ============================================
|
||||
|
||||
// Add static labels and forward journald logs to Loki
|
||||
loki.process "host_journal" {
|
||||
stage.static_labels {
|
||||
values = {
|
||||
cluster = sys.env("CLUSTER_NAME"),
|
||||
host = sys.env("HOSTNAME"),
|
||||
zone = sys.env("ZONE"),
|
||||
job = "journald",
|
||||
service_name = "journald",
|
||||
}
|
||||
}
|
||||
|
||||
forward_to = [loki.write.loki_srv.receiver]
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
// ============================================
|
||||
// Metrics: Prometheus connection config
|
||||
// ============================================
|
||||
|
||||
prometheus.remote_write "metrics_service" {
|
||||
endpoint {
|
||||
url = sys.env("PROMETHEUS_URL")
|
||||
}
|
||||
|
||||
// global labels
|
||||
external_labels = {
|
||||
cluster = sys.env("CLUSTER_NAME"),
|
||||
zone = sys.env("ZONE"),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// ============================================
|
||||
// Metrics: Prometheus Unix Exporter
|
||||
// ============================================
|
||||
|
||||
prometheus.exporter.unix "node_exporter" {
|
||||
rootfs_path = "/host"
|
||||
disable_collectors = ["ipvs", "btrfs", "infiniband", "xfs", "zfs"]
|
||||
enable_collectors = [
|
||||
"cpu",
|
||||
"meminfo",
|
||||
"vmstat",
|
||||
"loadavg",
|
||||
"filesystem",
|
||||
"netdev",
|
||||
"uname",
|
||||
]
|
||||
|
||||
filesystem {
|
||||
mount_points_exclude = "^(/host|/)(etc/resolv.conf|etc/hosts|etc/hostname|etc/machine-id|dataDisks/nvme01-ephemeral/.+|proc|sys|run/containerd/.+|run/k3s/containerd/.+|var/lib/docker/.+|var/lib/kubelet/.+)($|/)"
|
||||
fs_types_exclude = "^(tmpfs|shm|autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs|erofs)$"
|
||||
mount_timeout = "5s"
|
||||
}
|
||||
|
||||
netclass {
|
||||
ignored_devices = "^(veth.*|cali.*|[a-f0-9]{15})$"
|
||||
}
|
||||
|
||||
netdev {
|
||||
device_exclude = "^(veth.*|cali.*|[a-f0-9]{15})$"
|
||||
}
|
||||
}
|
||||
|
||||
prometheus.scrape "node_exporter" {
|
||||
targets = prometheus.exporter.unix.node_exporter.targets
|
||||
scrape_interval = "30s"
|
||||
|
||||
forward_to = [prometheus.relabel.node_exporter.receiver]
|
||||
}
|
||||
|
||||
prometheus.relabel "node_exporter" {
|
||||
forward_to = [prometheus.remote_write.metrics_service.receiver]
|
||||
|
||||
rule {
|
||||
action = "replace"
|
||||
replacement = sys.env("HOSTNAME")
|
||||
target_label = "nodename"
|
||||
}
|
||||
|
||||
rule {
|
||||
action = "replace"
|
||||
replacement = sys.env("HOSTNAME")
|
||||
target_label = "node"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
create: false
|
||||
name: alloy-node-scraper
|
||||
key: config.alloy
|
||||
extraEnv:
|
||||
- name: NODE_IP
|
||||
valueFrom:
|
||||
|
||||
@@ -3,6 +3,12 @@ kind: Kustomization
|
||||
namespace: monitoring
|
||||
resources:
|
||||
- helm-release.yaml
|
||||
|
||||
configMapGenerator:
|
||||
- name: alloy-node-scraper
|
||||
files:
|
||||
- config.alloy=config.alloy
|
||||
|
||||
secretGenerator:
|
||||
- name: alloy-node-scraper-helm-values
|
||||
files:
|
||||
|
||||
Reference in New Issue
Block a user