controller: type: 'deployment' nodeSelector: kubernetes.io/hostname: casa-minion-01 alloy: # https://grafana.com/docs/alloy/latest/collect/logs-in-kubernetes/ configMap: content: | // ============================================ // Discovery: Kubernetes - Nodes // used by: // - prometheus - Kube_node_metrics - cAdvisor // ============================================ discovery.kubernetes "kube_nodes" { role = "node" } // ============================================ // Discovery: Kubernetes - pods // used by: // - loki kubernetes pod logs // ============================================ discovery.kubernetes "kube_pods" { role = "pod" } // ============================================ // Discovery: Kubernetes - pods // used by: // - loki kubernetes pod logs // ============================================ // ============================================ // Discovery: Kubernetes - pods // used by: // - loki kubernetes kube state metrics // ============================================ discovery.kubernetes "kube_services" { role = "service" } // ============================================ // Logs: Loki config // ============================================ loki.write "loki_srv" { endpoint { url = env("LOKI_URL") } } loki.process "loki_srv" { stage.static_labels { values = { cluster = env("CLUSTER_NAME"), zone = env("ZONE"), } } forward_to = [loki.write.loki_srv.receiver] } // ============================================ // Loki.source kube_pod_logs // Requirements: // - discovery.relabel: kube_pod_logs // - discovery.process: loki_srv // ============================================ loki.source.kubernetes "kube_pod_logs" { targets = discovery.relabel.kube_pod_logs.output forward_to = [loki.process.loki_srv.receiver] } // ============================================ // Discovery:relabel: kube_pod_logs // Requirements: // - discovery.kubernetes.role.pod: kube_pods // ============================================ discovery.relabel "kube_pod_logs" { targets = discovery.kubernetes.kube_pods.targets rule { source_labels = ["__meta_kubernetes_namespace"] target_label = "namespace" } rule { source_labels = ["__meta_kubernetes_pod_name"] regex = "^(.+?)-[0-9a-zA-Z]{4,16}(?:-[0-9a-zA-Z]{4,16})?$" replacement = "$1" target_label = "pod" } rule { source_labels = ["__meta_kubernetes_pod_container_name"] target_label = "container" } rule { source_labels = ["__meta_kubernetes_node_name"] target_label = "node_name" } rule { source_labels = [ "__meta_kubernetes_pod_controller_name", ] regex = "([0-9a-z-.]+?)(-[0-9a-f]{8,10})?" target_label = "controller_name" } rule { source_labels = [ "__meta_kubernetes_pod_label_app_kubernetes_io_name", "__meta_kubernetes_pod_label_app", "controller_name", "__meta_kubernetes_pod_name", ] regex = "^;*([^;]+)(;.*)?$" target_label = "app" } rule { source_labels = [ "__meta_kubernetes_pod_label_app_kubernetes_io_component", "__meta_kubernetes_pod_label_component", ] regex = "^;*([^;]+)(;.*)?$" target_label = "component" } rule { source_labels = ["namespace", "app"] separator = "/" target_label = "job" } rule { source_labels = ["job", "container"] separator = ":" target_label = "instance" } rule { source_labels = ["__meta_kubernetes_pod_node_name"] separator = ":" target_label = "host" } } // ============================================ // Prometheus: prometheus_srv // ============================================ prometheus.remote_write "prometheus_srv" { endpoint { url = env("PROMETHEUS_URL") } // global labels external_labels = { cluster = env("CLUSTER_NAME"), zone = env("ZONE"), } } // =============================================================================== // prometheus / Metrics - Scrape kube_state_metrics // Requirements: // - discovery.relabel - kube_node_metrics (cAdviser) // =============================================================================== prometheus.scrape "kube_state_metrics" { job_name = "integrations/kubernetes/kube-state-metrics" targets = discovery.relabel.kube_state_metrics.output scheme = "http" forward_to = [prometheus.remote_write.prometheus_srv.receiver] } discovery.relabel "kube_state_metrics" { targets = discovery.kubernetes.kube_services.targets rule { source_labels = ["__meta_kubernetes_service_name"] regex = "kube-state-metrics" action = "keep" } rule { source_labels = ["__meta_kubernetes_node_name"] target_label = "node" } } // =============================================================================== // prometheus / Metrics - Scrape Kubernetes nodes / cAdvisor // Requirements: // - discovery.relabel - kube_node_metrics (cAdviser) // =============================================================================== prometheus.scrape "cadvisor" { job_name = "integrations/kubernetes/cadvisor" targets = discovery.relabel.kube_node_metrics.output scheme = "https" tls_config { server_name = "kubernetes" ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" insecure_skip_verify = true } bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" forward_to = [prometheus.remote_write.prometheus_srv.receiver] } // =========================================================== // Discovery:relabel: Kube_node_metrics (cAdviser) // Requirements: // - discovery.kubernetes.role.node: kube_nodes // =========================================================== discovery.relabel "kube_node_metrics" { targets = discovery.kubernetes.kube_nodes.targets rule { action = "replace" source_labels = ["__meta_kubernetes_node_name"] regex = "(.+)" target_label = "node" } rule { source_labels = ["node"] target_label = "host" } rule { source_labels = ["nodename"] target_label = "host" } rule { replacement = "/metrics/cadvisor" target_label = "__metrics_path__" } rule { source_labels = ["node"] regex = "(.+)" replacement = "$1:9100" target_label = "instance" } } extraEnv: - name: CLUSTER_NAME value: casa - name: ZONE value: casa-vlan - name: LOKI_URL valueFrom: secretKeyRef: name: alloy-global key: loki_url - name: PROMETHEUS_URL valueFrom: secretKeyRef: name: alloy-global key: prometheus_url