controller: type: 'deployment' nodeSelector: kubernetes.io/hostname: casa-minion-01 alloy: # https://grafana.com/docs/alloy/latest/collect/logs-in-kubernetes/ configMap: content: | // ============================================ // LOKI WRITE TARGET // ============================================ loki.write "loki_srv" { endpoint { url = env("LOKI_URL") } } // ============================================ // POD LOGS: DISCOVERY + RELABEL // ============================================ // Discover Kubernetes pods discovery.kubernetes "pod_logs" { role = "pod" } discovery.relabel "pod_logs" { targets = discovery.kubernetes.pod_logs.targets rule { source_labels = ["__meta_kubernetes_namespace"] target_label = "namespace" } rule { source_labels = ["__meta_kubernetes_pod_name"] regex = "^(.+?)-[0-9a-zA-Z]{4,16}(?:-[0-9a-zA-Z]{4,16})?$" replacement = "$1" target_label = "pod" } rule { source_labels = ["__meta_kubernetes_pod_container_name"] target_label = "container" } rule { source_labels = ["__meta_kubernetes_node_name"] target_label = "node_name" } rule { source_labels = [ "__meta_kubernetes_pod_controller_name", ] regex = "([0-9a-z-.]+?)(-[0-9a-f]{8,10})?" target_label = "controller_name" } rule { source_labels = [ "__meta_kubernetes_pod_label_app_kubernetes_io_name", "__meta_kubernetes_pod_label_app", "controller_name", "__meta_kubernetes_pod_name", ] regex = "^;*([^;]+)(;.*)?$" target_label = "app" } rule { source_labels = [ "__meta_kubernetes_pod_label_app_kubernetes_io_component", "__meta_kubernetes_pod_label_component", ] regex = "^;*([^;]+)(;.*)?$" target_label = "component" } rule { source_labels = ["namespace", "app"] separator = "/" target_label = "job" } rule { source_labels = ["job", "container"] separator = ":" target_label = "instance" } rule { source_labels = ["__meta_kubernetes_pod_node_name"] separator = ":" target_label = "host" } } // ============================================ // POD LOGS: SOURCE → PROCESS → WRITE // ============================================ // Tail logs from Kubernetes pods loki.source.kubernetes "pod_logs" { // targets = discovery.kubernetes.pod_logs.targets targets = discovery.relabel.pod_logs.output forward_to = [loki.process.pod_logs.receiver] } // Add static labels and forward logs to Loki loki.process "pod_logs" { stage.static_labels { values = { cluster = env("CLUSTER_NAME"), zone = env("ZONE"), } } forward_to = [loki.write.loki_srv.receiver] } // ============================================ // KUBERNETES EVENTS // ============================================ // Tail Kubernetes events and convert them to log entries loki.source.kubernetes_events "cluster_events" { job_name = "integrations/kubernetes/eventhandler" log_format = "logfmt" forward_to = [ loki.process.cluster_events.receiver, ] } // Add static labels and forward events to Loki loki.process "cluster_events" { forward_to = [loki.write.loki_srv.receiver] stage.static_labels { values = { cluster = env("CLUSTER_NAME"), host = "icarus-kube", job = "cluster_events", zone = env("ZONE"), } } // stage.labels { // values = { // kubernetes_cluster_events = "job", // } // } } // Metrics prometheus.remote_write "prometheus_srv" { endpoint { url = env("PROMETHEUS_URL") } } // discovery kubernetes discovery.kubernetes "k8s_nodes" { role = "node" } prometheus.scrape "cadvisor" { job_name = "integrations/kubernetes/cadvisor" targets = discovery.relabel.cadvisor.output scheme = "https" tls_config { server_name = "kubernetes" ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" insecure_skip_verify = true } bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" forward_to = [prometheus.remote_write.prometheus_srv.receiver] } discovery.relabel "cadvisor" { targets = discovery.kubernetes.k8s_nodes.targets rule { replacement = "/metrics/cadvisor" target_label = "__metrics_path__" } rule { target_label = "instance" replacement = env("NODE_IP") + ":10250" } rule { target_label = "node" replacement = env("HOSTNAME") } // rule { // target_label = "service" // replacement = "kube-scrapper" // } rule { target_label = "cluster" replacement = env("CLUSTER_NAME") } // rule { // target_label = "job" // replacement = "kubelet" // } } discovery.kubernetes "k8s_services" { role = "service" } discovery.relabel "kube_state_metrics" { targets = discovery.kubernetes.k8s_services.targets rule { source_labels = ["__meta_kubernetes_service_name"] regex = "kube-state-metrics" action = "keep" } rule { target_label = "node" replacement = env("HOSTNAME") } rule { target_label = "cluster" replacement = env("CLUSTER_NAME") } } prometheus.scrape "kube_state_metrics" { job_name = "integrations/kubernetes/kube-state-metrics" targets = discovery.relabel.kube_state_metrics.output scheme = "http" forward_to = [prometheus.remote_write.prometheus_srv.receiver] } // ============================================ // API Server discovery // ============================================ discovery.relabel "apiserver" { targets = discovery.kubernetes.k8s_services.targets rule { source_labels = ["__meta_kubernetes_service_name"] regex = "kubernetes" action = "keep" } rule { target_label = "node" replacement = env("HOSTNAME") } rule { target_label = "cluster" replacement = env("CLUSTER_NAME") } } prometheus.scrape "apiserver" { job_name = "integrations/kubernetes/apiserver" targets = discovery.relabel.apiserver.output scheme = "https" metrics_path = "/metrics" tls_config { server_name = "kubernetes" ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" insecure_skip_verify = true } bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" forward_to = [prometheus.remote_write.prometheus_srv.receiver] } // ============================================ // CoreDNS discovery // ============================================ discovery.relabel "coredns" { targets = discovery.kubernetes.k8s_services.targets rule { source_labels = ["__meta_kubernetes_pod_label_k8s_app"] regex = "kube-dns" action = "keep" } rule { target_label = "node" replacement = env("HOSTNAME") } rule { target_label = "cluster" replacement = env("CLUSTER_NAME") } } prometheus.scrape "coredns" { job_name = "integrations/kubernetes/coredns-pods" targets = discovery.relabel.coredns.output scheme = "http" metrics_path = "/metrics" forward_to = [prometheus.remote_write.prometheus_srv.receiver] } extraEnv: - name: NODE_IP valueFrom: fieldRef: fieldPath: status.hostIP - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - name: CLUSTER_NAME value: casa - name: ZONE value: casa-vlan - name: LOKI_URL valueFrom: secretKeyRef: name: alloy-global key: loki_url - name: PROMETHEUS_URL valueFrom: secretKeyRef: name: alloy-global key: prometheus_url