alloy: configMapGenerator setup config.alloy

services/monitoring/README.md

@@ -2,16 +2,12 @@
 
 ## alloy
 
-**promtail Encrypt secrets:**
+**Encrypt global secrets:**
 
 ``` bash
 sops -e deploy/alloy/global/secrets.dec.yaml > deploy/alloy/global/secrets.yaml 
 ```
 
-**Check agent:**
+[Official Helm chart default values.yaml](https://github.com/grafana/alloy/blob/main/operations/helm/charts/alloy/values.yaml)
-
-``` bash
-http://localhost:9090/agent
-```
 
 ## prometheus - kube-state-metrics

services/monitoring/deploy/alloy/kube-scraper/config.alloy

@@ -0,0 +1,435 @@
+logging {
+  level = "warn"
+}
+
+// ============================================
+// Discovery: Kubernetes - Nodes
+// used by:
+// - prometheus - Kube_node_metrics - cAdvisor 
+// ============================================
+
+discovery.kubernetes "kube_nodes" {
+  role = "node"
+}
+
+// ============================================
+// Discovery: Kubernetes - pods 
+// ============================================
+
+discovery.kubernetes "kube_pods" {
+  role = "pod"
+}
+
+// ============================================
+// Discovery: Kubernetes - services
+// used by:
+// - loki kubernetes kube state metrics
+// ============================================
+discovery.kubernetes "kube_services" {
+  role = "service"
+}
+
+// ============================================
+// Discovery: Kubernetes - endpoints
+// used by:
+// - loki kubernetes kube state metrics
+// ============================================
+discovery.kubernetes "kube_endpoints" {
+  role = "endpoints"
+}
+
+
+// ============================================
+// Logs: Loki config
+// ============================================
+
+loki.write "loki_srv" {
+  endpoint {
+    url = sys.env("LOKI_URL")
+  }
+}
+
+loki.process "loki_srv" {
+  stage.static_labels {
+    values = {
+      cluster = sys.env("CLUSTER_NAME"),
+      zone    = sys.env("ZONE"),
+      job     = "integrations/kubernetes/pod-logs",
+    }
+  }
+  forward_to = [loki.write.loki_srv.receiver]
+}
+
+
+// ============================================
+// Loki.source  kube_pod_logs
+// Requirements: 
+//  - discovery.relabel: kube_pod_logs
+//  - discovery.process: loki_srv
+// ============================================
+
+loki.source.kubernetes "kube_pod_logs" {
+  targets    = discovery.relabel.kube_pod_logs.output
+  forward_to = [loki.process.loki_srv.receiver]
+}
+
+
+// ============================================
+// Discovery:relabel: kube_pod_logs
+// Requirements: 
+//  - discovery.kubernetes.role.pod: kube_pods
+// ============================================
+
+discovery.relabel "kube_pod_logs" {
+  targets = discovery.kubernetes.kube_pods.targets
+  rule {
+    source_labels = ["__meta_kubernetes_namespace"]
+    target_label  = "namespace"
+  }
+
+  rule {
+    source_labels = ["__meta_kubernetes_pod_name"]
+    regex        = "^(.+?)-[0-9a-zA-Z]{4,16}(?:-[0-9a-zA-Z]{4,16})?$"
+    replacement  = "$1"
+    target_label = "pod"
+  }
+
+  rule {
+    source_labels = ["__meta_kubernetes_pod_container_name"]
+    target_label = "container"
+  }
+
+  rule {
+    source_labels = ["__meta_kubernetes_node_name"]
+    target_label  = "node_name"
+  }
+
+  rule {
+    source_labels = [
+      "__meta_kubernetes_pod_controller_name",
+    ]
+    regex        = "([0-9a-z-.]+?)(-[0-9a-f]{8,10})?"
+    target_label = "controller"
+  }
+
+  rule {
+    source_labels = [
+      "__meta_kubernetes_pod_label_app_kubernetes_io_name",
+      "__meta_kubernetes_pod_label_app",
+      "controller_name",
+      "__meta_kubernetes_pod_name",
+    ]
+    regex        = "^;*([^;]+)(;.*)?$"
+    target_label = "app"
+  }
+
+  rule {
+    source_labels = ["__meta_kubernetes_pod_node_name"]
+    separator     = ":"
+    target_label  = "host"
+  }
+}
+
+
+// ============================================
+// Metrics: Prometheus Remote Write 
+// ============================================
+prometheus.remote_write "prometheus_srv" {
+  endpoint {
+    url = sys.env("PROMETHEUS_URL")
+  }
+  external_labels = {
+    cluster = sys.env("CLUSTER_NAME"),
+    zone    = sys.env("ZONE"),
+  }
+}
+
+discovery.relabel "kubelet" {
+  targets = discovery.kubernetes.kube_nodes.targets
+
+  rule {
+    regex  = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$"
+    action = "drop"
+  }
+
+  rule {
+    regex  = "container_fs_.*|container_network_.*|container_hugetlb_.*|container_blkio_.*|container_tasks_state_.*"
+    action = "drop"
+  }
+
+
+  // duplicate metrics from cAdviser
+  rule {
+    regex  = "container_cpu_usage_seconds_total|container_memory_working_set_bytes|container_memory_rss|container_memory_cache"
+    action = "drop"
+  }
+
+  rule {
+    source_labels = ["__meta_kubernetes_node_address_InternalIP"]
+    target_label  = "__address__"
+    replacement   = "$1:10250"
+  }
+  
+  rule {
+    action        = "replace"
+    source_labels = ["__meta_kubernetes_node_name"]
+    regex         = "(.+)"
+    target_label  = "node"
+  }
+
+  rule {
+    source_labels = ["node"]
+    target_label = "host"
+  }
+
+
+  rule {
+    target_label = "__metrics_path__"
+    replacement  = "/metrics"
+  }
+
+  rule {
+    target_label = "cluster"
+    replacement  = sys.env("CLUSTER_NAME")
+  }
+
+  rule {
+    target_label = "zone"
+    replacement  = sys.env("ZONE")
+  }
+}
+
+
+prometheus.scrape "kubelet" {
+  job_name = "integrations/kubernetes/kubelet"
+  targets  = discovery.relabel.kubelet.output
+
+  scheme = "https"
+  metrics_path = "/metrics"
+
+  tls_config {
+    server_name          = "kubernetes"
+    ca_file              = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+    insecure_skip_verify = true
+  }
+
+  bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+
+  forward_to = [prometheus.remote_write.prometheus_srv.receiver]
+}
+
+// ===============================================================================
+// prometheus / Metrics - Scrape kube_state_metrics
+// Requirements: 
+//  - discovery.relabel - kube_state_metrics
+// ===============================================================================
+prometheus.scrape "kube_state_metrics" {
+  job_name = "integrations/kubernetes/kube-state-metrics"
+  targets  = discovery.relabel.kube_state_metrics.output
+
+  scheme = "http"
+
+  forward_to = [prometheus.remote_write.prometheus_srv.receiver]
+}
+
+discovery.relabel "kube_state_metrics" {
+  targets = discovery.kubernetes.kube_endpoints.targets
+
+  rule {
+    source_labels = ["__meta_kubernetes_service_name"]
+    regex         = "kube-state-metrics"
+    action        = "keep"
+  }
+
+  rule {
+    regex  = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$"
+    action = "drop"
+  }
+
+  rule {
+    target_label = "cluster"
+    replacement  = sys.env("CLUSTER_NAME")
+  }
+
+  rule {
+    target_label = "zone"
+    replacement  = sys.env("ZONE")
+  }
+}
+
+
+// ===============================================================================
+// Scrape Kubernetes pods containers metrics / cAdvisor
+// Requirements: 
+//  - discovery.relabel - kube_node_metrics
+// ===============================================================================
+
+prometheus.scrape "cadvisor" {
+  job_name = "integrations/kubernetes/cadvisor"
+  targets  = discovery.relabel.kube_node_metrics.output
+  scheme   = "https"
+
+  tls_config {
+    server_name          = "kubernetes"
+    ca_file              = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+    insecure_skip_verify = true
+  }
+
+  bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+  forward_to        = [prometheus.remote_write.prometheus_srv.receiver]
+}
+
+
+
+// ===========================================================
+// Discovery:relabel: kube_node_metrics (cAdviser)
+// Requirements: 
+//  - discovery.kubernetes.role.node: kube_nodes
+// ===========================================================
+
+  discovery.relabel "kube_node_metrics" {
+  targets = discovery.kubernetes.kube_nodes.targets
+
+  rule {
+    regex  = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$"
+    action = "drop"
+  }
+
+  rule {
+    action        = "replace"
+    source_labels = ["__meta_kubernetes_node_name"]
+    regex         = "(.+)"
+    target_label  = "node"
+  }
+
+  rule {
+    source_labels = ["node"]
+    target_label = "host"
+  }
+
+  rule {
+    replacement  = "/metrics/cadvisor"
+    target_label = "__metrics_path__"
+  }
+
+  rule {
+    target_label = "cluster"
+    replacement  = sys.env("CLUSTER_NAME")
+  }
+
+  rule {
+    target_label = "zone"
+    replacement  = sys.env("ZONE")
+  }
+}
+
+// ============================================
+// API Server discovery
+// ============================================
+
+discovery.relabel "apiserver" {
+  targets = discovery.kubernetes.kube_services.targets
+
+  rule {
+    source_labels = ["__meta_kubernetes_service_name"]
+    regex         = "kubernetes"
+    action        = "keep"
+  }
+
+  rule {
+    regex  = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$"
+    action = "drop"
+  }
+
+  rule {
+    regex  = "apiserver_flowcontrol_.*|apiserver_envelope_.*"
+    action = "drop"
+  }
+
+  rule {
+    target_label = "cluster"
+    replacement  = sys.env("CLUSTER_NAME")
+  }
+
+  rule {
+    target_label = "zone"
+    replacement  = sys.env("ZONE")
+  }
+}
+
+prometheus.scrape "apiserver" {
+  job_name = "integrations/kubernetes/apiserver"
+  targets  = discovery.relabel.apiserver.output
+
+  scheme = "https"
+  metrics_path = "/metrics"
+
+  tls_config {
+    server_name          = "kubernetes"
+    ca_file              = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+    insecure_skip_verify = true
+  }
+
+  bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+
+  forward_to = [prometheus.remote_write.prometheus_srv.receiver]
+}
+
+// ============================================
+// CoreDNS discovery (ENDPOINTS)
+// ============================================
+
+discovery.relabel "coredns" {
+  targets = discovery.kubernetes.kube_endpoints.targets
+
+  rule {
+    source_labels = ["__meta_kubernetes_service_name"]
+    regex         = "(kube-dns|coredns)"
+    action        = "keep"
+  }
+
+  rule {
+    regex  = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$"
+    action = "drop"
+  }
+
+  rule {
+    regex  = "coredns_cache_.*|coredns_forward_.*"
+    action = "drop"
+  }
+
+  rule {
+    source_labels = ["__meta_kubernetes_endpoint_node_name"]
+    target_label  = "node"
+  }
+
+  rule {
+    action        = "replace"
+    target_label  = "__address__"
+    replacement   = "$__meta_kubernetes_endpoint_address:9153"
+  }
+
+  rule {
+    target_label = "cluster"
+    replacement  = sys.env("CLUSTER_NAME")
+  }
+
+  rule {
+    target_label = "zone"
+    replacement  = sys.env("ZONE")
+  }
+
+}
+
+// ============================================
+// CoreDNS scrape
+// ============================================
+prometheus.scrape "coredns" {
+  job_name = "integrations/kubernetes/coredns"
+  targets  = discovery.relabel.coredns.output
+
+  scheme = "http"
+  metrics_path = "/metrics"
+
+  forward_to = [prometheus.remote_write.prometheus_srv.receiver]
+}

services/monitoring/deploy/alloy/kube-scraper/helm-values.yaml

@@ -6,352 +6,9 @@ alloy:
 
   # https://grafana.com/docs/alloy/latest/collect/logs-in-kubernetes/
   configMap:
-    content: |
+    create: false
-      logging {
+    name: alloy-kube-scraper 
-        level = "warn"
+    key: config.alloy
-      }
-
-      // ============================================
-      // Discovery: Kubernetes - Nodes
-      // used by:
-      // - prometheus - Kube_node_metrics - cAdvisor 
-      // ============================================
-      
-      discovery.kubernetes "kube_nodes" {
-        role = "node"
-      }
-
-      // ============================================
-      // Discovery: Kubernetes - pods
-      // used by:
-      // - loki kubernetes pod logs
-      // ============================================
-      
-      discovery.kubernetes "kube_pods" {
-        role = "pod"
-      }
-
-      // ============================================
-      // Discovery: Kubernetes - services
-      // used by:
-      // - loki kubernetes kube state metrics
-      // ============================================
-      discovery.kubernetes "kube_services" {
-        role = "service"
-      }
-      
-      // ============================================
-      // Discovery: Kubernetes - endpoints
-      // used by:
-      // - loki kubernetes kube state metrics
-      // ============================================
-      discovery.kubernetes "kube_endpoints" {
-        role = "endpoints"
-      }
-
-      // ============================================
-      // Metrics: Prometheus Remote Write 
-      // ============================================
-      prometheus.remote_write "prometheus_srv" {
-        endpoint {
-          url = sys.env("PROMETHEUS_URL")
-        }
-        external_labels = {
-          cluster = sys.env("CLUSTER_NAME"),
-          zone    = sys.env("ZONE"),
-        }
-      }
-
-      discovery.relabel "kubelet" {
-        targets = discovery.kubernetes.kube_nodes.targets
-
-        rule {
-          regex  = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$"
-          action = "drop"
-        }
-
-        rule {
-          regex  = "container_fs_.*|container_network_.*|container_hugetlb_.*|container_blkio_.*|container_tasks_state_.*"
-          action = "drop"
-        }
-
-  
-        // duplicate metrics from cAdviser
-        rule {
-          regex  = "container_cpu_usage_seconds_total|container_memory_working_set_bytes|container_memory_rss|container_memory_cache"
-          action = "drop"
-        }
-
-        rule {
-          source_labels = ["__meta_kubernetes_node_address_InternalIP"]
-          target_label  = "__address__"
-          replacement   = "$1:10250"
-        }
-        
-        rule {
-          action        = "replace"
-          source_labels = ["__meta_kubernetes_node_name"]
-          regex         = "(.+)"
-          target_label  = "node"
-        }
-
-        rule {
-          source_labels = ["node"]
-          target_label = "host"
-        }
-
-
-        rule {
-          target_label = "__metrics_path__"
-          replacement  = "/metrics"
-        }
-
-        rule {
-          target_label = "cluster"
-          replacement  = sys.env("CLUSTER_NAME")
-        }
-
-        rule {
-          target_label = "zone"
-          replacement  = sys.env("ZONE")
-        }
-      }
-
-
-      prometheus.scrape "kubelet" {
-        job_name = "integrations/kubernetes/kubelet"
-        targets  = discovery.relabel.kubelet.output
-
-        scheme = "https"
-        metrics_path = "/metrics"
-
-        tls_config {
-          server_name          = "kubernetes"
-          ca_file              = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
-          insecure_skip_verify = true
-        }
-
-        bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
-
-        forward_to = [prometheus.remote_write.prometheus_srv.receiver]
-      }
-
-      // ===============================================================================
-      // prometheus / Metrics - Scrape kube_state_metrics
-      // Requirements: 
-      //  - discovery.relabel - kube_state_metrics
-      // ===============================================================================
-      prometheus.scrape "kube_state_metrics" {
-        job_name = "integrations/kubernetes/kube-state-metrics"
-        targets  = discovery.relabel.kube_state_metrics.output
-
-        scheme = "http"
-
-        forward_to = [prometheus.remote_write.prometheus_srv.receiver]
-      }
-
-      discovery.relabel "kube_state_metrics" {
-        targets = discovery.kubernetes.kube_endpoints.targets
-
-        rule {
-          source_labels = ["__meta_kubernetes_service_name"]
-          regex         = "kube-state-metrics"
-          action        = "keep"
-        }
-
-        rule {
-          regex  = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$"
-          action = "drop"
-        }
-
-        rule {
-          target_label = "cluster"
-          replacement  = sys.env("CLUSTER_NAME")
-        }
-
-        rule {
-          target_label = "zone"
-          replacement  = sys.env("ZONE")
-        }
-      }
-
-
-      // ===============================================================================
-      // Scrape Kubernetes pods containers metrics / cAdvisor
-      // Requirements: 
-      //  - discovery.relabel - kube_node_metrics
-      // ===============================================================================
-
-      prometheus.scrape "cadvisor" {
-        job_name = "integrations/kubernetes/cadvisor"
-        targets  = discovery.relabel.kube_node_metrics.output
-        scheme   = "https"
-
-        tls_config {
-          server_name          = "kubernetes"
-          ca_file              = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
-          insecure_skip_verify = true
-        }
-
-        bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
-        forward_to        = [prometheus.remote_write.prometheus_srv.receiver]
-      }
-  
-
-
-      // ===========================================================
-      // Discovery:relabel: kube_node_metrics (cAdviser)
-      // Requirements: 
-      //  - discovery.kubernetes.role.node: kube_nodes
-      // ===========================================================
-
-       discovery.relabel "kube_node_metrics" {
-        targets = discovery.kubernetes.kube_nodes.targets
-
-        rule {
-          regex  = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$"
-          action = "drop"
-        }
-
-        rule {
-          action        = "replace"
-          source_labels = ["__meta_kubernetes_node_name"]
-          regex         = "(.+)"
-          target_label  = "node"
-        }
-
-        rule {
-          source_labels = ["node"]
-          target_label = "host"
-        }
-
-        rule {
-          replacement  = "/metrics/cadvisor"
-          target_label = "__metrics_path__"
-        }
-
-        rule {
-          target_label = "cluster"
-          replacement  = sys.env("CLUSTER_NAME")
-        }
-
-        rule {
-          target_label = "zone"
-          replacement  = sys.env("ZONE")
-        }
-      }
-
-      // ============================================
-      // API Server discovery
-      // ============================================
-
-      discovery.relabel "apiserver" {
-        targets = discovery.kubernetes.kube_services.targets
-
-        rule {
-          source_labels = ["__meta_kubernetes_service_name"]
-          regex         = "kubernetes"
-          action        = "keep"
-        }
-
-        rule {
-          regex  = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$"
-          action = "drop"
-        }
-
-        rule {
-          regex  = "apiserver_flowcontrol_.*|apiserver_envelope_.*"
-          action = "drop"
-        }
-     
-        rule {
-          target_label = "cluster"
-          replacement  = sys.env("CLUSTER_NAME")
-        }
-
-        rule {
-          target_label = "zone"
-          replacement  = sys.env("ZONE")
-        }
-      }
-
-      prometheus.scrape "apiserver" {
-        job_name = "integrations/kubernetes/apiserver"
-        targets  = discovery.relabel.apiserver.output
-
-        scheme = "https"
-        metrics_path = "/metrics"
-
-        tls_config {
-          server_name          = "kubernetes"
-          ca_file              = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
-          insecure_skip_verify = true
-        }
-
-        bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
-
-        forward_to = [prometheus.remote_write.prometheus_srv.receiver]
-      }
-      
-      // ============================================
-      // CoreDNS discovery (ENDPOINTS)
-      // ============================================
-
-      discovery.relabel "coredns" {
-        targets = discovery.kubernetes.kube_endpoints.targets
-
-        rule {
-          source_labels = ["__meta_kubernetes_service_name"]
-          regex         = "(kube-dns|coredns)"
-          action        = "keep"
-        }
-
-        rule {
-          regex  = ".*_uid$|.*_id$|^annotation_.*|^label_.*|^endpoint_.*|^pod_ip$"
-          action = "drop"
-        }
-
-        rule {
-          regex  = "coredns_cache_.*|coredns_forward_.*"
-          action = "drop"
-        }
-
-        rule {
-          source_labels = ["__meta_kubernetes_endpoint_node_name"]
-          target_label  = "node"
-        }
-
-        rule {
-          action        = "replace"
-          target_label  = "__address__"
-          replacement   = "$__meta_kubernetes_endpoint_address:9153"
-        }
-
-        rule {
-          target_label = "cluster"
-          replacement  = sys.env("CLUSTER_NAME")
-        }
-
-        rule {
-          target_label = "zone"
-          replacement  = sys.env("ZONE")
-        }
-
-      }
-
-      // ============================================
-      // CoreDNS scrape
-      // ============================================
-      prometheus.scrape "coredns" {
-        job_name = "integrations/kubernetes/coredns"
-        targets  = discovery.relabel.coredns.output
-
-        scheme = "http"
-        metrics_path = "/metrics"
-
-        forward_to = [prometheus.remote_write.prometheus_srv.receiver]
-      }
-
 
   extraEnv:
 

services/monitoring/deploy/alloy/kube-scraper/kustomization.yaml

@@ -3,6 +3,12 @@ kind: Kustomization
 namespace: monitoring
 resources:
   - helm-release.yaml
+
+configMapGenerator:
+  - name: alloy-kube-scraper
+    files:
+      - config.alloy=config.alloy
+
 secretGenerator:
   - name: alloy-kube-scraper-helm-values
     files:

services/monitoring/deploy/alloy/node-scraper/config.alloy

@@ -0,0 +1,145 @@
+logging {
+  level = "warn"
+}
+
+// ============================================
+// LOKI WRITE TARGET
+// ============================================
+loki.write "loki_srv" {
+  endpoint {
+    url = sys.env("LOKI_URL")
+  }
+}
+
+// ============================================
+// HOST JOURNALD SOURCE
+// ============================================
+
+// Read journald logs from the host
+loki.source.journal "journal" {
+  max_age   = "24h"
+  relabel_rules = discovery.relabel.host_journal.rules
+  forward_to    = [loki.process.host_journal.receiver]
+}
+
+// ============================================
+// HOST JOURNALD RELABELING
+// ============================================
+
+// Convert journald metadata into Loki labels
+discovery.relabel "host_journal" {
+  targets = []
+
+  // Systemd unit name
+  rule {
+    source_labels = ["__journal__systemd_unit"]
+    target_label  = "systemd_unit"
+  }
+
+  // Transport type (stdout, syslog, kernel, audit, etc.)
+  rule {
+    source_labels = ["__journal__transport"]
+    target_label  = "journal_transport"
+  }
+
+  // Priority (info, warning, error)
+  rule {
+    source_labels = ["__journal_priority_keyword"]
+    target_label  = "level"
+  }
+}
+
+// ============================================
+// HOST JOURNALD PROCESSING
+// ============================================
+
+// Add static labels and forward journald logs to Loki
+loki.process "host_journal" {
+  stage.static_labels {
+    values = {
+      cluster = sys.env("CLUSTER_NAME"),
+      host    = sys.env("HOSTNAME"),
+      zone    = sys.env("ZONE"),
+      job     = "integrations/journald",
+    }
+  }
+
+  forward_to = [loki.write.loki_srv.receiver]
+}
+
+
+
+
+// ============================================
+// Metrics: Prometheus connection config
+// ============================================
+
+prometheus.remote_write "metrics_service" {
+  endpoint {
+    url = sys.env("PROMETHEUS_URL")
+  }
+  
+  // global labels
+  external_labels = {
+    cluster     = sys.env("CLUSTER_NAME"),
+    zone        = sys.env("ZONE"),
+  }
+}
+
+
+// ============================================
+// Metrics: Prometheus Unix Exporter
+// ============================================
+
+prometheus.exporter.unix "node_exporter" {
+  rootfs_path = "/host"
+  disable_collectors = ["ipvs", "btrfs", "infiniband", "xfs", "zfs"]
+  enable_collectors = [
+    "cpu",
+    "meminfo",
+    "vmstat",
+    "loadavg",
+    "filesystem",
+    "netdev",
+    "uname",
+  ]
+
+  filesystem {
+    mount_points_exclude = "^(/host|/)(etc/resolv.conf|etc/hosts|etc/hostname|etc/machine-id|dataDisks/nvme01-ephemeral/.+|proc|sys|run/containerd/.+|run/k3s/containerd/.+|var/lib/docker/.+|var/lib/kubelet/.+)($|/)"
+    fs_types_exclude     = "^(tmpfs|shm|autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs|erofs)$"
+    mount_timeout        = "5s"
+  }
+
+  netclass {
+    ignored_devices = "^(veth.*|cali.*|[a-f0-9]{15})$"
+  }
+
+  netdev {
+    device_exclude = "^(veth.*|cali.*|[a-f0-9]{15})$"
+  }
+}
+
+prometheus.scrape "node_exporter" {
+  targets         = prometheus.exporter.unix.node_exporter.targets
+  scrape_interval = "30s"
+
+  forward_to = [prometheus.relabel.node_exporter.receiver]
+}
+
+prometheus.relabel "node_exporter" {
+  forward_to = [prometheus.remote_write.metrics_service.receiver]
+
+  rule {
+    action       = "replace"
+    replacement  = sys.env("HOSTNAME")
+    target_label = "nodename"
+  }
+
+  rule {
+    action       = "replace"
+    replacement  = sys.env("HOSTNAME")
+    target_label = "node"
+  }
+}
+
+

services/monitoring/deploy/alloy/node-scraper/helm-values.yaml

@@ -57,154 +57,9 @@ alloy:
         readOnly: true 
   # https://grafana.com/docs/alloy/latest/collect/logs-in-kubernetes/
   configMap:
-    content: |
+    create: false
-      logging {
+    name: alloy-node-scraper 
-        level = "warn"
+    key: config.alloy
-      }
-      
-      // ============================================
-      // LOKI WRITE TARGET
-      // ============================================
-      loki.write "loki_srv" {
-        endpoint {
-          url = sys.env("LOKI_URL")
-        }
-      }
-
-      // ============================================
-      // HOST JOURNALD SOURCE
-      // ============================================
-
-      // Read journald logs from the host
-      loki.source.journal "journal" {
-        max_age   = "24h"
-        relabel_rules = discovery.relabel.host_journal.rules
-        forward_to    = [loki.process.host_journal.receiver]
-      }
-
-      // ============================================
-      // HOST JOURNALD RELABELING
-      // ============================================
-
-      // Convert journald metadata into Loki labels
-      discovery.relabel "host_journal" {
-        targets = []
-
-        // Systemd unit name
-        rule {
-          source_labels = ["__journal__systemd_unit"]
-          target_label  = "systemd_unit"
-        }
-
-        // Transport type (stdout, syslog, kernel, audit, etc.)
-        rule {
-          source_labels = ["__journal__transport"]
-          target_label  = "journal_transport"
-        }
-
-        // Priority (info, warning, error)
-        rule {
-          source_labels = ["__journal_priority_keyword"]
-          target_label  = "level"
-        }
-      }
-
-      // ============================================
-      // HOST JOURNALD PROCESSING
-      // ============================================
-
-      // Add static labels and forward journald logs to Loki
-      loki.process "host_journal" {
-        stage.static_labels {
-          values = {
-            cluster = sys.env("CLUSTER_NAME"),
-            host    = sys.env("HOSTNAME"),
-            zone    = sys.env("ZONE"),
-            job     = "journald",
-            service_name = "journald",
-          }
-        }
-
-        forward_to = [loki.write.loki_srv.receiver]
-      }
-
-
-
-
-      // ============================================
-      // Metrics: Prometheus connection config
-      // ============================================
-
-      prometheus.remote_write "metrics_service" {
-        endpoint {
-          url = sys.env("PROMETHEUS_URL")
-        }
-        
-        // global labels
-        external_labels = {
-          cluster     = sys.env("CLUSTER_NAME"),
-          zone        = sys.env("ZONE"),
-        }
-      }
-
-
-      // ============================================
-      // Metrics: Prometheus Unix Exporter
-      // ============================================
-
-      prometheus.exporter.unix "node_exporter" {
-        rootfs_path = "/host"
-        disable_collectors = ["ipvs", "btrfs", "infiniband", "xfs", "zfs"]
-        enable_collectors = [
-          "cpu",
-          "meminfo",
-          "vmstat",
-          "loadavg",
-          "filesystem",
-          "netdev",
-          "uname",
-        ]
-
-        filesystem {
-          mount_points_exclude = "^(/host|/)(etc/resolv.conf|etc/hosts|etc/hostname|etc/machine-id|dataDisks/nvme01-ephemeral/.+|proc|sys|run/containerd/.+|run/k3s/containerd/.+|var/lib/docker/.+|var/lib/kubelet/.+)($|/)"
-          fs_types_exclude     = "^(tmpfs|shm|autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs|erofs)$"
-          mount_timeout        = "5s"
-        }
-
-        netclass {
-          ignored_devices = "^(veth.*|cali.*|[a-f0-9]{15})$"
-        }
-
-        netdev {
-          device_exclude = "^(veth.*|cali.*|[a-f0-9]{15})$"
-        }
-      }
-
-      prometheus.scrape "node_exporter" {
-        targets         = prometheus.exporter.unix.node_exporter.targets
-        scrape_interval = "30s"
-
-        forward_to = [prometheus.relabel.node_exporter.receiver]
-      }
-  
-      prometheus.relabel "node_exporter" {
-        forward_to = [prometheus.remote_write.metrics_service.receiver]
-
-        rule {
-          action       = "replace"
-          replacement  = sys.env("HOSTNAME")
-          target_label = "nodename"
-        }
-
-        rule {
-          action       = "replace"
-          replacement  = sys.env("HOSTNAME")
-          target_label = "node"
-        }
-      }
-
-
-
   extraEnv:
     - name: NODE_IP
       valueFrom:

services/monitoring/deploy/alloy/node-scraper/kustomization.yaml

@@ -3,6 +3,12 @@ kind: Kustomization
 namespace: monitoring
 resources:
   - helm-release.yaml
+  
+configMapGenerator:
+  - name: alloy-node-scraper
+    files:
+      - config.alloy=config.alloy
+
 secretGenerator:
   - name: alloy-node-scraper-helm-values
     files: