diff --git a/services/monitoring/deploy/alloy/global/kustomization.yaml b/services/monitoring/deploy/alloy/global/kustomization.yaml new file mode 100644 index 0000000..dfa4133 --- /dev/null +++ b/services/monitoring/deploy/alloy/global/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: monitoring +resources: + - secrets.yaml +generatorOptions: + disableNameSuffixHash: true \ No newline at end of file diff --git a/services/monitoring/deploy/promtail/helm-release.yaml b/services/monitoring/deploy/alloy/kube-scraper/helm-release.yaml similarity index 50% rename from services/monitoring/deploy/promtail/helm-release.yaml rename to services/monitoring/deploy/alloy/kube-scraper/helm-release.yaml index a4a5f87..c262e04 100644 --- a/services/monitoring/deploy/promtail/helm-release.yaml +++ b/services/monitoring/deploy/alloy/kube-scraper/helm-release.yaml @@ -1,22 +1,24 @@ apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: - name: promtail + name: alloy-kube-scraper spec: - releaseName: promtail + releaseName: alloy-kube-scraper interval: 1m chart: spec: - chart: promtail - version: 6.x.x + chart: alloy + version: 1.x.x sourceRef: kind: HelmRepository - name: grafana-legacy + name: grafana interval: 40h valuesFrom: - kind: Secret - name: promtail-helm-values + name: alloy-kube-scraper-helm-values valuesKey: values.yaml - - kind: Secret - name: promtail-helm-values - valuesKey: values.private.yaml \ No newline at end of file + + + + + diff --git a/services/monitoring/deploy/alloy/kube-scraper/helm-values.yaml b/services/monitoring/deploy/alloy/kube-scraper/helm-values.yaml new file mode 100644 index 0000000..be3697d --- /dev/null +++ b/services/monitoring/deploy/alloy/kube-scraper/helm-values.yaml @@ -0,0 +1,175 @@ +controller: + type: 'deployment' + tolerations: + - key: "infra.limbosolutions.com/dedicated" + operator: "Exists" + effect: "NoSchedule" + nodeSelector: + kubernetes.io/hostname: casa-minion-01 +alloy: + + # https://grafana.com/docs/alloy/latest/collect/logs-in-kubernetes/ + configMap: + content: | + // ============================================ + // LOKI WRITE TARGET + // ============================================ + loki.write "loki_srv" { + endpoint { + url = env("LOKI_URL") + } + } + + // ============================================ + // POD LOGS: DISCOVERY + RELABEL + // ============================================ + + // Discover Kubernetes pods + discovery.kubernetes "pod_logs" { + role = "pod" + } + + discovery.relabel "pod_logs" { + targets = discovery.kubernetes.pod_logs.targets + + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + + rule { + source_labels = ["__meta_kubernetes_pod_name"] + regex = "^(.+?)-[0-9a-zA-Z]{4,16}(?:-[0-9a-zA-Z]{4,16})?$" + replacement = "$1" + target_label = "pod" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + + rule { + source_labels = ["__meta_kubernetes_node_name"] + target_label = "node_name" + } + + rule { + source_labels = [ + "__meta_kubernetes_pod_controller_name", + ] + regex = "([0-9a-z-.]+?)(-[0-9a-f]{8,10})?" + target_label = "controller_name" + } + + rule { + source_labels = [ + "__meta_kubernetes_pod_label_app_kubernetes_io_name", + "__meta_kubernetes_pod_label_app", + "controller_name", + "__meta_kubernetes_pod_name", + ] + regex = "^;*([^;]+)(;.*)?$" + target_label = "app" + } + + rule { + source_labels = [ + "__meta_kubernetes_pod_label_app_kubernetes_io_component", + "__meta_kubernetes_pod_label_component", + ] + regex = "^;*([^;]+)(;.*)?$" + target_label = "component" + } + + rule { + source_labels = ["namespace", "app"] + separator = "/" + target_label = "job" + } + + rule { + source_labels = ["job", "container"] + separator = ":" + target_label = "instance" + } + rule { + source_labels = ["__meta_kubernetes_pod_node_name"] + separator = ":" + target_label = "host" + } + } + + + // ============================================ + // POD LOGS: SOURCE → PROCESS → WRITE + // ============================================ + + // Tail logs from Kubernetes pods + loki.source.kubernetes "pod_logs" { + // targets = discovery.kubernetes.pod_logs.targets + targets = discovery.relabel.pod_logs.output + forward_to = [loki.process.pod_logs.receiver] + } + + // Add static labels and forward logs to Loki + loki.process "pod_logs" { + + stage.static_labels { + values = { + cluster = env("CLUSTER_NAME"), + zone = env("ZONE"), + } + } + + forward_to = [loki.write.loki_srv.receiver] + } + + // ============================================ + // KUBERNETES EVENTS + // ============================================ + + // Tail Kubernetes events and convert them to log entries + loki.source.kubernetes_events "cluster_events" { + job_name = "integrations/kubernetes/eventhandler" + log_format = "logfmt" + forward_to = [ + loki.process.cluster_events.receiver, + ] + } + + // Add static labels and forward events to Loki + loki.process "cluster_events" { + forward_to = [loki.write.loki_srv.receiver] + + stage.static_labels { + values = { + cluster = env("CLUSTER_NAME"), + host = "casa-kube", + job = "cluster_events", + zone = env("ZONE"), + } + } + + // stage.labels { + // values = { + // kubernetes_cluster_events = "job", + // } + // } + } + + + extraEnv: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CLUSTER_NAME + value: casa + - name: ZONE + value: casa-vlan + - name: LOKI_URL + valueFrom: + secretKeyRef: + name: alloy-global + key: loki_url diff --git a/services/monitoring/deploy/promtail/kustomization.yaml b/services/monitoring/deploy/alloy/kube-scraper/kustomization.yaml similarity index 62% rename from services/monitoring/deploy/promtail/kustomization.yaml rename to services/monitoring/deploy/alloy/kube-scraper/kustomization.yaml index 22d6203..5298a73 100644 --- a/services/monitoring/deploy/promtail/kustomization.yaml +++ b/services/monitoring/deploy/alloy/kube-scraper/kustomization.yaml @@ -2,12 +2,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: monitoring resources: - - helm-repo.yaml - - helm-release.yaml + - helm-release.yaml secretGenerator: - - name: promtail-helm-values + - name: alloy-kube-scraper-helm-values files: - values.yaml=helm-values.yaml - - values.private.yaml=helm-values.private.yaml generatorOptions: disableNameSuffixHash: true \ No newline at end of file diff --git a/services/monitoring/deploy/alloy/kustomization.yaml b/services/monitoring/deploy/alloy/kustomization.yaml new file mode 100644 index 0000000..43eba25 --- /dev/null +++ b/services/monitoring/deploy/alloy/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: monitoring +resources: + - global + - node-scraper + - kube-scraper +generatorOptions: + disableNameSuffixHash: true \ No newline at end of file diff --git a/services/monitoring/deploy/alloy/node-scraper/helm-release.yaml b/services/monitoring/deploy/alloy/node-scraper/helm-release.yaml new file mode 100644 index 0000000..61f4079 --- /dev/null +++ b/services/monitoring/deploy/alloy/node-scraper/helm-release.yaml @@ -0,0 +1,24 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: alloy-node-scraper +spec: + releaseName: alloy-node-scraper + interval: 1m + chart: + spec: + chart: alloy + version: 1.x.x + sourceRef: + kind: HelmRepository + name: grafana + interval: 40h + valuesFrom: + - kind: Secret + name: alloy-node-scraper-helm-values + valuesKey: values.yaml + + + + + diff --git a/services/monitoring/deploy/alloy/node-scraper/helm-values.yaml b/services/monitoring/deploy/alloy/node-scraper/helm-values.yaml new file mode 100644 index 0000000..b38729e --- /dev/null +++ b/services/monitoring/deploy/alloy/node-scraper/helm-values.yaml @@ -0,0 +1,109 @@ +controller: + nameOverride: "alloy-node-scrapper" + volumes: + extra: + # requires host journal configuration storage = volatile + - name: run-log-journal + hostPath: + path: /run/log/journal + # Log scrapers (e.g., Alloy/Loki) read this ID from the journal metadata and use it to separate log streams per machine. + - name: etc-machine-id + hostPath: + path: /etc/machine-id + +alloy: + mounts: + varlog: true # Mounts /var/log from the host (persistent journal) + extra: + # requires host journal configuration storage = volatile + - name: run-log-journal + mountPath: /run/log/journal + readOnly: true + # required + - name: etc-machine-id + mountPath: /etc/machine-id + readOnly: true + + # https://grafana.com/docs/alloy/latest/collect/logs-in-kubernetes/ + configMap: + content: | + // ============================================ + // LOKI WRITE TARGET + // ============================================ + loki.write "loki_srv" { + endpoint { + url = env("LOKI_URL") + } + } + + // ============================================ + // HOST JOURNALD SOURCE + // ============================================ + + // Read journald logs from the host + loki.source.journal "journal" { + max_age = "24h" + relabel_rules = discovery.relabel.host_journal.rules + forward_to = [loki.process.host_journal.receiver] + } + + // ============================================ + // HOST JOURNALD RELABELING + // ============================================ + + // Convert journald metadata into Loki labels + discovery.relabel "host_journal" { + targets = [] + + // Systemd unit name + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "systemd_unit" + } + + // Transport type (stdout, syslog, kernel, audit, etc.) + rule { + source_labels = ["__journal__transport"] + target_label = "journal_transport" + } + + // Priority (info, warning, error) + rule { + source_labels = ["__journal_priority_keyword"] + target_label = "level" + } + } + + // ============================================ + // HOST JOURNALD PROCESSING + // ============================================ + + // Add static labels and forward journald logs to Loki + loki.process "host_journal" { + stage.static_labels { + values = { + cluster = env("CLUSTER_NAME"), + host = env("NODE_NAME"), + zone = env("ZONE"), + job = "journald", + service_name = "journald", + } + } + + forward_to = [loki.write.loki_srv.receiver] + } + + extraEnv: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CLUSTER_NAME + value: casa + - name: ZONE + value: casa-vlan + - name: LOKI_URL + valueFrom: + secretKeyRef: + name: alloy-global + key: loki_url diff --git a/services/monitoring/deploy/alloy/node-scraper/kustomization.yaml b/services/monitoring/deploy/alloy/node-scraper/kustomization.yaml new file mode 100644 index 0000000..f445d22 --- /dev/null +++ b/services/monitoring/deploy/alloy/node-scraper/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: monitoring +resources: + - helm-release.yaml +secretGenerator: + - name: alloy-node-scraper-helm-values + files: + - values.yaml=helm-values.yaml +generatorOptions: + disableNameSuffixHash: true \ No newline at end of file diff --git a/services/monitoring/deploy/flux/promtail-app-sync.yaml b/services/monitoring/deploy/flux/alloy-app-sync.yaml similarity index 86% rename from services/monitoring/deploy/flux/promtail-app-sync.yaml rename to services/monitoring/deploy/flux/alloy-app-sync.yaml index 05bcd5c..d65b2de 100644 --- a/services/monitoring/deploy/flux/promtail-app-sync.yaml +++ b/services/monitoring/deploy/flux/alloy-app-sync.yaml @@ -8,7 +8,7 @@ spec: kind: GitRepository name: casa namespace: casa-limbosolutions-com - path: services/monitoring/deploy/promtail + path: services/monitoring/deploy/alloy prune: true decryption: provider: sops diff --git a/services/monitoring/deploy/flux/kustomization.yaml b/services/monitoring/deploy/flux/kustomization.yaml index 78e1dad..1f5aca1 100644 --- a/services/monitoring/deploy/flux/kustomization.yaml +++ b/services/monitoring/deploy/flux/kustomization.yaml @@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: monitoring resources: - - promtail-app-sync.yaml + - alloy-app-sync.yaml - prometheus-app-sync.yaml secretGenerator: - name: flux-sops-age diff --git a/services/monitoring/deploy/promtail/helm-repo.yaml b/services/monitoring/deploy/promtail/helm-repo.yaml deleted file mode 100644 index 3a21612..0000000 --- a/services/monitoring/deploy/promtail/helm-repo.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: source.toolkit.fluxcd.io/v1 -kind: HelmRepository -metadata: - name: grafana-legacy -spec: - interval: 40h - url: https://grafana.github.io/helm-charts \ No newline at end of file diff --git a/services/monitoring/deploy/promtail/helm-values.private.yaml b/services/monitoring/deploy/promtail/helm-values.private.yaml deleted file mode 100644 index 02328ed..0000000 --- a/services/monitoring/deploy/promtail/helm-values.private.yaml +++ /dev/null @@ -1,18 +0,0 @@ -config: - clients: - - url: ENC[AES256_GCM,data:AarLpmfJTu63kYzATeKf4m+60h93G5unSf2e8BplmCws7iVRzeFYGdvp14caaFZiZwWXe5rsdrMBQRc=,iv:Se74MvPyIP5xDcjrKBv3/X4G3G+Q9AYmdK/5t4yDuZc=,tag:A64ERrlrlCgf7PiQMT9WuQ==,type:str] -sops: - age: - - enc: | - -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBYWjJxcitwdjR3QzQrSGVn - OVRFeTVrUGZMWlcycUREeWJWenpnVDVSbVJRClphakRuL2h2dEsvYWQ3VXJ4aHZL - YzQrU0FiRWttRUpmQkd5eVJFVVZBdVkKLS0tIDB3Y1FwQU5ndVlOQzNkZHA3V1Vl - bVpyTmhtUUhVTk9xZUFibHFyMVdqOEEKgoIrI9rJ1Q93AOZrP8r4rOggIGpSDv2H - uLp0yj1VqyyvtB/RHu4/Gyef2P5IwjTBnYYhZHbfX3AnYYWN58Riog== - -----END AGE ENCRYPTED FILE----- - recipient: age1f9e4pvp5y8gzuk8mz2s5xm85dd7znxhk56tcpuxqwn78qfjwja0qekwlju - encrypted_regex: ^(.*)$ - lastmodified: "2026-06-07T21:04:02Z" - mac: ENC[AES256_GCM,data:Ieh41SbHtPqOIT2ynSEnz+qwaCsEDo9cZOk63AyuiMqsT0vR8TR94gimOKrMgQhjLpJPREYg0hXALgq7x6BxMfzts8n+eRuDsVxah8e17Ad4Gk9Vq9RtHYL06RO4EhevhzuzX32W8N1jt2wJTSDA4Ztjh1QIAAd7YyNnvYOATBo=,iv:eNzc4ObZ7lplIDPjF8Ub4Rfq3AiWLyOGwAdMLY7ojvo=,tag:87y5KNeAYASA/wDs4ETWmw==,type:str] - version: 3.13.1 diff --git a/services/monitoring/deploy/promtail/helm-values.yaml b/services/monitoring/deploy/promtail/helm-values.yaml deleted file mode 100644 index 8653f5c..0000000 --- a/services/monitoring/deploy/promtail/helm-values.yaml +++ /dev/null @@ -1,52 +0,0 @@ -config: - clients: - - url: "????" #replaced values.local.yaml. Example: https://lokiserver/loki/api/v1/push - # by default all scrap configs had node_name - snippets: - extraRelabelConfigs: - - target_label: host - replacement: ${HOSTNAME} - - target_label: cluster - replacement: casa - - extraScrapeConfigs: | - #scrape config for syslog - - job_name: host-journald - journal: - json: true - max_age: 24h - path: /var/log/host/journal - labels: - job: journald - relabel_configs: - - source_labels: ['__journal__systemd_unit'] - target_label: 'journal_systemd_unit' - - source_labels: ['__journal_syslog_identifier'] - target_label: 'journal_syslog_identifier' - - source_labels: ['__journal__hostname'] - target_label: 'journal_hostname' - - target_label: 'host' - replacement: '${HOSTNAME}' - - target_label: 'cluster' - replacement: 'casa' - -extraArgs: - - --config.expand-env=true -extraVolumes: - - name: node-logs - hostPath: - path: /var/log - -extraVolumeMounts: - - name: node-logs - mountPath: /var/log/host - readOnly: true - -resources: - limits: - cpu: 200m - memory: 100Mi - requests: - cpu: 100m - memory: 50Mi -