Compare commits

...
14 Commits
Author SHA1 Message Date
Márcio Fernandes 7d5a8dc335 monitoring: add zone, and cluster name 2026-09-12 15:09:35 +00:00
Márcio Fernandes 703e560532 renamed: cluster/repositories/casa-limbosolutions-com/README.md -> cluster/repos/casa-limbosolutions-com/README.md
renamed:    cluster/repositories/casa-limbosolutions-com/certificate.yaml -> cluster/repos/casa-limbosolutions-com/certificate.yaml
renamed:    cluster/repositories/casa-limbosolutions-com/deploy/flux/.env.d/.gitignore -> cluster/repos/casa-limbosolutions-com/deploy/flux/.env.d/.gitignore
renamed:    cluster/repositories/casa-limbosolutions-com/deploy/flux/git-repo.yaml -> cluster/repos/casa-limbosolutions-com/deploy/flux/git-repo.yaml
renamed:    cluster/repositories/casa-limbosolutions-com/deploy/flux/kustomization.yaml -> cluster/repos/casa-limbosolutions-com/deploy/flux/kustomization.yaml
renamed:    cluster/repositories/myinfra-kubernetes-shared/git-repo.yaml -> cluster/repos/myinfra-kubernetes-shared/git-repo.yaml
renamed:    cluster/repositories/myinfra-kubernetes-shared/kustomization.yaml -> cluster/repos/myinfra-kubernetes-shared/kustomization.yaml
2026-09-12 14:57:15 +00:00
Márcio Fernandes 20128a2f59 modified: cluster/monitoring/alloy-global-sync.yaml 2026-09-12 14:51:34 +00:00
Márcio Fernandes 9be1d6b2bc cluster: add myinfra-kubernetes-shared-monitoring-sync 2026-09-12 14:33:24 +00:00
Márcio Fernandes 28022eb5f3 monitoring: moved to https://git.limbosolutions.com/myLimbo/myinfra.kubernetes.shared repo 2026-09-12 14:11:02 +00:00
Márcio Fernandes 097bbad83f alloy: configMapGenerator setup config.alloy 2026-09-12 12:01:15 +00:00
Márcio Fernandes 6d05d4c0a7 modified: services/monitoring/deploy/alloy/kube-scraper/helm-values.yaml
modified:   services/monitoring/deploy/alloy/node-scraper/helm-values.yaml
2026-09-12 10:04:46 +00:00
Márcio Fernandes 3274a2a3b4 modified: nodes/ansible/os-get-info.playbook.yaml
new file:   nodes/ansible/os-reboot.playbook.yaml
modified:   nodes/ansible/os-update.playbook.yaml
new file:   nodes/ansible/tasks/os-get-info.yaml
new file:   nodes/ansible/tasks/os-reboot.yaml
new file:   nodes/ansible/tasks/os-update.yaml
2026-09-12 10:04:18 +00:00
Márcio Fernandes a5d4882512 ally: kube and node scrapers, all dashboards working on grafana 2026-09-10 23:26:21 +00:00
Márcio Fernandes 7eb3469750 deleted: services/monitoring/deploy/alloy/alloy-loki-examples copy.config
deleted:    services/monitoring/deploy/alloy/alloy-prometheus-examples.config
modified:   services/monitoring/deploy/alloy/kube-scraper/helm-values.yaml
2026-09-07 22:49:51 +00:00
Márcio Fernandes b4a909e5b6 new file: services/monitoring/deploy/flux/flux-kustomization-prometheus.yaml
modified:   services/monitoring/deploy/flux/kustomization.yaml
2026-09-07 02:10:42 +00:00
Márcio Fernandes dcad0b31b7 modified: services/monitoring/deploy/alloy/kube-scraper/helm-values.yaml 2026-09-07 01:58:16 +00:00
Márcio Fernandes 33be97abcb modified: services/monitoring/deploy/alloy/kube-scraper/helm-values.yaml 2026-09-07 01:47:14 +00:00
Márcio Fernandes 0f8dc29331 modified: services/monitoring/deploy/alloy/kube-scraper/helm-values.yaml 2026-09-07 01:45:48 +00:00
36 changed files with 113 additions and 979 deletions
+1
View File
@@ -12,6 +12,7 @@
"source=${localWorkspaceFolder}/../myLimbo.Jellyfin,target=/workspaces/myLimbo.Jellyfin,type=bind",
"source=${localWorkspaceFolder}/../kb,target=/workspaces/kb,type=bind",
"source=${localWorkspaceFolder}/../pi.bluetooth.speaker,target=/workspaces/pi.bluetooth.speaker,type=bind",
"source=${localWorkspaceFolder}/../myinfra.kubernetes.shared,target=/workspaces/myinfra.kubernetes.shared,type=bind",
"source=${localWorkspaceFolder}/.env.d/kube,target=/home/vscode/.kube,type=bind",
"source=${localEnv:HOME}/.gitconfig,target=/home/vscode/.gitconfig,type=bind,consistency=cached",
"source=${localEnv:HOME}/.ssh,target=/home/vscode/.ssh,type=bind,consistency=cached"
+5
View File
@@ -18,6 +18,11 @@
"name": "Jellyfin",
"path": "../myLimbo.Jellyfin"
},
{
"name": "myinfra.kubernetes.shared",
"path": "../myinfra.kubernetes.shared"
},
{
"name": "kb",
"path": "../kb"
@@ -1,14 +1,14 @@
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: alloy
name: alloy-global
spec:
interval: 1m
sourceRef:
kind: GitRepository
name: casa
namespace: casa-limbosolutions-com
path: services/monitoring/deploy/alloy
path: services/monitoring/alloy-global
prune: true
decryption:
provider: sops
@@ -2,7 +2,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: monitoring
resources:
- flux-kustomization-alloy.yaml
- alloy-global-sync.yaml
- myinfra-kubernetes-shared-monitoring-sync.yaml
secretGenerator:
- name: flux-sops-age
files:
@@ -0,0 +1,12 @@
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: myinfra-kubernetes-shared-monitoring
spec:
interval: 1m
sourceRef:
kind: GitRepository
name: myinfra-kubernetes-shared
namespace: casa-limbosolutions-com
path: monitoring/overlays/complete-stack
prune: true
@@ -0,0 +1,9 @@
apiVersion: source.toolkit.fluxcd.io/v1
kind: GitRepository
metadata:
name: myinfra-kubernetes-shared
spec:
interval: 1m0s
url: https://git.limbosolutions.com/myLimbo/myinfra.kubernetes.shared.git
ref:
branch: main
@@ -1,9 +1,7 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: monitoring
namespace: casa-limbosolutions-com
resources:
- global
- node-scraper
- kube-scraper
- git-repo.yaml
generatorOptions:
disableNameSuffixHash: true
+4
View File
@@ -2,3 +2,7 @@
casa
casa-mini-minion-01
casa-minion-01
[maintenance-node-target]
casa-mini-minion-01
+3 -8
View File
@@ -1,12 +1,7 @@
- name: OS update playbook
- name: OS Info playbook
hosts:
- all
become: true
tasks:
- debug:
msg:
- "Distribution: {{ hostvars[inventory_hostname].ansible_distribution }}"
- "Major version: {{ hostvars[inventory_hostname].ansible_distribution_major_version }}"
- "Version: {{ hostvars[inventory_hostname].ansible_distribution_version }}"
- "kernel: {{ hostvars[inventory_hostname].ansible_kernel }}"
- ansible.builtin.import_tasks:
file: tasks/os-get-info.yaml
+8
View File
@@ -0,0 +1,8 @@
- name: Reboot system playbook
hosts:
- maintenance-node-target
become: true
tasks:
- ansible.builtin.import_tasks:
file: tasks/os-reboot.yaml
+3 -39
View File
@@ -1,44 +1,8 @@
- name: OS update playbook
hosts:
- casa
- casa-minion-01
- casa-mini-minion-01
- maintenance-node-target
become: true
tasks:
- debug:
msg:
- "ansible_distribution {{ hostvars[inventory_hostname].ansible_distribution }}"
- "major version {{ hostvars[inventory_hostname].ansible_distribution_major_version }}"
- "version {{ hostvars[inventory_hostname].ansible_distribution_version }}"
- "version {{ hostvars[inventory_hostname].ansible_kernel }}"
- ansible.builtin.import_tasks:
file: tasks/os-update.yaml
- name: Update all Packages to their latest version
ansible.builtin.apt:
update_cache: true
force_apt_get: true
- name: Upgrade all packages on servers
ansible.builtin.apt:
upgrade: dist
force_apt_get: true
- name: Check if a reboot is needed
register: reboot_required_file
stat: path=/var/run/reboot-required
- name: Reboot server
reboot:
msg: "Reboot initiated by Ansible due to kernel updates"
connect_timeout: 5
reboot_timeout: 300
pre_reboot_delay: 0
post_reboot_delay: 30
test_command: uptime
when: reboot_required_file.stat.exists
- debug:
msg:
- "ansible_distribution {{ hostvars[inventory_hostname].ansible_distribution }}"
- "major version {{ hostvars[inventory_hostname].ansible_distribution_major_version }}"
- "version {{ hostvars[inventory_hostname].ansible_distribution_version }}"
- "version {{ hostvars[inventory_hostname].ansible_kernel }}"
+6
View File
@@ -0,0 +1,6 @@
- debug:
msg:
- "Distribution: {{ hostvars[inventory_hostname].ansible_distribution }}"
- "Major version: {{ hostvars[inventory_hostname].ansible_distribution_major_version }}"
- "Version: {{ hostvars[inventory_hostname].ansible_distribution_version }}"
- "kernel: {{ hostvars[inventory_hostname].ansible_kernel }}"
+7
View File
@@ -0,0 +1,7 @@
- reboot:
msg: "Reboot initiated by Ansible due to kernel updates"
connect_timeout: 5
reboot_timeout: 300
pre_reboot_delay: 0
post_reboot_delay: 30
test_command: uptime
+23
View File
@@ -0,0 +1,23 @@
- ansible.builtin.import_tasks:
file: os-get-info.yaml
- name: Update all Packages to their latest version
ansible.builtin.apt:
update_cache: true
force_apt_get: true
- name: Upgrade all packages on servers
ansible.builtin.apt:
upgrade: dist
force_apt_get: true
- name: Check if a reboot is needed
register: reboot_required_file
stat: path=/var/run/reboot-required
- ansible.builtin.import_tasks:
file: os-reboot.yaml
when: reboot_required_file.stat.exists
- ansible.builtin.import_tasks:
file: os-get-info.yaml
+2 -10
View File
@@ -2,16 +2,8 @@
## alloy
**promtail Encrypt secrets:**
**Encrypt global secrets:**
``` bash
sops -e deploy/alloy/global/secrets.dec.yaml > deploy/alloy/global/secrets.yaml
sops -e alloy-global/secrets.dec.yaml > alloy-global/secrets.yaml
```
**Check agent:**
``` bash
http://localhost:9090/agent
```
## prometheus - kube-state-metrics
@@ -3,6 +3,5 @@ kind: Kustomization
namespace: monitoring
resources:
- secrets.yaml
- grafana-helm-repo.yaml
generatorOptions:
disableNameSuffixHash: true
@@ -0,0 +1,24 @@
apiVersion: v1
kind: Secret
metadata:
name: alloy-global
stringData:
loki_url: ENC[AES256_GCM,data:77WX9WmaQSCgXmU1NxBwjCyY+67innPetnjAvrCY2GRC9jCg3fH5ul0oq9tmh591MH+Jm3PqWyjaDew=,iv:x3sRiuVs3x4KKR8ZApvA87ZyiNzNrXDJ7/9p0KUMMqI=,tag:pdSxPi+XiGF/ZqjKYst/bA==,type:str]
prometheus_url: ENC[AES256_GCM,data:BBmLjjl0k7J1PkvDiIp3eHzcwqFjOHAGHOhAuNox+fGXwp+bBb/ZABqmCLTFEPMIDUc+FeCr4W3Q4J6AhQ==,iv:WnJAehA6+3l3Fu2T7Zg9OtTYFkwAC5w4MNya4hBiUpk=,tag:+LW3URk3gEUZfagwOhs5qg==,type:str]
zone: ENC[AES256_GCM,data:xPRm2s/Kc0lf,iv:YzimQNvQrEuA1A6saA977uHkGa83Z+9ey+QA6aYY22o=,tag:jg/2hix9bm6wkiXBgpd/AQ==,type:str]
cluster_name: ENC[AES256_GCM,data:kw0RVQ==,iv:Xf7GCffqPb9Zxj0+WfRLx13W7LU/7Cx8Skjui9bwRTs=,tag:wz+OTyhb3HQcMQqjG/Mreg==,type:str]
sops:
age:
- enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrdnZDb0lwc0ZvaWlIYzJs
ZVhJdnkvSVRlSUM1bUpocmNGRzkxUnZPdDJZClI2V2tLZ1pvY3U5NFRYYzE1VUdG
NFdpTE43U0tkTEF0cG5GRkhobEhteVEKLS0tIGZ2YVpjTHFrSVRhNURFZkJJQUxs
UTdKZGlLUkp2b21YRzVtdEQ2ZmdFN3MKjrJqtfMVd5t2qnDqio7cYRC3Ntpf2Gdk
NtPQw32lG8oFKBn1rDPI/ZkY/wqEPd2PLrnW9o5mzUcjltlVpWkKMw==
-----END AGE ENCRYPTED FILE-----
recipient: age1f9e4pvp5y8gzuk8mz2s5xm85dd7znxhk56tcpuxqwn78qfjwja0qekwlju
encrypted_regex: ^(data|stringData)$
lastmodified: "2026-09-12T15:08:17Z"
mac: ENC[AES256_GCM,data:8nLBCFCEICcqD4dxEFMPbpWJnVCKOH7NIfgQbsKix5rog1uUT5LR3/ZmEDpT3PWUmnCUEECWcwno4JZ7zfmNp0R3IpxAQESaA32SJBgiZwyMgQCKIORUBDIZG2vZ5miQmFgsBRIbS+o5HPILJWFBf9KOaEaE2w/24i5HCnhEO3c=,iv:Bp2wRfhV3YX+8IVW3mg7Shx0+pv18owVn0eMt0srABE=,tag:I1xMjQym63sT/iwErN4kRA==,type:str]
version: 3.13.1
@@ -1,144 +0,0 @@
// ============================================
// LOKI WRITE TARGET
// ============================================
loki.write "loki_srv" {
endpoint {
url = env("LOKI_URL")
}
}
// ============================================
// POD LOGS: DISCOVERY + RELABEL
// ============================================
// Discover Kubernetes pods
discovery.kubernetes "pod_logs" {
role = "pod"
}
discovery.relabel "pod_logs" {
targets = discovery.kubernetes.pod_logs.targets
rule {
source_labels = ["__meta_kubernetes_namespace"]
target_label = "namespace"
}
rule {
source_labels = ["__meta_kubernetes_pod_name"]
regex = "^(.+?)-[0-9a-zA-Z]{4,16}(?:-[0-9a-zA-Z]{4,16})?$"
replacement = "$1"
target_label = "pod"
}
rule {
source_labels = ["__meta_kubernetes_pod_container_name"]
target_label = "container"
}
rule {
source_labels = ["__meta_kubernetes_node_name"]
target_label = "node_name"
}
rule {
source_labels = [
"__meta_kubernetes_pod_controller_name",
]
regex = "([0-9a-z-.]+?)(-[0-9a-f]{8,10})?"
target_label = "controller_name"
}
rule {
source_labels = [
"__meta_kubernetes_pod_label_app_kubernetes_io_name",
"__meta_kubernetes_pod_label_app",
"controller_name",
"__meta_kubernetes_pod_name",
]
regex = "^;*([^;]+)(;.*)?$"
target_label = "app"
}
rule {
source_labels = [
"__meta_kubernetes_pod_label_app_kubernetes_io_component",
"__meta_kubernetes_pod_label_component",
]
regex = "^;*([^;]+)(;.*)?$"
target_label = "component"
}
rule {
source_labels = ["namespace", "app"]
separator = "/"
target_label = "job"
}
rule {
source_labels = ["job", "container"]
separator = ":"
target_label = "instance"
}
rule {
source_labels = ["__meta_kubernetes_pod_node_name"]
separator = ":"
target_label = "host"
}
}
// ============================================
// POD LOGS: SOURCE → PROCESS → WRITE
// ============================================
// Tail logs from Kubernetes pods
loki.source.kubernetes "pod_logs" {
// targets = discovery.kubernetes.pod_logs.targets
targets = discovery.relabel.pod_logs.output
forward_to = [loki.process.pod_logs.receiver]
}
// Add static labels and forward logs to Loki
loki.process "pod_logs" {
stage.static_labels {
values = {
cluster = env("CLUSTER_NAME"),
zone = env("ZONE"),
}
}
forward_to = [loki.write.loki_srv.receiver]
}
// ============================================
// KUBERNETES EVENTS
// ============================================
// Tail Kubernetes events and convert them to log entries
loki.source.kubernetes_events "cluster_events" {
job_name = "integrations/kubernetes/eventhandler"
log_format = "logfmt"
forward_to = [
loki.process.cluster_events.receiver,
]
}
// Add static labels and forward events to Loki
loki.process "cluster_events" {
forward_to = [loki.write.loki_srv.receiver]
stage.static_labels {
values = {
cluster = env("CLUSTER_NAME"),
host = "env("CLUSTER_NAME")",
job = "cluster_events",
zone = env("ZONE"),
}
}
}
@@ -1,146 +0,0 @@
// ============================================
// Metrics: Discovery Kubernetes Services
// ============================================
// discovery.kubernetes "k8s_services" {
// role = "service"
// }
// ============================================
// Metrics: Kube State Metrics (k8s_services)
// ============================================
// prometheus.scrape "kube_state_metrics" {
// job_name = "integrations/kubernetes/kube-state-metrics"
// targets = discovery.relabel.kube_state_metrics.output
//
// scheme = "http"
//
// forward_to = [prometheus.remote_write.prometheus_srv.receiver]
// }
// ============================================
// API Server discovery
// ============================================
// discovery.relabel "apiserver" {
// targets = discovery.kubernetes.k8s_services.targets
// rule {
// source_labels = ["__meta_kubernetes_service_name"]
// regex = "kubernetes"
// action = "keep"
// }
// rule {
// target_label = "cluster"
// replacement = env("CLUSTER_NAME")
// }
// }
// prometheus.scrape "apiserver" {
// job_name = "integrations/kubernetes/apiserver"
// targets = discovery.relabel.apiserver.output
// scheme = "https"
// metrics_path = "/metrics"
// tls_config {
// server_name = "kubernetes"
// ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
// insecure_skip_verify = true
// }
// bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
// forward_to = [prometheus.remote_write.prometheus_srv.receiver]
// }
// ============================================
// CoreDNS discovery
// ============================================
// discovery.relabel "coredns" {
// targets = discovery.kubernetes.k8s_services.targets
//
// rule {
// source_labels = ["__meta_kubernetes_pod_label_k8s_app"]
// regex = "kube-dns"
// action = "keep"
// }
// rule {
// target_label = "cluster"
// replacement = env("CLUSTER_NAME")
// }
// }
// prometheus.scrape "coredns" {
// job_name = "integrations/kubernetes/coredns-pods"
// targets = discovery.relabel.coredns.output
// scheme = "http"
// metrics_path = "/metrics"
//
// forward_to = [prometheus.remote_write.prometheus_srv.receiver]
// }
// ============================================
// Prometheus: prometheus_srv
// ============================================
prometheus.remote_write "prometheus_srv" {
endpoint {
url = env("PROMETHEUS_URL")
}
// global labels
external_labels = {
cluster = env("CLUSTER_NAME"),
zone = env("ZONE"),
}
}
// ===============================================================================
// prometheus / Metrics - Node exporter
// Requirements:
// - discovery.relabel - node_exporter
// ===============================================================================
prometheus.scrape "node_exporter" {
job_name = "integrations/kubernetes/node-exporter"
targets = discovery.relabel.node_exporter.output
scheme = "http"
forward_to = [prometheus.remote_write.prometheus_srv.receiver]
}
discovery.relabel "node_exporter" {
targets = discovery.kubernetes.kube_pods.targets
rule {
source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
regex = "node-exporter|prometheus-node-exporter"
action = "keep"
}
rule {
source_labels = ["__meta_kubernetes_pod_container_port_number"]
regex = "9100"
action = "keep"
}
rule {
source_labels = ["__meta_kubernetes_pod_node_name"]
target_label = "node"
}
rule {
source_labels = ["__meta_kubernetes_pod_node_name"]
target_label = "nodename"
}
rule {
source_labels = ["__meta_kubernetes_pod_node_name"]
target_label = "instance"
}
}
@@ -1,8 +0,0 @@
apiVersion: source.toolkit.fluxcd.io/v1
kind: HelmRepository
metadata:
name: grafana
spec:
interval: 24h
url: https://grafana.github.io/helm-charts
@@ -1,22 +0,0 @@
apiVersion: v1
kind: Secret
metadata:
name: alloy-global
stringData:
loki_url: ENC[AES256_GCM,data:2DeZZ+7deDbojQ/+YNgT8KwDmjFFyAqMcTX8l6itlF15hilXKgYP1p/SNoyw1JGJ7x5VZqUl0VIVhVY=,iv:SlFiLvEeIqj+zQ/8Jspc9FL/ElneIlWBzGsM+6mGZLc=,tag:9tIl6fi+xfmF+TJpMXgbKg==,type:str]
prometheus_url: ENC[AES256_GCM,data:k8Z3lvZRxOmlw6Wx4aLAgV93hPcEawtoprWU2Kn6j/WIe/YDHM9iuB+pRy7ght4I6jxyNFLotN0BaHE1yg==,iv:EEnRoS287e9EV8+25NtlEO31zN0eyfRBOUe9GVBah18=,tag:2K/MnM1QlWPbceQBqCwylw==,type:str]
sops:
age:
- enc: |
-----BEGIN AGE ENCRYPTED FILE-----
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBJYmx1dGJTSGZvbVdpV1di
Z3dKNUJPVGNkK1hBSnVoRXZrb0R4TmxnUjM4CmpyUXpGN2lneDViN3VPNC9Fa3d6
RURxZTFuV0o1SzF5ZEhzSVZyYkY5alkKLS0tIEF1T0tINnRUa0tDOGlaSVRJajF5
bTFTd0c4Z3p1aUNWWTZ0ZFVXQVdrMjAKku2YU048qf4E9vEPO19HNfAg78E3fSFQ
WBiUgiqtqG/T4cI5cf4CUWoUogtABawBG4z5G5+ud7sfVMzfqW+W1g==
-----END AGE ENCRYPTED FILE-----
recipient: age1f9e4pvp5y8gzuk8mz2s5xm85dd7znxhk56tcpuxqwn78qfjwja0qekwlju
encrypted_regex: ^(data|stringData)$
lastmodified: "2026-08-30T22:14:34Z"
mac: ENC[AES256_GCM,data:XQ2zRn6zGBONfBiyaip3DpwzIfDBN1mPI65fVfH3c7tsEeY1mV/ZRPD+5iJsiwxlnBa7Rq0iJNOr2f84JE+CiuqYz8UKuQMS02gsvqXuBSgUn5vDRcX/aBV3+1GSOzv3ilmwMkJI/jxfyyXL4YsxKl0fsFEOJRsa3VCCVF0OvrM=,iv:enk8h+TfYBIOFln3SlOw30arg2T/kSU3IMwusrm68Qc=,tag:yp9eQc56Ht6g3x3QnMqMZg==,type:str]
version: 3.13.1
@@ -1,24 +0,0 @@
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: alloy-kube-scraper
spec:
releaseName: alloy-kube-scraper
interval: 1m
chart:
spec:
chart: alloy
version: 1.x.x
sourceRef:
kind: HelmRepository
name: grafana
interval: 40h
valuesFrom:
- kind: Secret
name: alloy-kube-scraper-helm-values
valuesKey: values.yaml
@@ -1,265 +0,0 @@
controller:
type: 'deployment'
nodeSelector:
kubernetes.io/hostname: casa-minion-01
alloy:
# https://grafana.com/docs/alloy/latest/collect/logs-in-kubernetes/
configMap:
content: |
// ============================================
// Discovery: Kubernetes - Nodes
// used by:
// - prometheus - Kube_node_metrics - cAdvisor
// ============================================
discovery.kubernetes "kube_nodes" {
role = "node"
}
// ============================================
// Discovery: Kubernetes - pods
// used by:
// - loki kubernetes pod logs
// ============================================
discovery.kubernetes "kube_pods" {
role = "pod"
}
// ============================================
// Discovery: Kubernetes - endpoints
// used by:
// - loki kubernetes kube state metrics
// ============================================
discovery.kubernetes "kube_endpoints" {
role = "service"
}
// ============================================
// Logs: Loki config
// ============================================
loki.write "loki_srv" {
endpoint {
url = env("LOKI_URL")
}
}
loki.process "loki_srv" {
stage.static_labels {
values = {
cluster = env("CLUSTER_NAME"),
zone = env("ZONE"),
}
}
forward_to = [loki.write.loki_srv.receiver]
}
// ============================================
// Loki.source kube_pod_logs
// Requirements:
// - discovery.relabel: kube_pod_logs
// - discovery.process: loki_srv
// ============================================
loki.source.kubernetes "kube_pod_logs" {
targets = discovery.relabel.kube_pod_logs.output
forward_to = [loki.process.loki_srv.receiver]
}
// ============================================
// Discovery:relabel: kube_pod_logs
// Requirements:
// - discovery.kubernetes.role.pod: kube_pods
// ============================================
discovery.relabel "kube_pod_logs" {
targets = discovery.kubernetes.kube_pods.targets
rule {
source_labels = ["__meta_kubernetes_namespace"]
target_label = "namespace"
}
rule {
source_labels = ["__meta_kubernetes_pod_name"]
regex = "^(.+?)-[0-9a-zA-Z]{4,16}(?:-[0-9a-zA-Z]{4,16})?$"
replacement = "$1"
target_label = "pod"
}
rule {
source_labels = ["__meta_kubernetes_pod_container_name"]
target_label = "container"
}
rule {
source_labels = ["__meta_kubernetes_node_name"]
target_label = "node_name"
}
rule {
source_labels = [
"__meta_kubernetes_pod_controller_name",
]
regex = "([0-9a-z-.]+?)(-[0-9a-f]{8,10})?"
target_label = "controller_name"
}
rule {
source_labels = [
"__meta_kubernetes_pod_label_app_kubernetes_io_name",
"__meta_kubernetes_pod_label_app",
"controller_name",
"__meta_kubernetes_pod_name",
]
regex = "^;*([^;]+)(;.*)?$"
target_label = "app"
}
rule {
source_labels = [
"__meta_kubernetes_pod_label_app_kubernetes_io_component",
"__meta_kubernetes_pod_label_component",
]
regex = "^;*([^;]+)(;.*)?$"
target_label = "component"
}
rule {
source_labels = ["namespace", "app"]
separator = "/"
target_label = "job"
}
rule {
source_labels = ["job", "container"]
separator = ":"
target_label = "instance"
}
rule {
source_labels = ["__meta_kubernetes_pod_node_name"]
separator = ":"
target_label = "host"
}
}
// ===============================================================================
// prometheus / Metrics - Scrape kube_state_metrics
// Requirements:
// - discovery.relabel - kube_state_metrics
// ===============================================================================
prometheus.scrape "kube_state_metrics" {
job_name = "integrations/kubernetes/kube-state-metrics"
targets = discovery.relabel.kube_state_metrics.output
scheme = "http"
forward_to = [prometheus.remote_write.prometheus_srv.receiver]
}
discovery.relabel "kube_state_metrics" {
targets = discovery.kubernetes.kube_endpoints.targets
rule {
source_labels = ["__meta_kubernetes_service_name"]
regex = "kube-state-metrics"
action = "keep"
}
rule {
source_labels = ["__meta_kubernetes_endpoint_node_name"]
target_label = "node"
}
rule {
source_labels = ["__meta_kubernetes_endpoint_node_name"]
target_label = "host"
}
// rule {
// source_labels = ["host"]
// target_label = "nodename"
// }
}
// ===============================================================================
// Scrape Kubernetes pods containers metrics / cAdvisor
// Requirements:
// - discovery.relabel - kube_pods_metrics (cAdviser)
// ===============================================================================
prometheus.scrape "cadvisor" {
job_name = "integrations/kubernetes/cadvisor"
targets = discovery.relabel.kube_pods_metrics.output
scheme = "https"
tls_config {
server_name = "kubernetes"
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
insecure_skip_verify = true
}
bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
forward_to = [prometheus.remote_write.prometheus_srv.receiver]
}
// ===========================================================
// Discovery:relabel: kube_pods_metrics (cAdviser)
// Requirements:
// - discovery.kubernetes.role.node: kube_nodes
// ===========================================================
discovery.relabel "kube_node_metrics" {
targets = discovery.kubernetes.kube_nodes.targets
rule {
action = "replace"
source_labels = ["__meta_kubernetes_node_name"]
regex = "(.+)"
target_label = "node"
}
rule {
source_labels = ["node"]
target_label = "host"
}
// rule {
// source_labels = ["host"]
// target_label = "nodename"
// }
rule {
replacement = "/metrics/cadvisor"
target_label = "__metrics_path__"
}
}
extraEnv:
- name: CLUSTER_NAME
value: casa
- name: ZONE
value: casa-vlan
- name: LOKI_URL
valueFrom:
secretKeyRef:
name: alloy-global
key: loki_url
- name: PROMETHEUS_URL
valueFrom:
secretKeyRef:
name: alloy-global
key: prometheus_url
@@ -1,11 +0,0 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: monitoring
resources:
- helm-release.yaml
secretGenerator:
- name: alloy-kube-scraper-helm-values
files:
- values.yaml=helm-values.yaml
generatorOptions:
disableNameSuffixHash: true
@@ -1,24 +0,0 @@
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: alloy-node-scraper
spec:
releaseName: alloy-node-scraper
interval: 1m
chart:
spec:
chart: alloy
version: 1.x.x
sourceRef:
kind: HelmRepository
name: grafana
interval: 40h
valuesFrom:
- kind: Secret
name: alloy-node-scraper-helm-values
valuesKey: values.yaml
@@ -1,231 +0,0 @@
controller:
nameOverride: "alloy-node-scrapper"
volumes:
extra:
# requires host journal configuration storage = volatile
- name: run-log-journal
hostPath:
path: /run/log/journal
# Log scrapers (e.g., Alloy/Loki) read this ID from the journal metadata and use it to separate log streams per machine.
- name: etc-machine-id
hostPath:
path: /etc/machine-id
- name: rootfs
hostPath:
path: /
- name: proc
hostPath:
path: /proc
- name: sys
hostPath:
path: /sys
- name: dev
hostPath:
path: /dev
tolerations:
- key: "node-role.kubernetes.io/control-plane"
operator: "Exists"
effect: "NoSchedule"
- key: "infra.limbosolutions.com/dedicated"
operator: "Exists"
effect: "NoSchedule"
alloy:
mounts:
varlog: true # Mounts /var/log from the host (persistent journal)
extra:
# requires host journal configuration storage = volatile
- name: run-log-journal
mountPath: /run/log/journal
readOnly: true
# required
- name: etc-machine-id
mountPath: /etc/machine-id
readOnly: true
- name: rootfs
mountPath: /host
readOnly: true
- name: proc
mountPath: /host/proc
readOnly: true
- name: sys
mountPath: /host/sys
readOnly: true
- name: dev
mountPath: /host/dev
readOnly: true
# https://grafana.com/docs/alloy/latest/collect/logs-in-kubernetes/
configMap:
content: |
// ============================================
// LOKI WRITE TARGET
// ============================================
loki.write "loki_srv" {
endpoint {
url = env("LOKI_URL")
}
}
// ============================================
// HOST JOURNALD SOURCE
// ============================================
// Read journald logs from the host
loki.source.journal "journal" {
max_age = "24h"
relabel_rules = discovery.relabel.host_journal.rules
forward_to = [loki.process.host_journal.receiver]
}
// ============================================
// HOST JOURNALD RELABELING
// ============================================
// Convert journald metadata into Loki labels
discovery.relabel "host_journal" {
targets = []
// Systemd unit name
rule {
source_labels = ["__journal__systemd_unit"]
target_label = "systemd_unit"
}
// Transport type (stdout, syslog, kernel, audit, etc.)
rule {
source_labels = ["__journal__transport"]
target_label = "journal_transport"
}
// Priority (info, warning, error)
rule {
source_labels = ["__journal_priority_keyword"]
target_label = "level"
}
}
// ============================================
// HOST JOURNALD PROCESSING
// ============================================
// Add static labels and forward journald logs to Loki
loki.process "host_journal" {
stage.static_labels {
values = {
cluster = env("CLUSTER_NAME"),
host = env("HOSTNAME"),
zone = env("ZONE"),
job = "journald",
service_name = "journald",
}
}
forward_to = [loki.write.loki_srv.receiver]
}
// ============================================
// Metrics: Prometheus connection config
// ============================================
prometheus.remote_write "metrics_service" {
endpoint {
url = env("PROMETHEUS_URL")
}
// global labels
external_labels = {
cluster = env("CLUSTER_NAME"),
zone = env("ZONE"),
}
}
// ============================================
// Metrics: Prometheus Unix Exporter
// ============================================
prometheus.exporter.unix "node_exporter" {
rootfs_path = "/host"
disable_collectors = ["ipvs", "btrfs", "infiniband", "xfs", "zfs"]
enable_collectors = [
"cpu",
"meminfo",
"vmstat",
"loadavg",
"filesystem",
"netdev",
"uname",
]
filesystem {
mount_points_exclude = "^/(dev|proc|sys|run/containerd/.+|var/lib/docker/.+|var/lib/kubelet/.+)($|/)"
fs_types_exclude = "^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs|erofs)$"
mount_timeout = "5s"
}
netclass {
ignored_devices = "^(veth.*|cali.*|[a-f0-9]{15})$"
}
netdev {
device_exclude = "^(veth.*|cali.*|[a-f0-9]{15})$"
}
}
prometheus.scrape "node_exporter" {
targets = prometheus.exporter.unix.node_exporter.targets
scrape_interval = "30s"
forward_to = [prometheus.relabel.node_exporter.receiver]
}
prometheus.relabel "node_exporter" {
forward_to = [prometheus.remote_write.metrics_service.receiver]
rule {
action = "replace"
replacement = env("HOSTNAME")
target_label = "nodename"
}
rule {
action = "replace"
replacement = env("HOSTNAME")
target_label = "node"
}
}
extraEnv:
- name: NODE_IP
valueFrom:
fieldRef:
fieldPath: status.hostIP
- name: CLUSTER_NAME
value: casa
- name: JOB
value: node-exporter
- name: CONTAINER
value: node-exporter
- name: NS
value: monitoring
- name: POD
value: alloy-node-scraper
- name: SERVICE
value: alloy-node-scraper
- name: ZONE
value: casa-vlan
- name: LOKI_URL
valueFrom:
secretKeyRef:
name: alloy-global
key: loki_url
- name: PROMETHEUS_URL
valueFrom:
secretKeyRef:
name: alloy-global
key: prometheus_url
@@ -1,11 +0,0 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: monitoring
resources:
- helm-release.yaml
secretGenerator:
- name: alloy-node-scraper-helm-values
files:
- values.yaml=helm-values.yaml
generatorOptions:
disableNameSuffixHash: true
@@ -1,15 +0,0 @@
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: kube-state-metrics
spec:
releaseName: kube-state-metrics
interval: 1m
chart:
spec:
chart: kube-state-metrics
version: 8.x.x
sourceRef:
kind: HelmRepository
name: prometheus
interval: 40h
@@ -1,7 +0,0 @@
apiVersion: source.toolkit.fluxcd.io/v1
kind: HelmRepository
metadata:
name: prometheus
spec:
interval: 40h
url: https://prometheus-community.github.io/helm-charts
@@ -1,6 +0,0 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: monitoring
resources:
- helm-repo.yaml
- helm-release.yaml