diff --git a/services/monitoring/README.md b/services/monitoring/README.md index 55bd247..775c301 100644 --- a/services/monitoring/README.md +++ b/services/monitoring/README.md @@ -1,10 +1,5 @@ # monitoring - -## prometheus-stack - -- -- - + ## alloy **promtail Encrypt secrets:** diff --git a/services/monitoring/deploy/alloy/global/secrets.yaml b/services/monitoring/deploy/alloy/global/secrets.yaml index 493d6e0..afdf60b 100644 --- a/services/monitoring/deploy/alloy/global/secrets.yaml +++ b/services/monitoring/deploy/alloy/global/secrets.yaml @@ -3,20 +3,20 @@ kind: Secret metadata: name: alloy-global stringData: - loki_url: ENC[AES256_GCM,data:xuMvbrKhvwvJK38ES59SuD/X5CywK3qlnDfSsYmr0W3TLv0wB08wT9kCd57pctlvAqL5RNYTaTIgJ1w=,iv:YqPv1DBbfmVC8P1KgEJPiOtYzx+OcfySIxPjqMjCnhA=,tag:aw7iW8gljmVhPvkKSKh5mw==,type:str] - prometheus_url: ENC[AES256_GCM,data:MqNjAcGzi87gC2A2+G6xcn7FroYUczfkNHj0eCXZVi46AJH+dKjFU/pVwtMIrz1Aikz4BhX0ZviijrvYlw==,iv:IydlpoIOq4bipZV/xihvpmYO/sBerDX2z+8LGl3Gu6k=,tag:dJv6nUncoJGcuoR8eQFitg==,type:str] + loki_url: ENC[AES256_GCM,data:2DeZZ+7deDbojQ/+YNgT8KwDmjFFyAqMcTX8l6itlF15hilXKgYP1p/SNoyw1JGJ7x5VZqUl0VIVhVY=,iv:SlFiLvEeIqj+zQ/8Jspc9FL/ElneIlWBzGsM+6mGZLc=,tag:9tIl6fi+xfmF+TJpMXgbKg==,type:str] + prometheus_url: ENC[AES256_GCM,data:k8Z3lvZRxOmlw6Wx4aLAgV93hPcEawtoprWU2Kn6j/WIe/YDHM9iuB+pRy7ght4I6jxyNFLotN0BaHE1yg==,iv:EEnRoS287e9EV8+25NtlEO31zN0eyfRBOUe9GVBah18=,tag:2K/MnM1QlWPbceQBqCwylw==,type:str] sops: age: - enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBpemRxQTNtUTQ2a1dUcnpu - VjNlN29OeGRrdDN1bm8yWC9hYllBNVNjSVJvCnR5RUU2Z1B0RWFxREFoUlVIVStM - c1AvbUlXa0dOTDF3VkhJRGdGME1oS1UKLS0tIDdncERRSDFKRnBlMW5kMkQ1VFhs - bXJYbmZyUXJ4NU1YOEhpdmhTT2I3S2MK+Zx6ntTO08AxawYo+jKOPTfpMVXdsvlC - 5hNPs6bqxei+Fa6q87LZrFENwdxjeti6QuKoi45acoOQ6zI2TM9yNw== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBJYmx1dGJTSGZvbVdpV1di + Z3dKNUJPVGNkK1hBSnVoRXZrb0R4TmxnUjM4CmpyUXpGN2lneDViN3VPNC9Fa3d6 + RURxZTFuV0o1SzF5ZEhzSVZyYkY5alkKLS0tIEF1T0tINnRUa0tDOGlaSVRJajF5 + bTFTd0c4Z3p1aUNWWTZ0ZFVXQVdrMjAKku2YU048qf4E9vEPO19HNfAg78E3fSFQ + WBiUgiqtqG/T4cI5cf4CUWoUogtABawBG4z5G5+ud7sfVMzfqW+W1g== -----END AGE ENCRYPTED FILE----- recipient: age1f9e4pvp5y8gzuk8mz2s5xm85dd7znxhk56tcpuxqwn78qfjwja0qekwlju encrypted_regex: ^(data|stringData)$ - lastmodified: "2026-08-30T10:21:08Z" - mac: ENC[AES256_GCM,data:FZkuO1JP6Ds3XUAWUa0tpb2gVcnFC9Ki3owd+f7On+Nu0EvVRqVBzrSLBXxPVOQ69jj9HDiQVFmYq2fsRd5aAItCA+Jry/2M7m6j6BfyzPMmEChemTYSJQKvQy7l9r4FYwXqvBJ8TmPQv3wc+VPrhiYegWusF/YzcZzFvvUwVOE=,iv:gQUZEbK5l/3Gc6Xcm3RAF5kMaV14dYH/Og+SKrdhUtw=,tag:YVE3seBOcDl31huyDog+FA==,type:str] + lastmodified: "2026-08-30T22:14:34Z" + mac: ENC[AES256_GCM,data:XQ2zRn6zGBONfBiyaip3DpwzIfDBN1mPI65fVfH3c7tsEeY1mV/ZRPD+5iJsiwxlnBa7Rq0iJNOr2f84JE+CiuqYz8UKuQMS02gsvqXuBSgUn5vDRcX/aBV3+1GSOzv3ilmwMkJI/jxfyyXL4YsxKl0fsFEOJRsa3VCCVF0OvrM=,iv:enk8h+TfYBIOFln3SlOw30arg2T/kSU3IMwusrm68Qc=,tag:yp9eQc56Ht6g3x3QnMqMZg==,type:str] version: 3.13.1 diff --git a/services/monitoring/deploy/alloy/kube-scraper/helm-values.yaml b/services/monitoring/deploy/alloy/kube-scraper/helm-values.yaml index cbdd99e..63a4c5d 100644 --- a/services/monitoring/deploy/alloy/kube-scraper/helm-values.yaml +++ b/services/monitoring/deploy/alloy/kube-scraper/helm-values.yaml @@ -147,15 +147,182 @@ alloy: } } - // stage.labels { - // values = { - // kubernetes_cluster_events = "job", - // } - // } - } + discovery.kubernetes "k8s_nodes" { + role = "node" + } + + + discovery.relabel "cadvisor" { + targets = discovery.kubernetes.k8s_nodes.targets + + rule { + replacement = "/metrics/cadvisor" + target_label = "__metrics_path__" + } + rule { + target_label = "instance" + replacement = env("NODE_IP") + ":10250" + } + + rule { + target_label = "node" + replacement = env("HOSTNAME") + } + + rule { + target_label = "service" + replacement = "kube-scrapper" + } + + rule { + target_label = "cluster" + replacement = env("CLUSTER_NAME") + } + + rule { + target_label = "job" + replacement = "kubelet" + } + } + + prometheus.remote_write "metrics_service" { + + endpoint { + url = env("PROMETHEUS_URL") + } + + } + prometheus.scrape "cadvisor" { + job_name = "integrations/kubernetes/cadvisor" + targets = discovery.relabel.cadvisor.output + scheme = "https" + + tls_config { + server_name = "kubernetes" + ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + insecure_skip_verify = true + } + + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + forward_to = [prometheus.remote_write.metrics_service.receiver] + } + + + discovery.kubernetes "k8s_services" { + role = "service" + } + + discovery.relabel "ksm" { + targets = discovery.kubernetes.k8s_services.targets + + rule { + source_labels = ["__meta_kubernetes_service_name"] + regex = "kube-state-metrics" + action = "keep" + } + } + + prometheus.scrape "kube_state_metrics" { + job_name = "integrations/kubernetes/kube-state-metrics" + targets = discovery.relabel.ksm.output + + scheme = "http" + + forward_to = [prometheus.remote_write.metrics_service.receiver] + } + + // ============================================ + // CoreDNS discovery + // ============================================ + discovery.kubernetes "coredns_pod" { + role = "pod" + } + + discovery.relabel "coredns" { + targets = discovery.kubernetes.pod.targets + + rule { + source_labels = ["__meta_kubernetes_pod_label_k8s_app"] + regex = "kube-dns" + action = "keep" + } + rule { + target_label = "node" + replacement = env("HOSTNAME") + } + + rule { + target_label = "cluster" + replacement = env("CLUSTER_NAME") + } + } + + // ============================================ + // CoreDNS scrape + // ============================================ + prometheus.scrape "coredns" { + job_name = "integrations/kubernetes/coredns-pods" + targets = discovery.relabel.coredns.output + + scheme = "http" + metrics_path = "/metrics" + + forward_to = [prometheus.remote_write.metrics_service.receiver] + } + + // ============================================ + // API Server discovery + // ============================================ + discovery.kubernetes "apiserver_services" { + role = "service" + } + + discovery.relabel "apiserver" { + targets = discovery.kubernetes.apiserver_services.targets + + rule { + source_labels = ["__meta_kubernetes_service_name"] + regex = "kubernetes" + action = "keep" + } + rule { + target_label = "node" + replacement = env("HOSTNAME") + } + + rule { + target_label = "cluster" + replacement = env("CLUSTER_NAME") + } + } + + // ============================================ + // API Server scrape + // ============================================ + prometheus.scrape "apiserver" { + job_name = "integrations/kubernetes/apiserver" + targets = discovery.relabel.apiserver.output + + scheme = "https" + metrics_path = "/metrics" + + tls_config { + server_name = "kubernetes" + ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + insecure_skip_verify = true + } + + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + forward_to = [prometheus.remote_write.metrics_service.receiver] + } extraEnv: + - name: NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP - name: NODE_NAME valueFrom: fieldRef: @@ -169,3 +336,8 @@ alloy: secretKeyRef: name: alloy-global key: loki_url + - name: PROMETHEUS_URL + valueFrom: + secretKeyRef: + name: alloy-global + key: prometheus_url \ No newline at end of file diff --git a/services/monitoring/deploy/prometheus/helm-release.yaml b/services/monitoring/deploy/prometheus/helm-release.yaml index 2374c0b..764b60f 100644 --- a/services/monitoring/deploy/prometheus/helm-release.yaml +++ b/services/monitoring/deploy/prometheus/helm-release.yaml @@ -1,19 +1,15 @@ apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: - name: kube-prometheus-stack + name: kube-state-metrics spec: - releaseName: prometheus + releaseName: kube-state-metrics interval: 1m chart: spec: - chart: kube-prometheus-stack - version: 86.x.x + chart: kube-state-metrics + version: 8.x.x sourceRef: kind: HelmRepository name: prometheus interval: 40h - valuesFrom: - - kind: Secret - name: prometheus-stack-helm-values - valuesKey: values.yaml diff --git a/services/monitoring/deploy/prometheus/helm-values.yaml b/services/monitoring/deploy/prometheus/helm-values.yaml deleted file mode 100644 index c22480e..0000000 --- a/services/monitoring/deploy/prometheus/helm-values.yaml +++ /dev/null @@ -1,141 +0,0 @@ - -defaultRules: - create: false - -alertmanager: - enabled: false - -grafana: - enabled: false - -prometheus: - enabled: false - -kubeStateMetrics: - enabled: true - -nodeExporter: - enabled: false - -prometheus-node-exporter: - prometheus: - monitor: - enabled: false - -coreDns: - enabled: true - serviceMonitor: - relabelings: - - targetLabel: cluster - replacement: casa - -kubeApiServer: - enabled: true - serviceMonitor: - relabelings: - - targetLabel: cluster - replacement: casa - -kubeControllerManager: - enabled: false - -kubeEtcd: - enabled: false - -kubeProxy: - enabled: false - -kubeScheduler: - enabled: false - -kube-state-metrics: - podLabels: - role: worker-node - nodeSelector: - role: worker-node - prometheus: - monitor: - relabelings: - - targetLabel: cluster - replacement: casa - -prometheusOperator: - kubeletService: - enabled: true - enabled: true - createCustomResource: true - tls: - enabled: false - admissionWebhooks: - enabled: false - cleanupCustomResource: false - serviceMonitor: - selfMonitor: false - - -kubelet: - enabled: true - namespace: kube-system - serviceMonitor: - interval: 60s #WARN: Error on ingesting out-of-order samples. https://github.com/prometheus-community/helm-charts/issues/5483 - enabled: true - ## Enable scraping /metrics from kubelet's service - kubelet: true - - probesMetricRelabelings: - - targetLabel: cluster - replacement: casa - - sourceLabels: [__name__, image] - separator: ; - regex: container_([a-z_]+); - replacement: $1 - action: drop - - sourceLabels: [__name__] - separator: ; - regex: container_(network_tcp_usage_total|network_udp_usage_total|tasks_state|cpu_load_average_10s) - replacement: $1 - action: drop - - -# # RelabelConfigs to apply to samples before scraping -# # ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api-reference/api.md#relabelconfig -# # -# # metrics_path is required to match upstream rules and charts - cAdvisorRelabelings: - - targetLabel: cluster - replacement: casa - - action: replace - sourceLabels: [__metrics_path__] - targetLabel: metrics_path - - sourceLabels: [__meta_kubernetes_pod_node_name] - separator: ; - regex: ^(.*)$ - targetLabel: nodename - replacement: $1 - action: replace - -# # RelabelConfigs to apply to samples before scraping -# # ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api-reference/api.md#relabelconfig -# # - probesRelabelings: - - - targetLabel: cluster - replacement: casa - - action: replace - sourceLabels: [__metrics_path__] - targetLabel: metrics_path - - - sourceLabels: [__meta_kubernetes_pod_node_name] - separator: ; - regex: ^(.*)$ - targetLabel: nodename - replacement: $1 - action: replace - - resourceRelabelings: - - targetLabel: cluster - replacement: casa - - action: replace - sourceLabels: [__metrics_path__] - targetLabel: metrics_path - diff --git a/services/monitoring/deploy/prometheus/prometheus-agent.yaml b/services/monitoring/deploy/prometheus/prometheus-agent.yaml deleted file mode 100644 index 4ce07c2..0000000 --- a/services/monitoring/deploy/prometheus/prometheus-agent.yaml +++ /dev/null @@ -1,76 +0,0 @@ - -apiVersion: monitoring.coreos.com/v1alpha1 -kind: PrometheusAgent -metadata: - name: prometheus-agent -spec: - podMonitorNamespaceSelector: - matchLabels: - prometheus-monitoring: enabled - - podMonitorSelector: - matchLabels: - release: prometheus - - serviceMonitorNamespaceSelector: - matchLabels: - prometheus-monitoring: enabled - - serviceMonitorSelector: - matchLabels: - release: prometheus - - replicas: 1 - remoteWrite: - - url: https://prometheus.monitoring.limbosolutions.com/api/v1/write - scrapeInterval: 60s - resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 200m - memory: 512Mi # minimum stable - serviceAccountName: prometheus-agent - nodeSelector: - role: worker-node ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: prometheus-agent -rules: - - apiGroups: [""] - resources: ["nodes", "nodes/metrics", "nodes/proxy", "services", "endpoints", "pods"] - verbs: ["get", "list", "watch"] - - apiGroups: ["monitoring.coreos.com"] - resources: ["servicemonitors", "podmonitors"] - verbs: ["get", "list", "watch"] - - nonResourceURLs: - - /metrics - - /metrics/cadvisor - - /metrics/probes - verbs: ["get"] ---- - - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: prometheus-agent-monitoring -roleRef: - kind: ClusterRole - name: prometheus-agent - apiGroup: rbac.authorization.k8s.io -subjects: - - kind: ServiceAccount - name: prometheus-agent - namespace: monitoring - ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: prometheus-agent - namespace: monitoring \ No newline at end of file