343 lines
9.3 KiB
YAML
343 lines
9.3 KiB
YAML
controller:
|
|
type: 'deployment'
|
|
nodeSelector:
|
|
kubernetes.io/hostname: casa-minion-01
|
|
alloy:
|
|
|
|
# https://grafana.com/docs/alloy/latest/collect/logs-in-kubernetes/
|
|
configMap:
|
|
content: |
|
|
// ============================================
|
|
// LOKI WRITE TARGET
|
|
// ============================================
|
|
loki.write "loki_srv" {
|
|
endpoint {
|
|
url = env("LOKI_URL")
|
|
}
|
|
}
|
|
|
|
// ============================================
|
|
// POD LOGS: DISCOVERY + RELABEL
|
|
// ============================================
|
|
|
|
// Discover Kubernetes pods
|
|
discovery.kubernetes "pod_logs" {
|
|
role = "pod"
|
|
}
|
|
|
|
discovery.relabel "pod_logs" {
|
|
targets = discovery.kubernetes.pod_logs.targets
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_namespace"]
|
|
target_label = "namespace"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_pod_name"]
|
|
regex = "^(.+?)-[0-9a-zA-Z]{4,16}(?:-[0-9a-zA-Z]{4,16})?$"
|
|
replacement = "$1"
|
|
target_label = "pod"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_pod_container_name"]
|
|
target_label = "container"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_node_name"]
|
|
target_label = "node_name"
|
|
}
|
|
|
|
rule {
|
|
source_labels = [
|
|
"__meta_kubernetes_pod_controller_name",
|
|
]
|
|
regex = "([0-9a-z-.]+?)(-[0-9a-f]{8,10})?"
|
|
target_label = "controller_name"
|
|
}
|
|
|
|
rule {
|
|
source_labels = [
|
|
"__meta_kubernetes_pod_label_app_kubernetes_io_name",
|
|
"__meta_kubernetes_pod_label_app",
|
|
"controller_name",
|
|
"__meta_kubernetes_pod_name",
|
|
]
|
|
regex = "^;*([^;]+)(;.*)?$"
|
|
target_label = "app"
|
|
}
|
|
|
|
rule {
|
|
source_labels = [
|
|
"__meta_kubernetes_pod_label_app_kubernetes_io_component",
|
|
"__meta_kubernetes_pod_label_component",
|
|
]
|
|
regex = "^;*([^;]+)(;.*)?$"
|
|
target_label = "component"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["namespace", "app"]
|
|
separator = "/"
|
|
target_label = "job"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["job", "container"]
|
|
separator = ":"
|
|
target_label = "instance"
|
|
}
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_pod_node_name"]
|
|
separator = ":"
|
|
target_label = "host"
|
|
}
|
|
}
|
|
|
|
|
|
// ============================================
|
|
// POD LOGS: SOURCE → PROCESS → WRITE
|
|
// ============================================
|
|
|
|
// Tail logs from Kubernetes pods
|
|
loki.source.kubernetes "pod_logs" {
|
|
// targets = discovery.kubernetes.pod_logs.targets
|
|
targets = discovery.relabel.pod_logs.output
|
|
forward_to = [loki.process.pod_logs.receiver]
|
|
}
|
|
|
|
// Add static labels and forward logs to Loki
|
|
loki.process "pod_logs" {
|
|
|
|
stage.static_labels {
|
|
values = {
|
|
cluster = env("CLUSTER_NAME"),
|
|
zone = env("ZONE"),
|
|
}
|
|
}
|
|
|
|
forward_to = [loki.write.loki_srv.receiver]
|
|
}
|
|
|
|
// ============================================
|
|
// KUBERNETES EVENTS
|
|
// ============================================
|
|
|
|
// Tail Kubernetes events and convert them to log entries
|
|
loki.source.kubernetes_events "cluster_events" {
|
|
job_name = "integrations/kubernetes/eventhandler"
|
|
log_format = "logfmt"
|
|
forward_to = [
|
|
loki.process.cluster_events.receiver,
|
|
]
|
|
}
|
|
|
|
// Add static labels and forward events to Loki
|
|
loki.process "cluster_events" {
|
|
forward_to = [loki.write.loki_srv.receiver]
|
|
|
|
stage.static_labels {
|
|
values = {
|
|
cluster = env("CLUSTER_NAME"),
|
|
host = "casa-kube",
|
|
job = "cluster_events",
|
|
zone = env("ZONE"),
|
|
}
|
|
}
|
|
|
|
discovery.kubernetes "k8s_nodes" {
|
|
role = "node"
|
|
}
|
|
|
|
|
|
discovery.relabel "cadvisor" {
|
|
targets = discovery.kubernetes.k8s_nodes.targets
|
|
|
|
rule {
|
|
replacement = "/metrics/cadvisor"
|
|
target_label = "__metrics_path__"
|
|
}
|
|
rule {
|
|
target_label = "instance"
|
|
replacement = env("NODE_IP") + ":10250"
|
|
}
|
|
|
|
rule {
|
|
target_label = "node"
|
|
replacement = env("HOSTNAME")
|
|
}
|
|
|
|
rule {
|
|
target_label = "service"
|
|
replacement = "kube-scrapper"
|
|
}
|
|
|
|
rule {
|
|
target_label = "cluster"
|
|
replacement = env("CLUSTER_NAME")
|
|
}
|
|
|
|
rule {
|
|
target_label = "job"
|
|
replacement = "kubelet"
|
|
}
|
|
}
|
|
|
|
prometheus.remote_write "metrics_service" {
|
|
|
|
endpoint {
|
|
url = env("PROMETHEUS_URL")
|
|
}
|
|
|
|
}
|
|
prometheus.scrape "cadvisor" {
|
|
job_name = "integrations/kubernetes/cadvisor"
|
|
targets = discovery.relabel.cadvisor.output
|
|
scheme = "https"
|
|
|
|
tls_config {
|
|
server_name = "kubernetes"
|
|
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
|
|
insecure_skip_verify = true
|
|
}
|
|
|
|
bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
|
|
forward_to = [prometheus.remote_write.metrics_service.receiver]
|
|
}
|
|
|
|
|
|
discovery.kubernetes "k8s_services" {
|
|
role = "service"
|
|
}
|
|
|
|
discovery.relabel "ksm" {
|
|
targets = discovery.kubernetes.k8s_services.targets
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_service_name"]
|
|
regex = "kube-state-metrics"
|
|
action = "keep"
|
|
}
|
|
}
|
|
|
|
prometheus.scrape "kube_state_metrics" {
|
|
job_name = "integrations/kubernetes/kube-state-metrics"
|
|
targets = discovery.relabel.ksm.output
|
|
|
|
scheme = "http"
|
|
|
|
forward_to = [prometheus.remote_write.metrics_service.receiver]
|
|
}
|
|
|
|
// ============================================
|
|
// CoreDNS discovery
|
|
// ============================================
|
|
discovery.kubernetes "coredns_pod" {
|
|
role = "pod"
|
|
}
|
|
|
|
discovery.relabel "coredns" {
|
|
targets = discovery.kubernetes.pod.targets
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_pod_label_k8s_app"]
|
|
regex = "kube-dns"
|
|
action = "keep"
|
|
}
|
|
rule {
|
|
target_label = "node"
|
|
replacement = env("HOSTNAME")
|
|
}
|
|
|
|
rule {
|
|
target_label = "cluster"
|
|
replacement = env("CLUSTER_NAME")
|
|
}
|
|
}
|
|
|
|
// ============================================
|
|
// CoreDNS scrape
|
|
// ============================================
|
|
prometheus.scrape "coredns" {
|
|
job_name = "integrations/kubernetes/coredns-pods"
|
|
targets = discovery.relabel.coredns.output
|
|
|
|
scheme = "http"
|
|
metrics_path = "/metrics"
|
|
|
|
forward_to = [prometheus.remote_write.metrics_service.receiver]
|
|
}
|
|
|
|
// ============================================
|
|
// API Server discovery
|
|
// ============================================
|
|
discovery.kubernetes "apiserver_services" {
|
|
role = "service"
|
|
}
|
|
|
|
discovery.relabel "apiserver" {
|
|
targets = discovery.kubernetes.apiserver_services.targets
|
|
|
|
rule {
|
|
source_labels = ["__meta_kubernetes_service_name"]
|
|
regex = "kubernetes"
|
|
action = "keep"
|
|
}
|
|
rule {
|
|
target_label = "node"
|
|
replacement = env("HOSTNAME")
|
|
}
|
|
|
|
rule {
|
|
target_label = "cluster"
|
|
replacement = env("CLUSTER_NAME")
|
|
}
|
|
}
|
|
|
|
// ============================================
|
|
// API Server scrape
|
|
// ============================================
|
|
prometheus.scrape "apiserver" {
|
|
job_name = "integrations/kubernetes/apiserver"
|
|
targets = discovery.relabel.apiserver.output
|
|
|
|
scheme = "https"
|
|
metrics_path = "/metrics"
|
|
|
|
tls_config {
|
|
server_name = "kubernetes"
|
|
ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
|
|
insecure_skip_verify = true
|
|
}
|
|
|
|
bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
|
|
|
|
forward_to = [prometheus.remote_write.metrics_service.receiver]
|
|
}
|
|
|
|
|
|
extraEnv:
|
|
- name: NODE_IP
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: status.hostIP
|
|
- name: NODE_NAME
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: spec.nodeName
|
|
- name: CLUSTER_NAME
|
|
value: casa
|
|
- name: ZONE
|
|
value: casa-vlan
|
|
- name: LOKI_URL
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: alloy-global
|
|
key: loki_url
|
|
- name: PROMETHEUS_URL
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: alloy-global
|
|
key: prometheus_url |