diff --git a/services/monitoring/deploy/alloy/alloy-loki-examples copy.config b/services/monitoring/deploy/alloy/alloy-loki-examples copy.config new file mode 100644 index 0000000..1007fba --- /dev/null +++ b/services/monitoring/deploy/alloy/alloy-loki-examples copy.config @@ -0,0 +1,141 @@ + // ============================================ + // LOKI WRITE TARGET + // ============================================ + loki.write "loki_srv" { + endpoint { + url = env("LOKI_URL") + } + } + + // ============================================ + // POD LOGS: DISCOVERY + RELABEL + // ============================================ + + // Discover Kubernetes pods + discovery.kubernetes "pod_logs" { + role = "pod" + } + + discovery.relabel "pod_logs" { + targets = discovery.kubernetes.pod_logs.targets + + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + + rule { + source_labels = ["__meta_kubernetes_pod_name"] + regex = "^(.+?)-[0-9a-zA-Z]{4,16}(?:-[0-9a-zA-Z]{4,16})?$" + replacement = "$1" + target_label = "pod" + } + + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + + rule { + source_labels = ["__meta_kubernetes_node_name"] + target_label = "node_name" + } + + rule { + source_labels = [ + "__meta_kubernetes_pod_controller_name", + ] + regex = "([0-9a-z-.]+?)(-[0-9a-f]{8,10})?" + target_label = "controller_name" + } + + rule { + source_labels = [ + "__meta_kubernetes_pod_label_app_kubernetes_io_name", + "__meta_kubernetes_pod_label_app", + "controller_name", + "__meta_kubernetes_pod_name", + ] + regex = "^;*([^;]+)(;.*)?$" + target_label = "app" + } + + rule { + source_labels = [ + "__meta_kubernetes_pod_label_app_kubernetes_io_component", + "__meta_kubernetes_pod_label_component", + ] + regex = "^;*([^;]+)(;.*)?$" + target_label = "component" + } + + rule { + source_labels = ["namespace", "app"] + separator = "/" + target_label = "job" + } + + rule { + source_labels = ["job", "container"] + separator = ":" + target_label = "instance" + } + rule { + source_labels = ["__meta_kubernetes_pod_node_name"] + separator = ":" + target_label = "host" + } + } + + + // ============================================ + // POD LOGS: SOURCE → PROCESS → WRITE + // ============================================ + + // Tail logs from Kubernetes pods + loki.source.kubernetes "pod_logs" { + // targets = discovery.kubernetes.pod_logs.targets + targets = discovery.relabel.pod_logs.output + forward_to = [loki.process.pod_logs.receiver] + } + + // Add static labels and forward logs to Loki + loki.process "pod_logs" { + + stage.static_labels { + values = { + cluster = env("CLUSTER_NAME"), + zone = env("ZONE"), + } + } + + forward_to = [loki.write.loki_srv.receiver] + } + + // ============================================ + // KUBERNETES EVENTS + // ============================================ + + // Tail Kubernetes events and convert them to log entries + loki.source.kubernetes_events "cluster_events" { + job_name = "integrations/kubernetes/eventhandler" + log_format = "logfmt" + forward_to = [ + loki.process.cluster_events.receiver, + ] + } + + // Add static labels and forward events to Loki + loki.process "cluster_events" { + forward_to = [loki.write.loki_srv.receiver] + + stage.static_labels { + values = { + cluster = env("CLUSTER_NAME"), + host = "env("CLUSTER_NAME")", + job = "cluster_events", + zone = env("ZONE"), + } + } + + } \ No newline at end of file diff --git a/services/monitoring/deploy/alloy/alloy-prometheus-examples.config b/services/monitoring/deploy/alloy/alloy-prometheus-examples.config new file mode 100644 index 0000000..00e32ce --- /dev/null +++ b/services/monitoring/deploy/alloy/alloy-prometheus-examples.config @@ -0,0 +1,83 @@ + // ============================================ + // Metrics: Discovery Kubernetes Services + // ============================================ + + // discovery.kubernetes "k8s_services" { + // role = "service" + // } + + // ============================================ + // Metrics: Kube State Metrics (k8s_services) + // ============================================ + + // prometheus.scrape "kube_state_metrics" { + // job_name = "integrations/kubernetes/kube-state-metrics" + // targets = discovery.relabel.kube_state_metrics.output + // + // scheme = "http" + // + // forward_to = [prometheus.remote_write.prometheus_srv.receiver] + // } + + // ============================================ + // API Server discovery + // ============================================ + + // discovery.relabel "apiserver" { + // targets = discovery.kubernetes.k8s_services.targets + + // rule { + // source_labels = ["__meta_kubernetes_service_name"] + // regex = "kubernetes" + // action = "keep" + // } + // rule { + // target_label = "cluster" + // replacement = env("CLUSTER_NAME") + // } + // } + + // prometheus.scrape "apiserver" { + // job_name = "integrations/kubernetes/apiserver" + // targets = discovery.relabel.apiserver.output + + // scheme = "https" + // metrics_path = "/metrics" + + // tls_config { + // server_name = "kubernetes" + // ca_file = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + // insecure_skip_verify = true + // } + + // bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + + // forward_to = [prometheus.remote_write.prometheus_srv.receiver] + // } + + // ============================================ + // CoreDNS discovery + // ============================================ + // discovery.relabel "coredns" { + // targets = discovery.kubernetes.k8s_services.targets + // + // rule { + // source_labels = ["__meta_kubernetes_pod_label_k8s_app"] + // regex = "kube-dns" + // action = "keep" + // } + // rule { + // target_label = "cluster" + // replacement = env("CLUSTER_NAME") + // } + // } + + // prometheus.scrape "coredns" { + // job_name = "integrations/kubernetes/coredns-pods" + // targets = discovery.relabel.coredns.output + + // scheme = "http" + // metrics_path = "/metrics" + // + // forward_to = [prometheus.remote_write.prometheus_srv.receiver] + // } \ No newline at end of file diff --git a/services/monitoring/deploy/alloy/node-scraper/helm-values.yaml b/services/monitoring/deploy/alloy/node-scraper/helm-values.yaml index 86b9e2d..45e8737 100644 --- a/services/monitoring/deploy/alloy/node-scraper/helm-values.yaml +++ b/services/monitoring/deploy/alloy/node-scraper/helm-values.yaml @@ -139,7 +139,7 @@ alloy: external_labels = { cluster = env("CLUSTER_NAME"), zone = env("ZONE"), - node = env("HOSTNAME") + node = env("HOSTNAME"), } } diff --git a/services/monitoring/deploy/flux/kustomization.yaml b/services/monitoring/deploy/flux/kustomization.yaml index 1f5aca1..5154b23 100644 --- a/services/monitoring/deploy/flux/kustomization.yaml +++ b/services/monitoring/deploy/flux/kustomization.yaml @@ -3,7 +3,6 @@ kind: Kustomization namespace: monitoring resources: - alloy-app-sync.yaml - - prometheus-app-sync.yaml secretGenerator: - name: flux-sops-age files: diff --git a/services/monitoring/deploy/flux/prometheus-app-sync.yaml b/services/monitoring/deploy/flux/prometheus-app-sync.yaml deleted file mode 100644 index c1a570d..0000000 --- a/services/monitoring/deploy/flux/prometheus-app-sync.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: kustomize.toolkit.fluxcd.io/v1 -kind: Kustomization -metadata: - name: prometheus -spec: - interval: 1m - sourceRef: - kind: GitRepository - name: casa - namespace: casa-limbosolutions-com - path: services/monitoring/deploy/prometheus - prune: true - decryption: - provider: sops - secretRef: - name: flux-sops-age