monitoring - increase scrapeInterval to 120s, better selectors on Agent and memory decrease of agent from 360MB to 228MB

monitoring/prometheus/deploy/helm/02-kube-metrics.yaml

@@ -8,6 +8,7 @@ kube-state-metrics: # ok tested!
     role: worker-node
   prometheus:
     monitor:
+      interval: "60s"
       relabelings:
         - targetLabel: cluster
           replacement: casa

monitoring/prometheus/deploy/helm/03-node-exporter.yaml

@@ -8,6 +8,7 @@ prometheus-node-exporter:
   prometheus:
     monitor:
       enabled: true
+      interval: "60s"
       relabelings:
       # https://github.com/dotdc/grafana-dashboards-kubernetes
       - action: replace
@@ -17,3 +18,8 @@ prometheus-node-exporter:
       # identification of cluster
       - targetLabel: cluster
         replacement: casa
+
+      # it seams to be an timestamp can not be an label!
+      - action: labeldrop
+        regex: __meta_kubernetes_endpoints_annotation_endpoints_kubernetes_io_last_change_trigger_time
+

monitoring/prometheus/deploy/helm/04-kubelet.yaml

@@ -4,10 +4,11 @@ kubelet:
   enabled: true
   namespace: kube-system
   serviceMonitor:
-    interval: 30s #WARN: Error on ingesting out-of-order samples. https://github.com/prometheus-community/helm-charts/issues/5483
+    interval: 60s #WARN: Error on ingesting out-of-order samples. https://github.com/prometheus-community/helm-charts/issues/5483
     enabled: true 
     ## Enable scraping /metrics from kubelet's service
     kubelet: true
+    cAdvisor: true
     additionalLabels:
       app.kubernetes.io/name: prometheus-kubelet # !important: selector used by agent