modified: .devcontainer/devcontainer.json

modified:   services/monitoring/deploy/prometheus/prometheus-agent.yaml

.devcontainer/devcontainer.json

@@ -31,7 +31,8 @@
 				"streetsidesoftware.code-spell-checker",
 				"ms-azuretools.vscode-containers",
 				"eamodio.gitlens",
-				"bierner.markdown-preview-github-styles"
+				"bierner.markdown-preview-github-styles",
+				"weaveworks.vscode-gitops-tools"
 			]
 		}
 	}

casa.full-stack.code-workspace

@@ -46,11 +46,13 @@
 		"davidanson",
 		"eamodio",
 		"envsubst",
+		"kubelet",
 		"lmscommunity",
 		"localtime",
 		"LOGLEVEL",
 		"lyrionmusicserver",
 		"mtxr",
+		"nvme",
 		"rclone",
 		"reverseproxy",
 		"rogalmic",

cluster/README.md

@@ -13,6 +13,7 @@
 - [minion01 - worker node](#minion01---worker-node)
   - [Minion01 - proxmox vm](#minion01---proxmox-vm)
   - [casa-minion-01 - k3s - setup](#casa-minion-01---k3s---setup)
+  - [fstab](#fstab)
 - [casa-mini-minion-01 - worker node](#casa-mini-minion-01---worker-node)
   - [casa-mini-minion-01 - proxmox vm](#casa-mini-minion-01---proxmox-vm)
   - [casa-mini-minion-01 - k3s - setup](#casa-mini-minion-01---k3s---setup)
@@ -238,6 +239,7 @@ numa: 0
 onboot: 1
 ostype: l26
 scsi0: fastcore:vm-XXX-disk-0,iothread=1,size=8G,ssd=1
+scsi1: fluxcore:vm-xx-disk-0,iothread=1,size=32G,ssd=1 # used mainly by jellyfin cache
 scsi1: fastcore:vm-XXX-disk-2,iothread=1,size=32G,ssd=1 /dataDisk # k3s-data-dir /dataDisk/k3s
 scsihw: virtio-scsi-single
 smbios1: xxxx-xxxx-xxxx-xxxx-xxxx
@@ -260,7 +262,7 @@ sockets: 1
 # cat /var/lib/rancher/k3s/server/node-token
 
 K3S_TOKEN="????"
-curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.36.1+k3s1 INSTALL_K3S_EXEC="agent --data-dir /dataDisk/k3s --server https://casa.dev.lan:6443 --token ${K3S_TOKEN}" sh -s -
+curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.36.1+k3s1 INSTALL_K3S_EXEC="agent --data-dir /dataDisks/nvme00/k3s --server https://casa.dev.lan:6443 --token ${K3S_TOKEN}" sh -s -
 ```
 
 Change kubectl -n kube-system edit configmap local-path-config on kube-system to set path to provisioner.
@@ -269,7 +271,6 @@ Change kubectl -n kube-system edit configmap local-path-config on kube-system to
    config.json: |-
     {
       "nodePathMap":[
-      {
         "node":"DEFAULT_PATH_FOR_NON_LISTED_NODES",
         "paths":["/var/lib/rancher/k3s/storage"]
       },
@@ -287,6 +288,22 @@ Change kubectl -n kube-system edit configmap local-path-config on kube-system to
 kubectl label node casa-minion-01 role=worker-node
 ```
 
+### fstab
+
+```bash
+UUID=xxxx-xxxx-xxxx-xxxx-xxxx /                           ext4    errors=remount-ro 0       1
+UUID=xxxx-xxxx-xxxx-xxxx-xxxx /dataDisks/nvme00           ext4    errors=remount-ro 0       1
+UUID=xxxx-xxxx-xxxx-xxxx-xxxx /dataDisks/nvme01-ephemeral ext4    errors=remount-ro 0       1
+UUID=xxxx-xxxx-xxxx-xxxx-xxxx /dataDisks/ssd00            ext4    errors=remount-ro 0       1
+```
+
+``` bash
+# setup kubelet pods data to nvme01-ephemeral
+ln -s /dataDisks/nvme01-ephemeral/kubelet/pods /var/lib/kubelet/pods
+# setup k3s containerd  data to nvme01-ephemeral
+ln -s /dataDisks/nvme01-ephemeral/k3s/agent/containerd /dataDisks/nvme00/k3s/agent/containerd
+```
+
 ## casa-mini-minion-01 - worker node
 
 ### casa-mini-minion-01 - proxmox vm

services/cilium/deploy/README.md

@@ -30,7 +30,8 @@ kubectl get CiliumLoadBalancerIPPool -A
 **Force helm:**
 
 ```bash
-helm upgrade --install cilium oci://quay.io/cilium/cilium \
+helm repo add cilium https://helm.cilium.io/
+helm upgrade cilium cilium/cilium \
 --version 1.19.5 \
 -f ./deploy/app/helm-values.yaml \
 --namespace cilium 

services/cilium/deploy/app/helm-values.yaml

@@ -14,4 +14,23 @@ ipam:
 l2announcements:
   enabled: true
 loadBalancer:
-  mode: hybrid
+  mode: hybrid
+
+operator:
+  replicas: 1
+
+
+# fix error="listen tcp :30913: bind: address already in use" in agent      
+nodePort:
+  enableHealthCheck: false
+  enableHealthCheckLoadBalancerIP: false
+
+advertisement:
+  loadBalancerIPs: true
+  externalIPs: true
+
+hubble:
+  relay:
+    enabled: true
+  ui:
+    enabled: true

services/cilium/deploy/policies/traefik-l2.yaml

@@ -0,0 +1,23 @@
+apiVersion: cilium.io/v2alpha1
+kind: CiliumL2AnnouncementPolicy
+metadata:
+  name: casa-l2
+spec:
+  # nodeSelector:
+  #   matchLabels:
+  #     casa-lan: "true"
+  serviceSelector:
+    matchLabels:
+     casa-lan-l2-expose: "true"
+  interfaces:
+    - ens18
+  externalIPs: true
+  loadBalancerIPs: true
+---
+apiVersion: cilium.io/v2
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: casa-l2
+spec:
+  blocks:
+    - cidr: 192.168.14.0/24

services/fluxcd-system/README.md

@@ -23,3 +23,10 @@ helm upgrade flux-operator oci://ghcr.io/controlplaneio-fluxcd/charts/flux-opera
   --create-namespace
   kubectl apply -f ./deploy/fluxsystem.yaml
 ```
+
+**Allow prometheus scrapping:**
+
+```bash
+kubectl apply -f deploy/monitoring.yaml
+kubectl label ns flux-system prometheus-monitoring=enabled
+```

services/fluxcd-system/deploy/monitoring.yaml

@@ -0,0 +1,26 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: flux-system
+  namespace: flux-system
+  labels:
+    app.kubernetes.io/part-of: flux
+    app.kubernetes.io/component: monitoring
+    prometheus-monitoring: enabled
+spec:
+  namespaceSelector:
+    matchNames:
+      - flux-system
+  selector:
+    matchExpressions:
+      - key: app
+        operator: In
+        values:
+          - helm-controller
+          - source-controller
+          - kustomize-controller
+          - notification-controller
+          - image-automation-controller
+          - image-reflector-controller
+  podMetricsEndpoints:
+    - port: http-prom

services/monitoring/README.md

@@ -20,3 +20,9 @@ Using flux for reconciliation.
 ``` bash
 sops -e deploy/promtail/helm-values.private.dec.yaml > deploy/promtail/helm-values.private.yaml
 ```
+
+**Check agent:**
+
+``` bash
+http://localhost:9090/agent
+```

services/monitoring/deploy/prometheus/helm-values.yaml

@@ -1,6 +1,4 @@
-# values.yaml to install only Prometheus Operator and CRDs
 
-# Disable all components except the operator
 defaultRules:
   create: false
 
@@ -10,11 +8,15 @@ alertmanager:
 grafana:
   enabled: false
 
+prometheus:
+  enabled: false
+
 kubeStateMetrics:
   enabled: true
 
 nodeExporter:
   enabled: true
+
 prometheus-node-exporter:
   prometheus:
     monitor:
@@ -27,8 +29,7 @@ prometheus-node-exporter:
       - targetLabel: cluster
         replacement: casa
 
-prometheus:
+
-  enabled: false
 
 coreDns: 
   enabled: true

services/node-red/deploy/app/ingress.yaml

@@ -4,6 +4,7 @@ metadata:
   name: node-red
   namespace: node-red
 spec:
+  ingressClassName: traefik  
   entryPoints:
     - websecure
   routes:

services/node-red/deploy/app/pvc.yaml

@@ -1,3 +1,25 @@
+
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: node-red
+spec:
+  capacity:
+    storage: 1Gi
+  accessModes:
+    - ReadWriteOnce
+  hostPath:
+    path: /dataDisks/nvme00/k3s/storageStatic/pv-node-red
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - casa-minion-01
+---
+
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
@@ -9,4 +31,4 @@ spec:
   resources:
     requests:
       storage: 1Gi
-  storageClassName: local-path
+  storageClassName: ""

services/traefik/deploy/app/default/helm-release.yaml

@@ -34,9 +34,11 @@ spec:
       enabled: true
       type: LoadBalancer
       spec:
-        externalTrafficPolicy: Local
+        internalTrafficPolicy: Cluster
-        loadBalancerIP: 192.168.14.200
+        loadBalancerIP: 192.168.14.203
-        
+        allocateLoadBalancerNodePorts: false
+      labels:
+        casa-lan-l2-expose: "true"     
     ports:
       web:
         port: 80
@@ -56,14 +58,13 @@ spec:
         enabled: true
         allowCrossNamespace: false
         namespaces: []
-        ingressClass: traefik-icarus
+        ingressClass: traefik
       kubernetesIngress:
         enabled: true
-        ingressClass: traefik-icarus
+        ingressClass: traefik
 
     additionalArguments:
       - "--providers.kubernetesingress.ingressclass=traefik"
       - "--providers.kubernetescrd.ingressClass=traefik"
       - "--providers.kubernetescrd.allowEmptyServices=false"
-      - "--providers.kubernetesingress.ingressendpoint.ip=192.168.14.200"
 

services/traefik/deploy/app/icarus/helm-release.yaml

@@ -29,9 +29,10 @@ spec:
       enabled: true
       type: LoadBalancer
       spec:
-        externalTrafficPolicy: Local
+        externalTrafficPolicy: Cluster
         loadBalancerIP: 192.168.14.204
-        
+      labels:
+        casa-lan-l2-expose: "true"      
     ports:
       web:
         port: 80
@@ -60,4 +61,3 @@ spec:
       - "--providers.kubernetesingress.ingressclass=traefik-icarus"
       - "--providers.kubernetescrd.ingressClass=traefik-icarus"
       - "--providers.kubernetescrd.allowEmptyServices=false"
-      - "--providers.kubernetesingress.ingressendpoint.ip=192.168.14.204"

services/wyoming/deploy/app/kustomization.yaml

@@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: wyoming  
 resources:
-  - pvc.yaml
+  - storage.yaml
   - piper-deployment.yaml
   - piper-service.yaml
   - whisper-deployment.yaml

services/wyoming/deploy/app/piper-deployment.yaml

@@ -20,7 +20,7 @@ spec:
       containers:
         - name: wyoming-piper
           ### Maintained by flux - Image Update Automation
-          image: rhasspy/wyoming-piper:latest@sha256:c874e4a04657ae3381332ee5d0c8c70a310dae6722892840f530ac0890b44eb3 # {"$imagepolicy": "wyoming:piper"}
+          image: rhasspy/wyoming-piper:latest@sha256:69b7f797ae3a8c3c0202cbf97152fb795d78c2355de2a31655c20671247360d8 # {"$imagepolicy": "wyoming:piper"}
           ###
           volumeMounts:
             - mountPath: /data
@@ -34,15 +34,3 @@ spec:
         - name: data
           persistentVolumeClaim:
             claimName: wyoming-piper
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: wyoming-piper
-  namespace: wyoming
-spec:
-  accessModes:
-  - ReadWriteOnce
-  resources:
-    requests:
-      storage: 2Gi

services/wyoming/deploy/app/pvc.yaml

@@ -1,11 +0,0 @@
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: wyoming-whisper
-  namespace: wyoming
-spec:
-  accessModes:
-  - ReadWriteOnce
-  resources:
-    requests:
-      storage: 8Gi

services/wyoming/deploy/app/storage.yaml

@@ -0,0 +1,73 @@
+
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: wyoming-whisper
+  namespace: wyoming
+spec:
+  capacity:
+    storage: 8Gi
+  accessModes:
+    - ReadWriteOnce
+  hostPath:
+    path: /dataDisks/nvme00/k3s/storageStatic/pv-wyoming-whisper
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - casa-minion-01
+---
+
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: wyoming-whisper
+  namespace: wyoming
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 8Gi
+  storageClassName: ""
+---
+
+
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: wyoming-piper
+  namespace: wyoming
+spec:
+  capacity:
+    storage: 2Gi
+  accessModes:
+    - ReadWriteOnce
+  hostPath:
+    path: /dataDisks/nvme00/k3s/storageStatic/pv-wyoming-piper
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - casa-minion-01
+
+---
+
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: wyoming-piper
+  namespace: wyoming
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 2Gi
+  storageClassName: ""