modified: .devcontainer/devcontainer.json

modified:   services/monitoring/deploy/prometheus/prometheus-agent.yaml

.devcontainer/devcontainer.json

@@ -31,7 +31,8 @@
 				"streetsidesoftware.code-spell-checker",
 				"ms-azuretools.vscode-containers",
 				"eamodio.gitlens",
-				"bierner.markdown-preview-github-styles"
+				"bierner.markdown-preview-github-styles",
+				"weaveworks.vscode-gitops-tools"
 			]
 		}
 	}

casa.full-stack.code-workspace

@@ -46,11 +46,13 @@
 		"davidanson",
 		"eamodio",
 		"envsubst",
+		"kubelet",
 		"lmscommunity",
 		"localtime",
 		"LOGLEVEL",
 		"lyrionmusicserver",
 		"mtxr",
+		"nvme",
 		"rclone",
 		"reverseproxy",
 		"rogalmic",

cluster/README.md

@@ -13,6 +13,7 @@
 - [minion01 - worker node](#minion01---worker-node)
   - [Minion01 - proxmox vm](#minion01---proxmox-vm)
   - [casa-minion-01 - k3s - setup](#casa-minion-01---k3s---setup)
+  - [fstab](#fstab)
 - [casa-mini-minion-01 - worker node](#casa-mini-minion-01---worker-node)
   - [casa-mini-minion-01 - proxmox vm](#casa-mini-minion-01---proxmox-vm)
   - [casa-mini-minion-01 - k3s - setup](#casa-mini-minion-01---k3s---setup)
@@ -238,6 +239,7 @@ numa: 0
 onboot: 1
 ostype: l26
 scsi0: fastcore:vm-XXX-disk-0,iothread=1,size=8G,ssd=1
+scsi1: fluxcore:vm-xx-disk-0,iothread=1,size=32G,ssd=1 # used mainly by jellyfin cache
 scsi1: fastcore:vm-XXX-disk-2,iothread=1,size=32G,ssd=1 /dataDisk # k3s-data-dir /dataDisk/k3s
 scsihw: virtio-scsi-single
 smbios1: xxxx-xxxx-xxxx-xxxx-xxxx
@@ -260,7 +262,7 @@ sockets: 1
 # cat /var/lib/rancher/k3s/server/node-token
 
 K3S_TOKEN="????"
-curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.36.1+k3s1 INSTALL_K3S_EXEC="agent --data-dir /dataDisk/k3s --server https://casa.dev.lan:6443 --token ${K3S_TOKEN}" sh -s -
+curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.36.1+k3s1 INSTALL_K3S_EXEC="agent --data-dir /dataDisks/nvme00/k3s --server https://casa.dev.lan:6443 --token ${K3S_TOKEN}" sh -s -
 ```
 
 Change kubectl -n kube-system edit configmap local-path-config on kube-system to set path to provisioner.
@@ -269,7 +271,6 @@ Change kubectl -n kube-system edit configmap local-path-config on kube-system to
    config.json: |-
     {
       "nodePathMap":[
-      {
         "node":"DEFAULT_PATH_FOR_NON_LISTED_NODES",
         "paths":["/var/lib/rancher/k3s/storage"]
       },
@@ -287,6 +288,22 @@ Change kubectl -n kube-system edit configmap local-path-config on kube-system to
 kubectl label node casa-minion-01 role=worker-node
 ```
 
+### fstab
+
+```bash
+UUID=xxxx-xxxx-xxxx-xxxx-xxxx /                           ext4    errors=remount-ro 0       1
+UUID=xxxx-xxxx-xxxx-xxxx-xxxx /dataDisks/nvme00           ext4    errors=remount-ro 0       1
+UUID=xxxx-xxxx-xxxx-xxxx-xxxx /dataDisks/nvme01-ephemeral ext4    errors=remount-ro 0       1
+UUID=xxxx-xxxx-xxxx-xxxx-xxxx /dataDisks/ssd00            ext4    errors=remount-ro 0       1
+```
+
+``` bash
+# setup kubelet pods data to nvme01-ephemeral
+ln -s /dataDisks/nvme01-ephemeral/kubelet/pods /var/lib/kubelet/pods
+# setup k3s containerd  data to nvme01-ephemeral
+ln -s /dataDisks/nvme01-ephemeral/k3s/agent/containerd /dataDisks/nvme00/k3s/agent/containerd
+```
+
 ## casa-mini-minion-01 - worker node
 
 ### casa-mini-minion-01 - proxmox vm

services/cilium/deploy/README.md

@@ -30,7 +30,8 @@ kubectl get CiliumLoadBalancerIPPool -A
 **Force helm:**
 
 ```bash
-helm upgrade --install cilium oci://quay.io/cilium/cilium \
+helm repo add cilium https://helm.cilium.io/
+helm upgrade cilium cilium/cilium \
 --version 1.19.5 \
 -f ./deploy/app/helm-values.yaml \
 --namespace cilium 

services/cilium/deploy/app/helm-values.yaml

@@ -6,4 +6,31 @@ autoDirectNodeRoutes: true
 ipv4NativeRoutingCIDR: "10.42.0.0/16"
 bpf:
   masquerade: true
-  hostLegacyRouting: false # force use of bfp
+  hostLegacyRouting: false # force use of bfp
+ipam:
+  operator:
+    clusterPoolIPv4PodCIDR: "10.42.0.0/16"
+    clusterPoolIPv4MaskSize: 24
+l2announcements:
+  enabled: true
+loadBalancer:
+  mode: hybrid
+
+operator:
+  replicas: 1
+
+
+# fix error="listen tcp :30913: bind: address already in use" in agent      
+nodePort:
+  enableHealthCheck: false
+  enableHealthCheckLoadBalancerIP: false
+
+advertisement:
+  loadBalancerIPs: true
+  externalIPs: true
+
+hubble:
+  relay:
+    enabled: true
+  ui:
+    enabled: true

services/cilium/deploy/policies/traefik-l2.yaml

@@ -0,0 +1,23 @@
+apiVersion: cilium.io/v2alpha1
+kind: CiliumL2AnnouncementPolicy
+metadata:
+  name: casa-l2
+spec:
+  # nodeSelector:
+  #   matchLabels:
+  #     casa-lan: "true"
+  serviceSelector:
+    matchLabels:
+     casa-lan-l2-expose: "true"
+  interfaces:
+    - ens18
+  externalIPs: true
+  loadBalancerIPs: true
+---
+apiVersion: cilium.io/v2
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: casa-l2
+spec:
+  blocks:
+    - cidr: 192.168.14.0/24

services/fluxcd-system/README.md

@@ -23,3 +23,10 @@ helm upgrade flux-operator oci://ghcr.io/controlplaneio-fluxcd/charts/flux-opera
   --create-namespace
   kubectl apply -f ./deploy/fluxsystem.yaml
 ```
+
+**Allow prometheus scrapping:**
+
+```bash
+kubectl apply -f deploy/monitoring.yaml
+kubectl label ns flux-system prometheus-monitoring=enabled
+```

services/fluxcd-system/deploy/monitoring.yaml

@@ -0,0 +1,26 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: flux-system
+  namespace: flux-system
+  labels:
+    app.kubernetes.io/part-of: flux
+    app.kubernetes.io/component: monitoring
+    prometheus-monitoring: enabled
+spec:
+  namespaceSelector:
+    matchNames:
+      - flux-system
+  selector:
+    matchExpressions:
+      - key: app
+        operator: In
+        values:
+          - helm-controller
+          - source-controller
+          - kustomize-controller
+          - notification-controller
+          - image-automation-controller
+          - image-reflector-controller
+  podMetricsEndpoints:
+    - port: http-prom

services/monitoring/README.md

@@ -20,3 +20,9 @@ Using flux for reconciliation.
 ``` bash
 sops -e deploy/promtail/helm-values.private.dec.yaml > deploy/promtail/helm-values.private.yaml
 ```
+
+**Check agent:**
+
+``` bash
+http://localhost:9090/agent
+```

services/monitoring/deploy/prometheus/helm-values.yaml

@@ -1,6 +1,4 @@
-# values.yaml to install only Prometheus Operator and CRDs
 
-# Disable all components except the operator
 defaultRules:
   create: false
 
@@ -10,11 +8,15 @@ alertmanager:
 grafana:
   enabled: false
 
+prometheus:
+  enabled: false
+
 kubeStateMetrics:
   enabled: true
 
 nodeExporter:
   enabled: true
+
 prometheus-node-exporter:
   prometheus:
     monitor:
@@ -27,8 +29,7 @@ prometheus-node-exporter:
       - targetLabel: cluster
         replacement: casa
 
-prometheus:
-  enabled: false
+
 
 coreDns: 
   enabled: true

services/node-red/deploy/app/ingress.yaml

@@ -4,6 +4,7 @@ metadata:
   name: node-red
   namespace: node-red
 spec:
+  ingressClassName: traefik  
   entryPoints:
     - websecure
   routes:

services/node-red/deploy/app/pvc.yaml

@@ -1,3 +1,25 @@
+
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: node-red
+spec:
+  capacity:
+    storage: 1Gi
+  accessModes:
+    - ReadWriteOnce
+  hostPath:
+    path: /dataDisks/nvme00/k3s/storageStatic/pv-node-red
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - casa-minion-01
+---
+
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
@@ -9,4 +31,4 @@ spec:
   resources:
     requests:
       storage: 1Gi
-  storageClassName: local-path
+  storageClassName: ""

services/traefik-public/README.md

@@ -1,9 +0,0 @@
-# traefik
-
-## Setup
-
-Using flux for reconciliation.
-
-``` bash
-kubectl kustomize ./deploy/flux | kubectl apply -f -
-```

services/traefik-public/deploy/app/helm-release.yaml

@@ -1,24 +0,0 @@
-apiVersion: helm.toolkit.fluxcd.io/v2
-kind: HelmRelease
-metadata:
-  name: traefik
-spec:
-
-  releaseName: traefik
-  interval: 40h
-  chart:
-    spec:
-      chart: traefik
-      version: 40.x.x
-      sourceRef:
-        kind: HelmRepository
-        name: traefik
-        namespace: traefik-public
-      interval: 40h
-  values:
-
-    nodeSelector:
-    ingressClass:
-      enabled: true
-      isDefaultClass: false
-      name: traefik-icarus

services/traefik-public/deploy/flux/kustomization.yaml

@@ -1,5 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-namespace: traefik-public
-resources:
- - app-sync.yaml

services/traefik/deploy/app/default/helm-release.yaml

@@ -0,0 +1,70 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: traefik
+spec:
+
+  releaseName: traefik
+  interval: 1m
+  chart:
+    spec:
+      chart: traefik
+      version: 40.x.x
+      sourceRef:
+        kind: HelmRepository
+        name: traefik
+        namespace: traefik
+      interval: 40h
+  values:
+
+    nodeSelector:
+      kubernetes.io/hostname: casa-mini-minion-01
+    tolerations:
+    - key: "infra.limbosolutions.com/dedicated"
+      operator: "Equal"
+      value: "casa-mini-minion-01"
+      effect: "NoSchedule"
+
+    ingressClass:
+      enabled: true
+      isDefaultClass: true
+      name: traefik
+
+    service:
+      enabled: true
+      type: LoadBalancer
+      spec:
+        internalTrafficPolicy: Cluster
+        loadBalancerIP: 192.168.14.203
+        allocateLoadBalancerNodePorts: false
+      labels:
+        casa-lan-l2-expose: "true"     
+    ports:
+      web:
+        port: 80
+        exposedPort: 80
+        expose:
+          default: true
+        protocol: TCP
+      websecure:
+        port: 443
+        exposedPort: 443
+        expose:
+          default: true
+        protocol: TCP
+
+    providers:
+      kubernetesCRD:
+        enabled: true
+        allowCrossNamespace: false
+        namespaces: []
+        ingressClass: traefik
+      kubernetesIngress:
+        enabled: true
+        ingressClass: traefik
+
+    additionalArguments:
+      - "--providers.kubernetesingress.ingressclass=traefik"
+      - "--providers.kubernetescrd.ingressClass=traefik"
+      - "--providers.kubernetescrd.allowEmptyServices=false"
+

services/traefik/deploy/app/default/kustomization.yaml

@@ -1,6 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-namespace: kube-system
+namespace: traefik
 resources:
   - helm-release.yaml
   - helm-repo.yaml

services/traefik/deploy/app/helm-release.yaml

@@ -1,31 +0,0 @@
-apiVersion: helm.toolkit.fluxcd.io/v2
-kind: HelmRelease
-metadata:
-  name: traefik
-spec:
-
-  releaseName: traefik
-  interval: 40h
-  chart:
-    spec:
-      chart: traefik
-      version: 40.x.x
-      sourceRef:
-        kind: HelmRepository
-        name: traefik
-        namespace: kube-system
-      interval: 40h
-  values:
-
-    nodeSelector:
-      kubernetes.io/hostname: casa-mini-minion-01
-    tolerations:
-    - key: "infra.limbosolutions.com/dedicated"
-      operator: "Equal"
-      value: "casa-mini-minion-01"
-      effect: "NoSchedule"
-
-    ingressClass:
-      enabled: true
-      isDefaultClass: true
-      name: traefik

services/traefik/deploy/app/icarus/helm-release.yaml

@@ -0,0 +1,63 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: traefik
+spec:
+
+  releaseName: traefik
+  interval: 1m
+  chart:
+    spec:
+      chart: traefik
+      version: 40.x.x
+      sourceRef:
+        kind: HelmRepository
+        name: traefik
+        namespace: traefik-icarus
+      interval: 40h
+  values:
+
+    nodeSelector:
+      kubernetes.io/hostname: casa-minion-01
+
+    ingressClass:
+      enabled: true
+      isDefaultClass: false
+      name: traefik-icarus
+
+    service:
+      enabled: true
+      type: LoadBalancer
+      spec:
+        externalTrafficPolicy: Cluster
+        loadBalancerIP: 192.168.14.204
+      labels:
+        casa-lan-l2-expose: "true"      
+    ports:
+      web:
+        port: 80
+        exposedPort: 80
+        expose:
+          default: true
+        protocol: TCP
+      websecure:
+        port: 443
+        exposedPort: 443
+        expose:
+          default: true
+        protocol: TCP
+
+    providers:
+      kubernetesCRD:
+        enabled: true
+        allowCrossNamespace: false
+        namespaces: []
+        ingressClass: traefik-icarus
+      kubernetesIngress:
+        enabled: true
+        ingressClass: traefik-icarus
+
+    additionalArguments:
+      - "--providers.kubernetesingress.ingressclass=traefik-icarus"
+      - "--providers.kubernetescrd.ingressClass=traefik-icarus"
+      - "--providers.kubernetescrd.allowEmptyServices=false"

services/traefik/deploy/app/icarus/kustomization.yaml

@@ -1,6 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-namespace: traefik-public
+namespace: traefik-icarus
 resources:
   - helm-release.yaml
   - helm-repo.yaml

services/traefik/deploy/flux/default-sync.yaml

@@ -2,12 +2,13 @@ apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
   name: traefik
+  namespace: traefik
 spec:
   interval: 1m
   sourceRef:
     kind: GitRepository
     name: casa
     namespace: casa-limbosolutions-com
-  path: services/traefik-public/deploy/app
+  path: services/traefik/deploy/app/default
   prune: true
 

services/traefik/deploy/flux/icarus-sync.yaml

@@ -1,13 +1,13 @@
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
-  name: traefik
+  name: traefik-icarus
+  namespace: traefik-icarus
 spec:
   interval: 1m
   sourceRef:
     kind: GitRepository
     name: casa
     namespace: casa-limbosolutions-com
-  path: services/traefik/deploy/app
+  path: services/traefik/deploy/app/icarus
   prune: true
-

services/traefik/deploy/flux/kustomization.yaml

@@ -1,5 +1,5 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-namespace: kube-system
 resources:
- - app-sync.yaml
+ - default-sync.yaml
+ - icarus-sync.yaml

services/wyoming/deploy/app/kustomization.yaml

@@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: wyoming  
 resources:
-  - pvc.yaml
+  - storage.yaml
   - piper-deployment.yaml
   - piper-service.yaml
   - whisper-deployment.yaml

services/wyoming/deploy/app/piper-deployment.yaml

@@ -20,7 +20,7 @@ spec:
       containers:
         - name: wyoming-piper
           ### Maintained by flux - Image Update Automation
-          image: rhasspy/wyoming-piper:latest@sha256:c874e4a04657ae3381332ee5d0c8c70a310dae6722892840f530ac0890b44eb3 # {"$imagepolicy": "wyoming:piper"}
+          image: rhasspy/wyoming-piper:latest@sha256:69b7f797ae3a8c3c0202cbf97152fb795d78c2355de2a31655c20671247360d8 # {"$imagepolicy": "wyoming:piper"}
           ###
           volumeMounts:
             - mountPath: /data
@@ -34,15 +34,3 @@ spec:
         - name: data
           persistentVolumeClaim:
             claimName: wyoming-piper
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: wyoming-piper
-  namespace: wyoming
-spec:
-  accessModes:
-  - ReadWriteOnce
-  resources:
-    requests:
-      storage: 2Gi

services/wyoming/deploy/app/pvc.yaml

@@ -1,11 +0,0 @@
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: wyoming-whisper
-  namespace: wyoming
-spec:
-  accessModes:
-  - ReadWriteOnce
-  resources:
-    requests:
-      storage: 8Gi

services/wyoming/deploy/app/storage.yaml

@@ -0,0 +1,73 @@
+
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: wyoming-whisper
+  namespace: wyoming
+spec:
+  capacity:
+    storage: 8Gi
+  accessModes:
+    - ReadWriteOnce
+  hostPath:
+    path: /dataDisks/nvme00/k3s/storageStatic/pv-wyoming-whisper
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - casa-minion-01
+---
+
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: wyoming-whisper
+  namespace: wyoming
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 8Gi
+  storageClassName: ""
+---
+
+
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: wyoming-piper
+  namespace: wyoming
+spec:
+  capacity:
+    storage: 2Gi
+  accessModes:
+    - ReadWriteOnce
+  hostPath:
+    path: /dataDisks/nvme00/k3s/storageStatic/pv-wyoming-piper
+  nodeAffinity:
+    required:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: kubernetes.io/hostname
+              operator: In
+              values:
+                - casa-minion-01
+
+---
+
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: wyoming-piper
+  namespace: wyoming
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 2Gi
+  storageClassName: ""