myLimbo/casa
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Files
185bf165464721faad3ddfaa40e1d53b915f499b
casa/cluster/README.md
T
Márcio Fernandes 185bf16546 modified: casa.full-stack.code-workspace 
modified:   cluster/README.md
modified:   services/node-red/deploy/app/pvc.yaml
2026-07-13 13:08:07 +00:00

8.6 KiB
Raw Blame History

Casa k3s cluster

k3s version: v1.36.1+k3s1

Table of Contents:

Common

Disable swap:

swapoff -a
Edit /etc/fstab and comment out any swap entries:
# /swapfile none swap sw 0 0

Other Packages:

sudo apt update -y && \
sudo apt install -y \
curl \
btop

Update system:

sudo apt update -y && \
sudo apt upgrade -y && \
sudo apt auto-remove -y

name servers:

cat <<EOF > /etc/resolv.conf       
domain dev.lan
search dev.lan. lan.
nameserver 192.168.14.254
EOF

OCI Mirrors Cache

# /etc/rancher/k3s/registries.yaml

mirrors:
  # --- Docker Hub (all aliases → docker-mirror) ---
  docker.io:
    endpoint:
      - https://oci.limbosolutions.com/v2/docker-mirror
  registry-1.docker.io:
    endpoint:
      - https://oci.limbosolutions.com/v2/docker-mirror
  index.docker.io:
    endpoint:
      - https://oci.limbosolutions.com/v2/docker-mirror

  # --- GHCR ---
  ghcr.io:
    endpoint:
      - https://oci.limbosolutions.com/v2/ghcr-mirror

  # --- Quay.io ---
  quay.io:
    endpoint:
      - https://oci.limbosolutions.com/v2/quay-mirror

  # --- Kubernetes official registry ---
  registry.k8s.io:
    endpoint:
      - https://oci.limbosolutions.com/v2/k8s-mirror

  # --- Google Container Registry (your "grc-mirror") ---
  gcr.io:
    endpoint:
      - https://oci.limbosolutions.com/v2/grc-mirror

  # --- AWS Public ECR ---
  public.ecr.aws:
    endpoint:
      - https://oci.limbosolutions.com/v2/aws-mirror

  # --- Microsoft Container Registry ---
  mcr.microsoft.com:
    endpoint:
      - https://oci.limbosolutions.com/v2/mcr-proxy

  # --- GitLab Container Registry ---
  registry.gitlab.com:
    endpoint:
      - https://oci.limbosolutions.com/v2/gitlab-mirror

  # --- Bitnami (namespace under Docker Hub) ---
  docker.io/bitnami:
    endpoint:
      - https://oci.limbosolutions.com/v2/bitnami-mirror

  # --- Red Hat registry ---
  registry.redhat.io:
    endpoint:
      - https://oci.limbosolutions.com/v2/readheat-mirror
      

configs:
  oci.limbosolutions.com:
    auth:
      username: <username>
      password: <password>
    tls:
      insecure_skip_verify: false

casa - control Plane

Name Value
NAME casa.dev.lan
VLAN casa
IP 192.168.14.9
OS Debian GNU/Linux 12 (bookworm)
KERNEL 6.1.0-44-amd64 
cat /etc/os-release; uname -r

casa - control Plane - proxmox vm

hosted on gaia.

agent: 1
balloon: 0
boot: order=scsi0;ide2;net0
cores: 2
cpu: host
ide2: none,media=cdrom
memory: 2355
meta: creation-qemu=10.1.2,ctime=1762626497
name: casa
net0: virtio=BXX:XX:XX:XX:XX:XX,bridge=vmbr0,tag=xx
numa: 0
onboot: 1
ostype: l26
scsi0: local-lvm:vm-XXX-disk-0,iothread=1,size=24G,ssd=1
scsihw: virtio-scsi-single
smbios1: uuid=cxxxx-xxxx-xxxx-xxxx-xxxx
sockets: 1

casa - control Plane - network configuration

ip a # check ethernet name

# removes automatic configuration as dhcp client
sed -i '/ens18/d' /etc/network/interfaces

cat <<EOF > /etc/network/interfaces.d/ens18
# my network configuration
auto ens18
iface ens18 inet static
  address 192.168.14.9/24
  gateway 192.168.0.1
EOF

casa - control Plane - k3s setup

curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.36.1+k3s1 \
INSTALL_K3S_EXEC="\
--flannel-backend=none \
--disable-network-policy \
--disable=servicelb" \
 sh -

Taint NoSchedule on master node:

kubectl taint nodes node-role.kubernetes.io/control-plane=:NoSchedule

Example of selector/tolerations for pods, deployments:

...
nodeSelector:
  kubernetes.io/hostname: casa

tolerations:
  - key: "node-role.kubernetes.io/control-plane"
    operator: "Exists"
    effect: "NoSchedule"
...


kubectl taint nodes casa node-role.kubernetes.io/control-plane=:NoSchedule

minion01 - worker node

Minion01 - proxmox vm

hosted on gaia.

agent: 1
balloon: 0
boot: order=scsi0;ide2;net0
cores: 4
cpu: host
ide2: none,media=cdrom
memory: 4096
meta: creation-qemu=10.1.2,ctime=1763219351
name: casa-minion-01
net0: virtio=BXX:XX:XX:XX:XX:XX,bridge=vmbr0,tag=xx
numa: 0
onboot: 1
ostype: l26
scsi0: fastcore:vm-XXX-disk-0,iothread=1,size=8G,ssd=1
scsi1: fluxcore:vm-xx-disk-0,iothread=1,size=32G,ssd=1 # used mainly by jellyfin cache
scsi1: fastcore:vm-XXX-disk-2,iothread=1,size=32G,ssd=1 /dataDisk # k3s-data-dir /dataDisk/k3s
scsihw: virtio-scsi-single
smbios1: xxxx-xxxx-xxxx-xxxx-xxxx
sockets: 1
Name Value
NAME minion01
VLAN casa
IP 192.168.14.10
OS Debian GNU/Linux 12 (bookworm)
KERNEL 6.1.0-44-amd64

casa-minion-01 - k3s - setup

# install k3s as agent / worker node
# execute on server to get token
# cat /var/lib/rancher/k3s/server/node-token

K3S_TOKEN="????"
curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.36.1+k3s1 INSTALL_K3S_EXEC="agent --data-dir /dataDisks/nvme00/k3s --server https://casa.dev.lan:6443 --token ${K3S_TOKEN}" sh -s -

Change kubectl -n kube-system edit configmap local-path-config on kube-system to set path to provisioner.

   config.json: |-
    {
      "nodePathMap":[
        "node":"DEFAULT_PATH_FOR_NON_LISTED_NODES",
        "paths":["/var/lib/rancher/k3s/storage"]
      },
      {
          "node":"casa-minion-01",
          "paths":["/dataDisk/k3s/storage"]
      }
      ]
    }

Set node labels:

kubectl label node casa-minion-01 role=worker-node

fstab

UUID=xxxx-xxxx-xxxx-xxxx-xxxx /                           ext4    errors=remount-ro 0       1
UUID=xxxx-xxxx-xxxx-xxxx-xxxx /dataDisks/nvme00           ext4    errors=remount-ro 0       1
UUID=xxxx-xxxx-xxxx-xxxx-xxxx /dataDisks/nvme01-ephemeral ext4    errors=remount-ro 0       1
UUID=xxxx-xxxx-xxxx-xxxx-xxxx /dataDisks/ssd00            ext4    errors=remount-ro 0       1

# setup kubelet pods data to nvme01-ephemeral
ln -s /dataDisks/nvme01-ephemeral/kubelet/pods /var/lib/kubelet/pods
# setup k3s containerd  data to nvme01-ephemeral
ln -s /dataDisks/nvme01-ephemeral/k3s/agent/containerd /dataDisks/nvme00/k3s/agent/containerd

casa-mini-minion-01 - worker node

casa-mini-minion-01 - proxmox vm

*hosted on gaia.

agent: 1
balloon: 0
boot: order=scsi0;ide2;net0
cores: 4
cpu: x86-64-v2-AES
memory: 2048
meta: creation-qemu=11.0.0,ctime=1782516417
name: casa-mini-minion-01
net0: virtio=xx:xx:xx:xx:xx:xx,bridge=vmbr0,tag=14
numa: 0
ostype: l26
scsi0: fastcore:vm-112-disk-0,iothread=1,size=24G,ssd=1
scsihw: virtio-scsi-single
sockets: 1
usb0: host=1-1.1
Name Value
NAME casa-mini-minion-01
VLAN casa
IP 192.168.14.11
OS Debian GNU/Linux 12 (bookworm)
KERNEL 6.1.0-44-amd64

casa-mini-minion-01 - k3s - setup

# install k3s as agent / worker node
# execute on server to get token
# cat /var/lib/rancher/k3s/server/node-token

K3S_TOKEN="???"
curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.36.1+k3s1 INSTALL_K3S_EXEC="agent --server https://casa.dev.lan:6443 --token ${K3S_TOKEN}" sh -s -

Change kubectl -n kube-system edit configmap local-path-config on kube-system to set path to provisioner.

kubectl taint nodes casa-mini-minion-01 infra.limbosolutions.com/dedicated=casa-mini-minion-01:NoSchedule
Reference in New Issue View Git Blame Copy Permalink