backups added resources limits, set offsite backup size limit to 15GB

modified:   deploy/backups/backup-borg-offsite-cronjob.yaml

.devcontainer/devcontainer.json

@@ -0,0 +1,34 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/ubuntu
+{
+	"name": "git-limbosolutions-com-dev",
+	"image": "git.limbosolutions.com/mylimbo/devcontainers/devops:latest",
+	"runArgs": ["--hostname=git-limbosolutions-com-dev-container"],
+	"remoteUser": "vscode",
+	"mounts": [
+		"source=${localWorkspaceFolder}/.kube,target=/home/vscode/.kube,type=bind",
+		"source=${localEnv:HOME}/.gitconfig,target=/home/vscode/.gitconfig,type=bind",
+		"source=${localEnv:HOME}/.ssh,target=/home/vscode/.ssh,type=bind"
+	],
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"ms-kubernetes-tools.vscode-kubernetes-tools",
+				"redhat.ansible",
+				"mtxr.sqltools-driver-mysql",
+				"stateful.runme",
+				"yzhang.markdown-all-in-one",
+				"davidanson.vscode-markdownlint",
+				"eamodio.gitlens",
+				"m4ns0ur.base64",
+				"rogalmic.bash-debug",
+				"streetsidesoftware.code-spell-checker",
+				"ms-azuretools.vscode-containers",
+				"eamodio.gitlens",
+				"shd101wyy.markdown-preview-enhanced",
+				"bierner.markdown-preview-github-styles"
+
+			]
+		}
+	}
+}

.gitea/workflows/app-continous-deploy.yaml

@@ -0,0 +1,93 @@
+on:
+  schedule:
+    - cron: '0 9 * * 0' # every sunday 9 am
+  push:
+    branches:
+      - main
+  pull_request:
+      branches:
+      - main
+jobs:
+  continuous-deploy:
+    runs-on: ubuntu-latest
+    container:
+      image: git.limbosolutions.com/kb/gitea/act:latest-network-stack
+    env:
+      GITHUB_TEMP: ${{ runner.temp }} # fix missing GITHUB_TEMP on gitea
+    steps:
+
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: limbo public actions
+        env: 
+          WORKSPACE: "${{ gitea.workspace }}"
+        run: |
+          curl -fsSL https://git.limbosolutions.com/kb/gitea/raw/branch/main/cloud-scripts/setup-limbo-actions.sh | bash 2>&1
+
+
+      # limbo custom actions required https://git.limbosolutions.com/kb/gitea/raw/branch/main
+      - name: Configure kubectl config
+        uses: ./.gitea/limbo_actions/kubectl-setup
+        with:
+          kube_server: ${{ secrets.HOSTING_KUBE_SERVER }}
+          kube_ca_base64: ${{ secrets.HOSTING_KUBE_CA_BASE64 }}
+          kube_token: ${{ secrets.HOSTING_KUBE_TOKEN }}
+
+      - name: Deploy
+        shell: bash
+        env:
+          # cron jobs env
+          CRONJOBS_BACKUPS_SECRETS_PBS_REPOSITORY: ${{ secrets.CRONJOBS_BACKUPS_SECRETS_PBS_REPOSITORY }}
+          CRONJOBS_BACKUPS_SECRETS_PBS_PASSWORD: ${{ secrets.CRONJOBS_BACKUPS_SECRETS_PBS_PASSWORD }}
+          CRONJOBS_BACKUPS_SECRETS_PBS_FINGERPRINT: ${{ secrets.CRONJOBS_BACKUPS_SECRETS_PBS_FINGERPRINT }}
+          CRONJOBS_BACKUPS_SECRETS_BORG_REPO: ${{ secrets.CRONJOBS_BACKUPS_SECRETS_BORG_REPO }}
+          CRONJOBS_BACKUPS_SECRETS_BORG_PASSPHRASE: ${{ secrets.CRONJOBS_BACKUPS_SECRETS_BORG_PASSPHRASE }}
+          CRONJOBS_BACKUPS_SECRETS_OFFSITE_TARGET_FOLDER: ${{ secrets.CRONJOBS_BACKUPS_SECRETS_OFFSITE_TARGET_FOLDER }}
+          CRONJOBS_BACKUPS_SECRETS_ID_RSA: ${{ secrets.CRONJOBS_BACKUPS_SECRETS_ID_RSA }}
+          CRONJOBS_BACKUPS_SECRETS_BORG_KEY: ${{ secrets.CRONJOBS_BACKUPS_SECRETS_BORG_KEY }}
+
+          # helm chart values
+          APP_HELM_VALUE_VALKEY_GLOBAL_PASSWORD: ${{ secrets.APP_HELM_VALUE_VALKEY_GLOBAL_PASSWORD }}
+          APP_HELM_VALUE_POSTGRESQL_AUTH_POSTGRESPASSWORD: ${{ secrets.APP_HELM_VALUE_POSTGRESQL_AUTH_POSTGRESPASSWORD }}
+          APP_HELM_VALUE_POSTGRESQL_AUTH_PASSWORD: ${{ secrets.APP_HELM_VALUE_POSTGRESQL_AUTH_PASSWORD }}
+          APP_HELM_VALUE_POSTGRESQL_AUTH_DATABASE: ${{ secrets.APP_HELM_VALUE_POSTGRESQL_AUTH_DATABASE }}
+          APP_HELM_VALUE_POSTGRESQL_AUTH_USERNAME: ${{ secrets.APP_HELM_VALUE_POSTGRESQL_AUTH_USERNAME }}
+          APP_HELM_VALUE_GITEA_ADMIN_USERNAME: ${{ secrets.APP_HELM_VALUE_GITEA_ADMIN_USERNAME }}
+          APP_HELM_VALUE_GITEA_ADMIN_PASSWORD: ${{ secrets.APP_HELM_VALUE_GITEA_ADMIN_PASSWORD }}
+          APP_HELM_VALUE_GITEA_ADMIN_EMAIL: ${{ secrets.APP_HELM_VALUE_GITEA_ADMIN_EMAIL }}
+          APP_HELM_VALUE_GITEA_CONFIG_OAUTH2_JWT_SECRET: ${{ secrets.APP_HELM_VALUE_GITEA_CONFIG_OAUTH2_JWT_SECRET }}
+          APP_HELM_VALUE_GITEA_CONFIG_SERVER_LFS_JWT_SECRET: ${{ secrets.APP_HELM_VALUE_GITEA_CONFIG_SERVER_LFS_JWT_SECRET }}
+          APP_HELM_VALUE_GITEA_CONFIG_SECURITY_SECRET_KEY: ${{ secrets.APP_HELM_VALUE_GITEA_CONFIG_SECURITY_SECRET_KEY }}
+          APP_HELM_VALUE_GITEA_CONFIG_SECURITY_REVERSE_PROXY_TRUSTED_PROXIES: ${{ secrets.APP_HELM_VALUE_GITEA_CONFIG_SECURITY_REVERSE_PROXY_TRUSTED_PROXIES }}
+          APP_HELM_VALUE_GITEA_CONFIG_SECURITY_INTERNAL_TOKEN: ${{ secrets.APP_HELM_VALUE_GITEA_CONFIG_SECURITY_INTERNAL_TOKEN }}
+          APP_HELM_VALUE_GITEA_CONFIG_SECURITY_PASSWORD_HASH_ALGO: ${{ secrets.APP_HELM_VALUE_GITEA_CONFIG_SECURITY_PASSWORD_HASH_ALGO }}
+          APP_HELM_VALUE_GITEA_CONFIG_SERVICE_OAUTH2_JWT_SECRET: ${{ secrets.APP_HELM_VALUE_GITEA_CONFIG_SERVICE_OAUTH2_JWT_SECRET }}
+
+        run: |
+          set -euo pipefail
+
+          # ensure cleanup always runs
+          trap 'rm -f \
+                    deploy/app/cronjobs/backups/.env.d/secrets \
+                    deploy/app/cronjobs/backups/.env.d/id_rsa \
+                    deploy/app/cronjobs/backups/.env.d/borg_key' EXIT
+
+          # setup env for cronjobs backups
+          echo "PBS_REPOSITORY=${CRONJOBS_BACKUPS_SECRETS_PBS_REPOSITORY}" >> deploy/app/cronjobs/backups/.env.d/secrets
+          echo "PBS_PASSWORD=${CRONJOBS_BACKUPS_SECRETS_PBS_PASSWORD}" >> deploy/app/cronjobs/backups/.env.d/secrets
+          echo "PBS_FINGERPRINT=${CRONJOBS_BACKUPS_SECRETS_PBS_FINGERPRINT}" >> deploy/app/cronjobs/backups/.env.d/secrets
+          echo "BORG_REPO=${CRONJOBS_BACKUPS_SECRETS_BORG_REPO}" >> deploy/app/cronjobs/backups/.env.d/secrets
+          echo "BORG_PASSPHRASE=${CRONJOBS_BACKUPS_SECRETS_BORG_PASSPHRASE}" >> deploy/app/cronjobs/backups/.env.d/secrets
+          echo "OFFSITE_TARGET_FOLDER=${CRONJOBS_BACKUPS_SECRETS_OFFSITE_TARGET_FOLDER}" >> deploy/app/cronjobs/backups/.env.d/secrets
+          
+          echo "${CRONJOBS_BACKUPS_SECRETS_ID_RSA}" >> deploy/app/cronjobs/backups/.env.d/id_rsa
+          echo "${CRONJOBS_BACKUPS_SECRETS_BORG_KEY}" >> deploy/app/cronjobs/backups/.env.d/borg_key
+          
+          # enforce security
+          chmod 600 deploy/app/cronjobs/backups/.env.d/secrets
+          chmod 600 deploy/app/cronjobs/backups/.env.d/id_rsa
+          chmod 600 deploy/app/cronjobs/backups/.env.d/borg_key
+
+          # invoke deploy script
+          ops-scripts/apply-app.sh

.gitignore

@@ -1,4 +1,5 @@
 tmp
 **.env
 **.private.**
-**.local.**
+**.local.**
+.kube/**

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "cSpell.words": [
+        "valkey"
+    ]
+}

README.md

@@ -1,77 +1,44 @@
-# git.limbosolutions.com
+# git.limbosolutions.com
-
+
-Welcome to public repository of my [Git Server](https://git.limbosolutions.com)
+Welcome to public repository of my [Git Server](https://git.limbosolutions.com)
-
+
-Using [gitea](https://git.limbosolutions.com/kb/gitea) as git server.
+Using [gitea](https://git.limbosolutions.com/kb/gitea) as git server.
-
+
-## gitea  helm chart
+- [Deploy](#deploy)
-
+  - [App](#app)
-``` yaml
+  - [Continuous Deploy](#continuous-deploy)
-helm repo add gitea-charts https://dl.gitea.com/charts/
+  - [Infra](#infra)
-helm repo update
+- [Backups](#backups)
-helm upgrade --install gitea gitea-charts/gitea -f ./values.yaml -f ./values.private.yml --namespace=git-limbosolutions-com
+
-```
+## Deploy
-
+
-## Git Action
+### App
-
+
-## runners
+**Environment files:**
-
+
-## lxc container - proxmox
+- ./deploy/app/cronjobs/backups/.env.d/secrets [Example](./deploy/app/cronjobs/backups/.env.d/secrets.example)
-
+- ./deploy/app/cronjobs/backups/.env.d/borg_key [Example](./deploy/app/cronjobs/backups/.env.d/borg_key.example)
-Template : debian-12-turnkey-core_18.1-1_amd64.tar.gz
+- ./deploy/app/cronjobs/backups/.env.d/id_rsa [Example](./deploy/app/cronjobs/backups/.env.d/id_rsa.example)
-
+- ./deploy/helm/.env [Example](./deploy/helm/.env.example)
-- Unprivileged contrainer - Yes
+
-- Nesting -Yes
+```bash
-
+./ops-scripts/apply-app.sh
-```bash
+```
-# setup new lxc container with docker
+
-
+- [kustomization](/deploy/app/kustomization.yaml)
-apt update -y
+
-apt upgrade -y
+### Continuous Deploy
-curl -fsSL https://get.docker.com -o get-docker.sh
+
-sh ./get-docker.sh
+Executes [App Deploy](#app) using [Gitea workflow](./.gitea/workflows/app-deploy.yaml).
-```
+
-
+### Infra
-### act tunner
+
-
+```bash
-```bash
+./ops-scripts/apply-infra.sh
-nano setup-act-runners.sh
+```
-```
+
-
+- [kustomization](/deploy/infra/kustomization.yaml)
-```bash
+  
-#/bin/bash
+## Backups
-
+
-setup(){
+for more information [check readme](./docs/backups.md).
-
-CONTAINER_NAME=${OWNER}_act_runner
-
-docker container stop $CONTAINER_NAME
-docker container rm $CONTAINER_NAME
-
-docker run \
-    --restart=unless-stopped \
-    -v ${CONTAINER_NAME}_data:/data \
-    -v /var/run/docker.sock:/var/run/docker.sock \
-    -e GITEA_INSTANCE_URL=${INSTANCE_URL} \
-    -e GITEA_RUNNER_REGISTRATION_TOKEN=${REGISTRATION_TOKEN} \
-    -e GITEA_RUNNER_NAME=${OWNER}_ubuntu_default \
-    --name ${CONTAINER_NAME} \
-    -d gitea/act_runner
-}
-
-INSTANCE_URL=https://git.limbosolutions.com
-OWNER=kb
-REGISTRATION_TOKEN=???
-setup
-
-
-OWNER=????
-REGISTRATION_TOKEN=???
-setup
-
-```
-
-```bash
-chmod +x setup-act-runners.sh
-./setup-act-runners.sh
-```

backup/gitea-backup-secret-deploy.sh

@@ -1,8 +0,0 @@
-#/bin/bash
-
-# set source env variables
-. ./.env
-export $(cut -d= -f1 ./.env)
-
-
-envsubst < secret.yaml | kubectl apply -f -

backup/gitea-backup-secret.yaml

@@ -1,14 +0,0 @@
-
-apiVersion: v1
-kind: Secret
-metadata:
-  name: gitea-backup-secret
-  namespace: git-limbosolutions-com
-data:
-  PBS_REPOSITORY: ${PBS_REPOSITORY}
-  PBS_PASSWORD: ${PBS_PASSWORD}
-  PBS_FINGERPRINT: ${PBS_FINGERPRINT}
-  DB_USER: ${DB_USER}
-  DB_PASSWORD: ${DB_PASSWORD}
-  DB_NAME: ${DB_NAME}
-  DB_HOST: ${DB_HOST}

deploy/app/cronjobs/backups/.env.d/.gitignore

@@ -0,0 +1,3 @@
+**
+!.gitignore
+!*.example

deploy/app/cronjobs/backups/.env.d/borg_key.example

@@ -0,0 +1 @@
+BORG_KEY an valid borg key

deploy/app/cronjobs/backups/.env.d/id_rsa.example

@@ -0,0 +1,3 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+AND VALID PRIVATE SSH KEY WITH ACCESS TO SSH SERVER  
+-----END OPENSSH PRIVATE KEY-----

deploy/app/cronjobs/backups/.env.d/secrets.example

@@ -0,0 +1,7 @@
+PBS_REPOSITORY="pbs repository"
+PBS_PASSWORD="pbs access passwordd"
+PBS_FINGERPRINT="00:00:00:00:00" # the pbs finger print
+BORG_REPO="ssh://user@reposerver/path" # required by offsite babckup
+BORG_PASSPHRASE="borg passphare" # required by offsite babckup
+OFFSITE_TARGET_FOLDER="test:target_path" # follow rclone naming convension
+

deploy/app/cronjobs/backups/backup-borg-offsite-cronjob.yaml

@@ -0,0 +1,163 @@
+
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: backup-borg-offsite
+  namespace: git-limbosolutions-com
+spec:
+  schedule: "0 16 * * 0" #every sunday at 4pm
+  jobTemplate:
+    spec:
+      backoffLimit: 1
+      template:
+        metadata:
+          labels:
+            app: offsite-backup
+        spec:
+          restartPolicy: Never
+          initContainers:
+            - name: postgres-export
+              resources:
+                limits:
+                  memory: "512Mi"
+                  cpu: "500m"
+                requests:
+                  memory: "256Mi"
+                  cpu: "250m"
+              image: postgres:latest
+              command: ["sh", "-c"]
+              args:
+                - |
+                  set -e
+                  . /root/.gitea-inline-config/database
+                  export PGPASSWORD=$PASSWD
+                  pg_dump -h gitea-postgresql.git-limbosolutions-com.svc.cluster.local -U $USER -d $NAME > /data/postgresql-export/db_backup.sql
+  
+              volumeMounts:
+                
+              - name: backup-var-data
+                mountPath: /data/postgresql-export
+                subPath: postgresql-export
+
+              - name: gitea-inline-config
+                mountPath: /root/.gitea-inline-config
+                readOnly: true
+
+
+     
+          containers:
+            - name: borg-client
+              image: git.limbosolutions.com/kb/borg-backup:latest
+              imagePullPolicy: Always
+              resources:
+                limits:
+                  memory: "512Mi"
+                  cpu: "500m"
+                requests:
+                  memory: "256Mi"
+                  cpu: "250m"
+              env:
+                - name: BORG_REPO
+                  valueFrom:
+                    secretKeyRef:
+                      name: gitea-backup
+                      key: BORG_REPO
+
+                - name: BORG_PASSPHRASE
+                  valueFrom:
+                    secretKeyRef:
+                      name: gitea-backup
+                      key: BORG_PASSPHRASE
+
+          
+                - name: OFFSITE_TARGET_FOLDER
+                  valueFrom:
+                    secretKeyRef:
+                      name: gitea-backup
+                      key: OFFSITE_TARGET_FOLDER
+
+
+                - name: BORG_RSH
+                  value: ssh -p 2222 -o StrictHostKeyChecking=no -o LogLevel=ERROR
+
+                - name: REPO_SYNC_MAX_SIZE
+                  value: "16106127360" # 15GB
+
+                - name: MODE
+                  value: SHELL
+           
+
+              args:
+              - |
+                set -e
+                
+                SCRIPT_START_TIME=$(date +%s)
+
+                # while true; do
+                #   sleep 5
+                # done
+                
+                borg create ${BORG_REPO}::postgresql-export-$(date +%Y%m%d%H%M%S) /data/postgresql-export
+                borg create ${BORG_REPO}::gitea-data-$(date +%Y%m%d%H%M%S) /data/gitea-data
+
+                #cleanup
+                borg prune -v --list --keep-daily=10 --keep-weekly=7 --keep-monthly=-1 ${BORG_REPO} --glob-archives='gitea-data*'
+                borg prune -v --list --keep-daily=10 --keep-weekly=7 --keep-monthly=-1 ${BORG_REPO} --glob-archives='postgresql-export*'
+                borg compact ${BORG_REPO} 
+
+                # check repo size
+                REPO_SIZE_IN_BYTES=$(remote-get-folder-size)
+                echo "Repository size: $((REPO_SIZE_IN_BYTES / 1024 / 1024))MB"
+                
+                if [ $REPO_SIZE_IN_BYTES -gt $REPO_SYNC_MAX_SIZE ]; then \
+                  echo "ERROR: Repository size $((REPO_SIZE_IN_BYTES / 1024 / 1024))MB exceeds $((REPO_SYNC_MAX_SIZE / 1024 / 1024))MB";
+                  exit 1;
+                else 
+                  # Repository size is within limits for offsite sync
+                  # ssh to backup server and enforce rclone to onedrive
+                  remote-connect "rclone sync $SSH_FOLDER $OFFSITE_TARGET_FOLDER --stats=0" && \
+                  echo "INFO: Finished Backup of git.limbosolutions.com (offsite) ($((SCRIPT_DURATION / 60 / 60)):$((SCRIPT_DURATION / 60)):$((SCRIPT_DURATION % 60))) "
+                fi
+
+                #outputs info
+                borg info ${BORG_REPO}
+                #borg info ${BORG_REPO} --json
+
+              volumeMounts:
+                - name: gitea-data
+                  mountPath: /data/gitea-data
+
+                - name: backup-var-data
+                  mountPath: /data/postgresql-export
+                  subPath: postgresql-export
+              
+                - name: gitea-backup-secrets
+                  mountPath: /root/.ssh/id_rsa
+                  subPath: SSH_ID_RSA
+                  readOnly: true
+
+                - name: gitea-backup-secrets
+                  mountPath: /app/borg/key
+                  subPath: BORG_KEY
+
+          volumes:
+
+            - name: gitea-data
+              persistentVolumeClaim:
+                claimName: gitea-shared-storage
+
+            - name: gitea-inline-config
+              secret:
+                secretName: gitea-inline-config
+
+            - name: gitea-backup-secrets
+              secret:
+                secretName: gitea-backup
+                defaultMode: 0600
+
+            - name: backup-var-data
+              emptyDir: {}
+
+
+
+

deploy/app/cronjobs/backups/backup-pbs-cronjob.yaml

@@ -1,7 +1,7 @@
 apiVersion: batch/v1
 kind: CronJob
 metadata:
-  name: gitea-backup
+  name: backup-pbs
   namespace: git-limbosolutions-com
 spec:
   schedule: "0 1 * * *"
@@ -9,64 +9,86 @@ spec:
     spec:
       backoffLimit: 1
       template:
+        metadata:
+          labels:
+            app: pbs-backup
         spec:
           restartPolicy: Never
           initContainers:
             - name: postgres-export
               image: postgres:latest
+              resources:
+                limits:
+                  memory: "512Mi"
+                  cpu: "500m"
+                requests:
+                  memory: "256Mi"
+                  cpu: "250m"
               command: ["sh", "-c"]
               args:
                 - |
-                  echo "INFO: Starting export"
+                  #echo "INFO: Starting export"
-                  . /run/database-access-secret/database
+                  . /root/.gitea-inline-config/database
                   export PGPASSWORD=$PASSWD
-                  echo "INFO: Exporting database"
+                  #echo "INFO: Exporting database"
                   pg_dump -h gitea-postgresql.git-limbosolutions-com.svc.cluster.local -U $USER -d $NAME > /data/postgresql-export/db_backup.sql
                   if [ $? -ne 0 ]; then
                     echo "ERROR: Exporting database failed"
                     exit 1
-                  fi
+                  fi 
-                  echo "INFO: Exporting database finished"
+                  #echo "INFO: Exporting database finished"
 
               volumeMounts:
-              - name: db-postgresql-data
+
-                mountPath: /var/lib/postgresql/data
-                
               - name: backup-run-data
                 mountPath: /data/postgresql-export
                 subPath: postgresql-export
 
-              - name: database-access-secret
+              - name: gitea-inline-config
-                mountPath: /run/database-access-secret
+                mountPath: /root/.gitea-inline-config
                 readOnly: true
 
           containers:
             - name: gitea-pbs-client
               image: git.limbosolutions.com/kb/pbsclient
+              imagePullPolicy: Always
+              resources:
+                limits:
+                  memory: "512Mi"
+                  cpu: "500m"
+                requests:
+                  memory: "256Mi"
+                  cpu: "250m"
               env:  
                 - name: MODE
                   value: shell
                 - name: PBS_REPOSITORY
                   valueFrom:
                     secretKeyRef:
-                      name: gitea-backup-secret
+                      name: gitea-backup
                       key:  PBS_REPOSITORY
                 - name: PBS_PASSWORD
                   valueFrom:
                     secretKeyRef:
-                      name: gitea-backup-secret
+                      name: gitea-backup
                       key: PBS_PASSWORD
                 - name: PBS_FINGERPRINT
                   valueFrom:
                     secretKeyRef:
-                      name: gitea-backup-secret
+                      name: gitea-backup
                       key: PBS_FINGERPRINT
 
               command: ["bash", "-c"]
               args:
                 - |
+                  set -e
+                  #  while true; do
+                  #    sleep 1s
+                  #  done
+                  SCRIPT_START_TIME=$(date +%s)
                   proxmox-backup-client backup gitea-data.pxar:/data/gitea-data postgresql-data.pxar:/data/postgresql-data postgresql-export.pxar:/data/postgresql-export --include-dev /data/postgresql-data --include-dev /data/postgresql-export --include-dev /data/gitea-data --backup-id "gitea-full" -ns git.limbosolutions.com
-                  echo "INFO: git.limbosolutions.com finished"
+                  SCRIPT_DURATION=$(($(date +%s) - SCRIPT_START_TIME))
+                  echo "INFO: Finished Backup of git.limbosolutions.com ($((SCRIPT_DURATION / 60 / 60)):$((SCRIPT_DURATION / 60)):$((SCRIPT_DURATION % 60))) "
 
               volumeMounts:
               - name: gitea-shared-storage
@@ -96,6 +118,6 @@ spec:
             - name: backup-run-data
               emptyDir: {}
 
-            - name: database-access-secret
+            - name: gitea-inline-config
               secret:
                 secretName: gitea-inline-config

deploy/app/kustomization.yaml

@@ -0,0 +1,17 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - cronjobs/backups/backup-borg-offsite-cronjob.yaml
+  - cronjobs/backups/backup-pbs-cronjob.yaml
+
+secretGenerator:
+- name: gitea-backup
+  namespace: git-limbosolutions-com
+  envs:
+    - cronjobs/backups/.env.d/secrets
+  files:
+    - BORG_KEY=cronjobs/backups/.env.d/borg_key
+    - SSH_ID_RSA=cronjobs/backups/.env.d/id_rsa
+    
+generatorOptions:
+  disableNameSuffixHash: true

deploy/helm/.env.example

@@ -0,0 +1,15 @@
+APP_HELM_VALUE_VALKEY_GLOBAL_PASSWORD="????"
+APP_HELM_VALUE_POSTGRESQL_AUTH_POSTGRESPASSWORD="????"
+APP_HELM_VALUE_POSTGRESQL_AUTH_PASSWORD="????"
+APP_HELM_VALUE_POSTGRESQL_AUTH_DATABASE="????"
+APP_HELM_VALUE_POSTGRESQL_AUTH_USERNAME="????"
+APP_HELM_VALUE_GITEA_ADMIN_USERNAME="????"
+APP_HELM_VALUE_GITEA_ADMIN_PASSWORD="????"
+APP_HELM_VALUE_GITEA_ADMIN_EMAIL="????"
+APP_HELM_VALUE_GITEA_CONFIG_OAUTH2_JWT_SECRET="????"
+APP_HELM_VALUE_GITEA_CONFIG_SERVER_LFS_JWT_SECRET="????"
+APP_HELM_VALUE_GITEA_CONFIG_SECURITY_SECRET_KEY="????"
+APP_HELM_VALUE_GITEA_CONFIG_SECURITY_REVERSE_PROXY_TRUSTED_PROXIES="????"
+APP_HELM_VALUE_GITEA_CONFIG_SECURITY_INTERNAL_TOKEN="????"
+APP_HELM_VALUE_GITEA_CONFIG_SECURITY_PASSWORD_HASH_ALGO="????"
+APP_HELM_VALUE_GITEA_CONFIG_SERVICE_OAUTH2_JWT_SECRET="????"

deploy/helm/values.yaml

@@ -1,16 +1,34 @@
 image:
   registry: ""
   repository: gitea/gitea
-  imagePullPolicy: Always
+  pullPolicy: Always
   tag: "1"
-redis-cluster:
+
+cache:
   enabled: false
-  
+
-redis:
+valkey-cluster:
+  enabled: false
+
+valkey:
   enabled: true
+  architecture: standalone
+  global:
+    valkey:
+      password: "???"
+  master:
+    count: 1
+    service:
+      ports:
+        valkey: 6379
 
 postgresql:
   enabled: true
+  image:
+    registry: ""
+    repository: bitnami/postgresql
+    tag: 16
+    imagePullPolicy: IfNotPresent
   global:
     postgresql:
       auth:
@@ -20,14 +38,15 @@ postgresql:
         username: "???"
       service:
         ports:
-          postgresql: "???"
+          postgresql: 5432
   primary:
     persistence:
       size: 10Gi
   metrics:
     enabled: true
-
+    collectors:
-
+      wal: false
+ 
 postgresql-ha:
   enabled: false
 
@@ -39,14 +58,18 @@ service:
       port: 2222
       annotations:
         metallb.universe.tf/allow-shared-ip: test
-
+  http:
+    clusterIP: ""   # empty string → Kubernetes assigns a routable ClusterIP
+    type: ClusterIP
+    port: 3000
 gitea:
-
   admin:
     username: "???"
     password: "???"
     email: "???"
   config:
+    oauth2:
+      JWT_SECRET: "???"
     actions:
       ENABLED: true
     database:
@@ -59,7 +82,6 @@ gitea:
     server:
       DOMAIN: git.limbosolutions.com
       SSH_DOMAIN: git.limbosolutions.com
-      #HTTP_PORT: 3000
       ROOT_URL: https://git.limbosolutions.com
       DISABLE_SSH: false
       SSH_PORT: 2222
@@ -67,7 +89,7 @@ gitea:
       LFS_START_SERVER: true
       START_SSH_SERVER: true
       LFS_PATH: /data/git/lfs
-      #LFS_JWT_SECRET: "???"
+      LFS_JWT_SECRET: "???"
       OFFLINE_MODE: false
       #MFF 03/08/2024
       REPO_INDEXER_ENABLED: true
@@ -76,39 +98,52 @@ gitea:
       REPO_INDEXER_INCLUDE:
       REPO_INDEXER_EXCLUDE: resources/bin/**
       ####
-    SERVICE:
+    
+    service:
       DISABLE_REGISTRATION: true
       REQUIRE_SIGNIN_VIEW: false
       REGISTER_EMAIL_CONFIRM: false
       ENABLE_NOTIFY_MAIL: false
       ALLOW_ONLY_EXTERNAL_REGISTRATION: false
       ENABLE_CAPTCHA: true
-      DEFAULT_KEEP_EMAIL_PRIVATE : false
+      DEFAULT_KEEP_EMAIL_PRIVATE : true
       DEFAULT_ALLOW_CREATE_ORGANIZATION: true
-      DEFAULT_ENABLE_TIMETRACKING: true
+      DEFAULT_ENABLE_TIMETRACKING:  true
       NO_REPLY_ADDRESS: noreply.localhost
-
+      oauth2:
+          JWT_SECRET: "???"
     mailer:
       ENABLED: false
 
     openid:
-      ENABLE_OPENID_SIGNIN: true
+      ENABLE_OPENID_SIGNIN: false
-      ENABLE_OPENID_SIGNUP: true
+      ENABLE_OPENID_SIGNUP: false
+    
 
+    security:
+      INSTALL_LOCK: true
+      SECRET_KEY: "???"
+      REVERSE_PROXY_LIMIT: 1
+      REVERSE_PROXY_TRUSTED_PROXIES: "???"
+      INTERNAL_TOKEN: "???"
+      PASSWORD_HASH_ALGO: "???"
 
 ingress:
   enabled: true
   className: traefik
   annotations:
     kubernetes.io/ingress.class: traefik
+    cert-manager.io/cluster-issuer: "letsencrypt-prod"
+    traefik.ingress.kubernetes.io/router.entrypoints: websecure, public-https
   hosts:
     - host: git.limbosolutions.com
       paths:
         - path: /
           pathType: Prefix
   tls:
-    - secretName: limbosolutions-com-secret-tls
+    - secretName: limbosolutions-com-tls
       hosts:
        - "git.limbosolutions.com"
 
 
+

deploy/infra/cd-service-account-rbac.yaml

@@ -0,0 +1,43 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  namespace: git-limbosolutions-com
+  name: continuous-deploy
+rules:
+- apiGroups: [""]
+  resources: ["pods", "services", "secrets", "configmaps", "persistentvolumeclaims", "endpoints", "serviceaccounts"]
+  verbs: ["get", "watch", "list", "create", "update", "patch", "delete"]
+
+- apiGroups: ["apps"]
+  resources: ["deployments", "statefulsets"]
+  verbs: ["get", "watch", "list", "create", "update", "patch", "delete"]
+
+- apiGroups: ["batch"]
+  resources: ["cronjobs", "jobs"]
+  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+
+- apiGroups: ["networking.k8s.io"]
+  resources: ["networkpolicies", "ingresses"]
+  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+
+- apiGroups: ["policy"]
+  resources: ["poddisruptionbudgets"]
+  verbs: ["get", "list", "watch", "update", "patch"]
+
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: continuous-deploy
+  namespace:  git-limbosolutions-com
+subjects:
+- kind: ServiceAccount
+  name: continuous-deploy
+  namespace: git-limbosolutions-com
+roleRef:
+  kind: Role
+  name: continuous-deploy
+  apiGroup: rbac.authorization.k8s.io
+

deploy/infra/cd-service-account-token.yaml

@@ -0,0 +1,8 @@
+
+apiVersion: v1
+kind: Secret
+metadata:
+  name: continuous-deploy
+  annotations:
+    kubernetes.io/service-account.name: continuous-deploy
+type: kubernetes.io/service-account-token

deploy/infra/cd-service-account.yaml

@@ -0,0 +1,6 @@
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: continuous-deploy
+  namespace: git-limbosolutions-com

deploy/infra/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - cd-service-account.yaml
+  - cd-service-account-token.yaml
+  - cd-service-account-rbac.yaml
+

deploy/ops/bork-backup-sidekick/pod.yaml

@@ -0,0 +1,61 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: borg-backup-sidekick
+  namespace: git-limbosolutions-com
+  labels:
+    app: borg-backup-sidekick
+spec:
+  containers:
+      - name: borg-backup-sidekick
+        image: git.limbosolutions.com/kb/borg-backup:latest
+        imagePullPolicy: Always
+        resources:
+          limits:
+            memory: "512Mi"
+            cpu: "500m"
+          requests:
+            memory: "256Mi"
+            cpu: "250m"
+        env:
+          - name: BORG_REPO
+            valueFrom:
+              secretKeyRef:
+                name: gitea-backup-secret
+                key: borg_repo
+
+          - name: BORG_PASSPHRASE
+            valueFrom:
+              secretKeyRef:
+                name: gitea-backup-secret
+                key: borg_passphrase
+
+          - name: BORG_RSH
+            value: ssh -p 2222 -o StrictHostKeyChecking=no -o LogLevel=ERROR
+
+          - name: borg_key_file
+            value: /root/.borg/key
+
+        command: ["sh", "-c"]
+        args:
+        - |
+          while true; do
+             sleep 1s
+          done
+
+        volumeMounts:
+        
+        - name: gitea-backup-secrets
+          mountPath: /root/.ssh/id_rsa
+          subPath: ssh_id_rsa
+          readOnly: true
+
+        - name: gitea-backup-secrets
+          mountPath: /app/borg/key
+          subPath: borg_key
+  volumes:
+    - name: gitea-backup-secrets
+      secret:
+        secretName: gitea-backup
+        defaultMode: 0600
+

docs/act-runner.md

@@ -0,0 +1,63 @@
+# Git Action
+
+## runners
+
+## lxc container - proxmox
+
+Template : debian-12-turnkey-core_18.1-1_amd64.tar.gz
+
+- Unprivileged contrainer - Yes
+- Nesting -Yes
+
+```bash
+# setup new lxc container with docker
+
+apt update -y
+apt upgrade -y
+curl -fsSL https://get.docker.com -o get-docker.sh
+sh ./get-docker.sh
+```
+
+### act runner
+
+```bash
+nano setup-act-runners.sh
+```
+
+```bash
+#/bin/bash
+
+setup(){
+
+CONTAINER_NAME=${OWNER}_act_runner
+
+docker container stop $CONTAINER_NAME
+docker container rm $CONTAINER_NAME
+
+docker run \
+    --restart=unless-stopped \
+    -v ${CONTAINER_NAME}_data:/data \
+    -v /var/run/docker.sock:/var/run/docker.sock \
+    -e GITEA_INSTANCE_URL=${INSTANCE_URL} \
+    -e GITEA_RUNNER_REGISTRATION_TOKEN=${REGISTRATION_TOKEN} \
+    -e GITEA_RUNNER_NAME=${OWNER}_ubuntu_default \
+    --name ${CONTAINER_NAME} \
+    -d gitea/act_runner
+}
+
+INSTANCE_URL=https://git.limbosolutions.com
+OWNER=kb
+REGISTRATION_TOKEN=???
+setup
+
+
+OWNER=????
+REGISTRATION_TOKEN=???
+setup
+
+```
+
+```bash
+chmod +x setup-act-runners.sh
+./setup-act-runners.sh
+```

docs/backups.md

@@ -0,0 +1,36 @@
+# borgbackup sidekick
+
+**Create borgbackup-sidekick pod:**
+
+```bash
+kubectl apply -f deploy/ops/borg-backup-sidekick/pod.yaml
+```
+
+**Remove borgbackup-sidekick pod:**
+
+```bash
+# delete the sidekick pod after use
+kubectl delete pod -l app=borg-backup-sidekick
+```
+
+```bash
+# attach to borgbackup-sidekick
+POD_NAME=$(kubectl get pod -l app=borg-backup-sidekick -n git-limbosolutions-com -o jsonpath='{.items[0].metadata.name}')
+kubectl exec -it ${POD_NAME}  -- bash
+```
+
+```bash
+# list borg repo
+POD_NAME=$(kubectl get pod -l app=borg-backup-sidekick -n git-limbosolutions-com -o jsonpath='{.items[0].metadata.name}')
+kubectl exec -it ${POD_NAME} -- bash -c "\
+borg list ${BORG_REPO} \
+"
+```
+
+```bash
+# get borg info
+POD_NAME=$(kubectl get pod -l app=borg-backup-sidekick -n git-limbosolutions-com -o jsonpath='{.items[0].metadata.name}')
+kubectl exec -it ${POD_NAME} -- bash -c "\
+borg info ${BORG_REPO}\
+"
+```

gitea/helm-up.sh

@@ -1,4 +0,0 @@
-helm upgrade --install gitea gitea-charts/gitea \
--f ./values.yaml \
--f ./values.private.yml \
---namespace=git-limbosolutions-com

gitea/ssh-ingress.yaml

@@ -1,17 +0,0 @@
-apiVersion: traefik.containo.us/v1alpha1
-kind: IngressRouteTCP
-metadata:
-  name: git-limbosolutions-com-ssh-ingress
-  namespace: git-limbosolutions-com
-spec:
-  entryPoints:
-    - ssh-git
-  routes:
-    - match: HostSNI(`*`)
-      services:
-        - name: gitea-ssh
-          port: 2222
-          weight: 10
-          terminationDelay: 90000
-          proxyProtocol:
-            version: 1

ops-scripts/apply-app.sh

@@ -0,0 +1,36 @@
+#/bin/bash
+kubectl kustomize deploy/app | kubectl apply -f -
+
+if [ -f "deploy/helm/.env" ]; then
+  # Export all variables from the file
+  echo "export variables from file helm/.env" 
+  set -a
+  . deploy/helm/.env
+  set +a
+fi
+
+
+if [ -n "${APP_HELM_VALUE_GITEA_ADMIN_USERNAME:-}" ]; then
+  echo "Executing helm deploy."
+
+    helm repo add gitea-charts https://dl.gitea.com/charts/
+    helm repo update
+    helm upgrade --install gitea gitea-charts/gitea --version 12.4.0 \
+    --values deploy/helm/values.yaml \
+    --set valkey.global.valkey.password=${APP_HELM_VALUE_VALKEY_GLOBAL_PASSWORD} \
+    --set postgresql.global.postgresql.auth.postgresPassword=${APP_HELM_VALUE_POSTGRESQL_AUTH_POSTGRESPASSWORD} \
+    --set postgresql.global.postgresql.auth.password=${APP_HELM_VALUE_POSTGRESQL_AUTH_PASSWORD} \
+    --set postgresql.global.postgresql.auth.database=${APP_HELM_VALUE_POSTGRESQL_AUTH_DATABASE} \
+    --set postgresql.global.postgresql.auth.username=${APP_HELM_VALUE_POSTGRESQL_AUTH_USERNAME} \
+    --set gitea.admin.username=${APP_HELM_VALUE_GITEA_ADMIN_USERNAME} \
+    --set gitea.admin.password=${APP_HELM_VALUE_GITEA_ADMIN_PASSWORD} \
+    --set gitea.admin.email=${APP_HELM_VALUE_GITEA_ADMIN_EMAIL} \
+    --set gitea.config.oauth2.JWT_SECRET=${APP_HELM_VALUE_GITEA_CONFIG_OAUTH2_JWT_SECRET} \
+    --set gitea.config.server.LFS_JWT_SECRET=${APP_HELM_VALUE_GITEA_CONFIG_SERVER_LFS_JWT_SECRET} \
+    --set gitea.config.security.SECRET_KEY=${APP_HELM_VALUE_GITEA_CONFIG_SECURITY_SECRET_KEY} \
+    --set gitea.config.security.REVERSE_PROXY_TRUSTED_PROXIES=${APP_HELM_VALUE_GITEA_CONFIG_SECURITY_REVERSE_PROXY_TRUSTED_PROXIES} \
+    --set gitea.config.security.INTERNAL_TOKEN=${APP_HELM_VALUE_GITEA_CONFIG_SECURITY_INTERNAL_TOKEN} \
+    --set gitea.config.security.PASSWORD_HASH_ALGO=${APP_HELM_VALUE_GITEA_CONFIG_SECURITY_PASSWORD_HASH_ALGO} \
+    --set gitea.config.service.oauth2.JWT_SECRET=${APP_HELM_VALUE_GITEA_CONFIG_SERVICE_OAUTH2_JWT_SECRET} \
+    --namespace=git-limbosolutions-com
+fi

ops-scripts/apply-infra.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e
+echo "Executing infra deploy."
+
+kubectl kustomize deploy/infra | kubectl apply -f -
+
+