backups added resources limits, set offsite backup size limit to 15GB

.gitea/workflows/app-continous-deploy.yaml

@@ -0,0 +1,93 @@
+on:
+  schedule:
+    - cron: '0 9 * * 0' # every sunday 9 am
+  push:
+    branches:
+      - main
+  pull_request:
+      branches:
+      - main
+jobs:
+  continuous-deploy:
+    runs-on: ubuntu-latest
+    container:
+      image: git.limbosolutions.com/kb/gitea/act:latest-network-stack
+    env:
+      GITHUB_TEMP: ${{ runner.temp }} # fix missing GITHUB_TEMP on gitea
+    steps:
+
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: limbo public actions
+        env: 
+          WORKSPACE: "${{ gitea.workspace }}"
+        run: |
+          curl -fsSL https://git.limbosolutions.com/kb/gitea/raw/branch/main/cloud-scripts/setup-limbo-actions.sh | bash 2>&1
+
+
+      # limbo custom actions required https://git.limbosolutions.com/kb/gitea/raw/branch/main
+      - name: Configure kubectl config
+        uses: ./.gitea/limbo_actions/kubectl-setup
+        with:
+          kube_server: ${{ secrets.HOSTING_KUBE_SERVER }}
+          kube_ca_base64: ${{ secrets.HOSTING_KUBE_CA_BASE64 }}
+          kube_token: ${{ secrets.HOSTING_KUBE_TOKEN }}
+
+      - name: Deploy
+        shell: bash
+        env:
+          # cron jobs env
+          CRONJOBS_BACKUPS_SECRETS_PBS_REPOSITORY: ${{ secrets.CRONJOBS_BACKUPS_SECRETS_PBS_REPOSITORY }}
+          CRONJOBS_BACKUPS_SECRETS_PBS_PASSWORD: ${{ secrets.CRONJOBS_BACKUPS_SECRETS_PBS_PASSWORD }}
+          CRONJOBS_BACKUPS_SECRETS_PBS_FINGERPRINT: ${{ secrets.CRONJOBS_BACKUPS_SECRETS_PBS_FINGERPRINT }}
+          CRONJOBS_BACKUPS_SECRETS_BORG_REPO: ${{ secrets.CRONJOBS_BACKUPS_SECRETS_BORG_REPO }}
+          CRONJOBS_BACKUPS_SECRETS_BORG_PASSPHRASE: ${{ secrets.CRONJOBS_BACKUPS_SECRETS_BORG_PASSPHRASE }}
+          CRONJOBS_BACKUPS_SECRETS_OFFSITE_TARGET_FOLDER: ${{ secrets.CRONJOBS_BACKUPS_SECRETS_OFFSITE_TARGET_FOLDER }}
+          CRONJOBS_BACKUPS_SECRETS_ID_RSA: ${{ secrets.CRONJOBS_BACKUPS_SECRETS_ID_RSA }}
+          CRONJOBS_BACKUPS_SECRETS_BORG_KEY: ${{ secrets.CRONJOBS_BACKUPS_SECRETS_BORG_KEY }}
+
+          # helm chart values
+          APP_HELM_VALUE_VALKEY_GLOBAL_PASSWORD: ${{ secrets.APP_HELM_VALUE_VALKEY_GLOBAL_PASSWORD }}
+          APP_HELM_VALUE_POSTGRESQL_AUTH_POSTGRESPASSWORD: ${{ secrets.APP_HELM_VALUE_POSTGRESQL_AUTH_POSTGRESPASSWORD }}
+          APP_HELM_VALUE_POSTGRESQL_AUTH_PASSWORD: ${{ secrets.APP_HELM_VALUE_POSTGRESQL_AUTH_PASSWORD }}
+          APP_HELM_VALUE_POSTGRESQL_AUTH_DATABASE: ${{ secrets.APP_HELM_VALUE_POSTGRESQL_AUTH_DATABASE }}
+          APP_HELM_VALUE_POSTGRESQL_AUTH_USERNAME: ${{ secrets.APP_HELM_VALUE_POSTGRESQL_AUTH_USERNAME }}
+          APP_HELM_VALUE_GITEA_ADMIN_USERNAME: ${{ secrets.APP_HELM_VALUE_GITEA_ADMIN_USERNAME }}
+          APP_HELM_VALUE_GITEA_ADMIN_PASSWORD: ${{ secrets.APP_HELM_VALUE_GITEA_ADMIN_PASSWORD }}
+          APP_HELM_VALUE_GITEA_ADMIN_EMAIL: ${{ secrets.APP_HELM_VALUE_GITEA_ADMIN_EMAIL }}
+          APP_HELM_VALUE_GITEA_CONFIG_OAUTH2_JWT_SECRET: ${{ secrets.APP_HELM_VALUE_GITEA_CONFIG_OAUTH2_JWT_SECRET }}
+          APP_HELM_VALUE_GITEA_CONFIG_SERVER_LFS_JWT_SECRET: ${{ secrets.APP_HELM_VALUE_GITEA_CONFIG_SERVER_LFS_JWT_SECRET }}
+          APP_HELM_VALUE_GITEA_CONFIG_SECURITY_SECRET_KEY: ${{ secrets.APP_HELM_VALUE_GITEA_CONFIG_SECURITY_SECRET_KEY }}
+          APP_HELM_VALUE_GITEA_CONFIG_SECURITY_REVERSE_PROXY_TRUSTED_PROXIES: ${{ secrets.APP_HELM_VALUE_GITEA_CONFIG_SECURITY_REVERSE_PROXY_TRUSTED_PROXIES }}
+          APP_HELM_VALUE_GITEA_CONFIG_SECURITY_INTERNAL_TOKEN: ${{ secrets.APP_HELM_VALUE_GITEA_CONFIG_SECURITY_INTERNAL_TOKEN }}
+          APP_HELM_VALUE_GITEA_CONFIG_SECURITY_PASSWORD_HASH_ALGO: ${{ secrets.APP_HELM_VALUE_GITEA_CONFIG_SECURITY_PASSWORD_HASH_ALGO }}
+          APP_HELM_VALUE_GITEA_CONFIG_SERVICE_OAUTH2_JWT_SECRET: ${{ secrets.APP_HELM_VALUE_GITEA_CONFIG_SERVICE_OAUTH2_JWT_SECRET }}
+
+        run: |
+          set -euo pipefail
+
+          # ensure cleanup always runs
+          trap 'rm -f \
+                    deploy/app/cronjobs/backups/.env.d/secrets \
+                    deploy/app/cronjobs/backups/.env.d/id_rsa \
+                    deploy/app/cronjobs/backups/.env.d/borg_key' EXIT
+
+          # setup env for cronjobs backups
+          echo "PBS_REPOSITORY=${CRONJOBS_BACKUPS_SECRETS_PBS_REPOSITORY}" >> deploy/app/cronjobs/backups/.env.d/secrets
+          echo "PBS_PASSWORD=${CRONJOBS_BACKUPS_SECRETS_PBS_PASSWORD}" >> deploy/app/cronjobs/backups/.env.d/secrets
+          echo "PBS_FINGERPRINT=${CRONJOBS_BACKUPS_SECRETS_PBS_FINGERPRINT}" >> deploy/app/cronjobs/backups/.env.d/secrets
+          echo "BORG_REPO=${CRONJOBS_BACKUPS_SECRETS_BORG_REPO}" >> deploy/app/cronjobs/backups/.env.d/secrets
+          echo "BORG_PASSPHRASE=${CRONJOBS_BACKUPS_SECRETS_BORG_PASSPHRASE}" >> deploy/app/cronjobs/backups/.env.d/secrets
+          echo "OFFSITE_TARGET_FOLDER=${CRONJOBS_BACKUPS_SECRETS_OFFSITE_TARGET_FOLDER}" >> deploy/app/cronjobs/backups/.env.d/secrets
+          
+          echo "${CRONJOBS_BACKUPS_SECRETS_ID_RSA}" >> deploy/app/cronjobs/backups/.env.d/id_rsa
+          echo "${CRONJOBS_BACKUPS_SECRETS_BORG_KEY}" >> deploy/app/cronjobs/backups/.env.d/borg_key
+          
+          # enforce security
+          chmod 600 deploy/app/cronjobs/backups/.env.d/secrets
+          chmod 600 deploy/app/cronjobs/backups/.env.d/id_rsa
+          chmod 600 deploy/app/cronjobs/backups/.env.d/borg_key
+
+          # invoke deploy script
+          ops-scripts/apply-app.sh

README.md

@@ -16,13 +16,13 @@ Using [gitea](https://git.limbosolutions.com/kb/gitea) as git server.
 
 **Environment files:**
 
-- ./deploy/app/cronjobs/backups/.env.d/secrets [Example:](./deploy/app/cronjobs/backups/.env.d/secrets.example).
-- ./deploy/app/cronjobs/backups/.env.d/borg_key [Example:](./deploy/app/cronjobs/backups/.env.d/borg_key.example).
-- ./deploy/app/cronjobs/backups/.env.d/id_rsa [Example:](./deploy/app/cronjobs/backups/.env.d/id_rsa.example).
-- ./deploy/helm/.env [Example:](./deploy/helm/.env.example).
+- ./deploy/app/cronjobs/backups/.env.d/secrets [Example](./deploy/app/cronjobs/backups/.env.d/secrets.example)
+- ./deploy/app/cronjobs/backups/.env.d/borg_key [Example](./deploy/app/cronjobs/backups/.env.d/borg_key.example)
+- ./deploy/app/cronjobs/backups/.env.d/id_rsa [Example](./deploy/app/cronjobs/backups/.env.d/id_rsa.example)
+- ./deploy/helm/.env [Example](./deploy/helm/.env.example)
 
 ```bash
-./deploy/apply-app.sh
+./ops-scripts/apply-app.sh
 ```
 
 - [kustomization](/deploy/app/kustomization.yaml)
@@ -30,12 +30,11 @@ Using [gitea](https://git.limbosolutions.com/kb/gitea) as git server.
 ### Continuous Deploy
 
 Executes [App Deploy](#app) using [Gitea workflow](./.gitea/workflows/app-deploy.yaml).
-kubectl get secret continuous-deploy -o jsonpath='{.data.token}' | base64 -d
 
 ### Infra
 
 ```bash
-./deploy/apply-infra.sh
+./ops-scripts/apply-infra.sh
 ```
 
 - [kustomization](/deploy/infra/kustomization.yaml)

deploy/app/cronjobs/backups/backup-borg-offsite-cronjob.yaml

@@ -17,6 +17,13 @@ spec:
           restartPolicy: Never
           initContainers:
             - name: postgres-export
+              resources:
+                limits:
+                  memory: "512Mi"
+                  cpu: "500m"
+                requests:
+                  memory: "256Mi"
+                  cpu: "250m"
               image: postgres:latest
               command: ["sh", "-c"]
               args:
@@ -42,13 +49,13 @@ spec:
             - name: borg-client
               image: git.limbosolutions.com/kb/borg-backup:latest
               imagePullPolicy: Always
-              # resources:
-              #   limits:
-              #     memory: "512Mi"
-              #     cpu: "500m"
-              #   requests:
-              #     memory: "256Mi"
-              #     cpu: "250m"
+              resources:
+                limits:
+                  memory: "512Mi"
+                  cpu: "500m"
+                requests:
+                  memory: "256Mi"
+                  cpu: "250m"
               env:
                 - name: BORG_REPO
                   valueFrom:
@@ -74,7 +81,7 @@ spec:
                   value: ssh -p 2222 -o StrictHostKeyChecking=no -o LogLevel=ERROR
 
                 - name: REPO_SYNC_MAX_SIZE
-                  value: "10737418240" # 10GB
+                  value: "16106127360" # 15GB
 
                 - name: MODE
                   value: SHELL

deploy/app/cronjobs/backups/backup-pbs-cronjob.yaml

@@ -17,6 +17,13 @@ spec:
           initContainers:
             - name: postgres-export
               image: postgres:latest
+              resources:
+                limits:
+                  memory: "512Mi"
+                  cpu: "500m"
+                requests:
+                  memory: "256Mi"
+                  cpu: "250m"
               command: ["sh", "-c"]
               args:
                 - |
@@ -45,6 +52,13 @@ spec:
             - name: gitea-pbs-client
               image: git.limbosolutions.com/kb/pbsclient
               imagePullPolicy: Always
+              resources:
+                limits:
+                  memory: "512Mi"
+                  cpu: "500m"
+                requests:
+                  memory: "256Mi"
+                  cpu: "250m"
               env:  
                 - name: MODE
                   value: shell

deploy/apply-app.sh

@@ -1,36 +0,0 @@
-#/bin/bash
-kubectl kustomize deploy/app | kubectl apply -f -
-
-if [ -f "deploy/helm/.env" ]; then
-  # Export all variables from the file
-  echo "export variables from file helm/.env" 
-  set -a
-  . deploy/helm/.env
-  set +a
-fi
-
-
-if [ -n "${GITEA_ADMIN_USERNAME:-}" ]; then
-  echo "Executing helm deploy."
-
-    helm repo add gitea-charts https://dl.gitea.com/charts/
-    helm repo update
-    helm upgrade --install gitea gitea-charts/gitea \
-    --values deploy/helm/values.yaml \
-    --set valkey.global.valkey.password=${VALKEY_GLOBAL_PASSWORD} \
-    --set postgresql.global.postgresql.auth.postgresPassword=${POSTGRESQL_AUTH_POSTGRESPASSWORD} \
-    --set postgresql.global.postgresql.auth.password=${POSTGRESQL_AUTH_PASSWORD} \
-    --set postgresql.global.postgresql.auth.database=${POSTGRESQL_AUTH_DATABASE} \
-    --set postgresql.global.postgresql.auth.username=${POSTGRESQL_AUTH_USERNAME} \
-    --set gitea.admin.username=${GITEA_ADMIN_USERNAME} \
-    --set gitea.admin.password=${GITEA_ADMIN_PASSWORD} \
-    --set gitea.admin.email=${GITEA_ADMIN_EMAIL} \
-    --set gitea.config.oauth2.JWT_SECRET=${GITEA_CONFIG_OAUTH2_JWT_SECRET} \
-    --set gitea.config.server.LFS_JWT_SECRET=${GITEA_CONFIG_SERVER_LFS_JWT_SECRET} \
-    --set gitea.config.security.SECRET_KEY=${GITEA_CONFIG_SECURITY_SECRET_KEY} \
-    --set gitea.config.security.REVERSE_PROXY_TRUSTED_PROXIES=${GITEA_CONFIG_SECURITY_REVERSE_PROXY_TRUSTED_PROXIES} \
-    --set gitea.config.security.INTERNAL_TOKEN=${GITEA_CONFIG_SECURITY_INTERNAL_TOKEN} \
-    --set gitea.config.security.PASSWORD_HASH_ALGO=${GITEA_CONFIG_SECURITY_PASSWORD_HASH_ALGO} \
-    --set gitea.config.service.oauth2.JWT_SECRET=${GITEA_CONFIG_SERVICE_OAUTH2_JWT_SECRET} \
-    --namespace=git-limbosolutions-com
-fi

deploy/helm/.env.example

@@ -1,15 +1,15 @@
-VALKEY_GLOBAL_PASSWORD="????"
-POSTGRESQL_AUTH_POSTGRESPASSWORD="????"
-POSTGRESQL_AUTH_PASSWORD="????"
-POSTGRESQL_AUTH_DATABASE="????"
-POSTGRESQL_AUTH_USERNAME="????"
-GITEA_ADMIN_USERNAME=m"????"
-GITEA_ADMIN_PASSWORD="????"
-GITEA_ADMIN_EMAIL="????"
-GITEA_CONFIG_OAUTH2_JWT_SECRET=i"????"
-GITEA_CONFIG_SERVER_LFS_JWT_SECRET="????"
-GITEA_CONFIG_SECURITY_INTERNAL_TOKEN="????"
-GITEA_CONFIG_SECURITY_PASSWORD_HASH_ALGO="????"
-GITEA_CONFIG_SECURITY_REVERSE_PROXY_TRUSTED_PROXIES="????"
-GITEA_CONFIG_SECURITY_SECRET_KEY="????"
-GITEA_CONFIG_SERVICE_OAUTH2_JWT_SECRET="????"
+APP_HELM_VALUE_VALKEY_GLOBAL_PASSWORD="????"
+APP_HELM_VALUE_POSTGRESQL_AUTH_POSTGRESPASSWORD="????"
+APP_HELM_VALUE_POSTGRESQL_AUTH_PASSWORD="????"
+APP_HELM_VALUE_POSTGRESQL_AUTH_DATABASE="????"
+APP_HELM_VALUE_POSTGRESQL_AUTH_USERNAME="????"
+APP_HELM_VALUE_GITEA_ADMIN_USERNAME="????"
+APP_HELM_VALUE_GITEA_ADMIN_PASSWORD="????"
+APP_HELM_VALUE_GITEA_ADMIN_EMAIL="????"
+APP_HELM_VALUE_GITEA_CONFIG_OAUTH2_JWT_SECRET="????"
+APP_HELM_VALUE_GITEA_CONFIG_SERVER_LFS_JWT_SECRET="????"
+APP_HELM_VALUE_GITEA_CONFIG_SECURITY_SECRET_KEY="????"
+APP_HELM_VALUE_GITEA_CONFIG_SECURITY_REVERSE_PROXY_TRUSTED_PROXIES="????"
+APP_HELM_VALUE_GITEA_CONFIG_SECURITY_INTERNAL_TOKEN="????"
+APP_HELM_VALUE_GITEA_CONFIG_SECURITY_PASSWORD_HASH_ALGO="????"
+APP_HELM_VALUE_GITEA_CONFIG_SERVICE_OAUTH2_JWT_SECRET="????"

deploy/helm/values.yaml

@@ -58,7 +58,10 @@ service:
       port: 2222
       annotations:
         metallb.universe.tf/allow-shared-ip: test
-
+  http:
+    clusterIP: ""   # empty string → Kubernetes assigns a routable ClusterIP
+    type: ClusterIP
+    port: 3000
 gitea:
   admin:
     username: "???"
@@ -126,6 +129,21 @@ gitea:
       PASSWORD_HASH_ALGO: "???"
 
 ingress:
-  enabled: false
+  enabled: true
+  className: traefik
+  annotations:
+    kubernetes.io/ingress.class: traefik
+    cert-manager.io/cluster-issuer: "letsencrypt-prod"
+    traefik.ingress.kubernetes.io/router.entrypoints: websecure, public-https
+  hosts:
+    - host: git.limbosolutions.com
+      paths:
+        - path: /
+          pathType: Prefix
+  tls:
+    - secretName: limbosolutions-com-tls
+      hosts:
+       - "git.limbosolutions.com"
+
 
 

deploy/infra/cd-service-account-rbac.yaml

@@ -5,7 +5,7 @@ metadata:
   name: continuous-deploy
 rules:
 - apiGroups: [""]
-  resources: ["pods", "services", "secrets", "configmaps", "persistentvolumeclaims", "endpoints"]
+  resources: ["pods", "services", "secrets", "configmaps", "persistentvolumeclaims", "endpoints", "serviceaccounts"]
   verbs: ["get", "watch", "list", "create", "update", "patch", "delete"]
 
 - apiGroups: ["apps"]
@@ -15,6 +15,16 @@ rules:
 - apiGroups: ["batch"]
   resources: ["cronjobs", "jobs"]
   verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+
+- apiGroups: ["networking.k8s.io"]
+  resources: ["networkpolicies", "ingresses"]
+  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+
+- apiGroups: ["policy"]
+  resources: ["poddisruptionbudgets"]
+  verbs: ["get", "list", "watch", "update", "patch"]
+
+
 ---
 
 apiVersion: rbac.authorization.k8s.io/v1

deploy/infra/ingress.yaml

@@ -1,53 +0,0 @@
-apiVersion: traefik.containo.us/v1alpha1
-kind: IngressRouteTCP
-metadata:
-  name: git-limbosolutions-com-ssh
-  namespace: git-limbosolutions-com
-spec:
-  entryPoints:
-    - ssh-git
-  routes:
-    - match: HostSNI(`*`)
-      services:
-        - name: gitea-ssh
-          port: 2222
-          weight: 10
-          terminationDelay: 90000
-          proxyProtocol:
-            version: 1
----
-
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: gitea
-  namespace: git-limbosolutions-com
-  labels:
-    helm.sh/chart: gitea-12.4.0
-    app: gitea
-    app.kubernetes.io/name: gitea
-    app.kubernetes.io/instance: gitea
-    app.kubernetes.io/version: "1"
-    version: "1"
-    app.kubernetes.io/managed-by: Helm
-  annotations:
-      cert-manager.io/cluster-issuer: "letsencrypt-prod"
-      kubernetes.io/ingress.class: "traefik"
-      traefik.ingress.kubernetes.io/router.entrypoints: "websecure, public-https"
-spec:
-  ingressClassName: traefik
-  tls:
-    - hosts:
-        - "git.limbosolutions.com"
-      secretName: limbosolutions-com-tls
-  rules:
-    - host: "git.limbosolutions.com"
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: gitea-http
-                port:
-                  number: 3000

deploy/infra/kustomization.yaml

@@ -2,7 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - namespace.yaml
-  - ingress.yaml
   - cd-service-account.yaml
   - cd-service-account-token.yaml
   - cd-service-account-rbac.yaml

ops-scripts/apply-app.sh

@@ -0,0 +1,36 @@
+#/bin/bash
+kubectl kustomize deploy/app | kubectl apply -f -
+
+if [ -f "deploy/helm/.env" ]; then
+  # Export all variables from the file
+  echo "export variables from file helm/.env" 
+  set -a
+  . deploy/helm/.env
+  set +a
+fi
+
+
+if [ -n "${APP_HELM_VALUE_GITEA_ADMIN_USERNAME:-}" ]; then
+  echo "Executing helm deploy."
+
+    helm repo add gitea-charts https://dl.gitea.com/charts/
+    helm repo update
+    helm upgrade --install gitea gitea-charts/gitea --version 12.4.0 \
+    --values deploy/helm/values.yaml \
+    --set valkey.global.valkey.password=${APP_HELM_VALUE_VALKEY_GLOBAL_PASSWORD} \
+    --set postgresql.global.postgresql.auth.postgresPassword=${APP_HELM_VALUE_POSTGRESQL_AUTH_POSTGRESPASSWORD} \
+    --set postgresql.global.postgresql.auth.password=${APP_HELM_VALUE_POSTGRESQL_AUTH_PASSWORD} \
+    --set postgresql.global.postgresql.auth.database=${APP_HELM_VALUE_POSTGRESQL_AUTH_DATABASE} \
+    --set postgresql.global.postgresql.auth.username=${APP_HELM_VALUE_POSTGRESQL_AUTH_USERNAME} \
+    --set gitea.admin.username=${APP_HELM_VALUE_GITEA_ADMIN_USERNAME} \
+    --set gitea.admin.password=${APP_HELM_VALUE_GITEA_ADMIN_PASSWORD} \
+    --set gitea.admin.email=${APP_HELM_VALUE_GITEA_ADMIN_EMAIL} \
+    --set gitea.config.oauth2.JWT_SECRET=${APP_HELM_VALUE_GITEA_CONFIG_OAUTH2_JWT_SECRET} \
+    --set gitea.config.server.LFS_JWT_SECRET=${APP_HELM_VALUE_GITEA_CONFIG_SERVER_LFS_JWT_SECRET} \
+    --set gitea.config.security.SECRET_KEY=${APP_HELM_VALUE_GITEA_CONFIG_SECURITY_SECRET_KEY} \
+    --set gitea.config.security.REVERSE_PROXY_TRUSTED_PROXIES=${APP_HELM_VALUE_GITEA_CONFIG_SECURITY_REVERSE_PROXY_TRUSTED_PROXIES} \
+    --set gitea.config.security.INTERNAL_TOKEN=${APP_HELM_VALUE_GITEA_CONFIG_SECURITY_INTERNAL_TOKEN} \
+    --set gitea.config.security.PASSWORD_HASH_ALGO=${APP_HELM_VALUE_GITEA_CONFIG_SECURITY_PASSWORD_HASH_ALGO} \
+    --set gitea.config.service.oauth2.JWT_SECRET=${APP_HELM_VALUE_GITEA_CONFIG_SERVICE_OAUTH2_JWT_SECRET} \
+    --namespace=git-limbosolutions-com
+fi