feat: backups log included duration

feat: offsite included size limit
2025-04-18 21:01:51 +00:00
parent 4288e7fc03
commit 4fc469be77
5 changed files with 75 additions and 30 deletions
--- a/README.md
+++ b/README.md
@@ -10,8 +10,8 @@ Using [gitea](https://git.limbosolutions.com/kb/gitea) as git server.
  - [Update Helm charts](#update-helm-charts)
  - [Upgrade / Install](#upgrade--install)
 - [Backups](#backups)
-  - [kubernetes cron job - Proxmox Backup Server](#kubernetes-cron-job---proxmox-backup-server)
-  - [kubernetes cron job - Borg and Offsite sync](#kubernetes-cron-job---borg-and-offsite-sync)
+  - [Proxmox Backup Server (kubernetes cron job)](#proxmox-backup-server-kubernetes-cron-job)
+  - [Borg and Offsite sync (kubernetes cron job)](#borg-and-offsite-sync-kubernetes-cron-job)
  - [borgbackup sidekick](#borgbackup-sidekick)
    - [Create borgbackup-sidekick pod](#create-borgbackup-sidekick-pod)
    - [Remove borgbackup-sidekick pod](#remove-borgbackup-sidekick-pod)
@@ -53,25 +53,43 @@ helm upgrade --install gitea gitea-charts/gitea \

 ## Backups

-### kubernetes cron job - Proxmox Backup Server
+### Proxmox Backup Server (kubernetes cron job)
+
+```bash
+# deploy cronjon
+kubectl apply -f ./backups/backup-cronjob.yaml -n git-limbosolutions-com
+```

 [kubernetes cron job](/backups/backup-cronjob.yaml)

-### kubernetes cron job - Borg and Offsite sync
+### Borg and Offsite sync (kubernetes cron job)

 ```bash
-# deploy secrets      
-. ./backups/.env
-# set variables in env file as export
-export $(cut -d= -f1 ./backups/.env)
-envsubst < ./backups/gitea-backup-secret.yaml | kubectl apply -f -
+set -a
+source ./backups/.env
+set +a
+envsubst < ./backups/gitea-backup-secret.yaml | kubectl apply -n git-limbosolutions-com -f -

+SSH_ID_RSA=$(echo -n "$SSH_ID_RSA" | base64 -w 0)
+BORG_KEY_FILE=$(echo -n "$BORG_KEY_FILE" | base64 -w 0)
+
+kubectl patch secret gitea-backup-secret --patch "{\"data\":{\"SSH_ID_RSA\":\"$SSH_ID_RSA\"}}"
+kubectl patch secret gitea-backup-secret --patch "{\"data\":{\"BORG_KEY_FILE\":\"$BORG_KEY_FILE\"}}"
+
+# deploy cronjon
+kubectl apply -f ./backups/borgbackup-offsite-cronjob.yaml
 ```

 [kubernetes cron job](/backups/borgbackup-offsite-cronjob.yaml)

 ### borgbackup sidekick

+```bash
+# attach to borgbackup-sidekick
+POD_NAME=$(kubectl get pod -l app=borgbackup-sidekick -n git-limbosolutions-com -o jsonpath='{.items[0].metadata.name}')
+kubectl exec -it ${POD_NAME}  -- bash
+```
+
 ```bash
 # list borg repo
 POD_NAME=$(kubectl get pod -l app=borgbackup-sidekick -n git-limbosolutions-com -o jsonpath='{.items[0].metadata.name}')
@@ -88,6 +106,23 @@ borg info ${BORG_REPO}\
 "
 ```

+```bash
+POD_NAME=$(kubectl get pod -l app=borgbackup-sidekick -n git-limbosolutions-com -o jsonpath='{.items[0].metadata.name}')
+# Check if repository size is greater than 4GB
+kubectl exec -it ${POD_NAME} -- bash -c "\
+MAX_SIZE=\$((4 * 1024 * 1024 * 1024))
+SIZE_IN_BYTES=\$(\${BORG_RSH} mf@backupsrv01.dev.lan \"du -b ~/borg-repos/git.limbosolutions.com --max-depth=0 | cut -f1 \") && \
+echo \"Repository size: \$((SIZE_IN_BYTES / 1024 / 1024)) MB\" && \
+if [ \$SIZE_IN_BYTES -gt \$MAX_SIZE ]; then \
+   echo 'Repository size exceeds \$MAX_SIZE'; \
+   exit 1; \
+else \
+   echo 'Repository size is within limits for offsite sync'; \
+fi \
+
+"
+```
+
 #### Create borgbackup-sidekick pod

 ```bash
--- a/backups/backup-cronjob.yaml
+++ b/backups/backup-cronjob.yaml
@@ -63,8 +63,11 @@ spec:
              command: ["bash", "-c"]
              args:
                - |
+                  set -e
+                  SCRIPT_START_TIME=$(date +%s)
                  proxmox-backup-client backup gitea-data.pxar:/data/gitea-data postgresql-data.pxar:/data/postgresql-data postgresql-export.pxar:/data/postgresql-export --include-dev /data/postgresql-data --include-dev /data/postgresql-export --include-dev /data/gitea-data --backup-id "gitea-full" -ns git.limbosolutions.com
-                  echo "INFO: Backup git.limbosolutions.com finished"
+                  SCRIPT_DURATION=$(($(date +%s) - SCRIPT_START_TIME))
+                  echo "INFO: Finished Backup of git.limbosolutions.com ($((SCRIPT_DURATION / 60 / 60)):$((SCRIPT_DURATION / 60)):$((SCRIPT_DURATION % 60))) "

              volumeMounts:
              - name: gitea-shared-storage
--- a/backups/borgbackup-offsite-cronjob.yaml
+++ b/backups/borgbackup-offsite-cronjob.yaml
@@ -59,6 +59,10 @@ spec:
                - name: BORG_RSH
                  value: ssh -o StrictHostKeyChecking=no -o LogLevel=ERROR

+
+                - name: REPO_SYNC_MAX_SIZE
+                  value: "4294967296" # 4GB
+           
                - name: BORG_KEY_FILE
                  value: /root/.borg/key

@@ -66,24 +70,37 @@ spec:
              args:
              - |
                set -e
-                borg create ${BORG_REPO}::"postgresql-export-$(date +%Y-%m-%d_%H:%M:%S)" /data/postgresql-export
                
-                borg create ${BORG_REPO}::"gitea-data-$(date +%Y-%m-%d_%H:%M:%S)" /data/gitea-data
+                SCRIPT_START_TIME=$(date +%s)
+
                
-                # ssh to backup server and enforce rclone to onedrive
-                ${BORG_RSH} mf@backupsrv01.dev.lan \
-                "rclone sync ~/borg-repos/git.limbosolutions.com mf.onedrive:.backups/git.limbosolutions.com/borg" &&
-                echo "INFO: Backup git.limbosolutions.com (offsite) finished"
+                borg create ${BORG_REPO}::postgresql-export-$(date +%Y%m%d%H%M%S) /data/postgresql-export
+                borg create ${BORG_REPO}::gitea-data-$(date +%Y%m%d%H%M%S) /data/gitea-data

                #cleanup
                borg prune -v --list --keep-daily=10 --keep-weekly=7 --keep-monthly=-1 ${BORG_REPO} --glob-archives='gitea-data*'
                borg prune -v --list --keep-daily=10 --keep-weekly=7 --keep-monthly=-1 ${BORG_REPO} --glob-archives='postgresql-export*'
                borg compact ${BORG_REPO} 

+                # check repo size
+                REPO_SIZE_IN_BYTES=$(${BORG_RSH} mf@backupsrv01.dev.lan "du -b ~/borg-repos/git.limbosolutions.com --max-depth=0 | cut -f1 ")
+                echo "Repository size: $((REPO_SIZE_IN_BYTES / 1024 / 1024)) MB"
+                
+                if [ $REPO_SIZE_IN_BYTES -gt $REPO_SYNC_MAX_SIZE ]; then \
+                  echo 'Repository size exceeds $REPO_SYNC_MAX_SIZE';
+                  exit 1;
+                else 
+                  # Repository size is within limits for offsite sync
+                  # ssh to backup server and enforce rclone to onedrive
+                  ${BORG_RSH} mf@backupsrv01.dev.lan \
+                  "rclone sync ~/borg-repos/git.limbosolutions.com mf.onedrive:.backups/git.limbosolutions.com/borg" && \
+                  SCRIPT_DURATION=$(($(date +%s) - SCRIPT_START_TIME)) && \
+                  echo "INFO: Finished Backup of git.limbosolutions.com (offsite) ($((SCRIPT_DURATION / 60 / 60)):$((SCRIPT_DURATION / 60)):$((SCRIPT_DURATION % 60))) "
+                fi
+
                #outputs info
                borg info ${BORG_REPO}
                #borg info ${BORG_REPO} --json
-
              volumeMounts:
                - name: gitea-data
                  mountPath: /data/gitea-data
--- a/backups/gitea-backup-secret-deploy.sh
+++ b/backups/gitea-backup-secret-deploy.sh
@@ -1,11 +0,0 @@
-#/bin/bash
-
-# set source env variables
-. ./.env
-
-# set variables in env file as export
-export $(cut -d= -f1 ./.env)
-
-
-envsubst < gitea-backup-secret.yaml | kubectl apply -f -
-
--- a/backups/gitea-backup-secret.yaml
+++ b/backups/gitea-backup-secret.yaml
@@ -9,7 +9,8 @@ stringData:
  PBS_REPOSITORY: ${PBS_REPOSITORY}
  PBS_PASSWORD: ${PBS_PASSWORD}
  PBS_FINGERPRINT: ${PBS_FINGERPRINT}
-  BORG_KEY_FILE: ${BORG_KEY_FILE}
  BORG_REPO: ${BORG_REPO}
  BORG_PASSPHRASE: ${BORG_PASSPHRASE}
-  SSH_ID_RSA: ${SSH_ID_RSA}
+  #SSH_ID_RSA: ""
+  #BORG_KEY_FILE: ""
+