From 4fc469be773d5c2906b290b87f54989b09ebb97b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Fernandes?= Date: Fri, 18 Apr 2025 21:01:51 +0000 Subject: [PATCH] feat: backups log included duration feat: offsite included size limit --- README.md | 53 ++++++++++++++++++++----- backups/backup-cronjob.yaml | 5 ++- backups/borgbackup-offsite-cronjob.yaml | 31 +++++++++++---- backups/gitea-backup-secret-deploy.sh | 11 ----- backups/gitea-backup-secret.yaml | 5 ++- 5 files changed, 75 insertions(+), 30 deletions(-) delete mode 100755 backups/gitea-backup-secret-deploy.sh diff --git a/README.md b/README.md index 21e6097..df6a274 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,8 @@ Using [gitea](https://git.limbosolutions.com/kb/gitea) as git server. - [Update Helm charts](#update-helm-charts) - [Upgrade / Install](#upgrade--install) - [Backups](#backups) - - [kubernetes cron job - Proxmox Backup Server](#kubernetes-cron-job---proxmox-backup-server) - - [kubernetes cron job - Borg and Offsite sync](#kubernetes-cron-job---borg-and-offsite-sync) + - [Proxmox Backup Server (kubernetes cron job)](#proxmox-backup-server-kubernetes-cron-job) + - [Borg and Offsite sync (kubernetes cron job)](#borg-and-offsite-sync-kubernetes-cron-job) - [borgbackup sidekick](#borgbackup-sidekick) - [Create borgbackup-sidekick pod](#create-borgbackup-sidekick-pod) - [Remove borgbackup-sidekick pod](#remove-borgbackup-sidekick-pod) @@ -53,25 +53,43 @@ helm upgrade --install gitea gitea-charts/gitea \ ## Backups -### kubernetes cron job - Proxmox Backup Server +### Proxmox Backup Server (kubernetes cron job) + +```bash +# deploy cronjon +kubectl apply -f ./backups/backup-cronjob.yaml -n git-limbosolutions-com +``` [kubernetes cron job](/backups/backup-cronjob.yaml) -### kubernetes cron job - Borg and Offsite sync +### Borg and Offsite sync (kubernetes cron job) ```bash -# deploy secrets -. ./backups/.env -# set variables in env file as export -export $(cut -d= -f1 ./backups/.env) -envsubst < ./backups/gitea-backup-secret.yaml | kubectl apply -f - +set -a +source ./backups/.env +set +a +envsubst < ./backups/gitea-backup-secret.yaml | kubectl apply -n git-limbosolutions-com -f - +SSH_ID_RSA=$(echo -n "$SSH_ID_RSA" | base64 -w 0) +BORG_KEY_FILE=$(echo -n "$BORG_KEY_FILE" | base64 -w 0) + +kubectl patch secret gitea-backup-secret --patch "{\"data\":{\"SSH_ID_RSA\":\"$SSH_ID_RSA\"}}" +kubectl patch secret gitea-backup-secret --patch "{\"data\":{\"BORG_KEY_FILE\":\"$BORG_KEY_FILE\"}}" + +# deploy cronjon +kubectl apply -f ./backups/borgbackup-offsite-cronjob.yaml ``` [kubernetes cron job](/backups/borgbackup-offsite-cronjob.yaml) ### borgbackup sidekick +```bash +# attach to borgbackup-sidekick +POD_NAME=$(kubectl get pod -l app=borgbackup-sidekick -n git-limbosolutions-com -o jsonpath='{.items[0].metadata.name}') +kubectl exec -it ${POD_NAME} -- bash +``` + ```bash # list borg repo POD_NAME=$(kubectl get pod -l app=borgbackup-sidekick -n git-limbosolutions-com -o jsonpath='{.items[0].metadata.name}') @@ -88,6 +106,23 @@ borg info ${BORG_REPO}\ " ``` +```bash +POD_NAME=$(kubectl get pod -l app=borgbackup-sidekick -n git-limbosolutions-com -o jsonpath='{.items[0].metadata.name}') +# Check if repository size is greater than 4GB +kubectl exec -it ${POD_NAME} -- bash -c "\ +MAX_SIZE=\$((4 * 1024 * 1024 * 1024)) +SIZE_IN_BYTES=\$(\${BORG_RSH} mf@backupsrv01.dev.lan \"du -b ~/borg-repos/git.limbosolutions.com --max-depth=0 | cut -f1 \") && \ +echo \"Repository size: \$((SIZE_IN_BYTES / 1024 / 1024)) MB\" && \ +if [ \$SIZE_IN_BYTES -gt \$MAX_SIZE ]; then \ + echo 'Repository size exceeds \$MAX_SIZE'; \ + exit 1; \ +else \ + echo 'Repository size is within limits for offsite sync'; \ +fi \ + +" +``` + #### Create borgbackup-sidekick pod ```bash diff --git a/backups/backup-cronjob.yaml b/backups/backup-cronjob.yaml index c776b34..b01d6a5 100644 --- a/backups/backup-cronjob.yaml +++ b/backups/backup-cronjob.yaml @@ -63,8 +63,11 @@ spec: command: ["bash", "-c"] args: - | + set -e + SCRIPT_START_TIME=$(date +%s) proxmox-backup-client backup gitea-data.pxar:/data/gitea-data postgresql-data.pxar:/data/postgresql-data postgresql-export.pxar:/data/postgresql-export --include-dev /data/postgresql-data --include-dev /data/postgresql-export --include-dev /data/gitea-data --backup-id "gitea-full" -ns git.limbosolutions.com - echo "INFO: Backup git.limbosolutions.com finished" + SCRIPT_DURATION=$(($(date +%s) - SCRIPT_START_TIME)) + echo "INFO: Finished Backup of git.limbosolutions.com ($((SCRIPT_DURATION / 60 / 60)):$((SCRIPT_DURATION / 60)):$((SCRIPT_DURATION % 60))) " volumeMounts: - name: gitea-shared-storage diff --git a/backups/borgbackup-offsite-cronjob.yaml b/backups/borgbackup-offsite-cronjob.yaml index 989cfb7..a3a7a0b 100644 --- a/backups/borgbackup-offsite-cronjob.yaml +++ b/backups/borgbackup-offsite-cronjob.yaml @@ -59,6 +59,10 @@ spec: - name: BORG_RSH value: ssh -o StrictHostKeyChecking=no -o LogLevel=ERROR + + - name: REPO_SYNC_MAX_SIZE + value: "4294967296" # 4GB + - name: BORG_KEY_FILE value: /root/.borg/key @@ -66,24 +70,37 @@ spec: args: - | set -e - borg create ${BORG_REPO}::"postgresql-export-$(date +%Y-%m-%d_%H:%M:%S)" /data/postgresql-export - borg create ${BORG_REPO}::"gitea-data-$(date +%Y-%m-%d_%H:%M:%S)" /data/gitea-data + SCRIPT_START_TIME=$(date +%s) + - # ssh to backup server and enforce rclone to onedrive - ${BORG_RSH} mf@backupsrv01.dev.lan \ - "rclone sync ~/borg-repos/git.limbosolutions.com mf.onedrive:.backups/git.limbosolutions.com/borg" && - echo "INFO: Backup git.limbosolutions.com (offsite) finished" + borg create ${BORG_REPO}::postgresql-export-$(date +%Y%m%d%H%M%S) /data/postgresql-export + borg create ${BORG_REPO}::gitea-data-$(date +%Y%m%d%H%M%S) /data/gitea-data #cleanup borg prune -v --list --keep-daily=10 --keep-weekly=7 --keep-monthly=-1 ${BORG_REPO} --glob-archives='gitea-data*' borg prune -v --list --keep-daily=10 --keep-weekly=7 --keep-monthly=-1 ${BORG_REPO} --glob-archives='postgresql-export*' borg compact ${BORG_REPO} + # check repo size + REPO_SIZE_IN_BYTES=$(${BORG_RSH} mf@backupsrv01.dev.lan "du -b ~/borg-repos/git.limbosolutions.com --max-depth=0 | cut -f1 ") + echo "Repository size: $((REPO_SIZE_IN_BYTES / 1024 / 1024)) MB" + + if [ $REPO_SIZE_IN_BYTES -gt $REPO_SYNC_MAX_SIZE ]; then \ + echo 'Repository size exceeds $REPO_SYNC_MAX_SIZE'; + exit 1; + else + # Repository size is within limits for offsite sync + # ssh to backup server and enforce rclone to onedrive + ${BORG_RSH} mf@backupsrv01.dev.lan \ + "rclone sync ~/borg-repos/git.limbosolutions.com mf.onedrive:.backups/git.limbosolutions.com/borg" && \ + SCRIPT_DURATION=$(($(date +%s) - SCRIPT_START_TIME)) && \ + echo "INFO: Finished Backup of git.limbosolutions.com (offsite) ($((SCRIPT_DURATION / 60 / 60)):$((SCRIPT_DURATION / 60)):$((SCRIPT_DURATION % 60))) " + fi + #outputs info borg info ${BORG_REPO} #borg info ${BORG_REPO} --json - volumeMounts: - name: gitea-data mountPath: /data/gitea-data diff --git a/backups/gitea-backup-secret-deploy.sh b/backups/gitea-backup-secret-deploy.sh deleted file mode 100755 index 53132df..0000000 --- a/backups/gitea-backup-secret-deploy.sh +++ /dev/null @@ -1,11 +0,0 @@ -#/bin/bash - -# set source env variables -. ./.env - -# set variables in env file as export -export $(cut -d= -f1 ./.env) - - -envsubst < gitea-backup-secret.yaml | kubectl apply -f - - diff --git a/backups/gitea-backup-secret.yaml b/backups/gitea-backup-secret.yaml index aa9a50c..467eaf9 100644 --- a/backups/gitea-backup-secret.yaml +++ b/backups/gitea-backup-secret.yaml @@ -9,7 +9,8 @@ stringData: PBS_REPOSITORY: ${PBS_REPOSITORY} PBS_PASSWORD: ${PBS_PASSWORD} PBS_FINGERPRINT: ${PBS_FINGERPRINT} - BORG_KEY_FILE: ${BORG_KEY_FILE} BORG_REPO: ${BORG_REPO} BORG_PASSPHRASE: ${BORG_PASSPHRASE} - SSH_ID_RSA: ${SSH_ID_RSA} + #SSH_ID_RSA: "" + #BORG_KEY_FILE: "" + \ No newline at end of file