From 96fb2ddb6c0e52adccd769a34b40100e17bf4936 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Fernandes?= Date: Wed, 16 Apr 2025 22:37:05 +0000 Subject: [PATCH] borg offsite backup included so Maintenance actions (purge and compact) --- README.md | 82 ++++++-------------------- backup/README.md | 35 ++++++++++- backup/borgbackup-cockpit-deploy.yaml | 68 --------------------- backup/borgbackup-offsite-cronjob.yaml | 35 +++++------ docs/act-runner.md | 63 ++++++++++++++++++++ gitea/values.yaml | 24 ++++---- 6 files changed, 141 insertions(+), 166 deletions(-) delete mode 100644 backup/borgbackup-cockpit-deploy.yaml create mode 100644 docs/act-runner.md diff --git a/README.md b/README.md index a6e7737..f165eb2 100644 --- a/README.md +++ b/README.md @@ -4,74 +4,30 @@ Welcome to public repository of my [Git Server](https://git.limbosolutions.com) Using [gitea](https://git.limbosolutions.com/kb/gitea) as git server. -## gitea helm chart +- [gitea - helm chart](#gitea---helm-chart) + - [add chart](#add-chart) + - [update chart](#update-chart) + - [Upgrade / Install](#upgrade--install) + +## gitea - helm chart + +### add chart ``` yaml helm repo add gitea-charts https://dl.gitea.com/charts/ +``` + +### update chart + +``` yaml helm repo update -helm upgrade --install gitea gitea-charts/gitea -f ./values.yaml -f ./values.private.yml --namespace=git-limbosolutions-com ``` -## Git Action +### Upgrade / Install -## runners - -## lxc container - proxmox - -Template : debian-12-turnkey-core_18.1-1_amd64.tar.gz - -- Unprivileged contrainer - Yes -- Nesting -Yes - -```bash -# setup new lxc container with docker - -apt update -y -apt upgrade -y -curl -fsSL https://get.docker.com -o get-docker.sh -sh ./get-docker.sh -``` - -### act tunner - -```bash -nano setup-act-runners.sh -``` - -```bash -#/bin/bash - -setup(){ - -CONTAINER_NAME=${OWNER}_act_runner - -docker container stop $CONTAINER_NAME -docker container rm $CONTAINER_NAME - -docker run \ - --restart=unless-stopped \ - -v ${CONTAINER_NAME}_data:/data \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -e GITEA_INSTANCE_URL=${INSTANCE_URL} \ - -e GITEA_RUNNER_REGISTRATION_TOKEN=${REGISTRATION_TOKEN} \ - -e GITEA_RUNNER_NAME=${OWNER}_ubuntu_default \ - --name ${CONTAINER_NAME} \ - -d gitea/act_runner -} - -INSTANCE_URL=https://git.limbosolutions.com -OWNER=kb -REGISTRATION_TOKEN=??? -setup - - -OWNER=???? -REGISTRATION_TOKEN=??? -setup - -``` - -```bash -chmod +x setup-act-runners.sh -./setup-act-runners.sh +``` yaml +helm upgrade --install gitea gitea-charts/gitea \ +--values ./gitea/values.yaml \ +--values ./gitea/values.private.yaml \ +--namespace=git-limbosolutions-com ``` diff --git a/backup/README.md b/backup/README.md index d898089..97ee6cd 100644 --- a/backup/README.md +++ b/backup/README.md @@ -2,7 +2,36 @@ ## borgbackup cockpit (companion scripts) -``` bash -borg list ${BORG_REPO} -borg list ${BORG_REPO}::gitea-data-2025-03-12_02:00:06 +Not in use anymore, the cron job already performs the following actions. Keeping it for now for notes purposes. + +### list current backups + +```bash +POD_NAME=$(kubectl get pod -l app=borgbackup-cockpit -o jsonpath='{.items[0].metadata.name}') +kubectl exec -it ${POD_NAME} -- borg list ${BORG_REPO} +``` + +### purge backups + +```bash +POD_NAME=$(kubectl get pod -l app=borgbackup-cockpit -o jsonpath='{.items[0].metadata.name}') +kubectl exec -it ${POD_NAME} -- bash -c "\ +borg prune -v --list --keep-daily=10 --keep-weekly=7 --keep-monthly=-1 ${BORG_REPO} --glob-archives='gitea-data*' && \ +borg prune -v --list --keep-daily=10 --keep-weekly=7 --keep-monthly=-1 ${BORG_REPO} --glob-archives='postgresql-export*'" +``` + +### clean up space + +```bash +POD_NAME=$(kubectl get pod -l app=borgbackup-cockpit -o jsonpath='{.items[0].metadata.name}') +kubectl exec -it ${POD_NAME} -- bash -c "\ +borg compact ${BORG_REPO} --verbose" +``` + +### Get info + +```bash +POD_NAME=$(kubectl get pod -l app=borgbackup-cockpit -o jsonpath='{.items[0].metadata.name}') +kubectl exec -it ${POD_NAME} -- bash -c "\ +borg info ${BORG_REPO} --json" ``` diff --git a/backup/borgbackup-cockpit-deploy.yaml b/backup/borgbackup-cockpit-deploy.yaml deleted file mode 100644 index 8853874..0000000 --- a/backup/borgbackup-cockpit-deploy.yaml +++ /dev/null @@ -1,68 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: borgbackup-cockpit - namespace: git-limbosolutions-com - labels: - app: borgbackup-cockpit -spec: - replicas: 1 - selector: - matchLabels: - app: borgbackup-cockpit - template: - metadata: - labels: - app: borgbackup-cockpit - spec: - containers: - - name: borg-client - image: git.limbosolutions.com/kb/borg-backup:latest - resources: - limits: - memory: "512Mi" - cpu: "500m" - requests: - memory: "256Mi" - cpu: "250m" - env: - - name: BORG_REPO - valueFrom: - secretKeyRef: - name: gitea-backup-secret - key: BORG_REPO - - - name: BORG_PASSPHRASE - valueFrom: - secretKeyRef: - name: gitea-backup-secret - key: BORG_PASSPHRASE - - - name: BORG_RSH - value: ssh -o StrictHostKeyChecking=no - - - name: BORG_KEY_FILE - value: /root/.borg/key - - command: ["sh", "-c"] - args: - - | - while true; do - sleep 1s - done - - volumeMounts: - - - name: gitea-backup-secret - mountPath: /root/.borg/key - subPath: BORG_KEY_FILE - - - name: gitea-backup-secret - mountPath: /root/.ssh/id_rsa - subPath: SSH_ID_RSA - volumes: - - name: gitea-backup-secret - secret: - secretName: gitea-backup-secret - defaultMode: 0600 - diff --git a/backup/borgbackup-offsite-cronjob.yaml b/backup/borgbackup-offsite-cronjob.yaml index 5639d63..989cfb7 100644 --- a/backup/borgbackup-offsite-cronjob.yaml +++ b/backup/borgbackup-offsite-cronjob.yaml @@ -18,17 +18,11 @@ spec: command: ["sh", "-c"] args: - | - #echo "INFO: Starting export" + set -e . /root/.gitea-inline-config/database export PGPASSWORD=$PASSWD - #echo "INFO: Exporting database" pg_dump -h gitea-postgresql.git-limbosolutions-com.svc.cluster.local -U $USER -d $NAME > /data/postgresql-export/db_backup.sql - if [ $? -ne 0 ]; then - echo "ERROR: Exporting database failed" - exit 1 - fi - #echo "INFO: Exporting database finished" - + volumeMounts: - name: backup-var-data @@ -71,24 +65,25 @@ spec: command: ["sh", "-c"] args: - | - + set -e borg create ${BORG_REPO}::"postgresql-export-$(date +%Y-%m-%d_%H:%M:%S)" /data/postgresql-export - if [ $? -ne 0 ]; then - echo "ERROR: Borg backup failed" - exit 1 - fi - + borg create ${BORG_REPO}::"gitea-data-$(date +%Y-%m-%d_%H:%M:%S)" /data/gitea-data - if [ $? -ne 0 ]; then - echo "ERROR: Borg backup failed" - exit 1 - fi - - #ssh to backup server and enforce rclone to onedrive + + # ssh to backup server and enforce rclone to onedrive ${BORG_RSH} mf@backupsrv01.dev.lan \ "rclone sync ~/borg-repos/git.limbosolutions.com mf.onedrive:.backups/git.limbosolutions.com/borg" && echo "INFO: Backup git.limbosolutions.com (offsite) finished" + #cleanup + borg prune -v --list --keep-daily=10 --keep-weekly=7 --keep-monthly=-1 ${BORG_REPO} --glob-archives='gitea-data*' + borg prune -v --list --keep-daily=10 --keep-weekly=7 --keep-monthly=-1 ${BORG_REPO} --glob-archives='postgresql-export*' + borg compact ${BORG_REPO} + + #outputs info + borg info ${BORG_REPO} + #borg info ${BORG_REPO} --json + volumeMounts: - name: gitea-data mountPath: /data/gitea-data diff --git a/docs/act-runner.md b/docs/act-runner.md new file mode 100644 index 0000000..b8b46ce --- /dev/null +++ b/docs/act-runner.md @@ -0,0 +1,63 @@ +# Git Action + +## runners + +## lxc container - proxmox + +Template : debian-12-turnkey-core_18.1-1_amd64.tar.gz + +- Unprivileged contrainer - Yes +- Nesting -Yes + +```bash +# setup new lxc container with docker + +apt update -y +apt upgrade -y +curl -fsSL https://get.docker.com -o get-docker.sh +sh ./get-docker.sh +``` + +### act tunner + +```bash +nano setup-act-runners.sh +``` + +```bash +#/bin/bash + +setup(){ + +CONTAINER_NAME=${OWNER}_act_runner + +docker container stop $CONTAINER_NAME +docker container rm $CONTAINER_NAME + +docker run \ + --restart=unless-stopped \ + -v ${CONTAINER_NAME}_data:/data \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -e GITEA_INSTANCE_URL=${INSTANCE_URL} \ + -e GITEA_RUNNER_REGISTRATION_TOKEN=${REGISTRATION_TOKEN} \ + -e GITEA_RUNNER_NAME=${OWNER}_ubuntu_default \ + --name ${CONTAINER_NAME} \ + -d gitea/act_runner +} + +INSTANCE_URL=https://git.limbosolutions.com +OWNER=kb +REGISTRATION_TOKEN=??? +setup + + +OWNER=???? +REGISTRATION_TOKEN=??? +setup + +``` + +```bash +chmod +x setup-act-runners.sh +./setup-act-runners.sh +``` diff --git a/gitea/values.yaml b/gitea/values.yaml index d8dd790..5840c86 100644 --- a/gitea/values.yaml +++ b/gitea/values.yaml @@ -49,7 +49,6 @@ service: metallb.universe.tf/allow-shared-ip: test gitea: - admin: username: "???" password: "???" @@ -75,7 +74,7 @@ gitea: LFS_START_SERVER: true START_SSH_SERVER: true LFS_PATH: /data/git/lfs - #LFS_JWT_SECRET: "???" + LFS_JWT_SECRET: "???" OFFLINE_MODE: false #MFF 03/08/2024 REPO_INDEXER_ENABLED: true @@ -86,17 +85,18 @@ gitea: #### service: - DISABLE_REGISTRATION: true - REQUIRE_SIGNIN_VIEW: false - REGISTER_EMAIL_CONFIRM: false - ENABLE_NOTIFY_MAIL: false - ALLOW_ONLY_EXTERNAL_REGISTRATION: false - ENABLE_CAPTCHA: true - DEFAULT_KEEP_EMAIL_PRIVATE : false - DEFAULT_ALLOW_CREATE_ORGANIZATION: true - DEFAULT_ENABLE_TIMETRACKING: true + DISABLE_REGISTRATION: "???" + REQUIRE_SIGNIN_VIEW: "???" + REGISTER_EMAIL_CONFIRM: "???" + ENABLE_NOTIFY_MAIL: "???" + ALLOW_ONLY_EXTERNAL_REGISTRATION: "???" + ENABLE_CAPTCHA: "???" + DEFAULT_KEEP_EMAIL_PRIVATE : "???" + DEFAULT_ALLOW_CREATE_ORGANIZATION: "???" + DEFAULT_ENABLE_TIMETRACKING: "???" NO_REPLY_ADDRESS: noreply.localhost - + oauth2: + JWT_SECRET: "???" mailer: ENABLED: false