borg offsite backup included so Maintenance actions (purge and compact)

2025-04-16 22:37:05 +00:00
parent 7544fc8ced
commit 96fb2ddb6c
6 changed files with 141 additions and 166 deletions
--- a/README.md
+++ b/README.md
@@ -4,74 +4,30 @@ Welcome to public repository of my [Git Server](https://git.limbosolutions.com)

 Using [gitea](https://git.limbosolutions.com/kb/gitea) as git server.

-## gitea  helm chart
+- [gitea - helm chart](#gitea---helm-chart)
+  - [add chart](#add-chart)
+  - [update chart](#update-chart)
+  - [Upgrade / Install](#upgrade--install)
+
+## gitea - helm chart
+
+### add chart

 ``` yaml
 helm repo add gitea-charts https://dl.gitea.com/charts/
+```
+
+### update chart
+
+``` yaml
 helm repo update
-helm upgrade --install gitea gitea-charts/gitea -f ./values.yaml -f ./values.private.yml --namespace=git-limbosolutions-com
 ```

-## Git Action
+### Upgrade / Install

-## runners
-
-## lxc container - proxmox
-
-Template : debian-12-turnkey-core_18.1-1_amd64.tar.gz
-
- Unprivileged contrainer - Yes
- Nesting -Yes
-
-```bash
-# setup new lxc container with docker
-
-apt update -y
-apt upgrade -y
-curl -fsSL https://get.docker.com -o get-docker.sh
-sh ./get-docker.sh
-```
-
-### act tunner
-
-```bash
-nano setup-act-runners.sh
-```
-
-```bash
-#/bin/bash
-
-setup(){
-
-CONTAINER_NAME=${OWNER}_act_runner
-
-docker container stop $CONTAINER_NAME
-docker container rm $CONTAINER_NAME
-
-docker run \
-    --restart=unless-stopped \
-    -v ${CONTAINER_NAME}_data:/data \
-    -v /var/run/docker.sock:/var/run/docker.sock \
-    -e GITEA_INSTANCE_URL=${INSTANCE_URL} \
-    -e GITEA_RUNNER_REGISTRATION_TOKEN=${REGISTRATION_TOKEN} \
-    -e GITEA_RUNNER_NAME=${OWNER}_ubuntu_default \
-    --name ${CONTAINER_NAME} \
-    -d gitea/act_runner
-}
-
-INSTANCE_URL=https://git.limbosolutions.com
-OWNER=kb
-REGISTRATION_TOKEN=???
-setup
-
-
-OWNER=????
-REGISTRATION_TOKEN=???
-setup
-
-```
-
-```bash
-chmod +x setup-act-runners.sh
-./setup-act-runners.sh
+``` yaml
+helm upgrade --install gitea gitea-charts/gitea \
+--values ./gitea/values.yaml \
+--values ./gitea/values.private.yaml \
+--namespace=git-limbosolutions-com
 ```
--- a/backup/README.md
+++ b/backup/README.md
@@ -2,7 +2,36 @@

 ## borgbackup cockpit (companion scripts)

+Not in use anymore, the cron job already performs the following actions. Keeping it for now for notes purposes.
+
+### list current backups
+
 ```bash
-borg list ${BORG_REPO}
-borg list ${BORG_REPO}::gitea-data-2025-03-12_02:00:06 
+POD_NAME=$(kubectl get pod -l app=borgbackup-cockpit -o jsonpath='{.items[0].metadata.name}')
+kubectl exec -it ${POD_NAME} -- borg list ${BORG_REPO}
+```
+
+### purge backups
+
+```bash
+POD_NAME=$(kubectl get pod -l app=borgbackup-cockpit -o jsonpath='{.items[0].metadata.name}')
+kubectl exec -it ${POD_NAME} -- bash -c "\
+borg prune -v --list --keep-daily=10 --keep-weekly=7 --keep-monthly=-1 ${BORG_REPO}  --glob-archives='gitea-data*' && \
+borg prune -v --list --keep-daily=10 --keep-weekly=7 --keep-monthly=-1 ${BORG_REPO}  --glob-archives='postgresql-export*'"
+```
+
+### clean up space
+
+```bash
+POD_NAME=$(kubectl get pod -l app=borgbackup-cockpit -o jsonpath='{.items[0].metadata.name}')
+kubectl exec -it ${POD_NAME} -- bash -c "\
+borg compact ${BORG_REPO} --verbose"
+```
+
+### Get info
+
+```bash
+POD_NAME=$(kubectl get pod -l app=borgbackup-cockpit -o jsonpath='{.items[0].metadata.name}')
+kubectl exec -it ${POD_NAME} -- bash -c "\
+borg info ${BORG_REPO} --json"
 ```
--- a/backup/borgbackup-cockpit-deploy.yaml
+++ b/backup/borgbackup-cockpit-deploy.yaml
@@ -1,68 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: borgbackup-cockpit
-  namespace: git-limbosolutions-com
-  labels:
-    app: borgbackup-cockpit
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: borgbackup-cockpit
-  template:
-    metadata:
-      labels:
-        app: borgbackup-cockpit
-    spec:
-      containers:
-          - name: borg-client
-            image: git.limbosolutions.com/kb/borg-backup:latest
-            resources:
-              limits:
-                memory: "512Mi"
-                cpu: "500m"
-              requests:
-                memory: "256Mi"
-                cpu: "250m"
-            env:
-              - name: BORG_REPO
-                valueFrom:
-                  secretKeyRef:
-                    name: gitea-backup-secret
-                    key: BORG_REPO
-
-              - name: BORG_PASSPHRASE
-                valueFrom:
-                  secretKeyRef:
-                    name: gitea-backup-secret
-                    key: BORG_PASSPHRASE
-
-              - name: BORG_RSH
-                value: ssh -o StrictHostKeyChecking=no
-
-              - name: BORG_KEY_FILE
-                value: /root/.borg/key
-
-            command: ["sh", "-c"]
-            args:
-            - |
-              while true; do
-                 sleep 1s
-              done
-
-            volumeMounts:
-            
-              - name: gitea-backup-secret
-                mountPath: /root/.borg/key
-                subPath: BORG_KEY_FILE
-
-              - name: gitea-backup-secret
-                mountPath: /root/.ssh/id_rsa
-                subPath: SSH_ID_RSA
-      volumes:
-        - name: gitea-backup-secret
-          secret:
-            secretName: gitea-backup-secret
-            defaultMode: 0600
-
--- a/backup/borgbackup-offsite-cronjob.yaml
+++ b/backup/borgbackup-offsite-cronjob.yaml
@@ -18,16 +18,10 @@ spec:
              command: ["sh", "-c"]
              args:
                - |
-                  #echo "INFO: Starting export"
+                  set -e
                  . /root/.gitea-inline-config/database
                  export PGPASSWORD=$PASSWD
-                  #echo "INFO: Exporting database"
                  pg_dump -h gitea-postgresql.git-limbosolutions-com.svc.cluster.local -U $USER -d $NAME > /data/postgresql-export/db_backup.sql
-                  if [ $? -ne 0 ]; then
-                    echo "ERROR: Exporting database failed"
-                    exit 1
-                  fi 
-                  #echo "INFO: Exporting database finished"
  
              volumeMounts:
                
@@ -71,24 +65,25 @@ spec:
              command: ["sh", "-c"]
              args:
              - |
-               
+                set -e
                borg create ${BORG_REPO}::"postgresql-export-$(date +%Y-%m-%d_%H:%M:%S)" /data/postgresql-export
-                if [ $? -ne 0 ]; then
-                echo "ERROR: Borg backup failed"
-                exit 1
-                fi
                
                borg create ${BORG_REPO}::"gitea-data-$(date +%Y-%m-%d_%H:%M:%S)" /data/gitea-data
-                if [ $? -ne 0 ]; then
-                echo "ERROR: Borg backup failed"
-                exit 1
-                fi
                
                # ssh to backup server and enforce rclone to onedrive
                ${BORG_RSH} mf@backupsrv01.dev.lan \
                "rclone sync ~/borg-repos/git.limbosolutions.com mf.onedrive:.backups/git.limbosolutions.com/borg" &&
                echo "INFO: Backup git.limbosolutions.com (offsite) finished"

+                #cleanup
+                borg prune -v --list --keep-daily=10 --keep-weekly=7 --keep-monthly=-1 ${BORG_REPO} --glob-archives='gitea-data*'
+                borg prune -v --list --keep-daily=10 --keep-weekly=7 --keep-monthly=-1 ${BORG_REPO} --glob-archives='postgresql-export*'
+                borg compact ${BORG_REPO} 
+
+                #outputs info
+                borg info ${BORG_REPO}
+                #borg info ${BORG_REPO} --json
+
              volumeMounts:
                - name: gitea-data
                  mountPath: /data/gitea-data
--- a/docs/act-runner.md
+++ b/docs/act-runner.md
@@ -0,0 +1,63 @@
+# Git Action
+
+## runners
+
+## lxc container - proxmox
+
+Template : debian-12-turnkey-core_18.1-1_amd64.tar.gz
+
+- Unprivileged contrainer - Yes
+- Nesting -Yes
+
+```bash
+# setup new lxc container with docker
+
+apt update -y
+apt upgrade -y
+curl -fsSL https://get.docker.com -o get-docker.sh
+sh ./get-docker.sh
+```
+
+### act tunner
+
+```bash
+nano setup-act-runners.sh
+```
+
+```bash
+#/bin/bash
+
+setup(){
+
+CONTAINER_NAME=${OWNER}_act_runner
+
+docker container stop $CONTAINER_NAME
+docker container rm $CONTAINER_NAME
+
+docker run \
+    --restart=unless-stopped \
+    -v ${CONTAINER_NAME}_data:/data \
+    -v /var/run/docker.sock:/var/run/docker.sock \
+    -e GITEA_INSTANCE_URL=${INSTANCE_URL} \
+    -e GITEA_RUNNER_REGISTRATION_TOKEN=${REGISTRATION_TOKEN} \
+    -e GITEA_RUNNER_NAME=${OWNER}_ubuntu_default \
+    --name ${CONTAINER_NAME} \
+    -d gitea/act_runner
+}
+
+INSTANCE_URL=https://git.limbosolutions.com
+OWNER=kb
+REGISTRATION_TOKEN=???
+setup
+
+
+OWNER=????
+REGISTRATION_TOKEN=???
+setup
+
+```
+
+```bash
+chmod +x setup-act-runners.sh
+./setup-act-runners.sh
+```
--- a/gitea/values.yaml
+++ b/gitea/values.yaml
@@ -49,7 +49,6 @@ service:
        metallb.universe.tf/allow-shared-ip: test

 gitea:
-
  admin:
    username: "???"
    password: "???"
@@ -75,7 +74,7 @@ gitea:
      LFS_START_SERVER: true
      START_SSH_SERVER: true
      LFS_PATH: /data/git/lfs
-      #LFS_JWT_SECRET: "???"
+      LFS_JWT_SECRET: "???"
      OFFLINE_MODE: false
      #MFF 03/08/2024
      REPO_INDEXER_ENABLED: true
@@ -86,17 +85,18 @@ gitea:
      ####
    
    service:
-      DISABLE_REGISTRATION: true
-      REQUIRE_SIGNIN_VIEW: false
-      REGISTER_EMAIL_CONFIRM: false
-      ENABLE_NOTIFY_MAIL: false
-      ALLOW_ONLY_EXTERNAL_REGISTRATION: false
-      ENABLE_CAPTCHA: true
-      DEFAULT_KEEP_EMAIL_PRIVATE : false
-      DEFAULT_ALLOW_CREATE_ORGANIZATION: true
-      DEFAULT_ENABLE_TIMETRACKING: true
+      DISABLE_REGISTRATION: "???"
+      REQUIRE_SIGNIN_VIEW: "???"
+      REGISTER_EMAIL_CONFIRM: "???"
+      ENABLE_NOTIFY_MAIL: "???"
+      ALLOW_ONLY_EXTERNAL_REGISTRATION: "???"
+      ENABLE_CAPTCHA: "???"
+      DEFAULT_KEEP_EMAIL_PRIVATE : "???"
+      DEFAULT_ALLOW_CREATE_ORGANIZATION: "???"
+      DEFAULT_ENABLE_TIMETRACKING: "???"
      NO_REPLY_ADDRESS: noreply.localhost
-
+      oauth2:
+          JWT_SECRET: "???"
    mailer:
      ENABLED: false