From 3180d653d30253ab7acb4ebc1eed3b23181716f8 Mon Sep 17 00:00:00 2001 From: "marcio.fernandes" Date: Sun, 21 Sep 2025 19:23:37 +0000 Subject: [PATCH] borg container image feature/review-docker: pipelines and scripts revisions (#1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Márcio Fernandes Reviewed-on: https://git.limbosolutions.com/kb/borg/pulls/1 --- .../workflows/ docker-image.deploy.beta.yml | 33 +++ ...ploy.yml => docker-image.deploy.prod.yml} | 1 + .gitignore | 3 + README.md | 223 ++++++++++-------- docker-compose.dev.yaml | 32 +-- docker/Dockerfile | 39 ++- docker/app/README.md | 9 + docker/app/scripts/entrypoint.sh | 36 +++ docker/app/scripts/loadenv | 3 +- docker/app/scripts/remote-get-folder-size | 4 +- 10 files changed, 254 insertions(+), 129 deletions(-) create mode 100644 .gitea/workflows/ docker-image.deploy.beta.yml rename .gitea/workflows/{ docker-image.deploy.yml => docker-image.deploy.prod.yml} (97%) create mode 100644 docker/app/README.md create mode 100755 docker/app/scripts/entrypoint.sh diff --git a/.gitea/workflows/ docker-image.deploy.beta.yml b/.gitea/workflows/ docker-image.deploy.beta.yml new file mode 100644 index 0000000..7ca4b8c --- /dev/null +++ b/.gitea/workflows/ docker-image.deploy.beta.yml @@ -0,0 +1,33 @@ +on: + push: + branches: [ feature/* ] + paths: + - "docker/**" + - ".gitea/**" + schedule: + - cron: "0 02 * * *" +jobs: + + build-docker-image: + runs-on: ubuntu-latest + + steps: + + - name: Checkout code + uses: actions/checkout@v2 + + - name: Log in to git.limbosolutions.com docker registry + uses: docker/login-action@v3 + with: + registry: git.limbosolutions.com + username: ${{ secrets.GITLIMBO_DOCKER_REGISTRY_USERNAME }} + password: ${{ secrets.GITLIMBO_DOCKER_REGISTRY_PASSWORD }} + + - name: Build and push Docker images + id: push + uses: docker/build-push-action@v6 + with: + context: . + file: ${{gitea.workspace}}/docker/Dockerfile + push: true + tags: git.limbosolutions.com/kb/borg-backup:alpha \ No newline at end of file diff --git a/.gitea/workflows/ docker-image.deploy.yml b/.gitea/workflows/ docker-image.deploy.prod.yml similarity index 97% rename from .gitea/workflows/ docker-image.deploy.yml rename to .gitea/workflows/ docker-image.deploy.prod.yml index 3d421aa..3ccebba 100644 --- a/.gitea/workflows/ docker-image.deploy.yml +++ b/.gitea/workflows/ docker-image.deploy.prod.yml @@ -1,5 +1,6 @@ on: push: + branches: [ main ] paths: - "docker/**" - ".gitea/**" diff --git a/.gitignore b/.gitignore index 621cb7b..935f06a 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ **.local.** +local/** +.env + diff --git a/README.md b/README.md index 406ff97..ee77311 100644 --- a/README.md +++ b/README.md @@ -2,46 +2,51 @@ +- [container image](#container-image) + - [environment variables](#environment-variables) + - [borg init repo](#borg-init-repo) + - [creating a backup](#creating-a-backup) + - [using a bash script](#using-a-bash-script) +- [Using binding volumes](#using-binding-volumes) + - [dev](#dev) + ## container image -## environment variables +### environment variables -``` bash -docker run git.limbosolutions.com/kb/borg-backup:latest -``` +### borg init repo -## repo init - -```bash +```yaml services: borg-backup: image: git.limbosolutions.com/kb/borg-backup:latest restart: no - tty: true environment: - - BORG_REPO: ssh://user@server/home/user/borg-repo - - BORG_RSH: "-o StrictHostKeyChecking=no -o LogLevel=ERROR" + - BORG_REPO=${BORG_REPO} + - BORG_RSH=ssh -o StrictHostKeyChecking=no -p 2222 + - BORG_PASSPHRASE="${BORG_PASSPHRASE}" + - MODE=SHELL # Valid modes are: BORG, SCRIPT, SHELL, default is BORG configs: - - source: id_ed25519 # required for ssh client - target: /home/borg/.ssh/id_ed25519 + + - source: id_ed25519 + target: /root/.ssh/id_ed25519 + mode: 0400 + + command: + - | + borg init --encryption=keyfile-blake2 $BORG_REPO + cat /root/.config/borg/keys/* configs: - create.sh: - content: - while true; do - sleep 5 - done - # execute for example - #borg init --encryption=keyfile-blake2 $BORG_REPO - # dont forget to copy key file content on borg folder (/root/.borg/keys/*) and BORG_PASSPHRASE - -``` -### docker compose + id_ed25519: + file: ~/.ssh/id_ed25519 -Example of simple usage for creating a backup + ``` + +### creating a backup ```yaml services: @@ -54,127 +59,145 @@ services: - ./home/user:/mnt/user # Mount local folder to container environment: - - BORG_REPO=????? - - BORG_RSH: "-o StrictHostKeyChecking=no -o LogLevel=ERROR" - - BORG_PASSPHRASE=???? + - BORG_REPO=${BORG_REPO} + - BORG_RSH=ssh -o StrictHostKeyChecking=no -p 2222 + - BORG_PASSPHRASE="${BORG_PASSPHRASE}" configs: - source: id_ed25519 # required for ssh client - target: /home/borg/.ssh/id_ed25519 + mode: 0400 + target: /root/.ssh/id_ed25519 - source: borg_key # required for borg client target: /app/borg/key - + mode: 0400 configs: id_ed25519: - content: | - -----BEGIN OPENSSH PRIVATE KEY----- - ************** - ************** - -----END OPENSSH PRIVATE KEY----- + file: ~/.ssh/id_ed25519 borg_key: - content: | - BORG_KEY ??????? - ???????????????? - ???????????????? - - + content: | + ${BORG_KEY} ``` -Example using an bash script +### using a bash script ```yaml services: - borg-backup: - restart: no - image: git.limbosolutions.com/kb/borg-backup:latest - entrypoint: ["bash", "backup.sh"] + borg: + image: git.limbosolutions.com/kb/borg-backup:alpha + + environment: + - BORG_REPO=${BORG_REPO} + - BORG_RSH=${BORG_RSH} + - OFFSITE_TARGET_FOLDER=${OFFSITE_TARGET_FOLDER} + - BORG_PASSPHRASE="${BORG_PASSPHRASE}" + - REPO_SYNC_MAX_SIZE=10737418240 #10GB + - MODE=SCRIPT + volumes: + - "/home/mf/repos:/backup/repos" + configs: - source: backup_script - target: /backup.sh + target: /app/backup-scripts/backup + mode: 0400 + - source: id_ed25519 target: /root/.ssh/id_ed25519 mode: 0400 - source: borg_key target: /app/borg/key - environment: - BORG_REPO: ssh://user@server/path - BORG_RSH: "ssh -o StrictHostKeyChecking=no" - BORG_PASSPHRASE: ***** - REPO_SYNC_MAX_SIZE: 10737418240 #10GB - - - volumes: - - /home/mf/repos:/mnt/repos - + mode: 0400 configs: backup_script: content: | - source loadenv + + #!/bin/bash set -e - - # while true; do - # sleep 5 - # done - SCRIPT_START_TIME=$$(date +%s) - - borg create $${BORG_REPO}::repos-$$(date +%Y%m%d%H%M%S) /mnt/backup - - #cleanup - borg prune -v --list --keep-daily=10 --keep-weekly=7 --keep-monthly=-1 $${BORG_REPO} --glob-archives='backup*' - borg compact $${BORG_REPO} + borg create $${BORG_REPO}::backup-$$(date +%Y%m%d%H%M%S) /backup + borg prune -v --list --keep-daily=10 --keep-weekly=7 --keep-monthly=-1 $${BORG_REPO} --glob-archives='backup-*' + # check repo size - REPO_SIZE_IN_BYTES=$$(remote-connect du -b "$$SSH_FOLDER" -d 0 | awk '{print $$1}') + REPO_SIZE_IN_BYTES="$$(remote-get-folder-size)" echo "Repository size: $$((REPO_SIZE_IN_BYTES / 1024 / 1024)) MB" - echo "Repository max size: $$((REPO_SYNC_MAX_SIZE / 1024 / 1024)) MB" - if [ $$REPO_SIZE_IN_BYTES -gt $$REPO_SYNC_MAX_SIZE ]; then \ - echo "ERROR: Repository size exceeds $$REPO_SYNC_MAX_SIZE"; - exit 1; - else - # Repository size is within limits for offsite sync - # ssh to backup server and enforce rclone to onedrive - remote-connect "rclone sync $$SSH_FOLDER xxxxx:.backups/xxxxxx" && \ - SCRIPT_DURATION=$$(($(date +%s) - SCRIPT_START_TIME)) && \ - echo "INFO: Finished Backup (offsite) ($((SCRIPT_DURATION / 60 / 60)):$$((SCRIPT_DURATION / 60)):$$((SCRIPT_DURATION % 60))) " - fi - #outputs info - borg info ${BORG_REPO} + if [ $$REPO_SIZE_IN_BYTES -gt $$REPO_SYNC_MAX_SIZE ]; then \ + echo "ERROR: Repository size exceeds $$REPO_SYNC_MAX_SIZE"; + exit 1; + else + # Repository size is within limits for offsite sync + # ssh to backup server and enforce rclone to offsite + remote-connect "rclone sync $$SSH_FOLDER $$OFFSITE_TARGET_FOLDER --progress" && \ + echo "INFO: Backup offsite sync Finished.($$(date -u -d "@$$(($$(date +%s) - SCRIPT_START_TIME))" +%H:%M:%S))" + fi exit 0 - id_ed25519: - content: | - -----BEGIN OPENSSH PRIVATE KEY----- - `*****************************´ - -----END OPENSSH PRIVATE KEY----- + file: ~/.ssh/id_ed25519 borg_key: content: | - BORG_KEY ****** - *************** + ${BORG_KEY} +``` + +## Using binding volumes + +Creates folder ./backup-scripts +And file ./backup-scripts/backup. + +```yaml +services: + borg: + image: git.limbosolutions.com/kb/borg-backup:latest + + environment: + - BORG_REPO=${BORG_REPO} + - BORG_RSH=${BORG_RSH} + - BORG_PASSPHRASE="${BORG_PASSPHRASE}" + - MODE=SCRIPT + volumes: + - "./backup-scripts:/app/backup-scripts" + - "/home/mf/repos:/backup/repos" + + configs: + - source: id_ed25519 + target: /root/.ssh/id_ed25519 + mode: 0400 + - source: borg_key + target: /app/borg/key + mode: 0400 +configs: + + id_ed25519: + file: ~/.ssh/id_ed25519 + + borg_key: + content: | + ${BORG_KEY} + ``` ### dev +For development environment and testing this docker compose files. + ``` bash +BUILD="" + +# uncomment do force build +BUILD="--build" + +if [ ! -f ./docker-compose.dev.local.yaml ]; then + touch ./docker-compose.dev.local.yaml +EOF +fi + docker compose \ --project-name borg-backup-dev \ -f docker-compose.dev.yaml \ -f docker-compose.dev.local.yaml \ -up -``` - -Force Build: - -``` bash -docker compose \ ---project-name borg-backup-dev \ --f docker-compose.dev.yaml \ --f docker-compose.dev.local.yaml \ -up --build +up $BUILD ``` diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index e874d4a..4f7f74d 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -1,35 +1,35 @@ services: - borg: + borg-dev: tty: true stdin_open: true - # entrypoint: ["bash"] + build: dockerfile: docker/Dockerfile context: . - environment: - - BORG_REPO=??????? - - BORG_RSH="ssh -o StrictHostKeyChecking=no" - - BORG_PASSPHRASE=???? + environment: + - BORG_REPO=${BORG_REPO} + - BORG_RSH=${BORG_RSH} + - BORG_PASSPHRASE="${BORG_PASSPHRASE}" + - MODE=SCRIPT # Valid modes are: BORG, SCRIPT, SHELL, default is BORG configs: + - source: id_ed25519 target: /root/.ssh/id_ed25519 + mode: 0400 - source: borg_key target: /app/borg/key - + mode: 0400 volumes: - ./docker/app/scripts:/app/scripts + - ./docker/dev-backup-scripts:/app/backup-scripts configs: - + + + id_ed25519: - content: | - -----BEGIN OPENSSH PRIVATE KEY----- - ??????? - ??????? - -----END OPENSSH PRIVATE KEY----- + file: ~/.ssh/id_ed25519 borg_key: content: | - BORG_KEY ??????? - ???????????????? - ???????????????? \ No newline at end of file + ${BORG_KEY} \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 0478bd9..cc7a26c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,25 +1,48 @@ FROM alpine:latest -# Install BorgBackup and OpenSSH client +# Installs +# - BorgBackup +# - OpenSSH client +RUN echo "Installing packages." RUN apk update && apk add --no-cache \ borgbackup \ openssh \ bash \ tzdata +RUN echo "Copying app." COPY ./docker/app /app -RUN chmod +x /app/scripts -R + + + +# having some problems if app/scripts files not set with executable permissions +# this is not working, so please confirm files are executable on host building docker file +# setting app owner. +# RUN chown -R root:root /app +#RUN echo "Setting app permissions" +#RUN chown -R root:root /app/scripts && find /app/scripts -type f -exec chmod +x {} \; +# use this entrypoint to verify final permissions on container +#ENTRYPOINT ["ls", "-lah", "/app/scripts"] + + +RUN echo "creating symbolic links to app/scripts." RUN ln -s /app/scripts/loadenv /usr/local/bin/loadenv RUN ln -s /app/scripts/remote-connect /usr/local/bin/remote-connect -RUN ln -s /app/scripts/remote-connect /usr/local/bin/remote-get-folder-size +RUN ln -s /app/scripts/remote-get-folder-size /usr/local/bin/remote-get-folder-size -#RUN addgroup -g 1001 -S borg && adduser -u 1001 -S borg -G borg -#USER borg -#WORKDIR /app +RUN echo "creating .ssh folder." +RUN mkdir /root/.ssh +RUN echo "setting .ssh folder permissions." +RUN chmod 700 /root/.ssh + +# for files inside correct permission is chmod 600 /root/.ssh/key + +RUN echo "Setting loadenv to bashrc and bash_profile" RUN echo "source /usr/local/bin/loadenv" > /root/.bash_profile RUN echo "source /usr/local/bin/loadenv" > /root/.bashrc -ENTRYPOINT ["bash" , "-c", "borg $0"] +RUN echo "setting entrypoint." +ENTRYPOINT ["/app/scripts/entrypoint.sh"] -CMD ["--help"] \ No newline at end of file +#CMD ["--help"] \ No newline at end of file diff --git a/docker/app/README.md b/docker/app/README.md new file mode 100644 index 0000000..ade21f2 --- /dev/null +++ b/docker/app/README.md @@ -0,0 +1,9 @@ +# borg backup helper + +for mode SCRIPT, file /app/backup-scripts/backup must be created. + +- Docker Compose Configs Sections (check repo readme file for more information); +- binding mounts (Ex: ./backup-scripts:/app/backup-scripts); +- kubernetes secrets, configSections or even volumes; + +The file backup will be automatically executed if exists. diff --git a/docker/app/scripts/entrypoint.sh b/docker/app/scripts/entrypoint.sh new file mode 100755 index 0000000..a831bb4 --- /dev/null +++ b/docker/app/scripts/entrypoint.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +source /app/scripts/loadenv + +#fix if its an string instead of an array +IFS=' ' read -r -a rsh_parts <<< "$BORG_RSH" +export BORG_RSH="${rsh_parts[@]}" + +case "$MODE" in + + BORG) + echo "executing borg cli" + borg "${@:1}" # Forward all arguments except $0 to another CLI so first argument (the name of this scripts don't passthrough)" + ;; + + SCRIPT) + # check if file exists + if [ ! -f /app/backup-scripts/backup ]; then + cat /app/README.md + exit 1 + fi + echo "Executing: backup script with arguments: ${@:1} " + cd /app/backup-scripts && bash backup "${@:1}" + ;; + + SHELL) + echo "Executing: bash script." + bash -c "${@:1}" + ;; + + *) + echo "Unknown mode: $MODE." + echo "Valid modes are: BORG, SCRIPT, SHELL" + exit 1 + ;; +esac \ No newline at end of file diff --git a/docker/app/scripts/loadenv b/docker/app/scripts/loadenv index c28b179..19b3497 100755 --- a/docker/app/scripts/loadenv +++ b/docker/app/scripts/loadenv @@ -25,8 +25,7 @@ fi export SSH_COMMAND="$SSH_COMMAND $SSH_CONNECTION" - - + : "${MODE:=BORG}" # Set default if MODE is unset to borg cli diff --git a/docker/app/scripts/remote-get-folder-size b/docker/app/scripts/remote-get-folder-size index d02b5c6..10525ad 100755 --- a/docker/app/scripts/remote-get-folder-size +++ b/docker/app/scripts/remote-get-folder-size @@ -1,5 +1,3 @@ #/bin/bash source loadenv -repo_size_bytes=$(remote-connect du -b "$SSH_FOLDER" -d 0) -repo_size_bytes=$(echo "$repo_size_bytes" | awk '{print $1}') -echo "$repo_size_bytes" \ No newline at end of file +echo $(remote-connect du -b "$SSH_FOLDER" -d 0 | awk '{print $1}') \ No newline at end of file