Merge pull request #314 from rafsaf/release-7

Release 7.0

.env.example

@@ -1,9 +1,9 @@
 POSTGRESQL_DB_PG_16=host=localhost port=10016 password=password-_-12!@#%^&*()/;><.,]}{[ user=user-_-12!@#%^&*()/;><.,]}{[ db=database-_-12!@#%^&*()/;><.,]}{[ cron_rule=* * * * *
-MYSQL_DB_TEST_80=host=localhost port=9080 password=password-_-12!@#%^&*()/;><.,]}{[ user=user-_-12!@#%^&*()/;><.,]}{[ db=database-_-12!@#%^&*()/;><.,]}{[ cron_rule=* * * * *
 MARIADB_DB_TEST_1011=host=localhost port=12011 password=password-_-12!@#%^&*()/;><.,]}{[ user=user-_-12!@#%^&*()/;><.,]}{[ db=database-_-12!@#%^&*()/;><.,]}{[ cron_rule=* * * * *
 #SINGLEFILE_MY_TEST_FILE=abs_path=/home/somefolder/somefile cron_rule=* * * * *
 #DIRECTORY_MY_TEST_FOLDER=abs_path=/home/somefolder/someotherfolder cron_rule=* * * * *
 LOG_LEVEL=INFO
 BACKUP_MAX_NUMBER=2
-ZIP_ARCHIVE_PASSWORD=passwordchangeme
-BACKUP_PROVIDER=name=debug
+AGE_RECIPIENTS=age15taqgmuey3k678djxfh4xsgt4f0e8qv7kp75gj2d8gmq9hf9d4ysj07r4k
+BACKUP_PROVIDER=name=debug
+FAKE_GCS_PORT=4443

.github/PULL_REQUEST_TEMPLATE.md

@@ -5,13 +5,4 @@ Please mark the following items with an [x] if they apply to your PR.
 Leave the [ ] open if they are not applicable, or if you have not completed the item.
  --->
 
-- [ ] I have added or updated documentation in `/docs` for any user-facing features or additions.
-- [ ] I have ran acceptance tests for `Debug` provider to verify that the change works as expected.
-- [ ] I have ran acceptance tests for `Google Cloud Storage` provider to verify that the change works as expected.
-- [ ] I have ran acceptance tests for `Azure Blob Storage` provider to verify that the change works as expected.
-- [ ] I have ran acceptance tests for `AWS S3` provider to verify that the change works as expected.
-
-<!---
-acceptance tests are defined in CONTRIBUTING.md file
-see https://github.com/rafsaf/ogion/blob/main/CONTRIBUTING.md#acceptance-tests
---->
+- [ ] I have added or updated documentation in `/docs` for any user-facing features or additions.

.github/dependabot.yml

@@ -3,38 +3,20 @@ updates:
   - package-ecosystem: pip
     directory: /
     schedule:
-      interval: weekly
-    open-pull-requests-limit: 10
+      interval: monthly
     allow:
       - dependency-type: "all"
     groups:
-      dev-dependencies:
+      all-dependencies:
         patterns:
           - "*"
-        exclude-patterns:
-          - "croniter"
-          - "google-cloud-storage"
-          - "pydantic"
-          - "boto3"
-          - "pydantic-settings"
-          - "azure-storage-blob"
-          - "azure-identity"
-      main-dependencies:
-        patterns:
-          - "croniter"
-          - "google-cloud-storage"
-          - "pydantic"
-          - "boto3"
-          - "pydantic-settings"
-          - "azure-storage-blob"
-          - "azure-identity"
 
   - package-ecosystem: github-actions
     directory: /
     schedule:
-      interval: weekly
+      interval: monthly
 
   - package-ecosystem: docker
     directory: docker/
     schedule:
-      interval: weekly
+      interval: monthly

.github/workflows/dev_build.yml

@@ -28,12 +28,35 @@ jobs:
           username: ${{ secrets.DOCKER_USER }}
           password: ${{ secrets.DOCKER_PASS }}
 
+      - name: Cache
+        uses: actions/cache@v4
+        id: cache
+        with:
+          path: |
+            var-cache-pip
+            var-cache-apt
+            var-lib-apt
+          key: ${{ runner.os }}-cache-${{ hashFiles('docker/Dockerfile') }}
+
+      - name: Inject cache into docker
+        uses: reproducible-containers/[email protected]
+        with:
+          cache-map: |
+            {
+              "var-cache-pip": "/var/cache/pip",
+              "var-cache-apt": "/var/cache/apt",
+              "var-lib-apt": "/var/lib/apt"
+            }
+          skip-extraction: ${{ steps.cache.outputs.cache-hit }}
+
       - name: Build and push ogion image
         uses: docker/build-push-action@v6
         with:
           file: docker/Dockerfile
           context: .
           target: build
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
           push: true
           tags: rafsaf/ogion:dev
           platforms: linux/amd64,linux/arm64

.github/workflows/release_build.yml

@@ -26,12 +26,35 @@ jobs:
           username: ${{ secrets.DOCKER_USER }}
           password: ${{ secrets.DOCKER_PASS }}
 
+      - name: Cache
+        uses: actions/cache@v4
+        id: cache
+        with:
+          path: |
+            var-cache-pip
+            var-cache-apt
+            var-lib-apt
+          key: ${{ runner.os }}-cache-${{ hashFiles('docker/Dockerfile') }}
+
+      - name: Inject cache into docker
+        uses: reproducible-containers/[email protected]
+        with:
+          cache-map: |
+            {
+              "var-cache-pip": "/var/cache/pip",
+              "var-cache-apt": "/var/cache/apt",
+              "var-lib-apt": "/var/lib/apt"
+            }
+          skip-extraction: ${{ steps.cache.outputs.cache-hit }}
+
       - name: Build and push ogion image
         uses: docker/build-push-action@v6
         with:
           file: docker/Dockerfile
           context: .
           target: build
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
           push: true
           tags: rafsaf/ogion:${{ github.ref_name }},rafsaf/ogion:latest
           platforms: linux/amd64,linux/arm64

.github/workflows/tests.yml

@@ -29,7 +29,43 @@ jobs:
         uses: docker/setup-buildx-action@v3
 
       - name: Download and start db containers
-        run: make docker_dbs_setup_up
+        run: |
+          touch .env
+          make docker_setup_up
+
+      - name: Cache
+        uses: actions/cache@v4
+        id: cache
+        with:
+          path: |
+            var-cache-pip
+            var-cache-apt
+            var-lib-apt
+          key: ${{ runner.os }}-cache-${{ hashFiles('docker/Dockerfile') }}
+
+      - name: Inject cache into docker
+        uses: reproducible-containers/[email protected]
+        with:
+          cache-map: |
+            {
+              "var-cache-pip": "/var/cache/pip",
+              "var-cache-apt": "/var/cache/apt",
+              "var-lib-apt": "/var/lib/apt"
+            }
+          skip-extraction: ${{ steps.cache.outputs.cache-hit }}
+
+      - name: Build and cache images
+        uses: docker/build-push-action@v6
+        with:
+          file: docker/Dockerfile
+          context: .
+          target: tests
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          push: false
+          tags: rafsaf/ogion:dev
+          load: true
+          platforms: linux/${{ matrix.arch }}
 
       - name: Build image and run tests in container
         env:

.github/workflows/type_check.yml

@@ -19,7 +19,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.12.2"
+          python-version: "3.13.0"
 
       - name: Install Poetry
         uses: snok/install-poetry@v1
@@ -32,7 +32,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: .venv
-          key: venv-${{ runner.os }}-python3.12.2-${{ hashFiles('poetry.lock') }}
+          key: venv-${{ runner.os }}-python3.13.0-${{ hashFiles('poetry.lock') }}
 
       - name: Install dependencies and actiavte virtualenv
         if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'

.github/workflows/update_compose_dbs.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.12.2"
+          python-version: "3.13.0"
 
       - name: Install Poetry
         uses: snok/install-poetry@v1
@@ -29,7 +29,7 @@ jobs:
         uses: actions/cache@v4
         with:
           path: .venv
-          key: venv-${{ runner.os }}-python3.12.2-${{ hashFiles('poetry.lock') }}
+          key: venv-${{ runner.os }}-python3.13.0-${{ hashFiles('poetry.lock') }}
 
       - name: Install dependencies and actiavte virtualenv
         if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'

.gitignore

@@ -1,3 +1,6 @@
+# age debug key
+key.txt
+
 # ogion
 data/*
 conf/*

.vscode/settings.json

@@ -1,6 +1,5 @@
 {
   "[python]": {
-    "editor.defaultFormatter": "ms-python.black-formatter"
-  },
-  "python.analysis.extraPaths": ["./ogion/tools"]
+    "editor.defaultFormatter": "charliermarsh.ruff"
+  }
 }

CONTRIBUTING.md

@@ -15,11 +15,11 @@ Both upload providers and backup targets were created with possibility to easly
 
 ## Project requirements
 
-- Python 3.12.
+- Python 3.13.
 - Poetry [https://python-poetry.org/](https://python-poetry.org/).
 - Docker and docker compose plugin [https://docs.docker.com/get-docker/](https://docs.docker.com/get-docker/).
 - Debian/Ubuntu are known to work.
-- To work with databases, `postgresql-client` and `mariadb-client` packages installed on your system.
+- To work with databases natively, `postgresql-client` and `mariadb-client` packages installed on your system, there are scripts in `scripts` that can help.
 
 ## Setup steps
 
@@ -39,7 +39,7 @@ Both upload providers and backup targets were created with possibility to easly
 
 5. Setup databases
 
-   `make docker_dbs_setup_up`
+   `make docker_setup_up`
 
    Note, file `docker/docker-compose.dbs.yml` is automatically updated weekly keeping always latest tag per release cycle and adding new releases and removing those after EOL.
 
@@ -73,30 +73,25 @@ Note, exactly above is being run inside runners in tests github action.
 
 ## Acceptance tests
 
-The tests folder includes tests for cloud providers integrations, but for obvious reasons this cannot replace end-to-end tests with **real** credentials to storage in cloud and eventually costs involved, proceed with caution!
+The tests folder includes tests for cloud providers integrations, including fake gcs server, minio instance and azurite, but for obvious reasons this is only 99% replacement for full end-to-end tests with **real** credentials to storage in cloud and eventually costs involved, proceed with caution!
 
 For `Debug` provider acceptance tests, existing `.env.example` can be ok for `.env` file. To test other providers, you will need to change `BACKUP_PROVIDER` environment variable accordingly.
 
 Then `make acceptance_tests_amd64` (or even for arm64 `make acceptance_tests_arm64`) will build docker image target `build` and your local `.env` file, then fire single backup to upload provider and exit.
 
 ## Coding conventions
 
-We expect that all code contributions have been formatted using `black`. You can
-run `black .` to format your code.
+We expect that all code contributions have been formatted using `ruff format`. You can
+run `ruff format .` to format your code.
 
-The project use `ruff` for style, isort rules etc. This is also included in `pre-commit` rules, but you can use `ruff check .`
+The project use `ruff` for style, isort rules etc. This is also included in `pre-commit` rules, but you can use `ruff check . --fix`
 
 We also expect passing `mypy` static code analysis, that can be run with `mypy .`
 
 ## Commit message conventions
 
 There is no dedicated commit message convention, just use descriptive commit messages and useful branch/PR names.
 
-## Submitting changes
-
-Please create a new PR against the `main` branch which must be based on the
-project's [pull request template](.github/PULL_REQUEST_TEMPLATE.md).
-
 ## Releasing
 
 On every push to `main` branch, there is automated build that will create `dev` image tag. Then usually after at minimum 1-2 days after it is propagated to some real world ogion instances and working just fine, release is made manually using GitHub releases with autogenerated changelog. After new release and tag is created, for example `4.0`, docker image with the same tag is build and uploaded to dockerhub.

Makefile

@@ -1,40 +1,30 @@
 # to run tests with arm64 see https://docs.docker.com/build/building/multi-platform/
 export OGION_ARCH ?= amd64
 
-ogion/bin/7zip/amd64/7zzs:
-	rm -rf ogion/bin/7zip/amd64
-	mkdir -p ogion/bin/7zip/amd64
-	cd ogion/bin/7zip/amd64 && \
-	wget --quiet "https://www.7-zip.org/a/7z2301-linux-x64.tar.xz" && \
-	tar -xf "7z2301-linux-x64.tar.xz" && \
-	rm -f "7z2301-linux-x64.tar.xz" && \
-	rm -rf MANUAL
+ifdef CI
+BUILD := 
+else
+BUILD := "--build"
+endif
 
-ogion/bin/7zip/arm64/7zzs:
-	rm -rf ogion/bin/7zip/arm64
-	mkdir -p ogion/bin/7zip/arm64
-	cd ogion/bin/7zip/arm64 && \
-	wget --quiet "https://www.7-zip.org/a/7z2301-linux-arm64.tar.xz" && \
-	tar -xf "7z2301-linux-arm64.tar.xz" && \
-	rm -f "7z2301-linux-arm64.tar.xz" && \
-	rm -rf MANUAL
+.PHONY: docker_setup_up
+docker_setup_up:
+	docker compose -f docker/docker-compose.yml up -d gcs minio azurite
+	docker compose -f docker/docker-compose.dbs.yml up -d
 
-.PHONY: docker_dbs_setup_up
-docker_dbs_setup_up:
-	docker compose -f docker/docker-compose.dbs.yml up -d --remove-orphans
-
-.PHONY: docker_dbs_setup_down
-docker_dbs_setup_down:
-	docker compose -f docker/docker-compose.dbs.yml down --remove-orphans
+.PHONY: docker_setup_down
+docker_setup_down:
+	docker compose -f docker/docker-compose.yml down
+	docker compose -f docker/docker-compose.dbs.yml down
 
 .PHONY: unit_tests
 unit_tests:
-	$(MAKE) docker_dbs_setup_up
-	docker compose -f docker/docker-compose.yml run --rm --build ogion_unit_tests 
+	$(MAKE) docker_setup_up
+	docker compose -f docker/docker-compose.yml run --rm $(BUILD) ogion_unit_tests
 
 .PHONY: acceptance_tests
 acceptance_tests:
-	$(MAKE) docker_dbs_setup_up
+	$(MAKE) docker_setup_up
 	docker compose -f docker/docker-compose.yml run --rm --build ogion_acceptance_tests
 
 .PHONY: update_compose_db_file

README.md

@@ -1,5 +1,5 @@
 [![License](https://img.shields.io/github/license/rafsaf/ogion)](https://github.com/rafsaf/ogion/blob/main/LICENSE)
-[![Python 3.12](https://img.shields.io/badge/python-3.12-blue)](https://docs.python.org/3/whatsnew/3.12.html)
+[![Python 3.13](https://img.shields.io/badge/python-3.13-blue)](https://docs.python.org/3/whatsnew/3.13.html)
 [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
 [![Tests](https://github.com/rafsaf/ogion/actions/workflows/tests.yml/badge.svg)](https://github.com/rafsaf/ogion/actions/workflows/tests.yml)
 [![Type check](https://github.com/rafsaf/ogion/actions/workflows/type_check.yml/badge.svg)](https://github.com/rafsaf/ogion/actions/workflows/type_check.yml)
@@ -11,7 +11,7 @@
 
 A tool for performing scheduled database backups and transferring encrypted data to secure public clouds, for home labs, hobby projects, etc., in environments such as k8s, docker, vms.
 
-Backups are in `zip` format using [7-zip](https://www.7-zip.org/), with strong AES-256 encryption under the hood.
+Backups are in `age` format using [age](https://github.com/FiloSottile/age), with strong encryption under the hood. Why age? it's modern replacement for GnuPG, available for most architectures and systems.
 
 ## Documentation
 
@@ -20,8 +20,8 @@ Backups are in `zip` format using [7-zip](https://www.7-zip.org/), with strong A
 ## Supported backup targets
 
 - PostgreSQL ([all currently supported versions](https://endoflife.date/postgresql))
-- MySQL ([all currently supported versions](https://endoflife.date/mysql))
 - MariaDB ([all currently supported versions](https://endoflife.date/mariadb))
+- MySQL ([currently supported versions after 8.0](https://endoflife.date/mysql), version 8.0 does not work with current mariadb-client)
 - Single file
 - Directory
 
@@ -66,7 +66,7 @@ services:
     image: rafsaf/ogion:latest
     environment:
       - POSTGRESQL_PG16=host=db password=pwd cron_rule=0 0 5 * * port=5432
-      - ZIP_ARCHIVE_PASSWORD=change_me
+      - AGE_RECIPIENTS=age1q5g88krfjgty48thtctz22h5ja85grufdm0jly3wll6pr9f30qsszmxzm2
       - BACKUP_PROVIDER=name=debug
 ```