Merge remote-tracking branch 'upstream/master' into surface-job-status

.env-devel

@@ -18,6 +18,12 @@ AGENT_VOLUMES_CLEANUP_S3_PROVIDER=MINIO
 
 API_SERVER_DEV_FEATURES_ENABLED=0
 
+AUTOSCALING_DASK=null
+AUTOSCALING_EC2_ACCESS=null
+AUTOSCALING_EC2_INSTANCES=null
+AUTOSCALING_NODES_MONITORING=null
+AUTOSCALING_POLL_INTERVAL=10
+
 BF_API_KEY=none
 BF_API_SECRET=none
 
@@ -47,6 +53,7 @@ DIRECTOR_V2_DEV_FEATURES_ENABLED=0
 DYNAMIC_SIDECAR_IMAGE=${DOCKER_REGISTRY:-itisfoundation}/dynamic-sidecar:${DOCKER_IMAGE_TAG:-latest}
 DYNAMIC_SIDECAR_LOG_LEVEL=DEBUG
 DYNAMIC_SIDECAR_PROMETHEUS_SERVICE_LABELS={}
+DYNAMIC_SIDECAR_PROMETHEUS_MONITORING_NETWORKS=[]
 
 FUNCTION_SERVICES_AUTHORS='{"UN": {"name": "Unknown", "email": "[email protected]", "affiliation": "unknown"}}'
 
@@ -69,6 +76,7 @@ PAYMENTS_ACCESS_TOKEN_SECRET_KEY=2c0411810565e063309be1457009fb39ce023946f6a354e
 PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT=10000
 PAYMENTS_AUTORECHARGE_DEFAULT_TOP_UP_AMOUNT=100.0
 PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS=100
+PAYMENTS_AUTORECHARGE_ENABLED=1
 PAYMENTS_FAKE_COMPLETION_DELAY_SEC=10
 PAYMENTS_FAKE_COMPLETION=0
 PAYMENTS_GATEWAY_API_SECRET=replace-with-api-secret

.github/codeql/codeql-config.yml

@@ -3,6 +3,7 @@ name: "ospac-simcore CodeQL config"
 disable-default-queries: false
 
 paths:
+  - packages/aws-library/src
   - packages/dask-task-models-library/src
   - packages/models-library/src/models_library
   - packages/postgres-database/src/simcore_postgres_database

.github/workflows/ci-testing-deploy.yml

@@ -50,6 +50,7 @@ jobs:
     runs-on: ubuntu-latest
     # Set job outputs to values from filter step
     outputs:
+      aws-library: ${{ steps.filter.outputs.aws-library }}
       dask-task-models-library: ${{ steps.filter.outputs.dask-task-models-library }}
       models-library: ${{ steps.filter.outputs.models-library }}
       postgres-database: ${{ steps.filter.outputs.postgres-database }}
@@ -87,6 +88,12 @@ jobs:
         id: filter
         with:
           filters: |
+            aws-library:
+              - 'packages/aws-library/**'
+              - 'packages/pytest-simcore/**'
+              - 'services/docker-compose*'
+              - 'scripts/mypy/*'
+              - 'mypy.ini'
             dask-task-models-library:
               - 'packages/dask-task-models-library/**'
               - 'packages/pytest-simcore/**'
@@ -804,6 +811,45 @@ jobs:
         with:
           flags: unittests #optional
 
+  unit-test-aws-library:
+    needs: changes
+    if: ${{ needs.changes.outputs.aws-library == 'true' || github.event_name == 'push' }}
+    timeout-minutes: 18 # if this timeout gets too small, then split the tests
+    name: "[unit] aws-library"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python: ["3.10"]
+        os: [ubuntu-22.04]
+        docker_buildx: [v0.10.4]
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v4
+      - name: setup docker buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          version: ${{ matrix.docker_buildx }}
+          driver: docker-container
+      - name: setup python environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+          cache: "pip"
+          cache-dependency-path: "packages/aws-library/requirements/ci.txt"
+      - name: show system version
+        run: ./ci/helpers/show_system_versions.bash
+      - name: install
+        run: ./ci/github/unit-testing/aws-library.bash install
+      - name: typecheck
+        run: ./ci/github/unit-testing/aws-library.bash typecheck
+      - name: test
+        if: always()
+        run: ./ci/github/unit-testing/aws-library.bash test
+      - uses: codecov/[email protected]
+        with:
+          flags: unittests #optional
+
   unit-test-dask-task-models-library:
     needs: changes
     if: ${{ needs.changes.outputs.dask-task-models-library == 'true' || github.event_name == 'push' }}
@@ -1447,6 +1493,7 @@ jobs:
         unit-test-catalog,
         unit-test-clusters-keeper,
         unit-test-dask-sidecar,
+        unit-test-aws-library,
         unit-test-dask-task-models-library,
         unit-test-datcore-adapter,
         unit-test-director-v2,

.gitignore

@@ -178,5 +178,6 @@ Untitled*
 
 # service settings.schemas.json
 services/**/settings-schema.json
+services/payments/scripts/openapi.json
 
 tests/public-api/osparc_python_wheels/*

.pylintrc

@@ -243,7 +243,7 @@ contextmanager-decorators=contextlib.contextmanager
 # List of members which are set dynamically and missed by pylint inference
 # system, and so shouldn't trigger E1101 when accessed. Python regular
 # expressions are accepted.
-generated-members=
+generated-members=sh
 
 # Tells whether to warn about missing members when the owner of the attribute
 # is inferred to be None.

.vscode/extensions.json

@@ -1,11 +1,17 @@
 {
   "recommendations": [
+    "42Crunch.vscode-openapi",
     "charliermarsh.ruff",
     "eamodio.gitlens",
     "exiasr.hadolint",
+    "ms-azuretools.vscode-docker",
+    "ms-python.black-formatter",
+    "ms-python.pylint",
     "ms-python.python",
+    "ms-vscode.makefile-tools",
     "njpwerner.autodocstring",
     "samuelcolvin.jinjahtml",
     "timonwong.shellcheck",
+    "vscode-icons-team.vscode-icons",
   ]
 }

.vscode/settings.template.json

@@ -1,5 +1,7 @@
 // This is a template. Clone and replace extension ".template.json" by ".json"
 {
+  "autoDocstring.docstringFormat": "pep257",
+
   "editor.tabSize": 2,
   "editor.insertSpaces": true,
   "editor.detectIndentation": false,
@@ -27,10 +29,10 @@
     "**/.git/subtree-cache/**": true,
     "**/node_modules/*/**": true
   },
-  "python.formatting.autopep8Args": [
-    "--max-line-length 140"
-  ],
+  "python.analysis.autoImportCompletions": true,
+  "python.analysis.typeCheckingMode": "basic",
   "python.analysis.extraPaths": [
+    "./packages/aws-library/src",
     "./packages/models-library/src",
     "./packages/postgres-database/src",
     "./packages/postgres-database/tests",
@@ -54,26 +56,15 @@
   "[makefile]": {
     "editor.insertSpaces": false
   },
-  "python.testing.pytestEnabled": true,
-  "autoDocstring.docstringFormat": "pep257",
   "hadolint.hadolintPath": "${workspaceFolder}/scripts/hadolint.bash",
   "hadolint.cliOptions": [],
-  "shellcheck.executablePath": "${workspaceFolder}/scripts/shellcheck.bash",
-  "shellcheck.run": "onSave",
-  "shellcheck.enableQuickFix": true,
-  "python.formatting.provider": "black",
-  "isort.path": [
-    "${workspaceFolder}/.venv/bin/isort"
-  ],
-  "isort.args": [
-    "--settings-path=${workspaceFolder}/.isort.cfg"
-  ],
-  "python.analysis.autoImportCompletions": true,
-  "python.analysis.typeCheckingMode": "basic",
   "ruff.lint.args": [
     "--config=${workspaceFolder}/.ruff.toml"
   ],
   "ruff.path": [
     "${workspaceFolder}/.venv/bin/ruff"
-  ]
+  ],
+  "shellcheck.executablePath": "${workspaceFolder}/scripts/shellcheck.bash",
+  "shellcheck.run": "onSave",
+  "shellcheck.enableQuickFix": true
 }

README.md

@@ -4,7 +4,6 @@
 <img src="https://user-images.githubusercontent.com/32800795/61083844-ff48fb00-a42c-11e9-8e63-fa2d709c8baf.png" width="500">
 </p>
 
-
 <!-- BADGES: LINKS ON CLICK --------------------------------------------------------------->
 [![black_badge]](https://github.com/psf/black)
 [![ci_badge]](https://github.com/ITISFoundation/osparc-simcore/actions/workflows/ci-testing-deploy.yml)
@@ -28,7 +27,6 @@
 [osparc_status]:https://img.shields.io/badge/dynamic/json?label=osparc.io&query=%24.status.description&url=https%3A%2F%2Fstatus.osparc.io%2Fapi%2Fv2%2Fstatus.json
 <!------------------------------------------------------------------------------------------>
 
-
 The SIM-CORE, named **o<sup>2</sup>S<sup>2</sup>PARC** – **O**pen **O**nline **S**imulations for **S**timulating **P**eripheral **A**ctivity to **R**elieve **C**onditions – is one of the three integrative cores of the SPARC program’s Data Resource Center (DRC).
 The aim of o<sup>2</sup>S<sup>2</sup>PARC is to establish a comprehensive, freely accessible, intuitive, and interactive online platform for simulating peripheral nerve system neuromodulation/ stimulation and its impact on organ physiology in a precise and predictive manner.
 To achieve this, the platform will comprise both state-of-the art and highly detailed animal and human anatomical models with realistic tissue property distributions that make it possible to perform simulations ranging from the molecular scale up to the complexity of the human body.
@@ -72,15 +70,14 @@ Services are deployed in two stacks:``simcore-stack`` comprises all core-service
 To build and run:
 
 - git
-- docker
+- [docker](https://docs.docker.com/engine/install/ubuntu/#installation-methods)
 - make >=4.2
 - awk, jq (optional tools within makefiles)
 
 To develop, in addition:
 
-- python 3.10
-- nodejs for client part (this dependency will be deprecated soon)
-- swagger-cli (make sure to have a recent version of nodejs)
+- *python 3.10*: we recommend using the python manager [pyenv](https://brain2life.hashnode.dev/how-to-install-pyenv-python-version-manager-on-ubuntu-2004)
+- *nodejs* for client part: we recommend using the node manager [nvm](https://github.com/nvm-sh/nvm#install--update-script)
 - [vscode] (highly recommended)
 
 To verify current base OS, Docker and Python build versions have a look at:
@@ -135,7 +132,7 @@ To upgrade a single requirement named `fastapi`run:
 
 - [Git release workflow](docs/releasing-workflow-instructions.md)
 - Public [releases](https://github.com/ITISFoundation/osparc-simcore/releases)
-- Production in https://osparc.io
+- Production in <https://osparc.io>
 - [Staging instructions](docs/releasing-workflow-instructions.md#staging-example)
 - [User Manual](https://itisfoundation.github.io/osparc-manual/)
 
@@ -191,6 +188,5 @@ This project is licensed under the terms of the [MIT license](LICENSE).
 </p>
 
 <!-- ADD REFERENCES BELOW AND KEEP THEM IN ALPHABETICAL ORDER -->
-[chocolatey]:https://chocolatey.org/
 [vscode]:https://code.visualstudio.com/
 [WSL2]:https://docs.microsoft.com/en-us/windows/wsl

api/specs/web-server/_auth.py

@@ -266,7 +266,7 @@ async def email_confirmation(code: str):
         },
     },
 )
-async def list_api_keys(code: str):
+async def list_api_keys():
     """lists display names of API keys by this user"""
 
 

api/specs/web-server/_wallets.py

@@ -168,10 +168,10 @@ async def delete_payment_method(
 @router.post(
     "/wallets/{wallet_id}/payments-methods/{payment_method_id}:pay",
     response_model=Envelope[WalletPaymentInitiated],
-    response_description="Payment initialized",
+    response_description="Pay with payment-method",
     status_code=status.HTTP_202_ACCEPTED,
 )
-async def init_payment_with_payment_method(
+async def pay_with_payment_method(
     wallet_id: WalletID, payment_method_id: PaymentMethodID, _body: CreateWalletPayment
 ):
     ...

ci/github/unit-testing/aws-library.bash

@@ -0,0 +1,41 @@
+#!/bin/bash
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+set -o errexit  # abort on nonzero exitstatus
+set -o nounset  # abort on unbound variable
+set -o pipefail # don't hide errors within pipes
+IFS=$'\n\t'
+
+install() {
+  bash ci/helpers/ensure_python_pip.bash
+  make devenv
+  # shellcheck source=/dev/null
+  source .venv/bin/activate
+  pushd packages/aws-library
+  make install-ci
+  popd
+  .venv/bin/pip list --verbose
+}
+
+test() {
+  # shellcheck source=/dev/null
+  source .venv/bin/activate
+  pushd packages/aws-library
+  make tests-ci
+  popd
+}
+
+typecheck() {
+  pushd packages/aws-library
+  make mypy
+  popd
+}
+
+# Check if the function exists (bash specific)
+if declare -f "$1" >/dev/null; then
+  # call arguments verbatim
+  "$@"
+else
+  # Show a helpful error
+  echo "'$1' is not a known function name" >&2
+  exit 1
+fi

docs/env-vars.md

@@ -0,0 +1,10 @@
+# Environment variables management
+
+As a developer you will need to extend the current env vars for a service.
+
+The following rules must be followed:
+
+1. for each service that requires it, add it to the `services/docker-compose.yml` file (such as `MY_VAR=${MY_VAR}`)
+2. add a meaningful default value for development inside `.env-devel` so that developers can work, and that `osparc-simcore` is **self contained**.
+  - **NOTE** if the variable has a default inside the code, put the same value here
+3. inside the repo where devops keep all the secrets follow the instructions to add the new env var

packages/aws-library/Makefile

@@ -0,0 +1,49 @@
+#
+# Targets for DEVELOPMENT of aws Library
+#
+include ../../scripts/common.Makefile
+include ../../scripts/common-package.Makefile
+
+.PHONY: requirements
+requirements: ## compiles pip requirements (.in -> .txt)
+	@$(MAKE_C) requirements reqs
+
+
+.PHONY: install-dev install-prod install-ci
+install-dev install-prod install-ci: _check_venv_active ## install app in development/production or CI mode
+	# installing in $(subst install-,,$@) mode
+	pip-sync requirements/$(subst install-,,$@).txt
+
+
+.PHONY: tests tests-ci
+tests: ## runs unit tests
+	# running unit tests
+	@pytest \
+		--asyncio-mode=auto \
+		--color=yes \
+		--cov-config=../../.coveragerc \
+		--cov-report=term-missing \
+		--cov=aws_library \
+		--durations=10 \
+		--exitfirst \
+		--failed-first \
+		--pdb \
+		-vv \
+		$(CURDIR)/tests
+
+tests-ci: ## runs unit tests
+	# running unit tests
+	@pytest \
+		--asyncio-mode=auto \
+		--color=yes \
+		--cov-append \
+		--cov-config=../../.coveragerc \
+		--cov-report=term-missing \
+		--cov-report=xml \
+		--cov=aws_library \
+		--durations=10 \
+		--log-date-format="%Y-%m-%d %H:%M:%S" \
+    --log-format="%(asctime)s %(levelname)s %(message)s" \
+    --verbose \
+    -m "not heavy_load" \
+		$(CURDIR)/tests

packages/aws-library/README.md

@@ -0,0 +1,22 @@
+# simcore AWS library
+
+Provides a wrapper around AWS python libraries.
+
+Requirements to be compatible with the library:
+
+- only AWS-related code
+
+
+## Installation
+
+```console
+make help
+make install-dev
+```
+
+## Test
+
+```console
+make help
+make test-dev
+```

packages/aws-library/VERSION

@@ -0,0 +1 @@
+0.1.0