Add dev env

.env.test

@@ -0,0 +1,19 @@
+API_URL=http://pyro_api:5050
+CAM_USER=my_dummy_login
+CAM_PWD=my_dummy_pwd
+
+LOKI_URL=http://my-loki-service.com
+PROMTAIL_DEVICE_SCOPE=tower_scope
+PROMTAIL_DEVICE_NAME=tower_name
+
+SUPERADMIN_LOGIN=dummy_login
+SUPERADMIN_PWD=dummy&P@ssw0rd!
+
+POSTGRES_USER=dummy_pg_user
+POSTGRES_PASSWORD=dummy_pg_pwd
+POSTGRES_DB=dummy_pg_db
+
+S3_ENDPOINT_URL=http://localstack:4566
+S3_ACCESS_KEY=fake
+S3_SECRET_KEY=fake
+S3_REGION=us-east-1

.github/workflows/docker-compose.yaml

@@ -0,0 +1,60 @@
+name: Docker Compose Up and Health Check
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+  schedule:
+    - cron: "0 0 * * *"
+
+jobs:
+  docker-compose-up:
+    name: Docker Compose Up
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python: [3.9]
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Wait for containers to start
+      run: make build
+
+    - name: Run docker-compose up -d
+      run: make run
+
+      # Wait for containers to start up (adjust the sleep time as needed)
+    - name: Wait for containers to start
+      run: sleep 120
+
+    - name: Check container health
+      run: docker compose ps --filter "status=unhealthy" --quiet
+      continue-on-error: true
+      id: check_health
+
+    - name: Print container health status
+      run: |-
+        if [ -z "${{ steps.check_health.outputs.stdout }}" ]; then
+          echo "All containers are healthy"
+          exit 0
+        else
+          echo "Some containers are unhealthy"
+          exit 1
+        fi
+
+    - name: load tests requirements
+      run: pip install -r tests/requirements.txt
+
+    - name: launch tests
+      run: pytest tests/*
+
+    - name: Capture logs
+      run: docker compose logs --tail=40 > logs.txt
+
+    - name: Show logs
+      run: cat logs.txt
+
+    - name: Filter logs
+      run: cat logs.txt | grep -i -E "error|exception" | grep -v -i "hypercorn.error" || true

.github/workflows/pre-commit.yaml

@@ -0,0 +1,22 @@
+name: pre-commit
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  pre-commit:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python: [3.9]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python }}
+        architecture: x64
+    - uses: pre-commit/[email protected]

.gitignore

@@ -1,3 +1,7 @@
+*.onnx
+*.jpg
+data/images/
+*.pem
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -26,7 +30,7 @@ share/python-wheels/
 .installed.cfg
 *.egg
 MANIFEST
-
+venv_*
 # PyInstaller
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
@@ -102,8 +106,8 @@ celerybeat.pid
 *.sage.py
 
 # Environments
-.env
 .venv
+.env
 env/
 venv/
 ENV/
@@ -127,3 +131,10 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+model_metadata.json
+metadata.yaml
+model_ncnn.py
+model.ncnn.bin
+model.ncnn.param
+*.zip

.pre-commit-config.yaml

@@ -0,0 +1,44 @@
+default_language_version:
+  python: python3.9
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.4.0
+  hooks:
+  - id: check-yaml
+    exclude: .conda
+  - id: check-toml
+  - id: check-added-large-files
+  - id: end-of-file-fixer
+  - id: trailing-whitespace
+  - id: check-ast
+  - id: check-json
+  - id: check-merge-conflict
+  - id: no-commit-to-branch
+    args: [--branch, main]
+  - id: debug-statements
+    language_version: python3
+- repo: https://github.com/psf/black-pre-commit-mirror
+  rev: 23.3.0
+  hooks:
+  - id: black
+    language_version: python3.9
+- repo: https://github.com/charliermarsh/ruff-pre-commit
+  rev: v0.0.290
+  hooks:
+  - id: ruff
+    args:
+    - --fix
+- repo: https://github.com/Lucas-C/pre-commit-hooks-safety
+  rev: v1.3.1
+  hooks:
+  - id: python-safety-dependencies-check
+
+- repo: https://github.com/hadolint/hadolint
+  rev: v2.12.0
+  hooks:
+  - id: hadolint-docker
+    name: Lint Dockerfiles (Docker image)
+    description: Runs hadolint Docker image to lint Dockerfiles
+    language: docker_image
+    types: [dockerfile]
+    entry: ghcr.io/hadolint/hadolint hadolint

Makefile

@@ -0,0 +1,37 @@
+# Build the docker images contained in this repo
+
+build:
+	docker build -f containers/init_script/Dockerfile -t pyronear/pyro-api-init:latest containers/init_script/
+	docker build -f containers/dev_reolink/Dockerfile -t pyronear/dev-reolink:latest containers/dev_reolink/
+	docker build -f containers/reolink_dev2/Dockerfile -t pyronear/reolink-dev2:latest containers/reolink_dev2/
+
+build-external:
+	cd ../pyro-api/; make build
+	cd ../pyro-engine/; make build-lib
+	cd ../pyro-engine/; make build-app
+	cd ../pyro-platform/; make build
+
+
+build-all: build build-external
+
+run-api:
+	cp .env.test .env
+	docker compose up -d
+
+run-engine:
+	cp .env.test .env
+	docker compose --profile engine up -d
+
+run-etl:
+	cp .env.test .env
+	docker compose --profile etl up -d
+
+run:
+	cp .env.test .env
+	docker compose --profile front --profile engine up -d
+
+stop:
+	docker compose --profile front --profile engine --profile etl down
+
+test:
+	pytest -s tests/*

README.md

@@ -1,82 +1,58 @@
-# pyro-devops
+**README**
 
-Deployment and infrastructure management
-
-
-
-## Getting started
-
-## Structure
-
-The file docker-swarm.yml is used for the docker swarm
-The folder nginx is a demo for a image of a reverse proxy with nginx 
+This Docker Compose configuration sets up a development environment for Pyronear's API along with supporting services like a PostgreSQL database, LocalStack for S3 emulation, Pyro Engine, and Promtail for log shipping.
 
 ### Prerequisites
+- Docker and Docker Compose installed on your system.
+- Precommit hook installed on this repo
 
-
-- Docker swarm
-
-
 
-### Installation
+### Services
+1. **pyro-api**: Runs the Pyronear API using uvicorn.
+2. **db**: PostgreSQL database for the API.
+3. **localstack**: Emulates AWS S3 using LocalStack.
+4. **pyro-engine**: Pyro Engine service.
+5. **reolink_dev**: a service which imitate a reolink camera by sending back pictures of fire.
+6. **frontend**: our webapp available on the 8085 port.
 
-https://docs.docker.com/get-docker/
+### Usage
 
-https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/
+First you have to create a .env file, a simple copy of the .env.test would be enough.
+    ```
+    cp .env.test .env
+    ```
 
-
-## Security good practice
-https://docs.docker.com/engine/install/linux-postinstall/
-Log your infrastructure and your containers (portainer,...) 
-Run your ssh/administration on a private network (with bastion + vpn)
-https://www.stackrox.com/post/2019/09/docker-security-101/
-AppArmor/ SELinux,failtoban, iptable, waf
-Check your SLA, IT Disastery Recovery process
-Vulnerability assessment and management (VAM)
-Identity and Access Management
+Start the Docker services using the following command:
+    ```
+    make build
+    make run
+    ```
+Then, you will be able to connect to the API thanks to the credentials in the .env file
 
-## Usage
+If you want to launch only the engine and two dev-cameras you can use :
+    ```
+    make run-engine
+    ```
 
+you can check that everyhing is working thanks to the following commands :
+    ```
+    docker logs init
+    docker logs engine
+    ```
 
+### Accessing the API
+Once the services are up and running, you can access the Pyronear API at `http://localhost:5050/docs`.
 
 
-Export the variables/secret in your env file (if you don't have a Vault) 
-```
-export BUCKET_MEDIA_FOLDER=media
-...
-```
-
-If needed build your images (for exemple the mynginx image in the folder nginx) and push it in the local registry
+### Accessing the web-app
+Since Dash can be a bit capricious, you should launch a private window from you browser and access the web app at `http://localhost:8050`
 
+### Cleanup
+To stop and remove the Docker services, run:
 ```
-docker run -d -p 5000:5000 --restart=always --name registry registry:2 #start the local registry
-
-docker build -t pyro/mynginx .
-
-docker image tag pyro/mynginx localhost:5000/mynginx
-
-docker push localhost:5000/mynginx:latest
-
-docker pull localhost:5000/mynginx
+make stop
 ```
 
-and after deploy your docker swarm
-```
-
-docker stack deploy -c docker-swarm.yml my_node
-
-```
-
-You can check that the service is running with
-
-
-
-```
-
-docker service ls
-
-docker ps
-
-docker service logs xxxxxx
+### More images in the Reoling Dev Camera
 
-```
+you need to create a directory data/images before launching the env, with the images inside !

containers/dev_reolink/Dockerfile

@@ -0,0 +1,23 @@
+# Use the official Python image as the base image
+FROM python:3.9-slim
+
+# hadolint ignore=DL3008
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends curl \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set the working directory in the container
+WORKDIR /app
+
+# Install dependencies
+RUN pip install --no-cache-dir flask==3.0.3
+
+# Copy the Flask server files into the container
+COPY reolinkcamera.py /app/
+
+# Expose port 5000 for Flask server
+EXPOSE 443
+
+# Command to run the Flask server
+CMD ["python", "/app/reolinkcamera.py"]