refactor and streamline repository

.dockerignore

@@ -1,6 +1,6 @@
 wis2box-data
-tests/data/public
-tests/data/incoming
+wis2box-management/tests/data/public
+wis2box-management/tests/data/incoming
 docs
 examples
 .git

.github/workflows/tests-docker.yml

@@ -13,6 +13,7 @@ jobs:
       with:
         python-version: 3.8
     - name: install requirements 📦
+      working-directory: ./wis2box-management
       run: |
         pip3 install -r requirements-dev.txt
         python3 setup.py install
@@ -22,11 +23,12 @@ jobs:
         docker-compose version
         python3 -V
     - name: run unit tests ⚙️
+      working-directory: ./wis2box-management
       run: |
         pytest tests/unit
     - name: setup wis2box configuration
       run: |
-        cp tests/test.env dev.env
+        cp wis2box-management/tests/test.env dev.env
         cat dev.env
         python3 wis2box-ctl.py config
     - name: build wis2box
@@ -37,7 +39,7 @@ jobs:
       run: |
         python3 wis2box-ctl.py start
         python3 wis2box-ctl.py status -a
-        docker logs wis2box
+        docker logs wis2box-management
     - name: setup wis2box ⚙️
       run: |
         sleep 5
@@ -73,19 +75,21 @@ jobs:
       run: |
         python3 wis2box-ctl.py execute wis2box metadata station sync /data/wis2box/metadata/station/station_list.csv
     - name: run integration tests ⚙️
+      working-directory: ./wis2box-management
       run: |
         sleep 5
         pytest -s tests/integration
     - name: run flake8 ⚙️
+      working-directory: ./wis2box-management
       run: |
         find . -type f -name "*.py" | xargs flake8
     - name: build docs ⚙️
       uses: ammaraskar/sphinx-action@master
       with:
-        pre-build-command: "apt-get install -y pandoc && cp requirements-dev.txt docs/requirements.txt"
+        pre-build-command: "apt-get install -y pandoc && cp wis2box-management/requirements-dev.txt docs/requirements.txt"
         docs-folder: "docs/"
     - name: failed tests 🚩
       if: ${{ failure() }}
       run: |
-        docker-compose --file docker/docker-compose.yml --file docker/docker-compose.override.yml --file docker/docker-compose.monitoring.yml --env-file dev.env --project-name wis2box_project ps
-        docker-compose --file docker/docker-compose.yml --file docker/docker-compose.override.yml --file docker/docker-compose.monitoring.yml --env-file dev.env --project-name wis2box_project logs
+        docker-compose --file docker-compose.yml --file docker-compose.override.yml --file docker-compose.monitoring.yml --env-file dev.env --project-name wis2box_project ps
+        docker-compose --file docker-compose.yml --file docker-compose.override.yml --file docker-compose.monitoring.yml --env-file dev.env --project-name wis2box_project logs

docker/docker-compose.dev.yml → docker-compose.dev.yml

@@ -1,5 +1,5 @@
 services:
-  wis2box:
+  wis2box-management:
     volumes:
       - ${WIS2BOX_HOST_DATADIR}:/data/wis2box:rw
       - ./wis2box/wis2box.cron:/etc/cron.d/wis2box:ro
@@ -8,7 +8,7 @@ services:
 
 #  wis2box-api:
 #    volumes:
-#      - ../../wis2box-api/wis2box_api:/usr/local/lib/python3.8/dist-packages/wis2box_api-0.3.dev0-py3.8.egg/wis2box_api
+#      - ../wis2box-api/wis2box_api:/usr/local/lib/python3.8/dist-packages/wis2box_api-0.3.dev0-py3.8.egg/wis2box_api
 
   wis2box-auth:
     volumes:

docker/docker-compose.monitoring.yml → docker-compose.monitoring.yml

@@ -36,7 +36,7 @@ services:
     container_name: mqtt_metrics_collector
     env_file:
       - default.env
-      - ../dev.env
+      - dev.env
     #image: ghcr.io/wmo-im/wis2box-mqtt-metrics-collector:0.4.0
     build:
       context: ./mqtt_metrics_collector
@@ -59,7 +59,7 @@ services:
     <<: *logging
     container_name: grafana
     env_file:
-      - ../dev.env
+      - dev.env
     image: grafana/grafana-oss:9.0.3
     volumes:
       - ./grafana/dashboards:/etc/grafana/provisioning/dashboards
@@ -81,7 +81,7 @@ services:
       - GF_SERVER_SERVE_FROM_SUB_PATH=true
     ports:
       - 3000:3000
-  wis2box:
+  wis2box-management:
     <<: *logging
   mosquitto:
     <<: *logging

docker/docker-compose.yml → docker-compose.yml

@@ -5,7 +5,7 @@ services:
     restart: always
     env_file:
       - default.env
-      - ../dev.env
+      - dev.env
     depends_on:
       - wis2box-ui
     volumes:
@@ -16,7 +16,7 @@ services:
     image: ghcr.io/wmo-im/wis2box-ui:0.4.0
     env_file:
       - default.env
-      - ../dev.env
+      - dev.env
     depends_on:
       - wis2box-api
 
@@ -25,7 +25,7 @@ services:
     image: ghcr.io/wmo-im/wis2box-api:0.4.0
     env_file:
       - default.env
-      - ../dev.env
+      - dev.env
     depends_on:
       elasticsearch:
         condition: service_healthy
@@ -37,7 +37,7 @@ services:
     image: minio/minio
     env_file:
       - default.env
-      - ../dev.env
+      - dev.env
     command: server --console-address ":9001" /data
     # in a production-setup minio needs to be
     volumes:
@@ -83,19 +83,19 @@ services:
       context: ./mosquitto
     env_file:
       - default.env
-      - ../dev.env
+      - dev.env
 
-  wis2box:
-    container_name: wis2box
+  wis2box-management:
+    container_name: wis2box-management
     #image: ghcr.io/wmo-im/wis2box:0.4.0
     build:
-      context: ..
+      context: ./wis2box-management
       # args:
       #   WIS2BOX_PIP3_EXTRA_PACKAGES: ${WIS2BOX_PIP3_EXTRA_PACKAGES}
     #user: wis2box:wis2box
     env_file:
       - default.env
-      - ../dev.env
+      - dev.env
     volumes:
       - ${WIS2BOX_HOST_DATADIR}:/data/wis2box:rw
     depends_on:
@@ -114,11 +114,11 @@ services:
       context: ./wis2box-auth
     env_file:
       - default.env
-      - ../dev.env
+      - dev.env
     volumes:
       - ${WIS2BOX_HOST_DATADIR}:/data/wis2box:rw
     depends_on:
-      - wis2box
+      - wis2box-management
 
 volumes:
   es-data:

docs/source/administration.rst

@@ -73,8 +73,8 @@ Internal services
 Changing default ports
 ^^^^^^^^^^^^^^^^^^^^^^
 
-The ``docker/docker-compose.override.yml`` file provides definitions on utilized ports.  To change default
-ports, edit ``docker/default.env``  before stopping and starting wis2box for changes to take effect.
+The ``docker-compose.override.yml`` file provides definitions on utilized ports.  To change default
+ports, edit ``default.env``  before stopping and starting wis2box for changes to take effect.
 
 
 MQTT Quality of Service (QoS)

docs/source/conf.py

@@ -44,7 +44,7 @@
 
 # The full version, including alpha/beta/rc tags
 
-file_ = '../wis2box/__init__.py'
+file_ = '../wis2box-management/wis2box/__init__.py'
 filepath = os.path.join(os.path.abspath('..'), file_)
 
 with open(filepath) as fh:

docs/source/configuration.rst

@@ -10,9 +10,9 @@ Environment variables
 ---------------------
 
 wis2box configuration is driven primarily by a small set of environment variables. The runtime
-configuration is defined in the `Env`_ format in a plain text file named ``dev.env`` and ``docker/default.env``.
+configuration is defined in the `Env`_ format in a plain text file named ``dev.env`` and ``default.env``.
 
-Any values set in ``dev.env`` override the default environment variables in ``docker/default.env``. For further / specialized
+Any values set in ``dev.env`` override the default environment variables in ``default.env``. For further / specialized
 configuration, see the sections below.
 
 ``WIS2BOX_HOST_DATADIR``
@@ -160,7 +160,7 @@ A full configuration example can be found below:
 .. literalinclude:: ../../examples/config/wis2box.env
    :language: bash
 
-.. literalinclude:: ../../docker/default.env
+.. literalinclude:: ../../default.env
    :language: bash
 
 Docker Compose
@@ -177,4 +177,4 @@ At this point, you have defined the runtime configuration required to administer
 
 
 .. _`Env`: https://en.wikipedia.org/wiki/Env
-.. _`GitHub`: https://github.com/wmo-im/wis2box/blob/main/docker/default.env
+.. _`GitHub`: https://github.com/wmo-im/wis2box/blob/main/default.env

docs/source/extending-wis2box.rst

@@ -14,7 +14,7 @@ Building your own data plugin
 -----------------------------
 
 The heart of a wis2box data plugin is driven from the ``wis2box.data.base`` abstract base class (ABC)
-located in ``wis2box/data/base.py``. Any wis2box plugin needs to inherit from
+located in ``wis2box-management/wis2box/data/base.py``. Any wis2box plugin needs to inherit from
 ``wis2box.data.base.BaseAbstractData``. A minimal example can be found below:
 
 .. code-block:: python

docs/source/monitoring/grafana.rst

@@ -14,5 +14,5 @@ Go to http://localhost:3000 to see the home dashboard of wis2box once the stack
 
 .. note::
 
-   The dashboard configuration can be found in ``docker/grafana/dashboards/home.json``.
+   The dashboard configuration can be found in ``grafana/dashboards/home.json``.
 

docs/source/running/data-mappings.rst

@@ -6,7 +6,7 @@ Data mappings
 Once a topic hierarchy is defined, it needs to be included in the wis2box data mappings
 configuration. wis2box provides a default data mapping (in YAML format):
 
-.. literalinclude:: ../../../wis2box/resources/data-mappings.yml
+.. literalinclude:: ../../../wis2box-management/wis2box/resources/data-mappings.yml
    :language: yaml
 
 The data mappings are indicated by the ``data`` keyword, with each topic having a separate entry specifying:

docs/source/running/discovery-metadata.rst

@@ -19,7 +19,7 @@ Creating a discovery metadata record in wis2box is as easy as completing a YAML
 leverages the `pygeometa`_ project's `metadata control file (MCF)`_ format. Below is an example MCF file.
 
 
-.. literalinclude:: ../../../tests/data/metadata/discovery/mw-surface-weather-observations.yml
+.. literalinclude:: ../../../wis2box-management/tests/data/metadata/discovery/mw-surface-weather-observations.yml
    :language: yaml
 
 .. note::

docs/source/running/environment.rst

@@ -15,5 +15,5 @@ directory for all data managed in wis2box.
 
 The default enviroment variables are below.
 
-.. literalinclude:: ../../../docker/default.env
+.. literalinclude:: ../../../default.env
    :language: bash

docs/source/services.rst

@@ -115,7 +115,7 @@ for a samba share or FTP server. Key considerations for adding services:
 - Storage buckets can be found at http://minio:9000
 - Elasticsearch indexes can be found at the container/URL ``http://elasticsearch:9200``
 
-Examples of additional services can be found in ``docker/extras``.
+Examples of additional services can be found in ``extras``.
 
 
 .. _`OGC API - Features`: https://ogcapi.ogc.org/features

wis2box-ctl.py

@@ -25,9 +25,9 @@
 import subprocess
 
 DOCKER_COMPOSE_ARGS = """
-    --file docker/docker-compose.yml
-    --file docker/docker-compose.override.yml
-    --file docker/docker-compose.monitoring.yml
+    --file docker-compose.yml
+    --file docker-compose.override.yml
+    --file docker-compose.monitoring.yml
     --env-file dev.env
     --project-name wis2box_project
     """
@@ -67,7 +67,7 @@
     - build [containers]: build all services
     - start [containers]: start system
     - start-dev [containers]: start system in local development mode
-    - login [container]: login to the container (default: wis2box)
+    - login [container]: login to the container (default: wis2box-management)
     - login-root [container]: login to the container as root
     - stop: stop [container] system
     - update: update Docker images
@@ -140,13 +140,13 @@ def make(args) -> None:
     containers = "" if not args.args else ' '.join(args.args)
 
     # if there can be only one, default to wisbox
-    container = "wis2box" if not args.args else ' '.join(args.args)
+    container = "wis2box-management" if not args.args else ' '.join(args.args)
 
     if args.command == "config":
         run(args, split(f'docker-compose {DOCKER_COMPOSE_ARGS} config'))
     elif args.command == "build":
-        run(args, split(
-            f'docker-compose {DOCKER_COMPOSE_ARGS} build {containers}'))
+        print(split( f'docker-compose {DOCKER_COMPOSE_ARGS} build {containers}'))
+        run(args, split( f'docker-compose {DOCKER_COMPOSE_ARGS} build {containers}'))
     elif args.command in ["up", "start", "start-dev"]:
         run(args, split(
             'docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions > /dev/null 2>&1'))
@@ -156,11 +156,11 @@ def make(args) -> None:
             run(args, split(f"docker-compose {DOCKER_COMPOSE_ARGS} start {containers}"))
         else:
             if args.command == 'start-dev':
-                run(args, split(f'docker-compose {DOCKER_COMPOSE_ARGS} --file docker/docker-compose.dev.yml up'))
+                run(args, split(f'docker-compose {DOCKER_COMPOSE_ARGS} --file docker-compose.dev.yml up'))
             else:
                 run(args, split(f'docker-compose {DOCKER_COMPOSE_ARGS} up -d'))
     elif args.command == "execute":
-        run(args, ['docker', 'exec', '-i', 'wis2box', 'sh', '-c', containers])
+        run(args, ['docker', 'exec', '-i', 'wis2box-management', 'sh', '-c', containers])
     elif args.command == "login":
         run(args, split(f'docker exec -it {container} /bin/bash'))
     elif args.command == "login-root":
@@ -206,4 +206,4 @@ def make(args) -> None:
 
 
 if __name__ == "__main__":
-    make(args)
+    make(args)