Merge pull request #212 from FREVA-CLINT/fix-images

fix freva and freva-dev broken images, fix docs and tests CI issues, fix minor typo in API ref

.docker/Dockerfile

@@ -40,7 +40,7 @@ RUN set -x && \
   --shell /bin/bash --disabled-password freva && \
   usermod -aG solr,mysql freva &&\
   if [ "$binder" = "true" ]; then\
-     /opt/conda/bin/conda create -y -q -c conda-forge cftime make bash_kernel cartopy\
+     /opt/conda/bin/conda create -y -q -c conda-forge cftime make bash_kernel curl mysql-common==8.3.0 mysql-libs=8.3.0 dask==2024.8.0 cartopy\
      h5netcdf netcdf4 ffmpeg pip notebook jupyterlab jupyterhub freva -p /opt/evaluation_system &&\
      /opt/evaluation_system/bin/python -m pip install pint-xarray cf_xarray nc-time-axis &&\
      /opt/evaluation_system/bin/python -m ipykernel install --name freva &&\

.docker/docker-entrypoint.sh

@@ -4,27 +4,35 @@ if [ "$VERBOSE" == "yes" ];then
     set -xe
 fi
 source /usr/local/bin/docker-entrypoint.sh
+set -- mariadbd
 # call main bits of the mariadb entrypoint to have DB initialized
-docker_setup_env
-docker_create_db_directories
+docker_setup_env "$@"
+docker_create_db_directories "$@"
 # there's no database, so it needs to be initialized
 if [ -z "$DATABASE_ALREADY_EXISTS" ]; then
-        docker_verify_minimum_env
-
-        docker_mariadb_init "$@"
+    docker_verify_minimum_env "$@"
+    docker_mariadb_init "$@"
 elif _check_if_upgrade_is_needed; then
-        docker_mariadb_upgrade "$@"
+    docker_mariadb_upgrade "$@"
 fi
-mariadbd &
-until healthcheck.sh --connect --innodb_initialized;
+mariadbd --user=${MARIADB_USER} --datadir=${MYSQL_DATA_DIR} --socket=/tmp/mysql.sock --console &
+until mariadb-admin ping --user="${MARIADB_USER}" --password="${MARIADB_PASSWORD}" --socket="/tmp/mysql.sock" --silent;
 do
     echo "waiting for mariadb"
+    echo "${MARIADB_USER} ${MARIADB_PASSWORD}"
     sleep 1
 done
 
 solr start -s ${SOLR_HOME}
 if [ "${IS_BINDER}" = "true" ];then
-    python $EVAL_HOME/ingest_dummy_data.py /mnt/data4freva
+    # wait for solr to be up
+    solr_status_url="http://localhost:8983/solr/admin/cores?action=STATUS"
+    until curl -s "$solr_status_url" > /dev/null; do
+        echo "Waiting for Solr to be available..."
+        sleep 2
+    done
+    echo "Solr is up!"
+    python $EVAL_HOME/ingest_dummy_data.py /mnt/data4frev
     python $EVAL_HOME/dummy_user_data.py
     for i in 1 2 3 4 5; do
         freva plugin dummyplugin the_number=$i

.github/workflows/ci_job.yml

@@ -39,7 +39,7 @@ jobs:
         python-version: [3.7, 3.8, 3.9, "3.10", "3.11"]
     steps:
       -
-        name: Ceckout
+        name: Checkout
         uses: actions/checkout@v4
         with:
           submodules: recursive
@@ -61,11 +61,12 @@ jobs:
       -
         name: Install dependencies on python ${{matrix.python-version}}
         run: |
-          python3 -m pip install -e .[test]
+          conda create -c conda-forge -n tests -y -q python pip cartopy make mysql-common==8.3.0 mysql-libs=8.3.0 mysqlclient
+          conda run -n tests python -m pip install -e .[test] dask==2024.8.0
       -
         name: Running tests for python ${{ matrix.python-version }}
         run: |
-          make test_coverage
+          conda run -n tests make test_coverage
       -
         name: Upload coverage to Codecov
         uses: codecov/codecov-action@v1
@@ -94,7 +95,7 @@ jobs:
       -
         name: Install freva
         run: |
-          conda create -c conda-forge -n docs -y -q freva cartopy make
+          conda create -c conda-forge -n docs -y -q cartopy make mysql-common==8.3.0 mysql-libs=8.3.0 mysqlclient dask==2024.8.0 freva
       -
         name: Set up services
         run: |

dev-environment.yml

@@ -9,7 +9,11 @@ dependencies:
   - mamba
   - pint
   - pip
+  - mysql-common==8.3.0
+  - mysql-libs=8.3.0
+  - mysqlclient
   - pip:
+      - dask==2024.8.0
       - -e .[test]
   - freva
   - pytest

docs/source/api/APIRef.rst

@@ -236,7 +236,7 @@ Searching for datasets locations
             import requests
             response = requests.get(
                 "https://www.freva.dkrz.de/api/databrowser/data_search/freva/file",
-                pramas={"product": "EUR-11", "fs_type": "swift"}
+                params={"product": "EUR-11", "fs_type": "swift"}
             )
             data = list(response.iter_lines(decode_unicode=True))
 
@@ -468,7 +468,7 @@ Searching for metadata
             import requests
             response = requests.get(
                 "https://www.freva.dkrz.de/api/databrowser/metadata_search/freva/file",
-                pramas={"product": "EUR-11"}
+                params={"product": "EUR-11"}
             )
             data = response.json()
 
@@ -678,7 +678,7 @@ Generating an intake-esm catalogue
             import intake
             response = requests.get(
                 "https://www.freva.dkrz.de/api/databrowser/intake_catalogue/freva/file",
-                pramas={"product": "EUR-11"}
+                params={"product": "EUR-11"}
             )
             cat = intake.open_esm_datastore(cat)
 
@@ -834,7 +834,7 @@ Creating zarr endpoints for streaming data
            import intake
            response = requests.get(
                "https://www.freva.dkrz.de/api/databrowser/load/freva",
-               pramas={"dataset": "cmip6-fs"},
+               params={"dataset": "cmip6-fs"},
                headers={"Authorization": "Bearer YOUR_ACCESS_TOKEN"},
                stream=True,
            )