Merge pull request ckan#55 from qld-gov-au/QOLDEV-323-ckan-2.10

[QOLDEV-323] pin test container version to preserve Docker Compose

.ahoy.yml

@@ -28,51 +28,59 @@ commands:
     cmd: |
       ahoy line "Project                  : " ${PROJECT}
       ahoy line "Site local URL           : " ${LAGOON_LOCALDEV_URL}
-      ahoy line "DB port on host          : " $(docker port $(docker-compose ps -q postgres) 5432 | cut -d : -f 2)
-      ahoy line "Solr port on host        : " $(docker port $(docker-compose ps -q solr) 8983 | cut -d : -f 2)
+      ahoy line "DB port on host          : " $(docker port $(sh bin/docker-compose.sh ps -q postgres) 5432 | cut -d : -f 2)
+      ahoy line "Solr port on host        : " $(docker port $(sh bin/docker-compose.sh ps -q solr) 8983 | cut -d : -f 2)
       ahoy line "Mailhog URL              : " http://mailhog.docker.amazee.io/
 
   up:
     usage: Build and start Docker containers.
     cmd: |
       ahoy title "Building and starting Docker containers"
-      docker-compose up -d "$@"
+      sh bin/docker-compose.sh up -d "$@"
       echo "Initialising database schema"
       ahoy cli '$APP_DIR/bin/init.sh'
       echo "Waiting for containers to start listening..."
       ahoy cli "dockerize -wait tcp://ckan:5000 -timeout 1m"
-      if docker-compose logs | grep -q "\[Error\]"; then exit 1; fi
-      if docker-compose logs | grep -q "Exception"; then exit 1; fi
+      if sh bin/docker-compose.sh logs | grep -q "\[Error\]"; then exit 1; fi
+      if sh bin/docker-compose.sh logs | grep -q "Exception"; then exit 1; fi
       docker ps -a --filter name=^/${COMPOSE_PROJECT_NAME}_
       export DOCTOR_CHECK_CLI=0
 
   down:
     usage: Stop Docker containers and remove container, images, volumes and networks.
-    cmd: 'if [ -f "docker-compose.yml" ]; then docker-compose down --volumes; fi'
+    cmd: |
+      ahoy title 'Stopping and removing old containers, images, volumes, networks'
+      if [ -f "docker-compose.yml" ]; then sh bin/docker-compose.sh down --volumes; fi
 
   start:
     usage: Start existing Docker containers.
-    cmd: docker-compose start "$@"
+    cmd: sh bin/docker-compose.sh start "$@"
 
   stop:
     usage: Stop running Docker containers.
-    cmd: docker-compose stop "$@"
+    cmd: sh bin/docker-compose.sh stop "$@"
 
   restart:
     usage: Restart all stopped and running Docker containers.
-    cmd: docker-compose restart "$@"
+    cmd: sh bin/docker-compose.sh restart "$@"
 
   logs:
     usage: Show Docker logs.
-    cmd: docker-compose logs "$@"
+    cmd: sh bin/docker-compose.sh logs "$@"
 
   pull:
     usage: Pull latest docker images.
     cmd: if [ ! -z "$(docker image ls -q)" ]; then docker image ls --format \"{{.Repository}}:{{.Tag}}\" | grep ckan/ckan- | grep -v none | xargs -n1 docker pull | cat; fi
 
   cli:
     usage: Start a shell inside CLI container or run a command.
-    cmd: if \[ "${#}" -ne 0 \]; then docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/bin/activate; cd $APP_DIR;'" $*"; else docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/bin/activate && cd $APP_DIR && sh'; fi
+    cmd: |
+      CKAN_CONTAINER=$(sh bin/docker-compose.sh ps -q ckan)
+      if [ "${#}" -ne 0 \]; then
+        docker exec $CKAN_CONTAINER sh -c '. ${APP_DIR}/bin/activate; cd $APP_DIR;'" $*"
+      else
+        docker exec $CKAN_CONTAINER sh -c '. ${APP_DIR}/bin/activate && cd $APP_DIR && sh'
+      fi
 
   doctor:
     usage: Find problems with current project setup.
@@ -104,7 +112,7 @@ commands:
 
   flush-redis:
     usage: Flush Redis cache.
-    cmd: docker exec -i $(docker-compose ps -q redis) redis-cli flushall > /dev/null
+    cmd: docker exec -i $(sh bin/docker-compose.sh ps -q redis) redis-cli flushall > /dev/null
 
   lint:
     usage: Lint code.
@@ -116,8 +124,8 @@ commands:
   copy-local-files:
     usage: Update files from local repo.
     cmd: |
-      docker cp . $(docker-compose ps -q ckan):/srv/app/
-      docker cp bin/ckan_cli $(docker-compose ps -q ckan):/usr/bin/
+      docker cp . $(sh bin/docker-compose.sh ps -q ckan):/srv/app/
+      docker cp bin/ckan_cli $(sh bin/docker-compose.sh ps -q ckan):/usr/bin/
       ahoy cli 'chmod -v u+x /usr/bin/ckan_cli $APP_DIR/bin/*; cp -v .docker/test.ini $CKAN_INI'
 
   test-unit:
@@ -142,14 +150,14 @@ commands:
     usage: Starts CKAN background job worker
     cmd: |
       ahoy title 'Starting CKAN background job worker'
-      ahoy cli "ckan_cli jobs clear && \
-                ckan_cli jobs worker &"
+      ahoy cli "ckan_cli jobs worker &"
 
   stop-ckan-job-worker:
     usage: Stops CKAN background job worker
     cmd: |
       ahoy title 'Stopping CKAN background job worker'
-      ahoy cli "pkill -f 'jobs worker'"
+      ahoy cli "ckan_cli jobs clear;
+                (pgrep 'jobs worker' && pkill -f 'jobs worker') || true"
 
   # Utilities.
   title:

.env

@@ -13,7 +13,7 @@
 PROJECT="ckanext-validation"
 
 # Docker Compose project name. All containers will have this name.
-COMPOSE_PROJECT_NAME="$PROJECT"
+COMPOSE_PROJECT_NAME="ckanext-validation"
 
 # Flag to allow code linting failures. 0=enforce, 1=ignore
 ALLOW_LINT_FAIL=0

.github/workflows/test.yml

@@ -34,7 +34,7 @@ jobs:
 
     name: Continuous Integration build on CKAN ${{ matrix.ckan-version }}
     runs-on: ubuntu-latest
-    container: drevops/ci-builder
+    container: drevops/ci-builder:23.7.0
     env:
       CKAN_VERSION: ${{ matrix.ckan-version }}
 
@@ -53,10 +53,10 @@ jobs:
       - name: Retrieve logs
         if: always()
         run: ahoy logs
-        timeout-minutes: 5
+        timeout-minutes: 1
 
       - name: Retrieve screenshots
-        if: failure()
+        if: always()
         run: bin/process-artifacts.sh
         timeout-minutes: 1
 

bin/docker-compose.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+# Pass commands to Docker Compose v1 or v2 depending on what is present
+
+if (which docker-compose >/dev/null); then
+    # Docker Compose v1
+    docker-compose $*
+elif (docker compose ls >/dev/null); then
+    # Docker Compose v2
+    docker compose $*
+else
+    # Docker Compose not found
+    exit 1
+fi

bin/doctor.sh

@@ -31,7 +31,6 @@ main() {
 
   if [ "${DOCTOR_CHECK_TOOLS}" == "1" ]; then
     [ "$(command_exists docker)" == "1" ] && error "Please install Docker (https://www.docker.com/get-started)" && exit 1
-    [ "$(command_exists docker-compose)" == "1" ] && error "Please install docker-compose (https://docs.docker.com/compose/install/)" && exit 1
     [ "$(command_exists composer)" == "1" ] && error "Please install Composer (https://getcomposer.org/)" && exit 1
     [ "$(command_exists pygmy)" == "1" ] && error "Please install Pygmy (https://pygmy.readthedocs.io/)" && exit 1
     [ "$(command_exists ahoy)" == "1" ] && error "Please install Ahoy (https://ahoy-cli.readthedocs.io/)" && exit 1
@@ -55,7 +54,7 @@ main() {
 
   # Check that the stack is running.
   if [ "${DOCTOR_CHECK_CLI}" == "1" ]; then
-    if ! docker ps -q --no-trunc | grep "$(docker-compose ps -q ckan)" > /dev/null 2>&1; then
+    if ! docker ps -q --no-trunc | grep "$(sh bin/docker-compose.sh ps -q ckan)" > /dev/null 2>&1; then
       error "CLI container is not running. Run 'ahoy up'."
       exit 1
     fi
@@ -94,7 +93,7 @@ main() {
     fi
 
     # Check that the volume is mounted into CLI container.
-    if ! docker exec -i "$(docker-compose ps -q ckan)" sh -c "grep \"^/dev\" /etc/mtab|grep -q /tmp/amazeeio_ssh-agent"; then
+    if ! docker exec -i "$(sh bin/docker-compose.sh ps -q ckan)" sh -c "grep \"^/dev\" /etc/mtab|grep -q /tmp/amazeeio_ssh-agent"; then
       error "SSH key is added to Pygmy, but the volume is not mounted into container. Make sure that your your \"docker-compose.yml\" has the following lines:"
       error "volumes_from:"
       error "  - container:amazeeio-ssh-agent"
@@ -103,7 +102,7 @@ main() {
     fi
 
     # Check that ssh key is available in the container.
-    if ! docker exec -i "$(docker-compose ps -q ckan)" bash -c "ssh-add -L | grep -q 'ssh-rsa'" ; then
+    if ! docker exec -i "$(sh bin/docker-compose.sh ps -q ckan)" bash -c "ssh-add -L | grep -q 'ssh-rsa'" ; then
       error "SSH key was not added into container. Run 'ahoy up -- --build'."
       exit 1
     fi
@@ -113,7 +112,7 @@ main() {
 
 
   if [ "${DOCTOR_CHECK_WEBSERVER}" == "1" ]; then
-    host_app_port="$(docker port $(docker-compose ps -q ckan) $APP_PORT | cut -d : -f 2)"
+    host_app_port="$(docker port $(sh bin/docker-compose.sh ps -q ckan) $APP_PORT | cut -d : -f 2)"
     if ! curl -L -s -o /dev/null -w "%{http_code}" "${LAGOON_LOCALDEV_URL}:${host_app_port}" | grep -q 200; then
       error "Web server is not accessible at ${LAGOON_LOCALDEV_URL}:${host_app_port}"
       exit 1
@@ -122,7 +121,7 @@ main() {
   fi
 
   if [ "${DOCTOR_CHECK_BOOTSTRAP}" == "1" ]; then
-    host_app_port="$(docker port $(docker-compose ps -q ckan) $APP_PORT | cut -d : -f 2)"
+    host_app_port="$(docker port $(sh bin/docker-compose.sh ps -q ckan) $APP_PORT | cut -d : -f 2)"
     if ! curl -L -s -N "${LAGOON_LOCALDEV_URL}:${host_app_port}" | grep -q -i "meta name=\"generator\" content=\"ckan"; then
       error "Website is running, but cannot be bootstrapped. Try pulling latest container images with 'ahoy pull'"
       exit 1

bin/process-artifacts.sh

@@ -10,4 +10,4 @@ ahoy cli "mkdir -p test/screenshots"
 
 # Copy from the app container to the build host for storage.
 mkdir -p /tmp/artifacts/behave
-docker cp "$(docker-compose ps -q ckan)":/srv/app/test/screenshots /tmp/artifacts/behave/
+docker cp "$(sh bin/docker-compose.sh ps -q ckan)":/srv/app/test/screenshots /tmp/artifacts/behave/

requirements.txt

@@ -1,4 +1,4 @@
-ckantoolkit>=0.0.3
+ckantoolkit>=0.0.4
 goodtables==1.5.1
 six>=1.13.0
 -e git+https://github.com/ckan/[email protected]#egg=ckanext-scheming

setup.py

@@ -38,9 +38,9 @@
         # Specify the Python versions you support here. In particular, ensure
         # that you indicate whether you support Python 2, Python 3 or both.
         'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3',
     ],
 
-
     # What does your project relate to?
     keywords='CKAN',
 

test/features/environment.py

@@ -95,4 +95,5 @@ def before_scenario(context, scenario):
 
 
 def after_scenario(context, scenario):
+    os.system("ckan_cli jobs clear")
     benv.after_scenario(context, scenario)