diff --git a/.ahoy.yml b/.ahoy.yml index 09fe121c..51b36f8f 100644 --- a/.ahoy.yml +++ b/.ahoy.yml @@ -28,43 +28,45 @@ commands: cmd: | ahoy line "Project : " ${PROJECT} ahoy line "Site local URL : " ${LAGOON_LOCALDEV_URL} - ahoy line "DB port on host : " $(docker port $(docker-compose ps -q postgres) 5432 | cut -d : -f 2) - ahoy line "Solr port on host : " $(docker port $(docker-compose ps -q solr) 8983 | cut -d : -f 2) + ahoy line "DB port on host : " $(docker port $(sh bin/docker-compose.sh ps -q postgres) 5432 | cut -d : -f 2) + ahoy line "Solr port on host : " $(docker port $(sh bin/docker-compose.sh ps -q solr) 8983 | cut -d : -f 2) ahoy line "Mailhog URL : " http://mailhog.docker.amazee.io/ up: usage: Build and start Docker containers. cmd: | ahoy title "Building and starting Docker containers" - docker-compose up -d "$@" + sh bin/docker-compose.sh up -d "$@" echo "Initialising database schema" ahoy cli '$APP_DIR/bin/init.sh' echo "Waiting for containers to start listening..." ahoy cli "dockerize -wait tcp://ckan:5000 -timeout 1m" - if docker-compose logs | grep -q "\[Error\]"; then exit 1; fi - if docker-compose logs | grep -q "Exception"; then exit 1; fi + if sh bin/docker-compose.sh logs | grep -q "\[Error\]"; then exit 1; fi + if sh bin/docker-compose.sh logs | grep -q "Exception"; then exit 1; fi docker ps -a --filter name=^/${COMPOSE_PROJECT_NAME}_ export DOCTOR_CHECK_CLI=0 down: usage: Stop Docker containers and remove container, images, volumes and networks. - cmd: 'if [ -f "docker-compose.yml" ]; then docker-compose down --volumes; fi' + cmd: | + ahoy title 'Stopping and removing old containers, images, volumes, networks' + if [ -f "docker-compose.yml" ]; then sh bin/docker-compose.sh down --volumes; fi start: usage: Start existing Docker containers. - cmd: docker-compose start "$@" + cmd: sh bin/docker-compose.sh start "$@" stop: usage: Stop running Docker containers. - cmd: docker-compose stop "$@" + cmd: sh bin/docker-compose.sh stop "$@" restart: usage: Restart all stopped and running Docker containers. - cmd: docker-compose restart "$@" + cmd: sh bin/docker-compose.sh restart "$@" logs: usage: Show Docker logs. - cmd: docker-compose logs "$@" + cmd: sh bin/docker-compose.sh logs "$@" pull: usage: Pull latest docker images. @@ -72,7 +74,13 @@ commands: cli: usage: Start a shell inside CLI container or run a command. - cmd: if \[ "${#}" -ne 0 \]; then docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/bin/activate; cd $APP_DIR;'" $*"; else docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/bin/activate && cd $APP_DIR && sh'; fi + cmd: | + CKAN_CONTAINER=$(sh bin/docker-compose.sh ps -q ckan) + if [ "${#}" -ne 0 \]; then + docker exec $CKAN_CONTAINER sh -c '. ${APP_DIR}/bin/activate; cd $APP_DIR;'" $*" + else + docker exec $CKAN_CONTAINER sh -c '. ${APP_DIR}/bin/activate && cd $APP_DIR && sh' + fi doctor: usage: Find problems with current project setup. @@ -104,7 +112,7 @@ commands: flush-redis: usage: Flush Redis cache. - cmd: docker exec -i $(docker-compose ps -q redis) redis-cli flushall > /dev/null + cmd: docker exec -i $(sh bin/docker-compose.sh ps -q redis) redis-cli flushall > /dev/null lint: usage: Lint code. @@ -116,8 +124,8 @@ commands: copy-local-files: usage: Update files from local repo. cmd: | - docker cp . $(docker-compose ps -q ckan):/srv/app/ - docker cp bin/ckan_cli $(docker-compose ps -q ckan):/usr/bin/ + docker cp . $(sh bin/docker-compose.sh ps -q ckan):/srv/app/ + docker cp bin/ckan_cli $(sh bin/docker-compose.sh ps -q ckan):/usr/bin/ ahoy cli 'chmod -v u+x /usr/bin/ckan_cli $APP_DIR/bin/*; cp -v .docker/test.ini $CKAN_INI' test-unit: @@ -142,14 +150,14 @@ commands: usage: Starts CKAN background job worker cmd: | ahoy title 'Starting CKAN background job worker' - ahoy cli "ckan_cli jobs clear && \ - ckan_cli jobs worker &" + ahoy cli "ckan_cli jobs worker &" stop-ckan-job-worker: usage: Stops CKAN background job worker cmd: | ahoy title 'Stopping CKAN background job worker' - ahoy cli "pkill -f 'jobs worker'" + ahoy cli "ckan_cli jobs clear; + (pgrep 'jobs worker' && pkill -f 'jobs worker') || true" # Utilities. title: diff --git a/.env b/.env index 0edd4baf..c0802d5e 100644 --- a/.env +++ b/.env @@ -13,7 +13,7 @@ PROJECT="ckanext-validation" # Docker Compose project name. All containers will have this name. -COMPOSE_PROJECT_NAME="$PROJECT" +COMPOSE_PROJECT_NAME="ckanext-validation" # Flag to allow code linting failures. 0=enforce, 1=ignore ALLOW_LINT_FAIL=0 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dc9ff04b..38b3f635 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,7 +34,7 @@ jobs: name: Continuous Integration build on CKAN ${{ matrix.ckan-version }} runs-on: ubuntu-latest - container: drevops/ci-builder + container: drevops/ci-builder:23.7.0 env: CKAN_VERSION: ${{ matrix.ckan-version }} @@ -53,10 +53,10 @@ jobs: - name: Retrieve logs if: always() run: ahoy logs - timeout-minutes: 5 + timeout-minutes: 1 - name: Retrieve screenshots - if: failure() + if: always() run: bin/process-artifacts.sh timeout-minutes: 1 diff --git a/bin/docker-compose.sh b/bin/docker-compose.sh new file mode 100755 index 00000000..405bb7cd --- /dev/null +++ b/bin/docker-compose.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +# Pass commands to Docker Compose v1 or v2 depending on what is present + +if (which docker-compose >/dev/null); then + # Docker Compose v1 + docker-compose $* +elif (docker compose ls >/dev/null); then + # Docker Compose v2 + docker compose $* +else + # Docker Compose not found + exit 1 +fi diff --git a/bin/doctor.sh b/bin/doctor.sh index 9c7f4551..2a3ca3fc 100755 --- a/bin/doctor.sh +++ b/bin/doctor.sh @@ -31,7 +31,6 @@ main() { if [ "${DOCTOR_CHECK_TOOLS}" == "1" ]; then [ "$(command_exists docker)" == "1" ] && error "Please install Docker (https://www.docker.com/get-started)" && exit 1 - [ "$(command_exists docker-compose)" == "1" ] && error "Please install docker-compose (https://docs.docker.com/compose/install/)" && exit 1 [ "$(command_exists composer)" == "1" ] && error "Please install Composer (https://getcomposer.org/)" && exit 1 [ "$(command_exists pygmy)" == "1" ] && error "Please install Pygmy (https://pygmy.readthedocs.io/)" && exit 1 [ "$(command_exists ahoy)" == "1" ] && error "Please install Ahoy (https://ahoy-cli.readthedocs.io/)" && exit 1 @@ -55,7 +54,7 @@ main() { # Check that the stack is running. if [ "${DOCTOR_CHECK_CLI}" == "1" ]; then - if ! docker ps -q --no-trunc | grep "$(docker-compose ps -q ckan)" > /dev/null 2>&1; then + if ! docker ps -q --no-trunc | grep "$(sh bin/docker-compose.sh ps -q ckan)" > /dev/null 2>&1; then error "CLI container is not running. Run 'ahoy up'." exit 1 fi @@ -94,7 +93,7 @@ main() { fi # Check that the volume is mounted into CLI container. - if ! docker exec -i "$(docker-compose ps -q ckan)" sh -c "grep \"^/dev\" /etc/mtab|grep -q /tmp/amazeeio_ssh-agent"; then + if ! docker exec -i "$(sh bin/docker-compose.sh ps -q ckan)" sh -c "grep \"^/dev\" /etc/mtab|grep -q /tmp/amazeeio_ssh-agent"; then error "SSH key is added to Pygmy, but the volume is not mounted into container. Make sure that your your \"docker-compose.yml\" has the following lines:" error "volumes_from:" error " - container:amazeeio-ssh-agent" @@ -103,7 +102,7 @@ main() { fi # Check that ssh key is available in the container. - if ! docker exec -i "$(docker-compose ps -q ckan)" bash -c "ssh-add -L | grep -q 'ssh-rsa'" ; then + if ! docker exec -i "$(sh bin/docker-compose.sh ps -q ckan)" bash -c "ssh-add -L | grep -q 'ssh-rsa'" ; then error "SSH key was not added into container. Run 'ahoy up -- --build'." exit 1 fi @@ -113,7 +112,7 @@ main() { if [ "${DOCTOR_CHECK_WEBSERVER}" == "1" ]; then - host_app_port="$(docker port $(docker-compose ps -q ckan) $APP_PORT | cut -d : -f 2)" + host_app_port="$(docker port $(sh bin/docker-compose.sh ps -q ckan) $APP_PORT | cut -d : -f 2)" if ! curl -L -s -o /dev/null -w "%{http_code}" "${LAGOON_LOCALDEV_URL}:${host_app_port}" | grep -q 200; then error "Web server is not accessible at ${LAGOON_LOCALDEV_URL}:${host_app_port}" exit 1 @@ -122,7 +121,7 @@ main() { fi if [ "${DOCTOR_CHECK_BOOTSTRAP}" == "1" ]; then - host_app_port="$(docker port $(docker-compose ps -q ckan) $APP_PORT | cut -d : -f 2)" + host_app_port="$(docker port $(sh bin/docker-compose.sh ps -q ckan) $APP_PORT | cut -d : -f 2)" if ! curl -L -s -N "${LAGOON_LOCALDEV_URL}:${host_app_port}" | grep -q -i "meta name=\"generator\" content=\"ckan"; then error "Website is running, but cannot be bootstrapped. Try pulling latest container images with 'ahoy pull'" exit 1 diff --git a/bin/process-artifacts.sh b/bin/process-artifacts.sh index 6324f60a..e20a7e69 100755 --- a/bin/process-artifacts.sh +++ b/bin/process-artifacts.sh @@ -10,4 +10,4 @@ ahoy cli "mkdir -p test/screenshots" # Copy from the app container to the build host for storage. mkdir -p /tmp/artifacts/behave -docker cp "$(docker-compose ps -q ckan)":/srv/app/test/screenshots /tmp/artifacts/behave/ +docker cp "$(sh bin/docker-compose.sh ps -q ckan)":/srv/app/test/screenshots /tmp/artifacts/behave/ diff --git a/requirements.txt b/requirements.txt index ccb14fba..305711e0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ckantoolkit>=0.0.3 +ckantoolkit>=0.0.4 goodtables==1.5.1 six>=1.13.0 -e git+https://github.com/ckan/ckanext-scheming.git@release-3.0.0#egg=ckanext-scheming diff --git a/setup.py b/setup.py index aa28a505..8311e70b 100644 --- a/setup.py +++ b/setup.py @@ -38,9 +38,9 @@ # Specify the Python versions you support here. In particular, ensure # that you indicate whether you support Python 2, Python 3 or both. 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', ], - # What does your project relate to? keywords='CKAN', diff --git a/test/features/environment.py b/test/features/environment.py index 7a0afbe7..7a2a3a95 100644 --- a/test/features/environment.py +++ b/test/features/environment.py @@ -95,4 +95,5 @@ def before_scenario(context, scenario): def after_scenario(context, scenario): + os.system("ckan_cli jobs clear") benv.after_scenario(context, scenario)