diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..1b75842 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,12 @@ +name: CKAN deb packages build workflow + +on: + push: + branches: + - '**' + tags-ignore: + - v* + +jobs: + call-build-workflow: + uses: ./.github/workflows/reusable-build-package.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..12b8100 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,71 @@ +name: CKAN deb packages publish workflow + +on: + push: + tags: + - v* + +jobs: + call-build-workflow: + uses: ./.github/workflows/reusable-build-package.yml + + upload-to-s3: + needs: call-build-workflow + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} + steps: + - uses: actions/download-artifact@v4 + with: + pattern: python-ckan* + merge-multiple: true + - name: Generate hash and upload + run: | + # Download current md5sum file + aws s3 cp s3://${{ secrets.AWS_BUCKET }}/md5sum . + + for file in python-ckan*; do + # Remove current md5sum entry + sed -i "/$file/d" md5sum + + # Add updated entry to md5sum file + md5sum $file >> md5sum + + # Upload deb file + aws s3 cp $file s3://${{ secrets.AWS_BUCKET }}/staging/$file + done + + # Upload updated md5sum file + aws s3 cp md5sum s3://${{ secrets.AWS_BUCKET }}/staging/md5sum + + upload-to-release: + needs: call-build-workflow + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + pattern: python-ckan* + merge-multiple: true + - name: Create release and upload the deb files + env: + GH_TOKEN: ${{ github.token }} + run: | + VERSIONS=$(cat "VERSIONS.json") + + LIST=$(echo $VERSIONS | jq -r ' + (.[] | ["* \(.ckan_ref) on Ubuntu \(.ubuntu_version)"]) | + .[] + ') + + NOTES="This release includes deb packages for the following versions. + + $LIST + + Please check the relevant file in the Assets section below. + Packages are also available at https://packaging.ckan.org." + + gh release create ${{ github.ref_name }} ./python-ckan* --verify-tag --notes "$NOTES" diff --git a/.github/workflows/reusable-build-package.yml b/.github/workflows/reusable-build-package.yml new file mode 100644 index 0000000..9f2b62d --- /dev/null +++ b/.github/workflows/reusable-build-package.yml @@ -0,0 +1,61 @@ +name: Reusable CKAN deb packages build workflow + +on: + workflow_call: + +jobs: + get-build-versions: + runs-on: ubuntu-latest + outputs: + versions: ${{ steps.get-versions.outputs.versions }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Get versions to build from VERSIONS file + id: get-versions + run: | + # Remove whitespace and line breaks + VERSIONS=$(cat "VERSIONS.json" | tr -d '[:space:]\n') + echo "versions=$VERSIONS" >> $GITHUB_OUTPUT + + build-package: + needs: get-build-versions + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + build: ${{ fromJson(needs.get-build-versions.outputs.versions) }} + + name: CKAN ref ${{ matrix.build.ckan_ref }} on Ubuntu ${{ matrix.build.ubuntu_version }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build package + uses: docker/build-push-action@v6 + with: + push: false + outputs: "type=local,dest=." + build-args: | + CKAN_REF=${{ matrix.build.ckan_ref }} + UBUNTU_VERSION=${{ matrix.build.ubuntu_version }} + + - name: Rename file + # To remove patch version .e.g python-ckan_2.11.1b0-jammy_amd64.deb -> python-ckan_2.11-jammy_amd64.deb + run: | + for f in python-ckan_*; do mv "$f" "$(echo "$f" | sed 's/\([0-9]\+\.[0-9]\+\)[^-]*-/\1-/')"; done + OUTPUT_FILE=$(basename python-ckan*) + echo "OUTPUT_FILE=$OUTPUT_FILE" >> $GITHUB_ENV + echo "Generated file: $OUTPUT_FILE" + dpkg --info $OUTPUT_FILE + + - name: Upload deb file + uses: actions/upload-artifact@v4 + with: + name: ${{ env.OUTPUT_FILE }} + path: ${{ env.OUTPUT_FILE }} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..dab82a5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,108 @@ +ARG UBUNTU_VERSION=22.04 +FROM ubuntu:${UBUNTU_VERSION} AS builder + +ARG UBUNTU_VERSION=${UBUNTU_VERSION} +ARG CKAN_REF=dev-v2.11 +ARG DATAPUSHER_VERSION=0.0.21 +ARG ITERATION + +# Install Ubuntu packages +RUN apt-get update -q && \ + apt-get upgrade -y -q && \ + DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y -q install \ + lsb-release \ + git \ + python3-dev \ + python3-pip \ + python3-venv \ + libpq-dev \ + libxml2-dev \ + libxslt1-dev \ + build-essential \ + rubygems-integration \ + ruby-dev \ + nginx + +# Install FPM +RUN if [ "$UBUNTU_VERSION" = "20.04" ] ; then gem install dotenv -v 2.8.1 ; fi && \ + gem install fpm -- creates=/usr/local/bin/fpm + +# Create dirs +RUN mkdir -p /etc/ckan/default && \ + mkdir -p /etc/ckan/datapusher && \ + mkdir -p /etc/supervisor/conf.d && \ + mkdir /output + +# Create venv +RUN python3 -m venv /usr/lib/ckan/default + +# Pull CKAN source +RUN git clone --depth=1 --branch=${CKAN_REF} https://github.com/ckan/ckan /usr/lib/ckan/default/src/ckan && \ + rm -rf /usr/lib/ckan/default/src/ckan/.git/ + +# Install CKAN and its requirements +RUN /usr/lib/ckan/default/bin/pip install -U pip && \ + /usr/lib/ckan/default/bin/pip install -r /usr/lib/ckan/default/src/ckan/requirements.txt uwsgi && \ + /usr/lib/ckan/default/bin/pip install /usr/lib/ckan/default/src/ckan + +# Configure CKAN + uWSGI +RUN cp /usr/lib/ckan/default/src/ckan/wsgi.py /etc/ckan/default/wsgi.py && \ + cp /usr/lib/ckan/default/src/ckan/ckan-uwsgi.ini /etc/ckan/default/ckan-uwsgi.ini + +# Install DataPusher +RUN python3 -m venv /usr/lib/ckan/datapusher && \ + # Clone source + git clone --depth=1 --branch=${DATAPUSHER_VERSION} https://github.com/ckan/datapusher /usr/lib/ckan/datapusher/src/datapusher && \ + rm -rf /usr/lib/ckan/datapusher/src/datapusher/.git/ && \ + # Install requirements and datapusher + /usr/lib/ckan/datapusher/bin/pip install -U pip && \ + /usr/lib/ckan/datapusher/bin/pip install -r /usr/lib/ckan/datapusher/src/datapusher/requirements.txt uwsgi && \ + /usr/lib/ckan/datapusher/bin/pip install /usr/lib/ckan/datapusher/src/datapusher + +# Configure DataPusher + uWSGI +RUN cp /usr/lib/ckan/datapusher/src/datapusher/deployment/datapusher.wsgi /etc/ckan/datapusher && \ + cp /usr/lib/ckan/datapusher/src/datapusher/deployment/datapusher_settings.py /etc/ckan/datapusher && \ + cp /usr/lib/ckan/datapusher/src/datapusher/deployment/datapusher-uwsgi.ini /etc/ckan/datapusher && \ + # Enable threads in DataPusher uwsgi conf + echo "workers = 2\nthreads = 2\nlazy-apps = true" >> /etc/ckan/datapusher/datapusher-uwsgi.ini && \ + # Replace datapusher wsgi file path + sed -i 's/\/usr\/lib\/ckan\/datapusher\/src\/datapusher\/deployment\/datapusher.wsgi/\/etc\/ckan\/datapusher\/datapusher.wsgi/g' /etc/ckan/datapusher/datapusher-uwsgi.ini + +# Copy conf files +COPY common/etc/nginx/sites-available/ckan /etc/nginx/sites-available/ +COPY common/etc/cron.daily/remove_old_sessions /etc/cron.daily/ +COPY common/etc/supervisor/conf.d/ckan-uwsgi.conf /etc/supervisor/conf.d/ +COPY common/etc/supervisor/conf.d/ckan-worker.conf /etc/supervisor/conf.d/ +COPY common/etc/supervisor/conf.d/ckan-datapusher.conf /etc/supervisor/conf.d/ +COPY --chmod=744 common/usr/bin/ckan /usr/bin/ckan +COPY --chmod=744 common/after_web.sh /tmp/after_web.sh + +# Get the actual CKAN version and create the deb package +RUN DISTRIBUTION=$(lsb_release -c -s) && \ + CKAN_VERSION=$(/usr/lib/ckan/default/bin/python3 -c "import ckan; print(ckan.__version__)") && \ + fpm \ + -t deb -s dir \ + --package /output \ + --name python-ckan \ + --description='CKAN is an open-source DMS (data management system) for powering data hubs and data portals.' \ + --license='AGPL v3.0' \ + --maintainer='CKAN Tech Team ' \ + --vendor='CKAN Association' \ + --url='https://ckan.org' \ + --after-install=/tmp/after_web.sh \ + --iteration "$DISTRIBUTION""${ITERATION}" \ + --version "$CKAN_VERSION" \ + --depends nginx \ + --depends libpq5 \ + --config-files /etc/nginx/sites-available/ckan \ + /usr/lib/ckan/ \ + /etc/ckan/ \ + /usr/bin/ckan \ + /etc/cron.daily/remove_old_sessions \ + /etc/nginx/sites-available/ckan \ + /etc/supervisor/conf.d + +RUN ls -la /output + +FROM scratch AS export +COPY --from=builder /output . diff --git a/README.md b/README.md new file mode 100644 index 0000000..5985ace --- /dev/null +++ b/README.md @@ -0,0 +1,77 @@ +# CKAN Packaging Scripts + +These scripts are used to build Debian packages (deb files) for CKAN releases. + +> [!WARNING] These scripts are used by CKAN maintainers. If you want to install CKAN, including +> via Debian package, check the [installation documentation](https://docs.ckan.org/en/latest/maintaining/installing/index.html) + +> [!NOTE] As of December 2024, Docker is used as a building environment to create the deb packages. +> The previous Ansible / Vagrant setup is no longer used + +## Overview + +To create Debian packages of CKAN, use the `ckan-package` executable: + +``` +/ckan-package --help +usage: ckan-package [-h] [-i ITERATION] ref target + +Builds CKAN deb packages. This script essentially sets up the necessary vars and calls `docker buildx build`. + +positional arguments: + ref The CKAN branch or tag to build (e.g. master, dev-v2.11, ckan-2.10.6...) + target The Ubuntu version to target (e.g. 20.04, 22.04, 24.04...) + +optional arguments: + -h, --help show this help message and exit + -i ITERATION, --iteration ITERATION + The iteration number to add to the package name. +``` + +For instance: + + ./ckan-package ckan-2.11.1 24.04 + ./ckan-package dev-v2.11 22.04 + ./ckan-package master 24.04 + +The currently supported packages are: + +| CKAN version | Ubuntu version | +| ------------ | -------------- | +| 2.10 | 20.04 (focal) | +| 2.10 | 22.04 (jammy) | +| 2.11 | 22.04 (jammy) | +| 2.11 | 24.04 (noble) | + +Any other combination is not officially supported, although you might be able to +build it tweaking the parameters above. + +# How it works + +Under the hood, the `ckan-package` command just calls `docker buildx build`. You can +call it directly using the appropiate build arguments: + +``` +docker buildx build \ + --output type=local,dest=. \ + --build-arg CKAN_REF=ckan-2.11.0 \ + --build-arg UBUNTU_VERSION=24.04 \ + . +``` + +# Release process + +There are two separate workflows: + +* `build.yml` builds the deb packages (based on the versions supplied in `VERSIONS.json`) and stores them as artifacfts in the workflow run page. This is triggered on every push. + + +* Additionally, when a tag is pushed, `publish.yml` also builds the packages and: + 1. Uploads them to the S3 bucket powering https://packaging.ckan.org + 2. Creates a new GitHub release with the packages attached as assets. + +With this, the suggested release process is the following: + +* Whenever there is a new CKAN release in the works, or fixes need to be applied to the packages, a new branch and pull request is created. This will trigger the workflows that will create the packages for that version of the code. The `ckan_ref` should be the relvant development branch (e.g. `dev-v2.11`). +* The packages can be downloded from the workflow page to test locally. Once everthing looks fine the PR is merged. +* A new tag in the form `vYYYYMMDD` is pushed to trigger the publication of the packages and the creation of the release. diff --git a/README.rst b/README.rst deleted file mode 100644 index ba56499..0000000 --- a/README.rst +++ /dev/null @@ -1,116 +0,0 @@ -CKAN Packaging Scripts -====================== - -These scripts are used to build Debian packages (deb files) for CKAN releases. - -**Note:** This version of the scripts can only package CKAN 2.9 and higher. -For previous CKAN versions use the `build-ckan-2.8` tag. - - -Overview --------- - -To create Debian packages of CKAN, install Vagrant and Ansible and run:: - - ./ckan-package -v dev-v2.10 -i 1 -t jammy - -If you omit the parameters you will be prompted for them. - - CKAN 2.10 Python3 jammy, focal - CKAN 2.9 Python3 jammy, focal - - -The following combinations of packages are no longer supported:: - - CKAN 2.9 Python3, Python2 focal, bionic, xenial - CKAN 2.8 Python2 focal, bionic, xenial - CKAN 2.7 Python2 focal, bionic, xenial - - -Keep reading for more options and to learn how it works. - - -How it works ------------- - -This repository contains a `Vagrant `_ environment -configured in a `multi-machine `_ setup. - -Each machine is running one of the supported distributions that we target, currently: - -* Ubuntu 20.04 64bit (focal) -* Ubuntu 22.04 64bit (jammy) - -We use `Ansible `_ to provision the Vagrant machines, which -results in the creation of the package. Ansible is configured via -`playbooks `_. These are a set of -configurations and tasks that are run in the remote server. The file we use -is `package.yml` and it is quite straight-forward to follow what is going on. - -The actual packaging is done in the building maching by -`FPM `_, a tool that aims to make building -packages quickly and easily. - -To pass parameters like the CKAN version or the iteration to the Vagrant file and -ultimately to Ansible we need to use env vars. To make this a little bit more -convenient, the ``ckan-package`` script is included, which essentially sets up the -necessary env vars and calls ``vagrant up`` or ``vagrant provision`` as appropiate. - - -Install and setup ------------------ - -1. Install Vagrant. On an Ubuntu machine:: - - sudo apt-get install vagrant - - Or for an OSX machine use the installer at https://www.vagrantup.com/downloads.html - -2. Install Ansible. If you have a virtualenv you can install it in that:: - - pip install ansible - - Otherwise simply install it globally:: - - sudo easy_install ansible - -3. Checkout this repository in your virtualenv or elsewhere:: - - git clone https://github.com/ckan/ckan-packaging.git - cd ckan-packaging - -You are now ready to build packages. - - -Building a package ------------------- - -Packages are built using the ``ckan-package`` script. Check the following for a -full list of options and parameters:: - - ./ckan-package --help - -Most of the times you will want to run something like the following:: - - ./ckan-package -v dev-v2.10 -i 1 -t jammy - -Where: - - * -v (version) relates to the CKAN branch or tag to build, eg master, dev-v2.10, ckan-v2.9.8 - * -i (iteration) e.g. `beta1` for a beta or for a proper release use a number e.g. `1` - * -t (target) will build just for that version of Ubuntu - - -The first time that you run the build commands Vagrant will -need to download the OS images from the central repository, this might take a while. -After the first run, the image will be already on your machine so it won't take as much. -Most of the steps in the Ansible playbook (like installing required packages) will be also -skipped, so subsequent builds should be much faster. - -You will see the output of the different tasks, both for Vagrant and Ansible. -Once the process is finished, you should see a summary of the tasks. -All tasks should be `ok` or `changed`. - -The packages will be created in the same directory. - -TODO: upload the packages to S3 (http://packaging.ckan.org/build/). diff --git a/VERSIONS.json b/VERSIONS.json new file mode 100644 index 0000000..9848540 --- /dev/null +++ b/VERSIONS.json @@ -0,0 +1,18 @@ +[ + { + "ubuntu_version": "20.04", + "ckan_ref": "ckan-2.10.5" + }, + { + "ubuntu_version": "22.04", + "ckan_ref": "ckan-2.10.5" + }, + { + "ubuntu_version": "22.04", + "ckan_ref": "ckan-2.11.0" + }, + { + "ubuntu_version": "24.04", + "ckan_ref": "ckan-2.11.0" + } +] diff --git a/Vagrantfile b/Vagrantfile deleted file mode 100644 index de40ef5..0000000 --- a/Vagrantfile +++ /dev/null @@ -1,46 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! -VAGRANTFILE_API_VERSION = "2" - -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - - config.vm.define "focal" do |focal| - focal.vm.box = "ubuntu/focal64" - end - - config.vm.define "jammy" do |focal| - focal.vm.box = "ubuntu/jammy64" - end - - config.ssh.forward_agent = true - config.vm.provider "virtualbox" do |v| - v.memory = 1024 - v.customize ["modifyvm", :id, "--uart1", "0x3F8", "4"] - v.customize ["modifyvm", :id, "--uartmode1", "file", File::NULL] - - end - - config.vm.provision "ansible" do |ansible| - ansible.playbook = "package.yml" - ansible.host_key_checking = false - ansible.become = true - ansible.compatibility_mode = "2.0" - - ansible.verbose = ENV.fetch("CKAN_PACKAGE_ANSIBLE_VERBOSE", "vv") - ansible.extra_vars = {} - - ansible.extra_vars["datapusher"] = ENV.fetch("CKAN_PACKAGE_DATAPUSHER", "y") - - if ENV.has_key?("CKAN_PACKAGE_VERSION") - ansible.extra_vars["version"] = ENV["CKAN_PACKAGE_VERSION"] - end - if ENV.has_key?("PYTHON_VERSION") - ansible.extra_vars["pythonversion"] = ENV["PYTHON_VERSION"] - end - if ENV.has_key?("CKAN_PACKAGE_ITERATION") - ansible.extra_vars["iteration"] = ENV["CKAN_PACKAGE_ITERATION"] - end - end -end diff --git a/ckan-package b/ckan-package index 2e54b56..de75b54 100755 --- a/ckan-package +++ b/ckan-package @@ -1,152 +1,54 @@ #!/usr/bin/env python +import argparse +import json import subprocess -import re -import os import sys -import argparse - - -def set_env_vars(version, iteration, datapusher="y", ansible_verbose="vv"): - - os.environ["CKAN_PACKAGE_VERSION"] = version - os.environ["CKAN_PACKAGE_ITERATION"] = iteration - os.environ["CKAN_PACKAGE_DATAPUSHER"] = datapusher - os.environ["CKAN_PACKAGE_ANSIBLE_VERBOSE"] = ansible_verbose - - -def clear_env_vars(): - - for key in [ - "CKAN_PACKAGE_VERSION", - "CKAN_PACKAGE_ITERATION", - "CKAN_PACKAGE_DATAPUSHER", - "CKAN_PACKAGE_ANSIBLE_VERBOSE", - ]: - if key in os.environ: - del os.environ[key] - - -def get_status(out, target): - return re.search(target + "\s*(\w+)", out).group(1) - - -def run(target=None): - - status_jammy = None - status_focal = None - - print("Checking Vagrant machines status...") - out = subprocess.check_output(["vagrant", "status"]).decode("utf8") - - if target is None or target == "focal": - status_focal = get_status(out, "focal") - print('Machine "focal" is ' + status_focal) - if target is None or target == "jammy": - status_jammy = get_status(out, "jammy") - print('Machine "jammy" is ' + status_jammy) - - if (status_jammy and status_jammy != "running") or ( - status_focal and status_focal != "running" - ): - print("Starting up machine(s)") - command = ["vagrant", "up", "--provision"] - else: - print("Machine(s) already started, provisioning") - command = ["vagrant", "provision"] - - if target: - command.append(target) - - subprocess.call(command) - - -def minor_version(version_string): - pattern = r"\D+\D+(?P\d+\.\d+)" - version_number = re.search(pattern, version_string) - if version_number: - parts = version_number.groups(0)[0].split(".") - try: - if len(parts) == 3: - # ckan-2.8.5 - return int(parts[1]) - else: - # dev-v2.8 - return int(parts[-1]) - except ValueError: - pass - raise ValueError("Could not parse version number: %s" % version_string) - if __name__ == "__main__": description = """Builds CKAN deb packages. -This script essentially sets up the necessary env vars and calls `vagrant up` -or `vagrant provision` as appropiate.""" +This script essentially sets up the necessary vars and calls `docker buildx build`.""" parser = argparse.ArgumentParser(description=description) parser.add_argument( - "-v", - "--version", - help="""The CKAN branch or tag to build, eg master, dev-v2.6, release-v2.5.3. -If not provided you will be propmt for it""", - ) - parser.add_argument( - "-i", - "--iteration", - help="""The iteration number to add to the package name. -If not provided you will be prompt for it""", - ) - parser.add_argument( - "-d", - "--datapusher", - action="store_true", - default="y", - help="""Whether to add the DataPusher to the package, -defaults to true""", + "ref", + help="The CKAN branch or tag to build (e.g. master, dev-v2.11, ckan-2.10.6...)", ) parser.add_argument( - "-t", - "--target", - help="""The distribution to target (focal or jammy). -If omitted, all are built""", + "target", help="The Ubuntu distribution to target (e.g. 20.04, 22.04, 24.04...)" ) parser.add_argument( - "-a", - "--ansible-verbose", - default="vv", - help='Ansible verbosity level, defaults to "vv"', + "-i", + "--iteration", + default="1", + help="The iteration number to add to the package name.", ) - def _check_arg(args, var, prompt): - if getattr(args, var): - return getattr(args, var) - try: - var = raw_input("{0}:".format(prompt)) - except NameError: - var = input("{0}:".format(prompt)) - if not var: - print("Please provide a value for the {0}".format(prompt)) - sys.exit(1) - return var - args = parser.parse_args() - version = _check_arg(args, "version", "CKAN Version (branch or tag)") - iteration = _check_arg(args, "iteration", "Iteration") - datapusher = args.datapusher + ref = args.ref target = args.target + iteration = args.iteration - if minor_version(version) < 9: - print("This version of the scripts can only package CKAN 2.9 onwards") - sys.exit(1) + with open("VERSIONS.json") as f: + supported_versions = json.load(f) - if target and target not in ("focal", "jammy"): - print("Wrong target: " + target) + if target not in [v["ubuntu_version"] for v in supported_versions]: + print(f"Wrong target: {target}") sys.exit(1) - ansible_verbose = args.ansible_verbose + command = [ + "docker", + "buildx", + "build", + "--output", + "type=local,dest=.", + "--build-arg", + f"CKAN_REF={ref}", + "--build-arg", + f"ITERATION={iteration}", + "--build-arg", + f"UBUNTU_VERSION={target}", + ".", + ] - set_env_vars(version, iteration, datapusher, ansible_verbose) - try: - run(target) - finally: - clear_env_vars() + subprocess.call(command) diff --git a/common/after_web.j2 b/common/after_web.sh similarity index 100% rename from common/after_web.j2 rename to common/after_web.sh diff --git a/common/app.yml b/common/app.yml deleted file mode 100644 index 9a31f60..0000000 --- a/common/app.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- - - name: copy source who.ini to configuration folder - action: command cp /usr/lib/ckan/default/src/ckan/who.ini /etc/ckan/default/who.ini - - - name: copy executables - action: copy src=common/{{ item }} dest=/{{ item }} mode=744 - with_items: - - usr/bin/ckan diff --git a/common/build.yml b/common/build.yml deleted file mode 100644 index 761b650..0000000 --- a/common/build.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- - - name: create the /vagrant directory - file: - path: /vagrant - state: directory - - - name: build deb main - action: command chdir=/vagrant fpm - -t deb -s dir - --name python-ckan - --description='CKAN is an open-source DMS (data management system) for powering data hubs and data portals.' - --license='AGPL v3.0' - --maintainer='CKAN team ' - --vendor='CKAN Association' - --url='https://ckan.org' - --after-install=/tmp/after_web.sh - --iteration {{ ansible_distribution_release }}{{ iteration }} - --version {{ ckan_version.stdout }} - --depends nginx --depends libpq5 - --config-files /etc/nginx/sites-available/ckan --config-files /etc/ckan/default/who.ini - /usr/lib/ckan/ /etc/ckan/ /usr/bin/ckan /etc/cron.daily/remove_old_sessions /etc/nginx/sites-available/ckan - /etc/supervisor/conf.d diff --git a/common/web.yml b/common/web.yml deleted file mode 100644 index 92af25c..0000000 --- a/common/web.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- - - name: install uWSGI - pip: - name: uwsgi - virtualenv: /usr/lib/ckan/default - virtualenv_command: pyvenv - - - name: create directories - action: file path={{ item }} state=directory - with_items: - - /etc/ckan/default - - /etc/supervisor/conf.d - - - name: copy source wsgi.py to configuration folder - action: command cp /usr/lib/ckan/default/src/ckan/wsgi.py /etc/ckan/default/wsgi.py - - - name: copy source ckan-uwsgi.ini to configuration folder - action: command cp /usr/lib/ckan/default/src/ckan/ckan-uwsgi.ini /etc/ckan/default/ckan-uwsgi.ini - - - name: copy all common needed web (and other) files - action: copy src=common/{{ item }} dest=/{{ item }} - with_items: - - etc/nginx/sites-available/ckan - - etc/cron.daily/remove_old_sessions - - etc/supervisor/conf.d/ckan-uwsgi.conf - - etc/supervisor/conf.d/ckan-worker.conf diff --git a/package.yml b/package.yml deleted file mode 100644 index 03fc5dc..0000000 --- a/package.yml +++ /dev/null @@ -1,89 +0,0 @@ ---- -- hosts: all - vars_prompt: - - name: version - prompt: "CKAN Version (branch or tag)" - private: True - - name: iteration - prompt: "Iteration" - private: True - - name: datapusher - prompt: "Enable DataPusher? [Y/n]" - default: "y" - private: True - tasks: - - name: set regex for CKAN version - set_fact: - re: '\D+\D+(?P\d+\.\d+)' - - set_fact: - ckanver: "{{ version | regex_search(re, '\\g') }}" - register: ckanver - - debug: - var: ckanver[0] - - # Install Ubuntu apt packages - - name: Install Ubuntu apt Packages - include_tasks: "py3/apt.yml" - - - name: make sure nginx is installed - action: apt pkg=nginx state=present update_cache=yes - - - name: install fpm requirements - action: command gem install dotenv -v 2.8.1 - when: ansible_distribution_version == "20.04" - - - name: install fpm - action: command gem install -v 1.15.0 fpm -- creates=/usr/local/bin/fpm - - - name: delete old directories - action: file path={{ item }} state=absent - with_items: - - /etc/ckan - - /usr/lib/ckan - - /root/.cache/pip - - # Install pip and virtualenv - - name: Install apt Packages - include_tasks: "py3/virtualenv.yml" - - # Pull CKAN - - name: pull ckan software - action: git repo=https://github.com/ckan/ckan dest=/usr/lib/ckan/default/src/ckan version={{ version }} - - # Clean up CKAN distro - - name: Recursively remove .git directory - file: - path: /usr/lib/ckan/default/src/ckan/.git - state: absent - - # Install ckan and dependancies - - name: Install ckan and dependancies - include_tasks: "py3/installckan.yml" - - # Install datapusher - - name: Install datapusher - include_tasks: "py3/datapusher.yml" - when: datapusher == 'y' - - # Clean up datapusher distro - - name: Recursively remove .git directory - file: - path: /usr/lib/ckan/datapusher/src/datapusher/.git - state: absent - when: datapusher == 'y' - - # Install web files and dependancies - - name: Install web and dependancies - include_tasks: "common/web.yml" - - # Copy other application files CKAN - - name: Copy other application files for CKAN - include_tasks: "common/app.yml" - - # template the after web script - - name: template the after web script - action: template src=common/after_web.j2 dest=/tmp/after_web.sh mode=744 - - # Build deb package for CKAN - - name: Build deb package for CKAN - include_tasks: "common/build.yml" diff --git a/py3/apt.yml b/py3/apt.yml deleted file mode 100644 index d15a7b8..0000000 --- a/py3/apt.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- - - name: Make sure Python3 apt packages are installed - apt: - update_cache: yes - force_apt_get: yes - name: - - 'git-core' - - 'python3-dev' - - 'python3-pip' - - 'python3-venv' - - 'python3-distutils' - - 'postgresql' - - 'libpq-dev' - - 'libxml2-dev' - - 'libxslt1-dev' - - 'build-essential' - - 'rubygems-integration' - - 'ruby-dev' diff --git a/py3/datapusher.yml b/py3/datapusher.yml deleted file mode 100644 index 52b5a75..0000000 --- a/py3/datapusher.yml +++ /dev/null @@ -1,54 +0,0 @@ ---- - - name: pull datapusher version - action: git repo=https://github.com/ckan/datapusher dest=/usr/lib/ckan/datapusher/src/datapusher version=0.0.21 - - - name: Install Python3 Virtual Environment for datapusher for Python 3 - action: command python3 -m venv /usr/lib/ckan/datapusher - - - name: install requirements for datapusher for Python 3 - pip: - requirements: /usr/lib/ckan/datapusher/src/datapusher/requirements.txt - virtualenv: /usr/lib/ckan/datapusher - virtualenv_command: pyvenv - - - name: install uWSGI for DataPusher - pip: - name: uwsgi - virtualenv: /usr/lib/ckan/datapusher - virtualenv_command: pyvenv - - - name: run setup.py develop for datapusher - action: command chdir=/usr/lib/ckan/datapusher/src/datapusher/ ../../bin/python setup.py develop - - - name: create directories for DataPusher - action: file path={{ item }} state=directory - with_items: - - /etc/ckan/datapusher - - /etc/supervisor/conf.d - - - name: copy all deployment files needed for the DataPusher - action: command cp /usr/lib/ckan/datapusher/src/datapusher/deployment/{{ item }} /etc/ckan/datapusher/{{ item }} - with_items: - - datapusher.wsgi - - datapusher_settings.py - - datapusher-uwsgi.ini - - - name: enable threads in DataPusher uwsgi conf - blockinfile: - path: /etc/ckan/datapusher/datapusher-uwsgi.ini - insertafter: "EOF" - block: | - workers = 2 - threads = 2 - lazy-apps = true - - - name: copy all common needed files for DataPusher - action: copy src=common/{{ item }} dest=/{{ item }} - with_items: - - etc/supervisor/conf.d/ckan-datapusher.conf - - - name: replace datapusher wsgi file path - replace: - path: /etc/ckan/datapusher/datapusher-uwsgi.ini - regexp: '/usr/lib/ckan/datapusher/src/datapusher/deployment/datapusher.wsgi' - replace: '/etc/ckan/datapusher/datapusher.wsgi' diff --git a/py3/installckan.yml b/py3/installckan.yml deleted file mode 100644 index 7632bce..0000000 --- a/py3/installckan.yml +++ /dev/null @@ -1,29 +0,0 @@ ---- - - name: install requirements version for Python3 - pip: - requirements: /usr/lib/ckan/default/src/ckan/requirements.txt - virtualenv: /usr/lib/ckan/default - virtualenv_command: pyvenv - - - name: install setuptools version for Python3 - pip: - requirements: /usr/lib/ckan/default/src/ckan/requirement-setuptools.txt - virtualenv: /usr/lib/ckan/default - virtualenv_command: pyvenv - when: ckanver[0] == "2.9" - - - name: install setuptools version for Python3 - pip: - name: - - setuptools>44 - - wheel - virtualenv: /usr/lib/ckan/default - virtualenv_command: pyvenv - when: ckanver[0] != "2.9" - - - name: run setup.py develop for Python3 ckan - action: command chdir=/usr/lib/ckan/default/src/ckan/ ../../bin/python3 setup.py develop - - - name: get ckan version - command: /usr/lib/ckan/default/bin/python3 -c "import ckan; print(ckan.__version__)" - register: ckan_version diff --git a/py3/virtualenv.yml b/py3/virtualenv.yml deleted file mode 100644 index 7ff8719..0000000 --- a/py3/virtualenv.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- - - name: Get pip - action: get_url url=https://bootstrap.pypa.io/get-pip.py dest=/tmp - - - name: Install pip for Python3 - action: command python3 /tmp/get-pip.py - - - name: Install virtualenv for Python3 - action: command python3 -m venv /usr/lib/ckan/default