From 3e4d4b750bc571568bf5653b4fcaddabe7c12c9e Mon Sep 17 00:00:00 2001 From: Naftuli Kay Date: Mon, 9 Apr 2018 12:19:30 -0700 Subject: [PATCH] Remove Builder, Replace with Docker Image --- README.md | 14 +-- builder/.dockerignore | 1 - builder/Dockerfile | 10 -- builder/README.md | 21 ---- builder/build.sh | 18 --- examples/ci/circle/Makefile | 47 +++++++ examples/ci/circle/bin/build | 4 +- examples/ci/circle/bin/create-deployment | 153 ----------------------- examples/ci/travis/Makefile | 2 +- examples/ci/travis/bin/build | 4 +- examples/ci/travis/bin/create-deployment | 153 ----------------------- 11 files changed, 60 insertions(+), 367 deletions(-) delete mode 100644 builder/.dockerignore delete mode 100644 builder/Dockerfile delete mode 100644 builder/README.md delete mode 100755 builder/build.sh delete mode 100644 examples/ci/circle/bin/create-deployment delete mode 100644 examples/ci/travis/bin/create-deployment diff --git a/README.md b/README.md index c8911fe..0ea3808 100644 --- a/README.md +++ b/README.md @@ -69,10 +69,11 @@ experience has shown this is the best way to build Python 3.6 shared libraries. [@naftulikay][naftulikay] created a sample Rust build environment based on the upstream [`lambci/lambda:build-python3.6`][lambci/lambda] image at -[`naftulikay/circleci-lambda-rust`][naftulikay/circleci-lambda-rust]. Previously, +[`naftulikay/crowbar`][naftulikay/crowbar]. Previously, [`naftulikay/circleci-amazonlinux-rust`][naftulikay/circleci-amazonlinux-rust] was used and the aforementioned issues were encountered. Despite CircleCI being used in the name, the image is a fairly generic Rust build environment and -should be fairly portable and resuable. For Travis CI and CircleCI examples, please look in the `examples/ci` directory. +should be fairly portable and resuable. For Travis CI and CircleCI examples, please look in the +[`examples/ci`](./examples/ci) directory. Because you're building a dynamic library, other libraries that you're dynamically linking against need to also be in the Lambda execution environment. By using the [`lambci/lambda:build-python3.6`][lambci/lambda] image, the build @@ -87,11 +88,8 @@ LD_LIBRARY_PATH=/lib64:/usr/lib64:$LAMBDA_TASK_ROOT:$LAMBDA_TASK_ROOT/lib [@naftulikay][naftulikay] wrote a fairly naïve Python script which will recursively copy linked libraries into the deployment package under `lib/`. This ensures that any non-standard libraries will be available on the library path at -runtime. See the `examples/ci/{travis,circle}` directories for examples on how to use this. - -The `builder` directory of the [crowbar git repo](https://github.com/ilianaw/rust-crowbar) contains a `Dockerfile` with -Rust set up and a build script to dump a zip file containing a stripped shared library to stdout. Documentation for -using that is available at [ilianaw/crowbar-builder on Docker Hub](https://hub.docker.com/r/ilianaw/crowbar-builder/). +runtime. See the `examples/ci/{travis,circle}` directories for examples on how to use this, and see +[naftulikay/docker-crowbar][naftulikay/crowbar] for more information. ## Contributing @@ -111,5 +109,5 @@ please read it. [lambda-execution-environment]: https://docs.aws.amazon.com/lambda/latest/dg/current-supported-versions.html [naftulikay]: https://github.com/naftulikay [naftulikay/circleci-amazonlinux-rust]: https://github.com/naftulikay/docker-circleci-amazonlinux-rust - [naftulikay/circleci-lambda-rust]: https://github.com/naftulikay/docker-circleci-lambda-rust + [naftulikay/crowbar]: https://github.com/naftulikay/docker-crowbar [woes]: https://github.com/naftulikay/docker-circleci-lambda-rust#background diff --git a/builder/.dockerignore b/builder/.dockerignore deleted file mode 100644 index b43bf86..0000000 --- a/builder/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -README.md diff --git a/builder/Dockerfile b/builder/Dockerfile deleted file mode 100644 index 9d8fa04..0000000 --- a/builder/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM amazonlinux:latest -RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable && \ - PATH="$HOME/.cargo/bin:$PATH" rustup install stable && \ - PATH="$HOME/.cargo/bin:$PATH" rustup install nightly -RUN yum install -y bzip2-devel gcc gcc-c++ git openssl-devel python27-devel python36-devel readline-devel sqlite-devel zip \ - && yum clean all -ADD build.sh /usr/local/bin/ -VOLUME ["/code"] -WORKDIR /code -ENTRYPOINT ["/usr/local/bin/build.sh"] diff --git a/builder/README.md b/builder/README.md deleted file mode 100644 index 288039a..0000000 --- a/builder/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# Building Lambda functions with crowbar in Docker - -The `Dockerfile` and `build.sh` script here help you build Lambda functions against libraries in Amazon Linux (used for the Lambda execution environment). - -It's automatically built on Docker Hub as [ilianaw/crowbar-builder](https://hub.docker.com/r/ilianaw/crowbar-builder/) and can be invoked like this from your source tree: - -```bash -docker run --rm -v $(pwd):/code:ro ilianaw/crowbar-builder > lambda.zip -``` - -If you need extra packages, add them as arguments: - -```bash -docker run --rm -v $(pwd):/code:ro ilianaw/crowbar-builder openssl-devel > lambda.zip -``` - -If you need to build with a nightly compiler set the `TOOLCHAIN` environment variable: - -```bash -docker run --rm -v $(pwd):/code:ro -e TOOLCHAIN=nightly ilianaw/crowbar-builder > lambda.zip -``` diff --git a/builder/build.sh b/builder/build.sh deleted file mode 100755 index 86d0831..0000000 --- a/builder/build.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -set -euo pipefail -export TOOLCHAIN=${TOOLCHAIN:-stable} -export CARGO_TARGET_DIR=$(mktemp -d) - -( - if [[ $# -gt 0 ]]; then - yum install -y "$@" - fi - . $HOME/.cargo/env - cargo +$TOOLCHAIN build ${CARGO_FLAGS:-} --release -) 1>&2 -cd $CARGO_TARGET_DIR/release -( - strip liblambda.so - zip lambda.zip liblambda.so -) 1>&2 -exec cat lambda.zip diff --git a/examples/ci/circle/Makefile b/examples/ci/circle/Makefile index e69de29..9f9d2cc 100644 --- a/examples/ci/circle/Makefile +++ b/examples/ci/circle/Makefile @@ -0,0 +1,47 @@ +#!/usr/bin/make -f + +LAMBDA_FUNCTION_NAME:=YOUR_LAMBDA_NAME +CACHE_DIR:=$(HOME)/.cache/cargo +ZIP_FILE:=$(CACHE_DIR)/target/deploy/lambda.zip +# change this to your own container based on lambci/lambda:build-python3.6 +DOCKER_IMAGE:=naftulikay/crowbar:latest + +WORKDIR:=$(shell pwd) + +USER_UID:=$(shell id -u) +USER_GID:=$(shell id -g) + +IS_TTY:=$(shell test -t && echo '-t') + +DOCKER_INVOKE:=docker run -i $(IS_TTY) --rm \ + -e USER_UID=$(USER_UID) \ + -e USER_GID=$(USER_GID) \ + -v $(WORKDIR):/home/circleci/project \ + -v $(CACHE_DIR)/cargo/registry:/home/circleci/.cargo/registry \ + -v $(CACHE_DIR)/target:/home/circleci/project/target \ + -v $(WORKDIR)/bin:/home/circleci/.local/bin \ + $(DOCKER_IMAGE) + +init: + @mkdir -p target/deploy + +pull: + @docker pull $(DOCKER_IMAGE) + +build: init + @$(DOCKER_INVOKE) .local/bin/build + @cp $(ZIP_FILE) target/deploy + +test: init + @$(DOCKER_INVOKE) .local/bin/test + +clean: init + @$(DOCKER_INVOKE).local/bin/clean + +shell: init + @$(DOCKER_INVOKE) + +deploy: build + @# update the function + aws --region us-east-1 lambda update-function-code --function-name $(LAMBDA_FUNCTION_NAME) \ + --zip-file fileb://$(ZIP_FILE) diff --git a/examples/ci/circle/bin/build b/examples/ci/circle/bin/build index 5de4831..a526bd9 100644 --- a/examples/ci/circle/bin/build +++ b/examples/ci/circle/bin/build @@ -17,4 +17,6 @@ else cargo build --lib --release --color never fi -~/.local/bin/create-deployment -v target/release/liblambda.so target/deploy +# this is provided by naftulikay/crowbar to bundle a zip with all dependencies +# this will create target/deploy/lambda.zip +create-deployment -v target/release/liblambda.so target/deploy diff --git a/examples/ci/circle/bin/create-deployment b/examples/ci/circle/bin/create-deployment deleted file mode 100644 index df830b0..0000000 --- a/examples/ci/circle/bin/create-deployment +++ /dev/null @@ -1,153 +0,0 @@ - -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from __future__ import absolute_import, print_function - -import argparse -import logging -import os -import re -import shutil -import subprocess -import tempfile - -WHITESPACE = re.compile(r'\s+', re.I) -SHARED_LIBRARY = re.compile(r'so(\.\d+)*$') - -LOGGING_HANDLER = logging.StreamHandler() -LOGGING_HANDLER.setFormatter(logging.Formatter('%(asctime)s [%(levelname)-5s] %(name)s: %(message)s')) - -root = logging.getLogger() -root.addHandler(LOGGING_HANDLER) -logger = logging.getLogger('deploy') - - -class SharedLibrary(object): - - def __init__(self, path): - self.logger = logging.getLogger('deploy.sharedlib') - self.basename = os.path.basename(path) - self.path = os.path.realpath(path) - - def __eq__(self, other): - return self.path == other.path - - def __hash__(self): - return hash(self.path) - - def __repr__(self): - return "SharedLibrary({})".format(self.basename) - - def __str__(self): - return self.path - - def dependencies(self): - result = set() - - for line in subprocess.check_output(['ldd', self.path]).splitlines(): - contents = WHITESPACE.split(line) - - # if we don't have a fourth field, or it is a system link, continue - if len(contents) < 4 or contents[3][0] == '(': - continue - - # found a library - library = SharedLibrary(contents[3]) - - # if it isn't present, insert it and its dependencies - if not library in result: - result.add(library) - result.union(library.dependencies()) - - return result - - -def main(): - parser = argparse.ArgumentParser(description="Find and relocate linked shared libraries for a so.") - parser.add_argument('--verbose', '-v', action='count', help="Set logging verbosity. By default, it is set to WARN, " - "Passing this once will yield INFO, twice will yield DEBUG.") - parser.add_argument('library', type=argparse.FileType('r'), help="The shared library to source dependencies from.") - parser.add_argument('output_dir', help="The output directory to create the distribution in.") - args = parser.parse_args() - - # setup logging - if args.verbose in (0, None): - root.setLevel(logging.WARNING) - elif args.verbose == 1: - root.setLevel(logging.INFO) - else: - root.setLevel(logging.DEBUG) - - # create the output directory - if not os.path.isdir(args.output_dir): - os.makedirs(args.output_dir) - - # output file - output_file = os.path.join(args.output_dir, "lambda.zip") - - # create the temporory directory - workdir = tempfile.mkdtemp() - workdir_lib = os.path.join(workdir, 'lib') - os.mkdir(workdir_lib) - - logger.debug("created temporary work directory: %s", workdir) - - # copy the source library into the temporary directory - shutil.copy(args.library.name, workdir) - - library = os.path.join(workdir, os.path.basename(args.library.name)) - library_basename = os.path.basename(library) - - # find, dereference, copy linked libraries - for linked_lib in SharedLibrary(library).dependencies(): - # parse each linked library - logger.debug("found linked library: %s", linked_lib.path) - - # copy the shared library file - logger.debug('copying linked library: %s', linked_lib.basename) - shutil.copyfile(linked_lib.path, os.path.join(workdir_lib, linked_lib.basename)) - - # strip the shared library file - logger.debug('stripping linked library: %s', linked_lib.basename) - subprocess.check_call(['strip', os.path.join(workdir_lib, linked_lib.basename)]) - - # strip the target - logger.debug('stripping %s', library_basename) - subprocess.check_call(['strip', library]) - - package_files = list([library_basename] + [os.path.join('lib', f) for f in os.listdir(workdir_lib)]) - - # create zip deployment - logger.debug("removing previous deployment zip...") - if os.path.isfile(output_file): - os.remove(os.path.abspath(output_file)) - - logger.debug("zipping shared libraries: %s", ', '.join(package_files)) - p = subprocess.Popen(['zip', os.path.abspath(output_file)] + package_files, - cwd=workdir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - - stdout, _ = p.communicate() - - logger.debug('zip output: \n%s', stdout) - - # display debugging linking info - env = os.environ.copy() - env.update(LD_LIBRARY_PATH=workdir_lib) - p = subprocess.Popen(['ldd', library], env=env, stdout=subprocess.PIPE) - stdout, _ = p.communicate() - - logger.debug("linking info: \n%s", stdout) - - if not p.returncode == 0: - logger.error('Unable to create deployment package: %s', stdout) - else: - logger.info("Built deployment package to: %s", output_file) - - # remove temporary directory - logger.debug("cleaning temporary directory: %s", workdir) - shutil.rmtree(workdir) - - -if __name__ == "__main__": - main() diff --git a/examples/ci/travis/Makefile b/examples/ci/travis/Makefile index 2f16b3a..9f9d2cc 100644 --- a/examples/ci/travis/Makefile +++ b/examples/ci/travis/Makefile @@ -4,7 +4,7 @@ LAMBDA_FUNCTION_NAME:=YOUR_LAMBDA_NAME CACHE_DIR:=$(HOME)/.cache/cargo ZIP_FILE:=$(CACHE_DIR)/target/deploy/lambda.zip # change this to your own container based on lambci/lambda:build-python3.6 -DOCKER_IMAGE:=naftulikay/circleci-lambda-rust:latest +DOCKER_IMAGE:=naftulikay/crowbar:latest WORKDIR:=$(shell pwd) diff --git a/examples/ci/travis/bin/build b/examples/ci/travis/bin/build index 5de4831..a526bd9 100644 --- a/examples/ci/travis/bin/build +++ b/examples/ci/travis/bin/build @@ -17,4 +17,6 @@ else cargo build --lib --release --color never fi -~/.local/bin/create-deployment -v target/release/liblambda.so target/deploy +# this is provided by naftulikay/crowbar to bundle a zip with all dependencies +# this will create target/deploy/lambda.zip +create-deployment -v target/release/liblambda.so target/deploy diff --git a/examples/ci/travis/bin/create-deployment b/examples/ci/travis/bin/create-deployment deleted file mode 100644 index df830b0..0000000 --- a/examples/ci/travis/bin/create-deployment +++ /dev/null @@ -1,153 +0,0 @@ - -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from __future__ import absolute_import, print_function - -import argparse -import logging -import os -import re -import shutil -import subprocess -import tempfile - -WHITESPACE = re.compile(r'\s+', re.I) -SHARED_LIBRARY = re.compile(r'so(\.\d+)*$') - -LOGGING_HANDLER = logging.StreamHandler() -LOGGING_HANDLER.setFormatter(logging.Formatter('%(asctime)s [%(levelname)-5s] %(name)s: %(message)s')) - -root = logging.getLogger() -root.addHandler(LOGGING_HANDLER) -logger = logging.getLogger('deploy') - - -class SharedLibrary(object): - - def __init__(self, path): - self.logger = logging.getLogger('deploy.sharedlib') - self.basename = os.path.basename(path) - self.path = os.path.realpath(path) - - def __eq__(self, other): - return self.path == other.path - - def __hash__(self): - return hash(self.path) - - def __repr__(self): - return "SharedLibrary({})".format(self.basename) - - def __str__(self): - return self.path - - def dependencies(self): - result = set() - - for line in subprocess.check_output(['ldd', self.path]).splitlines(): - contents = WHITESPACE.split(line) - - # if we don't have a fourth field, or it is a system link, continue - if len(contents) < 4 or contents[3][0] == '(': - continue - - # found a library - library = SharedLibrary(contents[3]) - - # if it isn't present, insert it and its dependencies - if not library in result: - result.add(library) - result.union(library.dependencies()) - - return result - - -def main(): - parser = argparse.ArgumentParser(description="Find and relocate linked shared libraries for a so.") - parser.add_argument('--verbose', '-v', action='count', help="Set logging verbosity. By default, it is set to WARN, " - "Passing this once will yield INFO, twice will yield DEBUG.") - parser.add_argument('library', type=argparse.FileType('r'), help="The shared library to source dependencies from.") - parser.add_argument('output_dir', help="The output directory to create the distribution in.") - args = parser.parse_args() - - # setup logging - if args.verbose in (0, None): - root.setLevel(logging.WARNING) - elif args.verbose == 1: - root.setLevel(logging.INFO) - else: - root.setLevel(logging.DEBUG) - - # create the output directory - if not os.path.isdir(args.output_dir): - os.makedirs(args.output_dir) - - # output file - output_file = os.path.join(args.output_dir, "lambda.zip") - - # create the temporory directory - workdir = tempfile.mkdtemp() - workdir_lib = os.path.join(workdir, 'lib') - os.mkdir(workdir_lib) - - logger.debug("created temporary work directory: %s", workdir) - - # copy the source library into the temporary directory - shutil.copy(args.library.name, workdir) - - library = os.path.join(workdir, os.path.basename(args.library.name)) - library_basename = os.path.basename(library) - - # find, dereference, copy linked libraries - for linked_lib in SharedLibrary(library).dependencies(): - # parse each linked library - logger.debug("found linked library: %s", linked_lib.path) - - # copy the shared library file - logger.debug('copying linked library: %s', linked_lib.basename) - shutil.copyfile(linked_lib.path, os.path.join(workdir_lib, linked_lib.basename)) - - # strip the shared library file - logger.debug('stripping linked library: %s', linked_lib.basename) - subprocess.check_call(['strip', os.path.join(workdir_lib, linked_lib.basename)]) - - # strip the target - logger.debug('stripping %s', library_basename) - subprocess.check_call(['strip', library]) - - package_files = list([library_basename] + [os.path.join('lib', f) for f in os.listdir(workdir_lib)]) - - # create zip deployment - logger.debug("removing previous deployment zip...") - if os.path.isfile(output_file): - os.remove(os.path.abspath(output_file)) - - logger.debug("zipping shared libraries: %s", ', '.join(package_files)) - p = subprocess.Popen(['zip', os.path.abspath(output_file)] + package_files, - cwd=workdir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - - stdout, _ = p.communicate() - - logger.debug('zip output: \n%s', stdout) - - # display debugging linking info - env = os.environ.copy() - env.update(LD_LIBRARY_PATH=workdir_lib) - p = subprocess.Popen(['ldd', library], env=env, stdout=subprocess.PIPE) - stdout, _ = p.communicate() - - logger.debug("linking info: \n%s", stdout) - - if not p.returncode == 0: - logger.error('Unable to create deployment package: %s', stdout) - else: - logger.info("Built deployment package to: %s", output_file) - - # remove temporary directory - logger.debug("cleaning temporary directory: %s", workdir) - shutil.rmtree(workdir) - - -if __name__ == "__main__": - main()