Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use AWS CodeBuild to compile C++ binary #469

Merged
merged 1 commit into from
Jan 11, 2023
Merged

Use AWS CodeBuild to compile C++ binary #469

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
176 changes: 0 additions & 176 deletions .travis.yml
View file
Open in desktop

This file was deleted.

4 changes: 2 additions & 2 deletions bootstrap.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ silence() {
}

OPENSSL_VERSION="1.1.1s"
BOOST_VERSION="1.80.0"
BOOST_VERSION_UNDERSCORED="${BOOST_VERSION//\./_}" # convert from 1.80.0 to 1_80_0
BOOST_VERSION="1.76.0"
BOOST_VERSION_UNDERSCORED="${BOOST_VERSION//\./_}" # convert from 1.76.0 to 1_76_0
ZLIB_VERSION="1.2.13"
PROTOBUF_VERSION="3.11.4"
CURL_VERSION="7.86.0"
Expand Down
60 changes: 60 additions & 0 deletions buildspec-aarch64.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
version: 0.2
env:
variables:
CODEBUILD_OS_NAME: "linux"
MAVEN_VERSION: "3.8.7"
MAVEN_DOWNLOAD_LINK: "https://archive.apache.org/dist/maven/maven-3/3.8.7/binaries/apache-maven-3.8.7-bin.tar.gz"

phases:
install:
run-as: root
runtime-versions:
java: corretto8
commands:
- nohup /usr/local/bin/dockerd --host=unix:///var/run/docker.sock --host=tcp://127.0.0.1:2375 --storage-driver=overlay2 &
- timeout 15 sh -c "until docker info; do echo .; sleep 1; done"
pre_build:
run-as: root
commands:
- echo "################################################################################"
- echo "Installing Dependencies for Linux/arm64"
- echo "################################################################################"
- yum install -y sudo
- sudo yum -y update
- sudo yum install -y ca-certificates
- echo "Install aws cli v2"
- curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip"
- unzip awscliv2.zip
- sudo ./aws/install --bin-dir /usr/local/bin --install-dir /usr/local/aws-cli --update
- export PATH=~/bin:$PATH
- echo "Aws cli has been installed successfully"
- echo "Assuming the builder role to get credentials for building KPL"
- BUILDER_ROLE=$(aws sts assume-role --role-arn arn:aws:iam::056543101242:role/KPLCodeBuildAdminRole --role-session-name kpl-build-session)
- export BUILDER_ROLE
- export AWS_ACCESS_KEY_ID=$(echo "${BUILDER_ROLE}" | jq -r '.Credentials.AccessKeyId')
- export AWS_SECRET_ACCESS_KEY=$(echo "${BUILDER_ROLE}" | jq -r '.Credentials.SecretAccessKey')
- export AWS_SESSION_TOKEN=$(echo "${BUILDER_ROLE}" | jq -r '.Credentials.SessionToken')
- echo "Pulled the credentials for KPL builders"
- echo "################################################################################"
- echo "SUCCESS Installed Dependencies for Linux/arm64"
- echo "################################################################################"
build:
commands:
- echo "################################################################################"
- echo "Starting build phase for Linux/arm64"
- echo "################################################################################"
- aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 056543101242.dkr.ecr.us-west-2.amazonaws.com
- echo "Getting Docker image for AL2/Arm64 build of KPL..."
- docker pull public.ecr.aws/amazonlinux/amazonlinux:2
- ls
- df -h
- echo "Building KPL in current directory in the Container..."
- docker run --rm -m 15g --privileged -v $(pwd):/kpl public.ecr.aws/amazonlinux/amazonlinux:2 /bin/bash -c "export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY && export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID && export MAVEN_DOWNLOAD_LINK=$MAVEN_DOWNLOAD_LINK && export MAVEN_VERSION=$MAVEN_VERSION && cd /kpl && ls && yum install -y -q sudo cmake3 gcc gcc-c++ make file tar gzip which perl git libuuid libuuid-devel zlib zlib-devel zlib-static wget && sudo amazon-linux-extras enable corretto8 && sudo yum install -y java-1.8.0-amazon-corretto-devel && export JAVA_HOME=/usr/lib/jvm/java-1.8.0-amazon-corretto.aarch64 && java -version && echo \"Install Maven from $MAVEN_DOWNLOAD_LINK...\" && mkdir -p /usr/local/maven && wget $MAVEN_DOWNLOAD_LINK && tar xvf apache-maven-$MAVEN_VERSION-bin.tar.gz -C /usr/local/maven --strip-components=1 && ln -s /usr/local/maven/bin/mvn /usr/bin/mvn && echo \"Maven has been installed with $(mvn --version)\" && cd /kpl && ls && ./bootstrap.sh"
- cd java/amazon-kinesis-producer/src/main/resources/amazon-kinesis-producer-native-binaries/$CODEBUILD_OS_NAME-`uname -m`
- ls
- sudo zip kinesis-producer.zip kinesis_producer
- echo "Uploading build artifacts for KPL built on Linux arm64..."
- aws s3 cp kinesis-producer.zip s3://kpl-build-kinesis-internal/$KPL_VERSION-$HASH/$CODEBUILD_OS_NAME-`uname -m`/kinesis-producer.zip
- echo "################################################################################"
- echo "SUCCESS Build phase for Linux/arm64, complete. Check for any errors reported"
- echo "################################################################################"
57 changes: 57 additions & 0 deletions buildspec-x86_64.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
version: 0.2
env:
variables:
CODEBUILD_OS_NAME: "linux"

phases:
install:
run-as: root
runtime-versions:
java: corretto8
commands:
- nohup /usr/local/bin/dockerd --host=unix:///var/run/docker.sock --host=tcp://127.0.0.1:2375 --storage-driver=overlay2 &
- timeout 15 sh -c "until docker info; do echo .; sleep 1; done"
pre_build:
run-as: root
commands:
- echo "################################################################################"
- echo "Installing Dependencies for Linux/x86_64"
- echo "################################################################################"
- sudo apt update
- sudo apt install -y ca-certificates
- echo "Install aws cli v2"
- curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
- unzip awscliv2.zip
- sudo ./aws/install --bin-dir /usr/local/bin --install-dir /usr/local/aws-cli --update
- export PATH=~/bin:$PATH
- echo "Aws cli has been installed successfully"
- echo "Assuming the builder role to get credentials for building KPL"
- BUILDER_ROLE=$(aws sts assume-role --role-arn arn:aws:iam::056543101242:role/KPLCodeBuildAdminRole --role-session-name kpl-build-session)
- export BUILDER_ROLE
- export AWS_ACCESS_KEY_ID=$(echo "${BUILDER_ROLE}" | jq -r '.Credentials.AccessKeyId')
- export AWS_SECRET_ACCESS_KEY=$(echo "${BUILDER_ROLE}" | jq -r '.Credentials.SecretAccessKey')
- export AWS_SESSION_TOKEN=$(echo "${BUILDER_ROLE}" | jq -r '.Credentials.SessionToken')
- echo "Pulled the credentials for KPL builders"
- echo "################################################################################"
- echo "SUCCESS Installed Dependencies for Linux/x86_64"
- echo "################################################################################"
build:
commands:
- echo "################################################################################"
- echo "Starting build phase for Linux/x86_64"
- echo "################################################################################"
- aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin 056543101242.dkr.ecr.us-west-2.amazonaws.com
- echo "Getting Docker image for AL2012 build of KPL..."
- docker pull 056543101242.dkr.ecr.us-west-2.amazonaws.com/kinesis-producer-library-builder:latest
- ls
- df -h
- echo "Building KPL in current directory in the Container..."
- docker run --rm -m 15g -v $(pwd):/kpl 056543101242.dkr.ecr.us-west-2.amazonaws.com/kinesis-producer-library-builder:latest /bin/bash -c "export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY && export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID && cd /kpl && ls && ./bootstrap.sh"
- cd java/amazon-kinesis-producer/src/main/resources/amazon-kinesis-producer-native-binaries/$CODEBUILD_OS_NAME-`uname -m`
- ls
- sudo zip kinesis-producer.zip kinesis_producer
- echo "Uploading build artifacts for KPL built on Linux x86_64..."
- aws s3 cp kinesis-producer.zip s3://kpl-build-kinesis-internal/$KPL_VERSION-$HASH/$CODEBUILD_OS_NAME-`uname -m`/kinesis-producer.zip
- echo "################################################################################"
- echo "SUCCESS Build phase for Linux/x86_64, complete. Check for any errors reported"
- echo "################################################################################"