From 6c611cb9406f41c25fa102051bec7861d30fdec3 Mon Sep 17 00:00:00 2001 From: alvin-reyes Date: Fri, 6 May 2022 11:32:56 -0400 Subject: [PATCH 1/9] chore: add ARM build --- .circleci/config.yml | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 169b17f7..ec844a94 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,7 @@ orbs: executors: golang: docker: - - image: cimg/go:1.17.9 + - image: cimg/go:1.17.9-stretch resource_class: 2xlarge rust: docker: @@ -188,6 +188,10 @@ commands: default: false description: is a darwin build environment? type: boolean + arm: + default: false + description: is a arm build environment? + type: boolean steps: - checkout - when: @@ -213,6 +217,13 @@ commands: command: | curl https://sh.rustup.rs -sSf | sh -s -- -y rustc --version + - when: + condition: << parameters.arm >> + steps: + - run: sudo apt-get update + - run: apt-get install -y wget jq hwloc ocl-icd-opencl-dev git libhwloc-dev pkg-config make + - run: apt-get install -y cargo golang-go + - run: curl https://sh.rustup.rs -sSf | sh -s -- -y - run: name: Ensure appropriate toolchain is installed command: | @@ -269,6 +280,10 @@ commands: default: false description: is a Darwin build environment? type: boolean + arm: + default: false + description: is a ARM build environment? + type: boolean steps: - run: name: Configure environment variables @@ -280,7 +295,10 @@ commands: echo 'export RUST_LOG=info' >> $BASH_ENV echo 'export CIRCLE_ARTIFACTS="/tmp"' >> $BASH_ENV - when: - condition: << parameters.darwin >> + condition: + or: + - << parameters.darwin >> + - << parameters.arm >> steps: - run: name: Add a few more environment variables From 0e0626f1acf4292549698c59b78bfe28f441af80 Mon Sep 17 00:00:00 2001 From: alvin-reyes Date: Sat, 7 May 2022 10:26:10 -0400 Subject: [PATCH 2/9] chore: docker file to build on arm --- Dockerfile | 20 ++++++++++++++++++++ install-filcrypto | 6 +++++- 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..d4919055 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +#FROM --platform=arm64 golang@sha256:96e888160bd68f54a1165b23c66318aea4ff6a4726cb854276d6d776c14b8978 +FROM ubuntu:latest +#FROM rust:slim-buster@sha256:5b0c5f6cd5aa84f6dd3f2188ac84aa0e0d602b514ccc14e9d1ebd5b865aa7849 +#FROM golang:1.17.9-stretch + +RUN apt-get update && \ + apt-get install -y wget jq hwloc ocl-icd-opencl-dev git libhwloc-dev pkg-config make && \ + apt-get install -y cargo golang-go +WORKDIR /filecoin-ffi/ +COPY . /filecoin-ffi/ +#RUN curl https://sh.rustup.rs -sSf | bash -s -- -y +RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly -y +ENV PATH="/root/.cargo/bin:${PATH}" +RUN rustup +nightly target add aarch64-unknown-linux-gnu +#RUN cargo --help +#RUN make +RUN RUST_BACKTRACE=1 RUSTFLAGS="-C target-cpu=native -g" FFI_BUILD_FROM_SOURCE=1 make +#RUN git clone https://github.com/alvin-reyes/filecoin-ffi.git . && \ +# RUSTFLAGS="-C target-cpu=native -g" FFI_BUILD_FROM_SOURCE=1 make + \ No newline at end of file diff --git a/install-filcrypto b/install-filcrypto index 41adf5fd..b925ef25 100755 --- a/install-filcrypto +++ b/install-filcrypto @@ -158,10 +158,14 @@ build_from_source() { # For building on Darwin, we try to use cargo-lipo instead of cargo build. # Note that the cross compile works on x86_64 for m1, but doesn't work on m1. # For m1, we will build natively if building from source. + echo "$(uname -s)" if [ "$(uname -s)" = "Darwin" ] && [ "$(uname -m)" = "x86_64" ]; then build="lipo" additional_flags="--targets x86_64-apple-darwin,aarch64-apple-darwin " - else + elif [ "$(uname -m)" = "arm64" ]; then + build="build" + additional_flags="--target aarch64-unknown-linux-gnu " + else build="build" fi From 36000adc874f1bd4c54b7526b39c7c8eb63e8ab8 Mon Sep 17 00:00:00 2001 From: alvin-reyes Date: Wed, 11 May 2022 08:28:41 -0400 Subject: [PATCH 3/9] chore: use ubuntu image and add script to extract files from image to host --- Dockerfile | 7 ++++--- run_arm_build_docker.sh | 8 ++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) create mode 100755 run_arm_build_docker.sh diff --git a/Dockerfile b/Dockerfile index d4919055..1a75a6b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,8 +3,10 @@ FROM ubuntu:latest #FROM rust:slim-buster@sha256:5b0c5f6cd5aa84f6dd3f2188ac84aa0e0d602b514ccc14e9d1ebd5b865aa7849 #FROM golang:1.17.9-stretch +ARG DEBIAN_FRONTEND=noninteractive + RUN apt-get update && \ - apt-get install -y wget jq hwloc ocl-icd-opencl-dev git libhwloc-dev pkg-config make && \ + apt-get install -y wget jq hwloc ocl-icd-opencl-dev git libhwloc-dev pkg-config make curl && \ apt-get install -y cargo golang-go WORKDIR /filecoin-ffi/ COPY . /filecoin-ffi/ @@ -16,5 +18,4 @@ RUN rustup +nightly target add aarch64-unknown-linux-gnu #RUN make RUN RUST_BACKTRACE=1 RUSTFLAGS="-C target-cpu=native -g" FFI_BUILD_FROM_SOURCE=1 make #RUN git clone https://github.com/alvin-reyes/filecoin-ffi.git . && \ -# RUSTFLAGS="-C target-cpu=native -g" FFI_BUILD_FROM_SOURCE=1 make - \ No newline at end of file +# RUSTFLAGS="-C target-cpu=native -g" FFI_BUILD_FROM_SOURCE=1 make \ No newline at end of file diff --git a/run_arm_build_docker.sh b/run_arm_build_docker.sh new file mode 100755 index 00000000..c7731190 --- /dev/null +++ b/run_arm_build_docker.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +timestamp=$(date +%s) +name=arm_build_only_$timestamp +docker build -t $name . + +id=$(docker create $name) +docker cp $id:path/filecoin-ffi/ - > $name_filecoin_ffi.tar.gz From e44c2285947858461c722e16b9986222a7c9f0a9 Mon Sep 17 00:00:00 2001 From: alvin-reyes Date: Wed, 11 May 2022 08:31:00 -0400 Subject: [PATCH 4/9] chore: use filecoin-ffi directly --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1a75a6b4..b60bbe80 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,6 @@ ENV PATH="/root/.cargo/bin:${PATH}" RUN rustup +nightly target add aarch64-unknown-linux-gnu #RUN cargo --help #RUN make -RUN RUST_BACKTRACE=1 RUSTFLAGS="-C target-cpu=native -g" FFI_BUILD_FROM_SOURCE=1 make -#RUN git clone https://github.com/alvin-reyes/filecoin-ffi.git . && \ -# RUSTFLAGS="-C target-cpu=native -g" FFI_BUILD_FROM_SOURCE=1 make \ No newline at end of file +#RUN RUST_BACKTRACE=1 RUSTFLAGS="-C target-cpu=native -g" FFI_BUILD_FROM_SOURCE=1 make +RUN git clone https://github.com/filecoin-project/filecoin-ffi.git . && \ + RUSTFLAGS="-C target-cpu=native -g" FFI_BUILD_FROM_SOURCE=1 make \ No newline at end of file From a0d954c1e2feefa14c0b63c9a7fcda0b5cb2938e Mon Sep 17 00:00:00 2001 From: alvin-reyes Date: Wed, 11 May 2022 09:51:50 -0400 Subject: [PATCH 5/9] chore: added dedicated docker file for running filecoin-ffi build --- .circleci/config.yml | 24 +++------------------- builder/arm64/.DS_Store | Bin 0 -> 6148 bytes Dockerfile => builder/arm64/Dockerfile | 9 ++++----- builder/arm64/README.md | 27 +++++++++++++++++++++++++ builder/arm64/run_arm_build_docker.sh | 12 +++++++++++ run_arm_build_docker.sh | 8 -------- 6 files changed, 46 insertions(+), 34 deletions(-) create mode 100644 builder/arm64/.DS_Store rename Dockerfile => builder/arm64/Dockerfile (83%) create mode 100644 builder/arm64/README.md create mode 100755 builder/arm64/run_arm_build_docker.sh delete mode 100755 run_arm_build_docker.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index ec844a94..0a86c8eb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,7 @@ orbs: executors: golang: docker: - - image: cimg/go:1.17.9-stretch + - image: cimg/go:1.17.9 resource_class: 2xlarge rust: docker: @@ -188,10 +188,6 @@ commands: default: false description: is a darwin build environment? type: boolean - arm: - default: false - description: is a arm build environment? - type: boolean steps: - checkout - when: @@ -217,13 +213,6 @@ commands: command: | curl https://sh.rustup.rs -sSf | sh -s -- -y rustc --version - - when: - condition: << parameters.arm >> - steps: - - run: sudo apt-get update - - run: apt-get install -y wget jq hwloc ocl-icd-opencl-dev git libhwloc-dev pkg-config make - - run: apt-get install -y cargo golang-go - - run: curl https://sh.rustup.rs -sSf | sh -s -- -y - run: name: Ensure appropriate toolchain is installed command: | @@ -280,10 +269,6 @@ commands: default: false description: is a Darwin build environment? type: boolean - arm: - default: false - description: is a ARM build environment? - type: boolean steps: - run: name: Configure environment variables @@ -295,10 +280,7 @@ commands: echo 'export RUST_LOG=info' >> $BASH_ENV echo 'export CIRCLE_ARTIFACTS="/tmp"' >> $BASH_ENV - when: - condition: - or: - - << parameters.darwin >> - - << parameters.arm >> + condition: << parameters.darwin >> steps: - run: name: Add a few more environment variables @@ -359,4 +341,4 @@ commands: - save_cache: key: v28-proof-params-{{ arch }} paths: - - "~/filecoin-proof-parameters/" + - "~/filecoin-proof-parameters/" \ No newline at end of file diff --git a/builder/arm64/.DS_Store b/builder/arm64/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..0c1ea6291bcef48df4a5077fb8b8ce6a9958cfe7 GIT binary patch literal 6148 zcmeHK%}T>S5T0$TO{hW+3OxqAR;;xO#Y>3w1&ruHr8YKdFlI}C%%K!=))(?gd>&_Z zw_>Ticn~QwF#FBU&Scqd!_F=M2!Gt$2XFwuLM1HN*!&lP|sD!u|VaWAmoc3hq%W>Kps@y;iShi(% zoa$t9*l5+ $name.tar.gz +docker cp $id:/filecoin-ffi/ - > $name.tar.gz + +## extract +tar -xvf $name.tar.gz diff --git a/run_arm_build_docker.sh b/run_arm_build_docker.sh deleted file mode 100755 index c7731190..00000000 --- a/run_arm_build_docker.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -timestamp=$(date +%s) -name=arm_build_only_$timestamp -docker build -t $name . - -id=$(docker create $name) -docker cp $id:path/filecoin-ffi/ - > $name_filecoin_ffi.tar.gz From 4a345ad6d856117790cd045b564163a10b1a685d Mon Sep 17 00:00:00 2001 From: alvin-reyes Date: Wed, 11 May 2022 10:51:12 -0400 Subject: [PATCH 6/9] chore: added arm64 scripts --- builder/.DS_Store | Bin 0 -> 6148 bytes builder/arm64/run_arm_build_docker.sh | 4 +++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 builder/.DS_Store diff --git a/builder/.DS_Store b/builder/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..65d2ec78f49b8f1d71046d812abef92594374a07 GIT binary patch literal 6148 zcmeHK%}T>S5Z>*NO({YS3OxqA7Ob@vikA@U3mDOZN=-=7V45vSY7V84v%Zi|;`2DO zyMY#iM-e*%yWi~m>}Ed5{xHV4vkXocGZ|wBG(?U{g`m08Rnx(UT+T7{^K{`y!mpU- zZ<_Gio6KVgo3jzy{QeJr5rt{i?|<@Iz0urj87-q@Jh;yycQY?rrjD1~;pke3$S-xz zzX@mg)I7Klapr~bERhLeFoTf$+b|A9?uca^q%zl44x??fr)GDxIvb38)^Ipp_pH@; z&P%y4Q1qAf1O8^XTAL&Y`@>{4woUO4? Vh_j$wrUTMNKoUY7G4Klvd;ttJN!b7Z literal 0 HcmV?d00001 diff --git a/builder/arm64/run_arm_build_docker.sh b/builder/arm64/run_arm_build_docker.sh index ca165d8d..1f4504e5 100755 --- a/builder/arm64/run_arm_build_docker.sh +++ b/builder/arm64/run_arm_build_docker.sh @@ -4,9 +4,11 @@ timestamp=$(date +%s) name=arm_build_only_$timestamp docker build -t $name . +# get the container id id=$(docker create $name) docker cp $id:/filecoin-ffi/ - > $name.tar.gz -docker cp $id:/filecoin-ffi/ - > $name.tar.gz ## extract tar -xvf $name.tar.gz + +## Copy to wherever.. \ No newline at end of file From b7be0a3347b1392ebb1509ea8ffd8239c4768c02 Mon Sep 17 00:00:00 2001 From: Alvin Reyes Date: Wed, 11 May 2022 12:40:39 -0400 Subject: [PATCH 7/9] Update README.md --- builder/arm64/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builder/arm64/README.md b/builder/arm64/README.md index 3a8e549e..b46ef3bd 100644 --- a/builder/arm64/README.md +++ b/builder/arm64/README.md @@ -1,6 +1,6 @@ # filecoin-ffi build workaround -As of this writing, .circleci doens't support arm based images and as a workaround we need to try and build the filecoin-ffi in a containerized environment.This means that we need to create a docker image using an ARM64 based operating system and build the binaries there. +As of this writing, .circleci does not support arm based images and as a workaround we need to try and build the filecoin-ffi in a containerized environment. The workaround is to create a docker image using an ARM64 based operating system and build the binaries in the image. We can then pull (`cp`) the binaries to the host. ## Pre-requisites - docker @@ -24,4 +24,4 @@ FROM --platform=arm64 golang@sha256:96e888160bd68f54a1165b23c66318aea4ff6a4726cb FROM ubuntu:latest as builder FROM rust:slim-buster@sha256:5b0c5f6cd5aa84f6dd3f2188ac84aa0e0d602b514ccc14e9d1ebd5b865aa7849 as builder FROM golang:1.17.9-stretch as builder -``` \ No newline at end of file +``` From 7a12cc9eeb748f60d3b65194846f7c65c40ff085 Mon Sep 17 00:00:00 2001 From: Alvin Reyes Date: Wed, 11 May 2022 12:40:55 -0400 Subject: [PATCH 8/9] Update README.md --- builder/arm64/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/arm64/README.md b/builder/arm64/README.md index b46ef3bd..89fb6fa1 100644 --- a/builder/arm64/README.md +++ b/builder/arm64/README.md @@ -17,7 +17,7 @@ There are 2 files in this repo: ``` -## Trial and error +## Builders We can use some of these images as builders. They are all arm64 based. ``` FROM --platform=arm64 golang@sha256:96e888160bd68f54a1165b23c66318aea4ff6a4726cb854276d6d776c14b8978 as builder From f317fd540a58a30a4f9053af0edef45e21b9f26a Mon Sep 17 00:00:00 2001 From: alvin-reyes Date: Wed, 11 May 2022 12:49:45 -0400 Subject: [PATCH 9/9] chore: removed echo uname --- install-filcrypto | 1 - 1 file changed, 1 deletion(-) diff --git a/install-filcrypto b/install-filcrypto index b925ef25..ae4a7e9a 100755 --- a/install-filcrypto +++ b/install-filcrypto @@ -158,7 +158,6 @@ build_from_source() { # For building on Darwin, we try to use cargo-lipo instead of cargo build. # Note that the cross compile works on x86_64 for m1, but doesn't work on m1. # For m1, we will build natively if building from source. - echo "$(uname -s)" if [ "$(uname -s)" = "Darwin" ] && [ "$(uname -m)" = "x86_64" ]; then build="lipo" additional_flags="--targets x86_64-apple-darwin,aarch64-apple-darwin "