From bbbf0c9fe6e8e0064e296801fc30aa07526a331f Mon Sep 17 00:00:00 2001 From: buffalu <85544055+buffalu@users.noreply.github.com> Date: Sat, 11 Jun 2022 21:30:50 -0500 Subject: [PATCH] Add basic CI (#2) --- .dockerignore | 1 + .github/workflows/build.yaml | 38 ++++++++++++++++++++++++++ .github/workflows/clean_code.yaml | 35 ++++++++++++++++++++++++ .github/workflows/master.yaml | 41 ++++++++++++++++++++++++++++ .github/workflows/pr.yaml | 29 ++++++++++++++++++++ .github/workflows/push_images.yaml | 28 +++++++++++++++++++ .github/workflows/release.yaml | 43 ++++++++++++++++++++++++++++++ .github/workflows/test.yaml | 17 ++++++++++++ Dockerfile | 26 ++++++++++++++++++ b | 18 +++++++++++++ docker-compose.yaml | 30 +++++++++++++++++++++ env/.env.dev | 7 +++++ relayer/Cargo.toml | 1 - 13 files changed, 313 insertions(+), 1 deletion(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/build.yaml create mode 100644 .github/workflows/clean_code.yaml create mode 100644 .github/workflows/master.yaml create mode 100644 .github/workflows/pr.yaml create mode 100644 .github/workflows/push_images.yaml create mode 100644 .github/workflows/release.yaml create mode 100644 .github/workflows/test.yaml create mode 100644 Dockerfile create mode 100755 b create mode 100644 docker-compose.yaml create mode 100644 env/.env.dev diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000000000..2f7896d1d1365e --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +target/ diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 00000000000000..2f1bd2bcdb9739 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,38 @@ +name: Build +on: + workflow_call: + secrets: + ACCESS_TOKEN: + required: true + DOCKERHUB_ORG: + required: true + DEPLOYER_SSH_KEY: + required: true + inputs: + TAG: + required: true + type: string + +jobs: + build: + runs-on: self-hosted + steps: + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + ssh-key: ${{ secrets.DEPLOYER_SSH_KEY }} + # https://github.com/moby/buildkit/issues/1368#issuecomment-731466205 + # Seeing conflicting build caching between this and other repos in CI + - name: Docker prune + if: always() + run: docker builder prune -f + - name: Docker image prune + if: always() + run: docker image prune -f + - name: Build containers + run: docker compose --env-file ./env/.env.dev build --progress=plain + env: + COMPOSE_DOCKER_CLI_BUILD: 1 + DOCKER_BUILDKIT: 1 + ORG: ${{ secrets.DOCKERHUB_ORG }} + TAG: ${{ inputs.TAG }} diff --git a/.github/workflows/clean_code.yaml b/.github/workflows/clean_code.yaml new file mode 100644 index 00000000000000..bb23eadce00d56 --- /dev/null +++ b/.github/workflows/clean_code.yaml @@ -0,0 +1,35 @@ +name: Clean Code Check +on: + workflow_call: + secrets: + GHUB_TOKEN: + required: true + ACCESS_TOKEN: + required: true + DEPLOYER_SSH_KEY: + required: true +jobs: + clippy_and_udeps_check: + runs-on: self-hosted + steps: + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + ssh-key: ${{ secrets.DEPLOYER_SSH_KEY }} + - uses: actions-rs/toolchain@v1 + with: + toolchain: nightly + override: true + components: clippy + - uses: actions-rs/clippy-check@v1 + with: + token: ${{ secrets.GHUB_TOKEN }} + args: --all-features + - uses: actions-rs/cargo@v1 + with: + command: install + args: cargo-udeps --locked + - name: cargo udeps + uses: actions-rs/cargo@v1 + with: + command: udeps diff --git a/.github/workflows/master.yaml b/.github/workflows/master.yaml new file mode 100644 index 00000000000000..b67d2deaddba3b --- /dev/null +++ b/.github/workflows/master.yaml @@ -0,0 +1,41 @@ +name: Master +on: + push: + branches: + - master + +jobs: + clean_code_check: + uses: ./.github/workflows/clean_code.yaml + secrets: + GHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} + DEPLOYER_SSH_KEY: ${{ secrets.DEPLOYER_SSH_KEY }} + + build_images: + needs: clean_code_check + uses: ./.github/workflows/build.yaml + with: + TAG: "latest" + secrets: + ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} + DOCKERHUB_ORG: ${{ secrets.DOCKERHUB_ORG }} + DEPLOYER_SSH_KEY: ${{ secrets.DEPLOYER_SSH_KEY }} + + run_tests: + needs: build_images + uses: ./.github/workflows/test.yaml + secrets: + DOCKERHUB_ORG: ${{ secrets.DOCKERHUB_ORG }} + with: + TAG: "latest" + + push_images: + needs: run_tests + uses: ./.github/workflows/push_images.yaml + secrets: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_PWD: ${{ secrets.DOCKERHUB_PWD }} + DOCKERHUB_ORG: ${{ secrets.DOCKERHUB_ORG }} + with: + TAG: "latest" diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml new file mode 100644 index 00000000000000..808b3fc99dceb7 --- /dev/null +++ b/.github/workflows/pr.yaml @@ -0,0 +1,29 @@ +name: Pull Request +on: + pull_request: + +jobs: + clean_code_check: + uses: ./.github/workflows/clean_code.yaml + secrets: + GHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} + DEPLOYER_SSH_KEY: ${{ secrets.DEPLOYER_SSH_KEY }} + + build_images: + needs: clean_code_check + uses: ./.github/workflows/build.yaml + with: + TAG: ${{ github.sha }} + secrets: + ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} + DOCKERHUB_ORG: ${{ secrets.DOCKERHUB_ORG }} + DEPLOYER_SSH_KEY: ${{ secrets.DEPLOYER_SSH_KEY }} + + run_tests: + needs: build_images + uses: ./.github/workflows/test.yaml + secrets: + DOCKERHUB_ORG: ${{ secrets.DOCKERHUB_ORG }} + with: + TAG: ${{ github.sha }} diff --git a/.github/workflows/push_images.yaml b/.github/workflows/push_images.yaml new file mode 100644 index 00000000000000..b9900a2e9b8c27 --- /dev/null +++ b/.github/workflows/push_images.yaml @@ -0,0 +1,28 @@ +name: Push Images +on: + workflow_call: + secrets: + DOCKERHUB_USER: + required: true + DOCKERHUB_PWD: + required: true + DOCKERHUB_ORG: + required: true + inputs: + TAG: + required: true + type: string + +jobs: + push: + name: Docker Push + runs-on: self-hosted + steps: + # login to registry + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_PWD }} + - name: Push relayer image + run: docker push ${{ secrets.DOCKERHUB_ORG }}/jito-transaction-relayer:${{ inputs.TAG }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 00000000000000..da6de3518d5a1d --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,43 @@ +name: Release +on: + push: + tags: + - 'v*.*.*' +env: + TAG: "$(git rev-parse --short HEAD)" + +jobs: + clean_code_check: + uses: ./.github/workflows/clean_code.yaml + secrets: + GHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} + DEPLOYER_SSH_KEY: ${{ secrets.DEPLOYER_SSH_KEY }} + + build_images: + needs: clean_code_check + uses: ./.github/workflows/build.yaml + with: + TAG: ${{ github.ref_name }}-${{ github.sha }} + secrets: + ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }} + DOCKERHUB_ORG: ${{ secrets.DOCKERHUB_ORG }} + DEPLOYER_SSH_KEY: ${{ secrets.DEPLOYER_SSH_KEY }} + + run_tests: + needs: build_images + uses: ./.github/workflows/test.yaml + secrets: + DOCKERHUB_ORG: ${{ secrets.DOCKERHUB_ORG }} + with: + TAG: ${{ github.ref_name }}-${{ github.sha }} + + push_images: + needs: run_tests + uses: ./.github/workflows/push_images.yaml + secrets: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_PWD: ${{ secrets.DOCKERHUB_PWD }} + DOCKERHUB_ORG: ${{ secrets.DOCKERHUB_ORG }} + with: + TAG: ${{ github.ref_name }}-${{ github.sha }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 00000000000000..407ba2e3afd301 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,17 @@ +name: Test +on: + workflow_call: + secrets: + DOCKERHUB_ORG: + required: true + inputs: + TAG: + required: true + type: string + +jobs: + test: + runs-on: self-hosted + steps: + - name: Run tests + run: RUST_LOG=info cargo test diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000000000..f47eab72376b26 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +# syntax=docker/dockerfile:1.4.0 +FROM rust:1.59.0-slim-buster as builder + +RUN apt-get update && apt-get install -y libudev-dev clang pkg-config libssl-dev build-essential cmake +RUN rustup component add rustfmt +RUN update-ca-certificates + +ENV HOME=/home/root +WORKDIR $HOME/app +COPY . . + +RUN --mount=type=cache,mode=0777,target=/home/root/app/target \ + --mount=type=cache,mode=0777,target=/usr/local/cargo/registry \ + cargo build --release && cp target/release/jito-* ./ + +FROM debian:buster-slim as jito-transaction-relayer + +# needed for HTTPS +RUN \ + apt-get update && \ + apt-get -y install ca-certificates && \ + apt-get clean + +WORKDIR /app +COPY --from=builder /home/root/app/jito-transaction-relayer ./ +ENTRYPOINT ./jito-transaction-relayer diff --git a/b b/b new file mode 100755 index 00000000000000..40a1a861456a61 --- /dev/null +++ b/b @@ -0,0 +1,18 @@ +#!/usr/bin/env sh +# Build the relayer container +set -e + +TAG=$(git describe --match=NeVeRmAtCh --always --abbrev=8 --dirty) +ORG="jitolabs" + +COMPOSE_DOCKER_CLI_BUILD=1 \ + DOCKER_BUILDKIT=1 \ + TAG="${TAG}" \ + ORG="${ORG}" \ + docker-compose --env-file ./env/.env.dev build --progress=plain + +COMPOSE_DOCKER_CLI_BUILD=1 \ + DOCKER_BUILDKIT=1 \ + TAG="${TAG}" \ + ORG="${ORG}" \ + docker-compose --env-file ./env/.env.dev up --remove-orphans diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 00000000000000..6884b6b1dce623 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,30 @@ +# See ./b for running instructions +version: "3.8" +services: + transaction_relayer: + image: $ORG/jito-transaction-relayer:$TAG + container_name: jito-transaction-relayer + build: + context: . + dockerfile: Dockerfile + target: jito-transaction-relayer + environment: + - RUST_LOG=info + - TPU_BIND_IP=0.0.0.0 + - TPU_PORT=$TPU_PORT + - TPU_FWD_PORT=$TPU_FWD_PORT + - TPU_QUIC_PORT=$TPU_QUIC_PORT + - TPU_QUIC_FWD_PORT=$TPU_QUIC_FWD_PORT + - GRPC_BIND_IP=0.0.0.0 + - GRPC_BIND_PORT=$GRPC_BIND_PORT + - NUM_TPU_BINDS=32 + - NUM_TPU_FWD_BINDS=16 + - RPC_SERVERS=$RPC_SERVERS + - WEBSOCKET_SERVERS=$WEBSOCKET_SERVERS + restart: on-failure + ports: + - "$TPU_PORT:$TPU_PORT/udp" + - "$TPU_FWD_PORT:$TPU_FWD_PORT/udp" + - "$TPU_QUIC_PORT:$TPU_QUIC_PORT/udp" + - "$TPU_QUIC_FWD_PORT:$TPU_QUIC_FWD_PORT/udp" + - "$GRPC_BIND_PORT:$GRPC_BIND_PORT/tcp" diff --git a/env/.env.dev b/env/.env.dev new file mode 100644 index 00000000000000..41dfe654847b65 --- /dev/null +++ b/env/.env.dev @@ -0,0 +1,7 @@ +TPU_PORT=8005 +TPU_FWD_PORT=8006 +TPU_QUIC_PORT=8007 +TPU_QUIC_FWD_PORT=8008 +GRPC_BIND_PORT=42069 +RPC_SERVERS="https://api.mainnet-beta.solana.com" +WEBSOCKET_SERVERS="wss://api.mainnet-beta.solana.com" diff --git a/relayer/Cargo.toml b/relayer/Cargo.toml index a0d2a40b891505..4e079875f5fa37 100644 --- a/relayer/Cargo.toml +++ b/relayer/Cargo.toml @@ -17,4 +17,3 @@ solana-core = "1.10.24" tokio-stream = "0.1.9" tokio = "1.14.1" tonic = "0.7.2" -uuid = { version = "1.1.2", features = ["v4"] } \ No newline at end of file