config.yml

version: 2.1

parameters:
  nightly-toolchain:
    type: string
    default: "nightly-2022-03-10"

executors:
  default:
    docker:
      - image: filecoin/rust:latest
    working_directory: /mnt/crate
    resource_class: 2xlarge+
  gpu:
    machine:
      image: ubuntu-2004-cuda-11.2:202103-01
    working_directory: ~/gpuci
    resource_class: gpu.nvidia.medium
  arm:
    machine:
      image: ubuntu-2004:202101-01
    resource_class: arm.large

setup-env: &setup-env
  FIL_PROOFS_PARAMETER_CACHE: "/tmp/filecoin-proof-parameters/"
  RUST_LOG: info


jobs:
  ensure_groth_parameters_and_keys_linux:
    executor: default
    environment: *setup-env
    steps:
      - checkout
      - restore_parameter_cache
      - ensure_filecoin_parameters
      - save_parameter_cache

  cargo_fetch:
    executor: default
    environment: *setup-env
    steps:
      - checkout
      - run:
          name: Calculate dependencies
          command: cargo generate-lockfile
          no_output_timeout: 30m
      - restore_rustup_cache
      - run: rustup install $(cat rust-toolchain)
      - run: rustup default $(cat rust-toolchain)
      - run: cargo update
      - run: cargo fetch
      - run: rustc +$(cat rust-toolchain) --version
      - run: rustup toolchain list --verbose
      - persist_to_workspace:
          root: "."
          paths:
            - Cargo.lock
      - save_rustup_cache
  test:
    executor: default
    environment: *setup-env
    parameters:
      crate:
        type: string
    steps:
      - checkout
      - attach_workspace:
          at: "."
      - restore_rustup_cache
      - restore_parameter_cache
      - run:
          name: Test (<< parameters.crate >>)
          command: cargo +$(cat rust-toolchain) test --verbose --package << parameters.crate >>
          no_output_timeout: 30m

  test_release:
    executor: default
    environment: *setup-env
    steps:
      - checkout
      - attach_workspace:
          at: "."
      - restore_rustup_cache
      - restore_parameter_cache
      - run:
          name: Test in release profile
          command: |
            ulimit -n 20000
            ulimit -u 20000
            ulimit -n 20000
            cargo test --verbose --release --workspace --all-targets
            # Some `storage-proofs-update` tests need to run sequentially due
            # to their high memory usage.
            cargo test -p storage-proofs-update --features isolated-testing --release -- --test-threads=1
          no_output_timeout: 30m

  test_ignored_release:
    executor: default
    environment: *setup-env
    parameters:
      crate:
        type: string
    steps:
      - checkout
      - attach_workspace:
          at: "."
      - restore_rustup_cache
      - restore_parameter_cache
      - run:
          name: Test ignored in release profile
          command: |
              ulimit -n 20000
              ulimit -u 20000
              ulimit -n 20000
              cd << parameters.crate >>
              cargo test --release -- --ignored --nocapture
          environment:
            RUST_TEST_THREADS: 1
          no_output_timeout: 30m


  # Running with `use_multicore_sdr=true` should be integrated directly into the test code. For now we
  # just re-run the lifecycle tests to exercise the use_multicore_sdr code path with that setting set.
  test_multicore_sdr:
    executor: default
    environment: *setup-env
    steps:
      - checkout
      - attach_workspace:
          at: "."
      - restore_rustup_cache
      - restore_parameter_cache
      - run:
          name: Test with use_multicore_sdr
          command: |
            ulimit -n 20000
            ulimit -u 20000
            ulimit -n 20000
            cargo test --all --verbose --release lifecycle -- --ignored --nocapture
            cargo test -p storage-proofs-porep --features isolated-testing --release checkout_cores -- --test-threads=1
            cargo test -p storage-proofs-porep --features isolated-testing --release test_parallel_generation_and_read_partial_range_v1_0
            cargo test -p storage-proofs-porep --features isolated-testing --release test_parallel_generation_and_read_partial_range_v1_1
          no_output_timeout: 30m
          environment:
            RUST_TEST_THREADS: 1
            FIL_PROOFS_USE_MULTICORE_SDR: true

  test_gpu_tree_building:
    executor: gpu
    environment: *setup-env
    parameters:
      cargo-args:
        description: Addtional arguments for the cargo command
        type: string
        default: ""
      test-args:
        description: Additional arguments for the test executable (after the `--`)
        type: string
        default: ""
    steps:
      - checkout
      - attach_workspace:
          at: "."
      - restore_rustup_cache
      - restore_parameter_cache
      - run:
          name: Set the PATH env variable
          command: |
            echo 'export PATH="~/.cargo/bin:/usr/local/cuda-11.2/bin:$PATH"' | tee --append $BASH_ENV
            source $BASH_ENV
      - run:
          name: Install required libraries for GPU support
          command: |
            sudo apt update
            sudo apt install -y ocl-icd-opencl-dev libhwloc-dev
      - run:
          name: Test with GPU column and tree builders.
          command: |
            ulimit -n 20000
            ulimit -u 20000
            ulimit -n 20000
            cargo test --verbose --release << parameters.cargo-args >> -- --nocapture << parameters.test-args >>
          no_output_timeout: 30m
          environment:
            FIL_PROOFS_USE_GPU_COLUMN_BUILDER: true
            FIL_PROOFS_USE_GPU_TREE_BUILDER: true
            BELLMAN_CUDA_NVCC_ARGS: --fatbin --gpu-architecture=sm_75 --generate-code=arch=compute_75,code=sm_75
            NEPTUNE_CUDA_NVCC_ARGS: --fatbin --gpu-architecture=sm_75 --generate-code=arch=compute_75,code=sm_75

  test_no_gpu:
    executor: default
    environment: *setup-env
    steps:
      - checkout
      - attach_workspace:
          at: "."
      - restore_rustup_cache
      - restore_parameter_cache
      - run:
          name: Test with no gpu
          command: |
            cargo test --all --verbose --release --no-default-features
          no_output_timeout: 30m

  test_arm_no_gpu:
    executor: arm
    environment: *setup-env
    steps:
      - checkout
      - attach_workspace:
          at: "."
      - restore_rustup_cache
      - restore_parameter_cache
      - run:
          name: Install Rust
          command: |
            curl https://sh.rustup.rs -sSf | sh -s -- -y
      - run: rustup install $(cat rust-toolchain)
      - run: rustup default $(cat rust-toolchain)
      - run: rustup install << pipeline.parameters.nightly-toolchain >>
      - run: cargo update
      - run: cargo fetch
      - run:
          name: Install required libraries
          command: |
            sudo apt-get update -y
            sudo apt install -y libhwloc-dev
      - run:
          name: Test arm with no gpu
          command: |
            cargo +<< pipeline.parameters.nightly-toolchain >> -Zpackage-features test --release --all --verbose --no-default-features
          no_output_timeout: 90m

  bench:
    executor: default
    environment: *setup-env
    steps:
      - checkout
      - attach_workspace:
          at: "."
      - restore_rustup_cache
      - restore_parameter_cache
      - run:
          name: Benchmarks
          command: cargo +$(cat rust-toolchain) build --benches --verbose --workspace
          no_output_timeout: 15m

  rustfmt:
    executor: default
    environment: *setup-env
    steps:
      - checkout
      - attach_workspace:
          at: "."
      - restore_rustup_cache
      - run:
          name: Run cargo fmt
          command: cargo fmt --all -- --check

  clippy:
    executor: default
    environment: *setup-env
    steps:
      - checkout
      - attach_workspace:
          at: "."
      - restore_rustup_cache
      - run:
          name: Run cargo clippy
          command: cargo +$(cat rust-toolchain) clippy --all-targets --workspace -- -D warnings
  test_darwin:
    macos:
      xcode: "12.5.0"
    working_directory: ~/crate
    resource_class: large
    environment: *setup-env
    steps:
      - checkout
      - run:
          name: Install hwloc with Homebrew
          command: HOMEBREW_NO_AUTO_UPDATE=1 brew install hwloc
      - run:
          name: Install Rust
          command: |
            curl https://sh.rustup.rs -sSf | sh -s -- -y
      - run: cargo update
      - run: cargo fetch
      # On CircleCI you cannot create a cache on Linux and restore it on macOS.
      # Hence we need a separate cache for macOS that we create and restore. It
      # is not a separate job (like `ensure_groth_parameters_and_keys_linux`,
      # as there is only this single job that runs on macOS.
      - restore_parameter_cache
      - ensure_filecoin_parameters
      - save_parameter_cache
      - run:
          name: Test Darwin
          command: |
            sudo ulimit -n 20000
            sudo ulimit -u 20000
            ulimit -n 20000
            cargo test --release --verbose --workspace -- --nocapture
          no_output_timeout: 2h

commands:
  ensure_filecoin_parameters:
    steps:
      - run:
          name: Create parameters directory
          command: mkdir -p ${FIL_PROOFS_PARAMETER_CACHE}
      - run:
          name: Build paramcache if it doesn't already exist
          command: |
            set -x; test -f /tmp/paramcache.awesome \
            || (cargo build --release --bin paramcache && find . -type f -name paramcache | xargs -I {} mv {} /tmp/paramcache.awesome)
      - run:
          name: Obtain filecoin groth parameters
          command: /tmp/paramcache.awesome --sector-sizes='2048,4096,16384,32768'
          no_output_timeout: 60m
      - run:
          name: Obtain filecoin ipp parameter file
          command: |
            # paramfetch is using `ipget` which currently always times out on
            # CI, hence get this file via HTTP instead.
            #cargo run --release --bin paramfetch -- -a -j srs-inner-product.json
            # `--insecure` is needed due to an outdated base systems.
            curl --insecure https://proofs.filecoin.io/v28-fil-inner-product-v1.srs --output ${FIL_PROOFS_PARAMETER_CACHE}v28-fil-inner-product-v1.srs
      - run:
          name: Make the parameters world readable
          command: chmod -R 755 ${FIL_PROOFS_PARAMETER_CACHE}
  save_rustup_cache:
    steps:
      # Move things from the home directory to `/tmp` first, so that it can be
      # restored on executors that have a different home directory.
      - run: cp -R ~/.cargo ~/.rustup /tmp/
      - save_cache:
          name: "Save rustup cache"
          key: cargo-v28-e-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}
          paths:
            - /tmp/.cargo
            - /tmp/.rustup
  restore_rustup_cache:
    steps:
      - restore_cache:
          name: "Restore rustup cache"
          key: cargo-v28-e-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}
      # Cache might not be created yet, hence ignore if the move fails
      - run: cp -R /tmp/.cargo /tmp/.rustup ~/ || true
  save_parameter_cache:
    steps:
      # Make sure we have different cache for Linux and Darwin.
      - run: uname > os.txt
      - save_cache:
          name: "Save parameter cache"
          key: proof-params-v28-k-{{ checksum "os.txt" }}-{{ checksum "filecoin-proofs/parameters.json" }}
          paths:
            - "/tmp/paramcache.awesome"
            - "/tmp/filecoin-proof-parameters/"
  restore_parameter_cache:
    steps:
      # Make sure we have different cache for Linux and Darwin.
      - run: uname > os.txt
      - restore_cache:
          name: "Restore parameter cache"
          key: proof-params-v28-k-{{ checksum "os.txt" }}-{{ checksum "filecoin-proofs/parameters.json" }}

workflows:
  version: 2.1
  test_all:
    jobs:
      - ensure_groth_parameters_and_keys_linux
      - cargo_fetch
      - rustfmt:
          requires:
            - cargo_fetch
      - clippy:
          requires:
            - cargo_fetch

      - test_release:
          requires:
            - cargo_fetch
            - ensure_groth_parameters_and_keys_linux
            
      - test_ignored_release:
          name: test_ignored_release_storage_proofs_post
          crate: "storage-proofs-post"
          requires:
            - cargo_fetch
            - ensure_groth_parameters_and_keys_linux
            
      - test_ignored_release:
          name: test_ignored_release_storage_proofs_core
          crate: "storage-proofs-core"
          requires:
            - cargo_fetch
            - ensure_groth_parameters_and_keys_linux
            
      - test_ignored_release:
          name: test_ignored_release_storage_proofs_porep
          crate: "storage-proofs-porep"
          requires:
            - cargo_fetch
            - ensure_groth_parameters_and_keys_linux
            
      - test_ignored_release:
          name: test_ignored_release_filecoin_proofs
          crate: "filecoin-proofs"
          requires:
            - cargo_fetch
            - ensure_groth_parameters_and_keys_linux

      - test_gpu_tree_building:
          name: test_gpu_tree_building_opencl (regular)
          cargo-args: "--workspace"
          requires:
            - cargo_fetch
            - ensure_groth_parameters_and_keys_linux

      - test_gpu_tree_building:
          name: test_gpu_tree_building_opencl (ignored, filecoin-proofs, storage-proofs-update)
          cargo-args: "--package filecoin-proofs --package storage-proofs-update"
          test-args: "--ignored"
          requires:
            - cargo_fetch
            - ensure_groth_parameters_and_keys_linux

      - test_gpu_tree_building:
          name: test_gpu_tree_building_opencl (ignored, rest)
          cargo-args: "--workspace --exclude filecoin-proofs --exclude storage-proofs-update"
          test-args: "--ignored"
          requires:
            - cargo_fetch
            - ensure_groth_parameters_and_keys_linux

      - test_gpu_tree_building:
          name: test_gpu_tree_building_cuda (regular)
          cargo-args: "--workspace --features cuda"
          requires:
            - cargo_fetch
            - ensure_groth_parameters_and_keys_linux

      - test_gpu_tree_building:
          name: test_gpu_tree_building_cuda (ignored, filecoin-proofs, storage-proofs-update)
          cargo-args: "--package filecoin-proofs --package storage-proofs-update --features cuda"
          test-args: "--ignored"
          requires:
            - cargo_fetch
            - ensure_groth_parameters_and_keys_linux

      - test_gpu_tree_building:
          name: test_gpu_tree_building_cuda (ignored, rest)
          cargo-args: "--workspace --exclude filecoin-proofs --exclude storage-proofs-update --features cuda"
          test-args: "--ignored"
          requires:
            - cargo_fetch
            - ensure_groth_parameters_and_keys_linux

      - test_multicore_sdr:
          requires:
            - cargo_fetch
            - ensure_groth_parameters_and_keys_linux

      - test:
          name: test_filecoin_proofs
          crate: "filecoin-proofs"
          requires:
            - cargo_fetch
            - ensure_groth_parameters_and_keys_linux

      - test:
          name: test_storage_proofs_core
          crate: "storage-proofs-core"
          requires:
            - cargo_fetch
            - ensure_groth_parameters_and_keys_linux

      - test:
          name: test_storage_proofs_post
          crate: "storage-proofs-post"
          requires:
            - cargo_fetch
            - ensure_groth_parameters_and_keys_linux

      - test:
          name: test_storage_proofs_porep
          crate: "storage-proofs-porep"
          requires:
            - cargo_fetch
            - ensure_groth_parameters_and_keys_linux

      - test:
          name: test_fil_proofs_tooling
          crate: "fil-proofs-tooling"
          requires:
            - cargo_fetch
            - ensure_groth_parameters_and_keys_linux

      - test:
          name: test_sha2raw
          crate: "sha2raw"
          requires:
            - cargo_fetch
            - ensure_groth_parameters_and_keys_linux

      - test:
          name: test_filecoin_hashers
          crate: "filecoin-hashers"
          requires:
            - cargo_fetch
            - ensure_groth_parameters_and_keys_linux


      - test:
          name: test_fil_proofs_param
          crate: "fil-proofs-param"
          requires:
            - cargo_fetch
            - ensure_groth_parameters_and_keys_linux

      - test_no_gpu:
          name: test_no_gpu
          requires:
            - cargo_fetch
            - ensure_groth_parameters_and_keys_linux

      - test_arm_no_gpu:
          name: test_arm_no_gpu
          requires:
            - cargo_fetch
            - ensure_groth_parameters_and_keys_linux

      - bench:
          requires:
            - cargo_fetch
            - ensure_groth_parameters_and_keys_linux

      - test_darwin

      - test:
          name: test_fr32
          crate: "fr32"
          requires:
            - cargo_fetch

      - test:
          name: test_fil_store_proofs_update
          crate: "storage-proofs-update"
          requires:
            - cargo_fetch
            - ensure_groth_parameters_and_keys_linux