Merge #4120

4120: Cherry picked network changes for cardano-node-1.35.5 release r=coot a=coot This cherry-picked patches from the following PRs: * #3794 * #3844 * #3785 * #3904 * #3915 * #3852 * #3970 * #3979 * #4015 * #4067 * #4004 * #4086 * #4113 * #4106 * #4127 * #4103 Also cherry-picked almost all the commits which modify GitHub actions: * 18c5244 Run GitHub Actions on pull requests * 3adf5a9 Use newer version of io-sim * ee9b7a6 Fix GH Actions Windows CI: switch from pkgconf to pkg-config * e6cf074 github-actions: use `ubuntu-latest` * 9a8b959 Updated versions of github actions * fc8f8f0 Fix GH Actions Windows CI caching * 7f07c40 Windows Github Actions now use MSYS2 * b21a7ce Fix chocolatey CI error * #4134 TODO: * [x] bump versions of packages * [x] input-output-hk/cardano-haskell-packages#84 Co-authored-by: Mark Tullsen <[email protected]> Co-authored-by: Marcin Szamotulski <[email protected]>
IntersectMBO · Nov 9, 2022 · e933084 · e933084
2 parents b6ab4ca + 8e3b963
commit e933084
Show file tree

Hide file tree

Showing 139 changed files with 5,520 additions and 2,597 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -1,222 +1,185 @@
 name: Haskell CI
 
-on: [push]
+on:
+  pull_request:
+  push:
+    branches:
+      - 'bors/*'
 
 jobs:
   build:
     runs-on: ${{ matrix.os }}
 
     defaults:
       run:
-        shell: bash
+        shell: ${{ matrix.os == 'windows-latest' && 'msys2 {0}' || 'bash' }}
 
     strategy:
       fail-fast: false
       matrix:
         ghc: ["8.10.7"]
-        os: [ubuntu-20.04, windows-latest]
+        os: [ubuntu-latest, windows-latest]
 
     env:
       # current ref from: 27.02.2022
       SECP256K1_REF: ac83be33d0956faf6b7f61a60ab524ef7d6a473a
 
     steps:
-    - name: Install Haskell
-      uses: haskell/actions/setup@v1
+    - name: "WIN: Setup MSYS2 and libraries"
+      if: runner.os == 'Windows'
+      uses: msys2/setup-msys2@v2
+      with:
+        update: true
+        install: >-
+          base-devel
+          autoconf-wrapper
+          autoconf
+          automake
+          libtool
+          make
+          git
+          mingw-w64-x86_64-toolchain
+          mingw-w64-x86_64-libsodium
+          mingw-w64-x86_64-jq
+
+    - name: "WIN: Setup pkg-config"
+      if: runner.os == 'Windows'
+      run: |
+        pacman --noconfirm -S mingw-w64-x86_64-pkg-config
+
+    - name: "LINUX: Setup haskell"
+      if: runner.os != 'Windows'
+      uses: haskell/actions/setup@v2
       id: setup-haskell
       with:
         ghc-version: ${{ matrix.ghc }}
         cabal-version: 3.6.2.0
 
-    - name: Show Haskell tool versions
-      run: |
-        ghc --version
-        cabal --version
-
-    - name: Select build directory
+    - name: "WIN: Setup Haskell"
+      id: win-setup-haskell
+      if: runner.os == 'Windows'
       run: |
-        if [ "$OS" == Windows_NT ]; then
-          CABAL_BUILDDIR="D:\\a\\_temp\\dist"
-        else
-          CABAL_BUILDDIR="dist-newstyle"
-        fi
-
-        echo "CABAL_BUILDDIR=$CABAL_BUILDDIR"
-        echo "CABAL_BUILDDIR=$CABAL_BUILDDIR" >> $GITHUB_ENV
+        # see https://gitlab.haskell.org/haskell/ghcup-hs/-/blob/master/scripts/bootstrap/bootstrap-haskell
+        curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | \
+          BOOTSTRAP_HASKELL_NONINTERACTIVE=1 \
+          BOOTSTRAP_HASKELL_ADJUST_CABAL_CONFIG=1 \
+          BOOTSTRAP_HASKELL_ADJUST_BASHRC=1 \
+          BOOTSTRAP_HASKELL_GHC_VERSION="${{ matrix.ghc }}" \
+          BOOTSTRAP_HASKELL_CABAL_VERSION="3.6.2.0" \
+          sh
 
-    - name: Set cache version
-      run: echo "CACHE_VERSION=9w76Z3Q" >> $GITHUB_ENV
+        # MSYS2 doesn't inherit $GITHUB_PATH so this is needed
+        cat <(echo "source /c/ghcup/env") ~/.bashrc > ~/.bashrc.new
+        mv ~/.bashrc.new ~/.bashrc
 
-    - name: Install pkgconfiglite
-      if: matrix.os == 'windows-latest'
-      run: choco install -y pkgconfiglite
+        source ~/.bashrc
 
-    - name: Install libsodium (Windows)
-      if: matrix.os == 'windows-latest'
-      env:
-        RUNNER_TEMP: ${{ runner.temp }}
-      run: |
-        echo "RUNNER_TEMP=$RUNNER_TEMP"
-        cd "$RUNNER_TEMP"
+        # There is an issue with crt libraries, fixed by prepending the ghc
+        # mingw32 libraries directory to every other library directory.
+        echo "# Original cabal config extra-lib-dirs"
+        grep extra-lib-dirs /c/cabal/config
 
-        RUNNER_TEMP_FWD="$(echo "$RUNNER_TEMP" | sed 's|\\|/|g')"
+        sed -i 's/C:\\msys64\\mingw64\\lib/C:\\ghcup\\ghc\\8.10.7\\mingw\\x86_64-w64-mingw32\\lib, C:\\msys64\\mingw64\\lib/g' /c/cabal/config
 
-        curl -Ls \
-          --connect-timeout 5 \
-          --max-time 10 \
-          --retry 5 \
-          --retry-delay 0 \
-          --retry-max-time 40 \
-          https://download.libsodium.org/libsodium/releases/libsodium-1.0.18-mingw.tar.gz -o libsodium-1.0.18-mingw.tar.gz
+        echo "# Modified cabal config extra-lib-dirs"
+        grep extra-lib-dirs /c/cabal/config
 
-        tar zxvf libsodium-1.0.18-mingw.tar.gz
-        sed -i "s|/d/a/1/s/|$RUNNER_TEMP_FWD\/|g" libsodium-win64/lib/pkgconfig/libsodium.pc
+        ghc --version
+        cabal --version
+        echo "cabal-store=$(dirname $(cabal --help | tail -1 | tr -d ' '))\\store" >> $GITHUB_OUTPUT
 
-        export PKG_CONFIG_PATH="$(readlink -f libsodium-win64/lib/pkgconfig | sed 's|^/d|D:|g' | tr / '\\')"
-        echo "PKG_CONFIG_PATH=$PKG_CONFIG_PATH"
-        echo "PKG_CONFIG_PATH=$PKG_CONFIG_PATH" >> $GITHUB_ENV
+    - name: Update Hackage index
+      run: cabal update
 
-        export LIBSODIUM_PATH="$(readlink -f libsodium-win64/bin | sed 's|^/d|D:|g' | tr / '\\')"
-        echo "LIBSODIUM_PATH=$LIBSODIUM_PATH"
-        echo "$LIBSODIUM_PATH" >> $GITHUB_PATH
+    - name: Set cache version
+      run: echo "CACHE_VERSION=20220719" >> $GITHUB_ENV
 
-    - name: Install build environment
-      if: matrix.os == 'ubuntu-20.04'
+    - name: "LINUX: Install build environment (apt-get)"
+      if: runner.os == 'Linux'
       run: |
         sudo apt-get update
         sudo apt-get -y install libsodium23 libsodium-dev
         sudo apt-get -y install libsystemd0 libsystemd-dev
         sudo apt-get -y remove --purge software-properties-common
         sudo apt-get -y autoremove
 
-    - name: Install secp256k1 (Linux)
-      if: matrix.os == 'ubuntu-20.04'
+    - name: "LINUX: Install build environment (for secp256k1)"
+      if: runner.os == 'Linux'
       run: |
         sudo apt-get -y install autoconf automake libtool
-        mkdir secp256k1-sources
-        cd secp256k1-sources
-        git clone https://github.com/bitcoin-core/secp256k1.git
-        cd secp256k1
-        git reset --hard $SECP256K1_REF
-        ./autogen.sh
-        ./configure --prefix=/usr --enable-module-schnorrsig --enable-experimental
-        make
-        make check
-        sudo make install
-        cd ../..
-
-    - name: Install secp256k1 (Windows)
-      if: matrix.os == 'windows-latest'
-      env:
-        RUNNER_TEMP: ${{ runner.temp }}
+        echo "CI_SECP_FLAGS=--prefix=/usr" >> $GITHUB_ENV
+        echo "CI_SECP_INSTALL_CMD=sudo" >> $GITHUB_ENV
+
+    - name: "Install secp256k1"
       run: |
-        echo "RUNNER_TEMP=$RUNNER_TEMP"
-        cd "$RUNNER_TEMP"
-
-        RUNNER_TEMP_FWD="$(echo "$RUNNER_TEMP" | sed 's|\\|/|g')"
-
-        curl -Ls \
-          --connect-timeout 5 \
-          --max-time 10 \
-          --retry 5 \
-          --retry-delay 0 \
-          --retry-max-time 40 \
-          https://hydra.iohk.io/job/Cardano/haskell-nix/windows-secp256k1/latest/download/1 -o secp256k1.zip
-        mkdir secp256k1
-        cd secp256k1
-        unzip ../secp256k1.zip
-        cd ..
-        export PKG_CONFIG_PATH="$PKG_CONFIG_PATH;$(readlink -f secp256k1/lib/pkgconfig | sed 's|^/d|D:|g' | tr / '\\')"
-        echo "PKG_CONFIG_PATH=$PKG_CONFIG_PATH"
-        echo "PKG_CONFIG_PATH=$PKG_CONFIG_PATH" >> $GITHUB_ENV
-        export SECP256K1_PATH="$(readlink -f secp256k1/bin | sed 's|^/d|D:|g' | tr / '\\')"
-        echo "SECP256K1_PATH=$SECP256K1_PATH"
-        echo "$SECP256K1_PATH" >> $GITHUB_PATH
-
-    - name: Set up temp directory
-      env:
-        RUNNER_TEMP: ${{ runner.temp }}
+        git clone https://github.com/bitcoin-core/secp256k1
+        ( cd secp256k1
+          git switch $SECP256K1_REF --detach
+          ./autogen.sh
+          ./configure $CI_SECP_FLAGS --enable-module-schnorrsig --enable-experimental
+          make
+          make check
+          $CI_SECP_INSTALL_CMD make install
+        )
+
+    - uses: actions/checkout@v3
+
+    - name: "Configure cabal.project.local"
+      if: runner.os != 'Windows'
       run: |
-        echo "TMPDIR=$RUNNER_TEMP"  >> $GITHUB_ENV
-        echo "TMP=$RUNNER_TEMP"     >> $GITHUB_ENV
+        cp .github/workflows/cabal.project.local.Linux cabal.project.local
 
-    - uses: actions/checkout@v2
-
-    - name: Update Hackage index
-      run: cabal update
-
-    - name: Cabal Configure
-      run: cabal --builddir="$CABAL_BUILDDIR" configure --enable-tests
+    - name: "Configure cabal.project.local Windows"
+      if: runner.os == 'Windows'
+      run: |
+        cp .github/workflows/cabal.project.local.Windows cabal.project.local
 
     - name: Record dependencies
+      id: record-deps
       run: |
         cabal build all --dry-run
         cat dist-newstyle/cache/plan.json | jq -r '."install-plan"[].id' | sort | uniq > dependencies.txt
-        date > date.txt
+        echo "weeknum=$(/bin/date -u "+%W")" >> $GITHUB_OUTPUT
 
-    - uses: actions/cache@v2
-      name: Cache cabal store
+    - uses: actions/cache@v3
+      name: "Cache `cabal store`"
       with:
-        path: ${{ steps.setup-haskell.outputs.cabal-store }}
-        key: cache-${{ env.CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }}-${{ hashFiles('dependencies.txt') }}
-        restore-keys: |
-          cache-${{ env.CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }}-${{ hashFiles('dependencies.txt') }}
-          cache-${{ env.CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }}
-
-    - name: Use cabal.project.local.ci
-      run: |
-        cat ./cabal.project.local.ci >> ./cabal.project.local
-
-        cat >> cabal.project.local <<EOF
-        package cardano-crypto-praos
-          flags: -external-libsodium-vrf
-        EOF
+        path: ${{ runner.os == 'Windows' && steps.win-setup-haskell.outputs.cabal-store || steps.setup-haskell.outputs.cabal-store }}
+        key: cache-dependencies-${{ env.CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }}-${{ hashFiles('dependencies.txt') }}
+        restore-keys: cache-dependencies-${{ env.CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }}
 
-        cat ./cabal.project.local
+    - uses: actions/cache@v3
+      name: "Cache `dist-newstyle`"
+      with:
+        path: |
+          dist-newstyle
+          !dist-newstyle/**/.git
+        key: cache-dist-${{ env.CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }}-${{ steps.record-deps.outputs.weeknum }}
+        restore-keys: cache-dist-${{ env.CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }}
 
-    - name: Install happy
-      run: |
-        cabal --builddir="$CABAL_BUILDDIR" install happy --install-method=copy
 
     - name: Build dependencies
-      run: cabal --builddir="$CABAL_BUILDDIR" build --only-dependencies all
-
-    - name: Print logs
-      if: always()
-      run: |
-        for x in $(find 'C:\Users\runneradmin\AppData\Roaming\cabal\logs' -name '*.log'); do
-          echo "==== $x ===="
-          cat "$x"
-        done
-
-    - name: Set checkinvariant flag
-      run: |
-        cat >> cabal.project.local <<EOF
-        package io-classes
-          flags: +checktvarinvariant
-        EOF
-
-        cat ./cabal.project.local
+      run: cabal build --only-dependencies all
 
     - name: Build projects [build]
-      run: cabal --builddir="$CABAL_BUILDDIR" build all
+      run: cabal build all
 
     # Test network packages
 
     - name: ntp-client [test]
-      run: cabal --builddir="$CABAL_BUILDDIR" run ntp-client:test
+      run: cabal run ntp-client:test
 
     - name: network-mux [test]
-      run: cabal --builddir="$CABAL_BUILDDIR" run network-mux:test
+      run: cabal run network-mux:test
 
     - name: ourobors-network-testing [test]
-      run: cabal --builddir="$CABAL_BUILDDIR" run ouroboros-network-testing:test
+      run: cabal run ouroboros-network-testing:test
 
     # issue: #1818
     - name: ourobors-network-framework [test]
-      run: cabal --builddir="$CABAL_BUILDDIR" run ouroboros-network-framework:test -- -p '$0 != "typed-protocols.Socket.socket send receive IPv4"'
+      run: cabal run ouroboros-network-framework:test -- -p '$0 != "typed-protocols.Socket.socket send receive IPv4"'
 
     - name: ouroboros-network [test]
-      run: cabal --builddir="$CABAL_BUILDDIR" run ouroboros-network:test
+      run: cabal run ouroboros-network:test
 
     # Consensus tests take too long on GitHub's Azure machines
diff --git a/cabal.project.local.ci → .github/workflows/cabal.project.local.Linux b/cabal.project.local.ci → .github/workflows/cabal.project.local.Linux
@@ -1,5 +1,7 @@
 max-backjumps: 5000
 reorder-goals: True
+ignore-project: False
+tests: True
 
 package Win32-network
   ghc-options: -Werror
@@ -9,14 +11,14 @@ package ntp-client
   ghc-options: -Werror
   flags: +demo
 
-package io-sim-classes
-  ghc-options: -Werror
-  flags: +asserts
-
 package io-sim
   ghc-options: -Werror
   flags: +asserts
 
+-- Add strcittvar check
+package io-classes
+  flags: +checktvarinvariant
+
 package monoidal-synchronisation
   ghc-options: -Werror
 
@@ -43,3 +45,6 @@ package ouroboros-network
 
 package cardano-client
   ghc-options: -Werror
+
+package cardano-crypto-praos
+  flags: -external-libsodium-vrf