From f23f29d1ef256cb860469351a2be9b847fd87933 Mon Sep 17 00:00:00 2001 From: Varun Agrawal Date: Thu, 5 Aug 2021 11:22:27 -0400 Subject: [PATCH 1/8] speed up boost install --- .github/scripts/boost.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/boost.sh b/.github/scripts/boost.sh index 647a846281..3c7e012748 100644 --- a/.github/scripts/boost.sh +++ b/.github/scripts/boost.sh @@ -9,10 +9,10 @@ tar -zxf ${BOOST_FOLDER}.tar.gz # Bootstrap cd ${BOOST_FOLDER}/ -./bootstrap.sh +./bootstrap.sh --with-libraries=serialization,filesystem,thread,system,atomic,date_time,timer,chrono,program_options,regex # Build and install -sudo ./b2 install +sudo ./b2 -j$(nproc) install # Rebuild ld cache sudo ldconfig From f9b8de3876bf5128091f970f1563fa6fc49a7d28 Mon Sep 17 00:00:00 2001 From: Varun Agrawal Date: Fri, 6 Aug 2021 15:53:40 -0400 Subject: [PATCH 2/8] improvements to CI files --- .github/workflows/build-linux.yml | 11 ++++------- .github/workflows/build-macos.yml | 9 +++------ .github/workflows/build-python.yml | 2 +- .github/workflows/build-special.yml | 2 +- .github/workflows/build-windows.yml | 20 +++++++++----------- 5 files changed, 18 insertions(+), 26 deletions(-) diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index 5483c32cfd..f52e5eec36 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -47,8 +47,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 - - name: Install (Linux) - if: runner.os == 'Linux' + - name: Install Dependencies run: | # LLVM (clang) 9 is not in Bionic's repositories so we add the official LLVM repository. if [ "${{ matrix.compiler }}" = "clang" ] && [ "${{ matrix.version }}" = "9" ]; then @@ -63,7 +62,7 @@ jobs: fi sudo apt-get -y update - sudo apt-get install cmake build-essential pkg-config libpython-dev python-numpy libicu-dev + sudo apt-get -y install cmake build-essential pkg-config libpython-dev python-numpy libicu-dev if [ "${{ matrix.compiler }}" = "gcc" ]; then sudo apt-get install -y g++-${{ matrix.version }} g++-${{ matrix.version }}-multilib @@ -79,7 +78,5 @@ jobs: run: | bash .github/scripts/boost.sh - - name: Build and Test (Linux) - if: runner.os == 'Linux' - run: | - bash .github/scripts/unix.sh -t + - name: Build and Test + run: bash .github/scripts/unix.sh -t diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index ed8f8563b7..462723222f 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -34,8 +34,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 - - name: Install (macOS) - if: runner.os == 'macOS' + - name: Install Dependencies run: | brew install cmake ninja brew install boost @@ -48,7 +47,5 @@ jobs: echo "CC=clang" >> $GITHUB_ENV echo "CXX=clang++" >> $GITHUB_ENV fi - - name: Build and Test (macOS) - if: runner.os == 'macOS' - run: | - bash .github/scripts/unix.sh -t + - name: Build and Test + run: bash .github/scripts/unix.sh -t diff --git a/.github/workflows/build-python.yml b/.github/workflows/build-python.yml index addde8c64c..3fc2d662f8 100644 --- a/.github/workflows/build-python.yml +++ b/.github/workflows/build-python.yml @@ -81,7 +81,7 @@ jobs: fi sudo apt-get -y update - sudo apt install cmake build-essential pkg-config libpython-dev python-numpy libboost-all-dev + sudo apt-get -y install cmake build-essential pkg-config libpython-dev python-numpy libboost-all-dev if [ "${{ matrix.compiler }}" = "gcc" ]; then sudo apt-get install -y g++-${{ matrix.version }} g++-${{ matrix.version }}-multilib diff --git a/.github/workflows/build-special.yml b/.github/workflows/build-special.yml index 6427e13bc8..3bb3fa2988 100644 --- a/.github/workflows/build-special.yml +++ b/.github/workflows/build-special.yml @@ -70,7 +70,7 @@ jobs: fi sudo apt-get -y update - sudo apt install cmake build-essential pkg-config libpython-dev python-numpy libicu-dev + sudo apt-get -y install cmake build-essential pkg-config libpython-dev python-numpy libicu-dev if [ "${{ matrix.compiler }}" = "gcc" ]; then sudo apt-get install -y g++-${{ matrix.version }} g++-${{ matrix.version }}-multilib diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index a564dade9d..8af23d76b0 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -18,15 +18,14 @@ jobs: # Github Actions requires a single row to be added to the build matrix. # See https://help.github.com/en/articles/workflow-syntax-for-github-actions. name: [ - #TODO This build keeps timing out, need to understand why. - # windows-2016-cl, - windows-2019-cl, - ] + #TODO This build keeps timing out, need to understand why. + # windows-2016-cl, + windows-2019-cl, + ] build_type: [Debug, Release] build_unstable: [ON] include: - #TODO This build keeps timing out, need to understand why. # - name: windows-2016-cl # os: windows-2016 @@ -37,10 +36,7 @@ jobs: compiler: cl steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Install (Windows) - if: runner.os == 'Windows' + - name: Install shell: powershell run: | Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://get.scoop.sh') @@ -72,8 +68,10 @@ jobs: boost_version: 1.72.0 toolset: msvc14.2 - - name: Build (Windows) - if: runner.os == 'Windows' + - name: Checkout + uses: actions/checkout@v2 + + - name: Build env: BOOST_ROOT: ${{ steps.install-boost.outputs.BOOST_ROOT }} run: | From 19f13f3b72544eb43bfa8a803c5d93c8408be3b3 Mon Sep 17 00:00:00 2001 From: Varun Agrawal Date: Fri, 6 Aug 2021 16:04:40 -0400 Subject: [PATCH 3/8] Install boost directly from binary --- .github/workflows/build-windows.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 8af23d76b0..d03a951004 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -12,6 +12,8 @@ jobs: CTEST_PARALLEL_LEVEL: 2 CMAKE_BUILD_TYPE: ${{ matrix.build_type }} GTSAM_BUILD_UNSTABLE: ${{ matrix.build_unstable }} + BOOST_ROOT: C:\hostedtoolcache\windows\Boost\1.72.0\x86_64 + strategy: fail-fast: false matrix: @@ -61,19 +63,18 @@ jobs: # Scoop modifies the PATH so we make the modified PATH global. echo "$env:PATH" >> $GITHUB_PATH - - name: Download and install Boost - uses: MarkusJx/install-boost@v1.0.1 - id: install-boost - with: - boost_version: 1.72.0 - toolset: msvc14.2 + - name: Install Boost + run: | + # Snippet from: https://github.com/actions/virtual-environments/issues/2667 + # Use the boost_1_72_0-msvc-14.1-64.exe for Windows 2016 + $Url = "https://sourceforge.net/projects/boost/files/boost-binaries/1.72.0/boost_1_72_0-msvc-14.2-64.exe" + (New-Object System.Net.WebClient).DownloadFile($Url, "$env:TEMP\boost.exe") + Start-Process -Wait -FilePath "$env:TEMP\boost.exe" "/SILENT","/SP-","/SUPPRESSMSGBOXES","/DIR=C:\hostedtoolcache\windows\Boost\1.72.0\x86_64" - name: Checkout uses: actions/checkout@v2 - name: Build - env: - BOOST_ROOT: ${{ steps.install-boost.outputs.BOOST_ROOT }} run: | cmake -E remove_directory build cmake -B build -S . -DGTSAM_BUILD_EXAMPLES_ALWAYS=OFF -DBOOST_ROOT="${env:BOOST_ROOT}" -DBOOST_INCLUDEDIR="${env:BOOST_ROOT}\boost\include" -DBOOST_LIBRARYDIR="${env:BOOST_ROOT}\lib" From 2d590656fc92a0e6ed292b8909697e093af7a49f Mon Sep 17 00:00:00 2001 From: Varun Agrawal Date: Fri, 6 Aug 2021 20:16:33 -0400 Subject: [PATCH 4/8] use Boost 1.67.0 for Windows --- .github/workflows/build-windows.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index d03a951004..20408b0f92 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -12,7 +12,7 @@ jobs: CTEST_PARALLEL_LEVEL: 2 CMAKE_BUILD_TYPE: ${{ matrix.build_type }} GTSAM_BUILD_UNSTABLE: ${{ matrix.build_unstable }} - BOOST_ROOT: C:\hostedtoolcache\windows\Boost\1.72.0\x86_64 + BOOST_ROOT: C:\hostedtoolcache\windows\Boost\1.67.0\x86_64 strategy: fail-fast: false @@ -66,10 +66,10 @@ jobs: - name: Install Boost run: | # Snippet from: https://github.com/actions/virtual-environments/issues/2667 - # Use the boost_1_72_0-msvc-14.1-64.exe for Windows 2016 - $Url = "https://sourceforge.net/projects/boost/files/boost-binaries/1.72.0/boost_1_72_0-msvc-14.2-64.exe" + # Use the boost_1_67_0-msvc-14.1-64.exe for Windows 2016 + $Url = "https://sourceforge.net/projects/boost/files/boost-binaries/1.67.0/boost_1_67_0-msvc-14.1-64.exe" (New-Object System.Net.WebClient).DownloadFile($Url, "$env:TEMP\boost.exe") - Start-Process -Wait -FilePath "$env:TEMP\boost.exe" "/SILENT","/SP-","/SUPPRESSMSGBOXES","/DIR=C:\hostedtoolcache\windows\Boost\1.72.0\x86_64" + Start-Process -Wait -FilePath "$env:TEMP\boost.exe" "/SILENT","/SP-","/SUPPRESSMSGBOXES","/DIR=C:\hostedtoolcache\windows\Boost\1.67.0\x86_64" - name: Checkout uses: actions/checkout@v2 From d34555f45d148cb57c6f14bb231bcd68c484a4ed Mon Sep 17 00:00:00 2001 From: Varun Agrawal Date: Sun, 8 Aug 2021 14:14:01 -0400 Subject: [PATCH 5/8] use env variables and fix setting of GITHUB_ENV --- .github/workflows/build-windows.yml | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 20408b0f92..0a4774744b 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -12,7 +12,8 @@ jobs: CTEST_PARALLEL_LEVEL: 2 CMAKE_BUILD_TYPE: ${{ matrix.build_type }} GTSAM_BUILD_UNSTABLE: ${{ matrix.build_unstable }} - BOOST_ROOT: C:\hostedtoolcache\windows\Boost\1.67.0\x86_64 + BOOST_VERSION: 1.67.0 + BOOST_EXE: boost_1_67_0-msvc-14.1-64.exe strategy: fail-fast: false @@ -38,7 +39,7 @@ jobs: compiler: cl steps: - - name: Install + - name: Install Dependencies shell: powershell run: | Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://get.scoop.sh') @@ -57,19 +58,25 @@ jobs: echo "CC=clang" >> $GITHUB_ENV echo "CXX=clang++" >> $GITHUB_ENV } else { - echo "CC=${{ matrix.compiler }}" >> $GITHUB_ENV - echo "CXX=${{ matrix.compiler }}" >> $GITHUB_ENV + echo "CC=${{ matrix.compiler }}" >> $env:GITHUB_ENV + echo "CXX=${{ matrix.compiler }}" >> $env:GITHUB_ENV } # Scoop modifies the PATH so we make the modified PATH global. - echo "$env:PATH" >> $GITHUB_PATH + echo "$env:PATH" >> $env:GITHUB_PATH - name: Install Boost + shell: powershell run: | # Snippet from: https://github.com/actions/virtual-environments/issues/2667 - # Use the boost_1_67_0-msvc-14.1-64.exe for Windows 2016 - $Url = "https://sourceforge.net/projects/boost/files/boost-binaries/1.67.0/boost_1_67_0-msvc-14.1-64.exe" - (New-Object System.Net.WebClient).DownloadFile($Url, "$env:TEMP\boost.exe") - Start-Process -Wait -FilePath "$env:TEMP\boost.exe" "/SILENT","/SP-","/SUPPRESSMSGBOXES","/DIR=C:\hostedtoolcache\windows\Boost\1.67.0\x86_64" + $BOOST_PATH = "C:\hostedtoolcache\windows\Boost\$env:BOOST_VERSION\x86_64" + + # Use the prebuilt binary for Windows + $Url = "https://sourceforge.net/projects/boost/files/boost-binaries/$env:BOOST_VERSION/$env:BOOST_EXE" + (New-Object System.Net.WebClient).DownloadFile($Url, "$env:TEMP\boost.exe") + Start-Process -Wait -FilePath "$env:TEMP\boost.exe" "/SILENT","/SP-","/SUPPRESSMSGBOXES","/DIR=$BOOST_PATH" + + # Set the BOOST_ROOT variable + echo "BOOST_ROOT=$BOOST_PATH" >> $env:GITHUB_ENV - name: Checkout uses: actions/checkout@v2 From 517ff4391abcf62f8c20cf8423ae979c5d5939b4 Mon Sep 17 00:00:00 2001 From: Varun Agrawal Date: Mon, 9 Aug 2021 19:50:40 -0400 Subject: [PATCH 6/8] use more cores for make on unix systems --- .github/scripts/unix.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/scripts/unix.sh b/.github/scripts/unix.sh index 55a8ac372b..6abbb55965 100644 --- a/.github/scripts/unix.sh +++ b/.github/scripts/unix.sh @@ -92,7 +92,11 @@ function build () configure - make -j2 + if [ "$(uname)" == "Linux" ]; then + make -j$(nproc) + elif [ "$(uname)" == "Darwin" ]; then + make -j$(sysctl -n hw.physicalcpu) + fi finish } @@ -105,8 +109,12 @@ function test () configure - # Actual build: - make -j2 check + # Actual testing + if [ "$(uname)" == "Linux" ]; then + make -j$(nproc) + elif [ "$(uname)" == "Darwin" ]; then + make -j$(sysctl -n hw.physicalcpu) + fi finish } From 47c0f717ae1bf2512aeebb66c510f2beae6c86a5 Mon Sep 17 00:00:00 2001 From: Varun Agrawal Date: Mon, 9 Aug 2021 19:50:56 -0400 Subject: [PATCH 7/8] improvements to windows builds --- .github/workflows/build-windows.yml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 0a4774744b..ab15d64bb7 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -13,7 +13,7 @@ jobs: CMAKE_BUILD_TYPE: ${{ matrix.build_type }} GTSAM_BUILD_UNSTABLE: ${{ matrix.build_unstable }} BOOST_VERSION: 1.67.0 - BOOST_EXE: boost_1_67_0-msvc-14.1-64.exe + BOOST_EXE: boost_1_67_0-msvc-14.1 strategy: fail-fast: false @@ -21,7 +21,7 @@ jobs: # Github Actions requires a single row to be added to the build matrix. # See https://help.github.com/en/articles/workflow-syntax-for-github-actions. name: [ - #TODO This build keeps timing out, need to understand why. + #TODO This build fails, need to understand why. # windows-2016-cl, windows-2019-cl, ] @@ -29,24 +29,29 @@ jobs: build_type: [Debug, Release] build_unstable: [ON] include: - #TODO This build keeps timing out, need to understand why. + #TODO This build fails, need to understand why. # - name: windows-2016-cl # os: windows-2016 # compiler: cl + # platform: 32 - name: windows-2019-cl os: windows-2019 compiler: cl + platform: 64 steps: - name: Install Dependencies shell: powershell run: | Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://get.scoop.sh') + scoop install cmake --global # So we don't get issues with CMP0074 policy scoop install ninja --global + if ("${{ matrix.compiler }}".StartsWith("clang")) { scoop install llvm --global } + if ("${{ matrix.compiler }}" -eq "gcc") { # Chocolatey GCC is broken on the windows-2019 image. # See: https://github.com/DaanDeMeyer/doctest/runs/231595515 @@ -54,13 +59,17 @@ jobs: scoop install gcc --global echo "CC=gcc" >> $GITHUB_ENV echo "CXX=g++" >> $GITHUB_ENV + } elseif ("${{ matrix.compiler }}" -eq "clang") { echo "CC=clang" >> $GITHUB_ENV echo "CXX=clang++" >> $GITHUB_ENV + } else { echo "CC=${{ matrix.compiler }}" >> $env:GITHUB_ENV echo "CXX=${{ matrix.compiler }}" >> $env:GITHUB_ENV + } + # Scoop modifies the PATH so we make the modified PATH global. echo "$env:PATH" >> $env:GITHUB_PATH @@ -71,7 +80,7 @@ jobs: $BOOST_PATH = "C:\hostedtoolcache\windows\Boost\$env:BOOST_VERSION\x86_64" # Use the prebuilt binary for Windows - $Url = "https://sourceforge.net/projects/boost/files/boost-binaries/$env:BOOST_VERSION/$env:BOOST_EXE" + $Url = "https://sourceforge.net/projects/boost/files/boost-binaries/$env:BOOST_VERSION/$env:BOOST_EXE-${{matrix.platform}}.exe" (New-Object System.Net.WebClient).DownloadFile($Url, "$env:TEMP\boost.exe") Start-Process -Wait -FilePath "$env:TEMP\boost.exe" "/SILENT","/SP-","/SUPPRESSMSGBOXES","/DIR=$BOOST_PATH" From 103b1985694abf33b5e3a861f37a8b7f443577fc Mon Sep 17 00:00:00 2001 From: Varun Agrawal Date: Tue, 10 Aug 2021 13:04:22 -0400 Subject: [PATCH 8/8] Boost 1.72.0 for Windows --- .github/workflows/build-windows.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index ab15d64bb7..5dfdcd0132 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -12,8 +12,8 @@ jobs: CTEST_PARALLEL_LEVEL: 2 CMAKE_BUILD_TYPE: ${{ matrix.build_type }} GTSAM_BUILD_UNSTABLE: ${{ matrix.build_unstable }} - BOOST_VERSION: 1.67.0 - BOOST_EXE: boost_1_67_0-msvc-14.1 + BOOST_VERSION: 1.72.0 + BOOST_EXE: boost_1_72_0-msvc-14.2 strategy: fail-fast: false