From 7c758200f5273dbdaacd0f51fbc3372eb34d5af2 Mon Sep 17 00:00:00 2001 From: Paul Berberian Date: Fri, 2 Feb 2024 17:42:55 +0100 Subject: [PATCH 1/2] [WIP] Auto-install Rust compiler toolchain for CI and JS people Since we brought the WebAssembly MPD parser to fix performance issues we had with some contents at canal+, we added another language (Rust) to the project which is not compatible with most of the JS-world tools we and other JS developers usually deal with. Now that we added the `MULTI_THREAD` experimental feature and embeds, the WebAssembly generation from that other language is even a required step in our main build script (`npm run build`). Consequently, when another developer or a CI needs to build an RxPlayer (e.g. to test a development before a release is made), they have to install: 1. NodeJS + npm 2. our JavaScript dependencies 3. A Rust toolchain able to output WebAssembly. We generally go through `rustup` here basically an helper tool to install Rust toolchains 4. `binaryen`, which is a collection of tools to transform that WebAssembly file. (1) and (2) are very common use cases that both JS people and CI workflows do very easily: they mostly already have nodejs + npm installed locally (or it's very simple to set-up) and the second is only an `npm install` call away. (3) and (4) however, though not hard (you have to install rustup and binaryen which is pretty simple on all platforms we relied on), are more unusual dependencies that JS developers might be reticent to install and that are more work to set-up on a CI. Consequently I'm doing here a proof-of-concept where those tools are auto-installed locally if not found when doing an `npm run build` call, letting both people and CI produce a build easily without having to deal with the dual-language complexity. --- For rustup it's pretty straightforward as their install script already take care of the platform detection, target installation and so on. All I had to do is to make sure binaries were installed locally (`./tmp`) and did not pollute the developers environment (e.g. no `PATH` modification). For binaryen however, it's less simple as they don't propose such auto-install system. however, I noticed that they propose prebuilt binaries for the most usual platforms on their github's release page: https://github.com/WebAssembly/binaryen/releases The idea here (not yet finished), is to automatically detect the platform of the user and fetch the right binaryen binary locally to then rely on both it and the local rustup to produce builds in case the command isn't available globally. Also, because it will be faster and less error-prone for frequent developers to install both of those globally once and for all, we print a notice explaining what we do when that auto-install script is run. --- .github/workflows/checks.yml | 4 - .gitignore | 1 + package.json | 2 +- scripts/build_wasm_release.sh | 64 +++++++++++++ scripts/install_rust_toolchain.sh | 146 ++++++++++++++++++++++++++++++ 5 files changed, 212 insertions(+), 5 deletions(-) create mode 100755 scripts/build_wasm_release.sh create mode 100755 scripts/install_rust_toolchain.sh diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 81e3f66119..6ce203daa6 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -64,13 +64,9 @@ jobs: with: node-version: ${{ matrix.node-version }} cache: 'npm' - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable # needed for integration & memory tests codecs support - run: sudo add-apt-repository multiverse && sudo apt update && sudo apt install -y ubuntu-restricted-extras - run: npm install - - run: rustup target add wasm32-unknown-unknown - run: npm run build # Firefox seems to have issue with integration tests on GitHub actions only # TODO to check diff --git a/.gitignore b/.gitignore index cb4569f74c..b725239919 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +/tmp /*.keys /*.log /.scannerwork diff --git a/package.json b/package.json index 5939f8598d..a09ad8d307 100644 --- a/package.json +++ b/package.json @@ -132,7 +132,7 @@ "build:dev": "./scripts/generate_build.mjs --dev-mode", "build:all": "npm run build:wasm:release && npm run bundle && npm run bundle:min && npm run build", "build:wasm:debug": "mkdir -p dist && cd ./src/parsers/manifest/dash/wasm-parser && cargo build --target wasm32-unknown-unknown && cp target/wasm32-unknown-unknown/debug/mpd_node_parser.wasm ../../../../../dist/mpd-parser.wasm", - "build:wasm:release": "mkdir -p dist && cd ./src/parsers/manifest/dash/wasm-parser && cargo build --target wasm32-unknown-unknown --release && node ../../../../../scripts/wasm-optimize.mjs target/wasm32-unknown-unknown/release/mpd_node_parser.wasm ../../../../../dist/mpd-parser.wasm && cd ../../../../../ && npm run wasm-strip", + "build:wasm:release": "./scripts/build_wasm_release.sh", "bundle": "webpack --progress --config webpack.config.mjs --env production", "bundle:min": "webpack --progress --config webpack.config.mjs --env minify --env production", "bundle:min:watch": "webpack --progress --config webpack.config.mjs -w --env production --env minify", diff --git a/scripts/build_wasm_release.sh b/scripts/build_wasm_release.sh new file mode 100755 index 0000000000..53cf737641 --- /dev/null +++ b/scripts/build_wasm_release.sh @@ -0,0 +1,64 @@ +#!/bin/sh + +# TODO documentation + +print_toolchain_installation_notice() { + echo " +----------------------------------------------------------------------------------+" + echo " | A rust toolchain will be installed locally in a temporary directory (./tmp). |" + echo " | |" + echo " | If you intend to develop on the RxPlayer regularly, it is recommended that you |" + echo " | install globally rustup (with the \"wasm32-unknown-unknown\" target) as well as |" + echo " | binaryen. Once done, please remove this \".tmp\" directory. |" + echo " +----------------------------------------------------------------------------------+" +} + +has_local_cargo=false +has_local_wasmopt=false +has_installed=false + +mkdir -p dist + +if [ -f tmp/cargo/bin/cargo ]; then + has_local_cargo=true +elif ! cargo_loc="$(type -p "cargo")" || [[ -z $cargo_loc ]]; then + echo "WARNING: cargo command not found." + print_toolchain_installation_notice + sleep 1 + ./scripts/install_rust_toolchain.sh + has_local_cargo=true + has_installed=true +fi + +if [ -f tmp/binaryen/bin/wasm-opt ]; then + has_local_wasmopt=true +elif ! wasmopt_loc="$(type -p "wasm-opt")" || [[ -z $wasmopt_loc ]]; then + if $has_installed; then + >&2 echo "Error: did not succeed to install binaryen dependency. Please install it manually." + exit 1 + fi + + echo "WARNING: wasm-opt command not found." + print_toolchain_installation_notice + sleep 1 + ./scripts/install_rust_toolchain.sh + has_local_wasmopt=true + has_installed=true +fi + +# Move to MPD parser directory +cd ./src/parsers/manifest/dash/wasm-parser +echo "Building mpd-parser WebAssembly file with Cargo..." +if $has_local_cargo; then + echo "⚠️ NOTE ⚠️ : Relying on local cargo in ./tmp/cargo/bin/cargo" + ../../../../../tmp/cargo/bin/cargo build --target wasm32-unknown-unknown --release +else + cargo build --target wasm32-unknown-unknown --release +fi + +echo "Optimizing mpd-parser WebAssembly build..." +if $has_local_wasmopt; then + echo "⚠️ NOTE ⚠️ : Relying on local wasm-opt in ./tmp/binaryen/bin/wasm-opt" + ../../../../../tmp/binaryen/bin/wasm-opt target/wasm32-unknown-unknown/release/mpd_node_parser.wasm --signext-lowering --strip-dwarf -O4 -o ../../../../../dist/mpd-parser.wasm +else + wasm-opt target/wasm32-unknown-unknown/release/mpd_node_parser.wasm --signext-lowering --strip-dwarf -O4 -o ../../../../../dist/mpd-parser.wasm +fi diff --git a/scripts/install_rust_toolchain.sh b/scripts/install_rust_toolchain.sh new file mode 100755 index 0000000000..703b66ddd2 --- /dev/null +++ b/scripts/install_rust_toolchain.sh @@ -0,0 +1,146 @@ +#!/bin/sh + +# TODO Documentation + +# Log a line to stdout, prefixing it with the name of this script +log() { + printf 'install_rust_toolchain: %s\n' "$1" +} + +# Log a line to sterr, prefixing it with the name of this script +err() { + log "ERROR: $1" >&2 + echo "" + echo "Please install a rust toolchain (with the \"wasm32-unknown-unknown\" target) and binaryen manually" >&2 + exit 1 +} + +# Checks that the command in argument exists, exits after printing the issue to +# stderr if that's not the case +requires_cmd() { + if ! command -v "$1" > /dev/null 2>&1; then + err "Need '$1' (command not found)" + fi +} + +# Run a command that should never fail. If the command fails execution +# will immediately terminate with an error showing the failing +# command. +ensure() { + if ! "$@"; then + err "Command failed: $*" + fi +} + +log "This script will only install dependencies locally in a ./tmp directory" + +requires_cmd curl +requires_cmd tar + +ensure mkdir -p tmp + +# Install RustUp in tmp directory with the right target +log "Fetching rustup..." +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | CARGO_HOME=./tmp/cargo RUSTUP_HOME=./tmp/rustup RUSTUP_INIT_SKIP_PATH_CHECK=yes sh -s -- --no-modify-path --profile minimal --target wasm32-unknown-unknown -y +if ! [ $? -eq 0 ]; then + err "Failed to install rustup" +fi + +if ! [ -f tmp/cargo/bin/cargo ]; then + err "Error while installing rustup: Cargo not available in ./tmp/cargo/bin/cargo" +fi + +# Should normally not be needed but CI have shown weird results +ensure tmp/cargo/bin/rustup target add wasm32-unknown-unknown + +ostype="$(uname -s)" +cpuarch="$(uname -m)" + +if [ "$ostype" = Linux ]; then + if [ "$(uname -o)" = Android ]; then + err "Unhandled OS type (Android), please install binaryen manually" + fi +fi + +if [ "$ostype" = Darwin ] && [ "$cpuarch" = i386 ]; then + # Darwin `uname -m` lies + if sysctl hw.optional.x86_64 | grep -q ': 1'; then + cpuarch=x86_64 + fi +fi + +case "$ostype" in + Linux) + ;; + Darwin) + ;; + MINGW* | MSYS* | CYGWIN* | Windows_NT) + ostype=Windows + ;; + *) + err "Unhandled OS type ($ostype), please install binaryen manually" + ;; +esac + +case "$cpuarch" in + aarch64 | arm64) + cpuarch=aarch64 + ;; + x86_64 | x86-64 | x64 | amd64) + cpuarch=x86_64 + ;; + *) + err "Unhandled CPU type ($cpuarch), please install binaryen manually" +esac + +# TODO automatically download last binaryen? +# We might need to detect which build is available. Targeting version 116 is +# good enough for now +if [ "${ostype}" = Darwin ]; then + if [ "${cpuarch}" = aarch64 ]; then + log "Architecture detected -> MacOS ARM" + binaryen_url=https://github.com/WebAssembly/binaryen/releases/download/version_116/binaryen-version_116-arm64-macos.tar.gz + else + log "Architecture detected -> MacOS x86_64" + binaryen_url=https://github.com/WebAssembly/binaryen/releases/download/version_116/binaryen-version_116-x86_64-macos.tar.gz + fi +elif [ "${ostype}" = Linux ]; then + if [ "${cpuarch}" != x86_64 ]; then + err "For Linux, only x86_64 is supported by our auto-install script. Please install binaryen manually." + fi + log "Architecture detected -> Linux x86_64" + binaryen_url=https://github.com/WebAssembly/binaryen/releases/download/version_116/binaryen-version_116-x86_64-linux.tar.gz +elif [ "${ostype}" = Windows ]; then + if [ "${cpuarch}" != x86_64 ]; then + err "For Windows, only x86_64 is supported by our auto-install script. Please install binaryen manually." + fi + log "Architecture detected -> Windows x86_64" + binaryen_url=https://github.com/WebAssembly/binaryen/releases/download/version_116/binaryen-version_116-x86_64-windows.tar.gz +fi + +log "Fetching binaryen 116..." +curl -L $binaryen_url > tmp/binaryen.tar.gz +if ! [ $? -eq 0 ]; then + err "Failed to fetch binaryen" +fi + + +cd tmp +ensure tar xzf binaryen.tar.gz + +ensure mv binaryen-version_116 binaryen + +# TODO I don't know my windows, does that still work as an executable or is it just dumb? +if [ "${ostype}" = Windows ]; then + ensure cp binaryen/bin/wasm-opt.exe binaryen/bin/wasm-opt +fi + +rm binaryen.tar.gz +cd .. + +if ! [ -f tmp/binaryen/bin/wasm-opt ]; then + err "Error after installing binaryen: wasm-opt not available in ./tmp/binaryen/bin/wasm-opt" +fi + +echo "" +echo "All dependencies have been installed with success!" From 630f4a105db32556623fe415dc05c00a07c2ef0e Mon Sep 17 00:00:00 2001 From: Paul Berberian Date: Fri, 2 Feb 2024 17:42:55 +0100 Subject: [PATCH 2/2] [WIP] Auto-install Rust compiler toolchain for CI and JS people Since we brought the WebAssembly MPD parser to fix performance issues we had with some contents at canal+, we added another language (Rust) to the project which is not compatible with most of the JS-world tools we and other JS developers usually deal with. Now that we added the `MULTI_THREAD` experimental feature and embeds, the WebAssembly generation from that other language is even a required step in our main build script (`npm run build`). Consequently, when another developer or a CI needs to build an RxPlayer (e.g. to test a development before a release is made), they have to install: 1. NodeJS + npm 2. our JavaScript dependencies 3. A Rust toolchain able to output WebAssembly. We generally go through `rustup` here basically an helper tool to install Rust toolchains 4. `binaryen`, which is a collection of tools to transform that WebAssembly file. (1) and (2) are very common use cases that both JS people and CI workflows do very easily: they mostly already have nodejs + npm installed locally (or it's very simple to set-up) and the second is only an `npm install` call away. (3) and (4) however, though not hard (you have to install rustup and binaryen which is pretty simple on all platforms we relied on), are more unusual dependencies that JS developers might be reticent to install and that are more work to set-up on a CI. Consequently I'm doing here a proof-of-concept where those tools are auto-installed locally if not found when doing an `npm run build` call, letting both people and CI produce a build easily without having to deal with the dual-language complexity. --- For rustup it's pretty straightforward as their install script already take care of the platform detection, target installation and so on. All I had to do is to make sure binaries were installed locally (`./tmp`) and did not pollute the developers environment (e.g. no `PATH` modification). For binaryen however, it's less simple as they don't propose such auto-install system. however, I noticed that they propose prebuilt binaries for the most usual platforms on their github's release page: https://github.com/WebAssembly/binaryen/releases The idea here (not yet finished), is to automatically detect the platform of the user and fetch the right binaryen binary locally to then rely on both it and the local rustup to produce builds in case the command isn't available globally. Also, because it will be faster and less error-prone for frequent developers to install both of those globally once and for all, we print a notice explaining what we do when that auto-install script is run. --- scripts/build_wasm_release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_wasm_release.sh b/scripts/build_wasm_release.sh index 53cf737641..8c1a9b8d96 100755 --- a/scripts/build_wasm_release.sh +++ b/scripts/build_wasm_release.sh @@ -8,7 +8,7 @@ print_toolchain_installation_notice() { echo " | |" echo " | If you intend to develop on the RxPlayer regularly, it is recommended that you |" echo " | install globally rustup (with the \"wasm32-unknown-unknown\" target) as well as |" - echo " | binaryen. Once done, please remove this \".tmp\" directory. |" + echo " | binaryen. Once done, please remove this \"tmp\" directory. |" echo " +----------------------------------------------------------------------------------+" }