Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rebase, improve WASM code #1

Merged
merged 3 commits into from
Jun 13, 2022
Merged

Rebase, improve WASM code #1

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3488894
Split WASM build factories
tiran Jun 13, 2022
8f0af91
parallel testing now works
tiran Jun 13, 2022
bebac25
use config.guess directly
tiran Jun 13, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion master/custom/builders.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@
MacOSArmWithBrewBuild,
WindowsARM64Build,
WindowsARM64ReleaseBuild,
Wasm32EmscriptenNodeBuild,
Wasm32EmscriptenBrowserBuild,
Wasm32WASIBuild,
)

STABLE = "stable"
Expand All @@ -49,7 +52,7 @@
def get_builders(settings):
# Override with a default simple worker if we are using local workers
if settings.use_local_worker:
return [("Test Builder", "local-worker", UnixBuild, STABLE)]
return [("Test Builder", "local-worker", UnixBuild, STABLE, NO_TIER)]

return [
# -- Stable builders --
Expand Down Expand Up @@ -207,6 +210,11 @@ def get_builders(settings):
("ARM64 Windows Non-Debug", "linaro-win-arm64", WindowsARM64ReleaseBuild, STABLE, NO_TIER),
("ARM64 Windows Azure", "linaro2-win-arm64", WindowsARM64Build, UNSTABLE, NO_TIER),
("ARM64 Windows Non-Debug Azure", "linaro2-win-arm64", WindowsARM64ReleaseBuild, UNSTABLE, NO_TIER),

# WebAssembly
("wasm32-emscripten node (threaded)", "bcannon-wasm", Wasm32EmscriptenNodeBuild, UNSTABLE, NO_TIER),
("wasm32-emscripten browser (dynamic linking)", "bcannon-wasm", Wasm32EmscriptenBrowserBuild, UNSTABLE, NO_TIER),
("wasm32-wasi", "bcannon-wasm", Wasm32WASIBuild, UNSTABLE, NO_TIER),
]


Expand Down
138 changes: 83 additions & 55 deletions master/custom/factories.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,6 @@

from buildbot.plugins import util

import re

from .steps import (
Test,
Clean,
Expand Down Expand Up @@ -592,14 +590,7 @@ class WindowsARM64ReleaseBuild(WindowsARM64Build):
##############################################################################

def extract_build_triple(_rc, stdout, _stderr):
lines = stdout.splitlines()
if re.match(r'GNU Make \d+\.\d+(\.\d+)?', lines[0]) is None:
return {}
triple_line = re.match(r'Built for (.*)', lines[1])
if len(triple_line.groups()) != 1:
return {}
else:
return {'build_triple': triple_line.group(1)}
return {'build_triple': stdout.strip()}
emmatyping marked this conversation as resolved.
Show resolved Hide resolved


class UnixCrossBuild(UnixBuild):
Expand All @@ -610,7 +601,8 @@ class UnixCrossBuild(UnixBuild):
extra_configure_flags = []
host_configure_cmd = ["../../configure"]
host = None
host_make_cmd = "make"
host_make_cmd = ["make"]
can_execute_python = True

def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
assert self.host is not None, "Must set self.host on cross builds"
Expand All @@ -623,8 +615,8 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
self.addStep(
SetPropertyFromCommand(
name="Gather build triple from worker",
description="Get the build triple from make",
command=["make", "-v"],
description="Get the build triple config.guess",
command=["./config.guess"],
extract_fn=extract_build_triple,
tiran marked this conversation as resolved.
Show resolved Hide resolved
warnOnFailure=True,
)
Expand Down Expand Up @@ -710,9 +702,9 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
]

if parallel:
compile = [self.host_make_cmd, parallel, self.makeTarget]
compile = self.host_make_cmd + [parallel, self.makeTarget]
else:
compile = [self.host_make_cmd, self.makeTarget]
compile = self.host_make_cmd + [self.makeTarget]
self.addStep(
Compile(
name="Compile host Python",
Expand All @@ -721,55 +713,91 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
workdir=oot_host_path,
)
)
pyinfo_command = [
"make",
"pythoninfo",
]
self.addStep(
ShellCommand(
name="pythoninfo",
description="pythoninfo",
command=pyinfo_command,
warnOnFailure=True,
env=self.test_environ,
workdir=oot_host_path,
if self.can_execute_python:
self.addStep(
ShellCommand(
name="pythoninfo",
description="pythoninfo",
command=["make", "pythoninfo"],
warnOnFailure=True,
env=self.test_environ,
workdir=oot_host_path,
)
)
)
self.addStep(
Test(
command=test,
timeout=self.test_timeout,
usePTY=test_with_PTY,
env=self.test_environ,
workdir=oot_host_path,
self.addStep(
Test(
command=test,
timeout=self.test_timeout,
usePTY=test_with_PTY,
env=self.test_environ,
workdir=oot_host_path,
)
)
)
if branch not in ("3",) and "-R" not in self.testFlags:
filename = os.path.join(oot_host_path, "test-results.xml")
self.addStep(UploadTestResults(branch, filename=filename))
if branch not in ("3",) and "-R" not in self.testFlags:
filename = os.path.join(oot_host_path, "test-results.xml")
self.addStep(UploadTestResults(branch, filename=filename))
self.addStep(Clean(workdir=oot_build_path))
self.addStep(Clean(workdir=oot_host_path))


class WASMNodeBuild(UnixCrossBuild):
extra_configure_flags = [
"--with-emscripten-target=node",
"--enable-wasm-dynamic-linking=no",
"--enable-wasm-pthreads",
]
class Wasm32EmscriptenBuild(UnixCrossBuild):
"""wasm32-emscripten builder

* Emscripten SDK >= 3.1.12 must be installed
* ccache must be installed
* Emscripten PATHs must be pre-pended to PATH
* ``which node`` must be equal $EMSDK_NODE
"""
factory_tags = ["wasm", "emscripten"]
compile_environ = {
"CONFIG_SITE": "../../Tools/wasm/config.site-wasm32-emscripten",
"EM_COMPILER_WRAPPER": "ccache",
}

host = "wasm32-unknown-emscripten"
host_configure_cmd = ["emconfigure", "../../configure"]
host_make_cmd = ["emmake", "make"]

def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
self.addStep(
ShellCommand(
name="Install Emscripten ports of dependencies",
description="Install wasm dependencies",
command=["embuilder", "build", "zlib", "bzip2"],
warnOnFailure=True,
)
)
super().setup(parallel, branch, test_with_PTY, **kwargs)

class Wasm32EmscriptenNodeBuild(Wasm32EmscriptenBuild):
extra_configure_flags = [
"--with-emscripten-target=node",
"--disable-wasm-dynamic-linking",
"--enable-wasm-pthreads",
]


class Wasm32EmscriptenBrowserBuild(Wasm32EmscriptenBuild):
extra_configure_flags = [
"--with-emscripten-target=browser",
"--enable-wasm-dynamic-linking",
"--disable-wasm-pthreads",
]
# browser builds do not accept argv from CLI
can_execute_python = False


class Wasm32WASIBuild(UnixCrossBuild):
"""wasm32-wasi builder

* WASI SDK >= 16 must be installed to default path /opt/wasi-sdk
* WASIX must be installed to /opt/wasix
* ccache must be installed
* wasmtime must be installed and on PATH
"""
factory_tags = ["wasm", "wasi"]
extra_configure_flags = [
"--disable-ipv6",
]
wasi_sdk = "/opt/wasi-sdk"
wasix = "/opt/wasix"
compile_environ = {
"CONFIG_SITE": "../../Tools/wasm/config.site-wasm32-wasi",
"CC": f"ccache {wasi_sdk}/bin/clang",
"LDSHARED": f"{wasi_sdk}/bin/wasm-ld",
"AR": f"{wasi_sdk}/bin/llvm-ar",
"CFLAGS": f"-isystem {wasix}/include",
"LDLAGS": f"-L{wasix}/lib -lwasix",
}
host = "wasm32-unknown-wasi"
host_configure_cmd = ["../../configure"]
7 changes: 6 additions & 1 deletion master/custom/workers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -289,5 +289,10 @@ def get_workers(settings):
tags=['windows', 'arm64'],
parallel_tests=2,
),

cpw(
name="bcannon-wasm",
tags=['wasm', 'emscripten', 'wasi'],
branches=['3.11', '3.x'],
parallel_tests=4,
),
]