Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

always produce .a files at the beginning of a build #3385

Merged
merged 38 commits into from
Dec 20, 2022
Merged
Show file tree
Hide file tree
Changes from 19 commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
d6cb38c
always produce .a files at the beginning of a build
matloob Nov 23, 2022
784c1c0
fix a bad cleanup in stdlib.bzl
matloob Dec 6, 2022
dc93dfb
redefine transition in terms of what's kept
matloob Dec 6, 2022
36c5314
Merge branch 'bazelbuild:master' into go
matloob Dec 7, 2022
04c752c
use go_reset_target for the builder
matloob Dec 7, 2022
b81c415
remove previosly deleted code that came back in a merge`
matloob Dec 7, 2022
8421750
fix buildifier issue
matloob Dec 7, 2022
c45ec9c
try to use tmpdir on WINDOWS
matloob Dec 7, 2022
106e275
try another way of getting gocache dirs for windows
matloob Dec 7, 2022
c3b34ad
add output
matloob Dec 7, 2022
7a79e59
debug
matloob Dec 7, 2022
de594a0
debug2
matloob Dec 7, 2022
78a8e2e
debug 3
matloob Dec 7, 2022
2b05009
debug 4
matloob Dec 7, 2022
0d37f96
debug 5
matloob Dec 7, 2022
f0e29ba
debug 6
matloob Dec 7, 2022
3a35f5b
only compile stdlib .a files for Go 1.20+
matloob Dec 13, 2022
f757ef4
set version to 1.20rc1 for CI testing purposes
matloob Dec 13, 2022
24e601f
fix libs attribute of Go sdk to allow zero libs files
matloob Dec 13, 2022
b252e0e
fix coverage test
matloob Dec 13, 2022
3b8dab0
get only digits of the part before the next dot
matloob Dec 13, 2022
490ec4b
allow empty
matloob Dec 13, 2022
0d3250c
Merge branch 'master' of github.com:matloob/rules_go into go
matloob Dec 13, 2022
25e52fd
set GOEXPERIMENT=nocoverageredesign
matloob Dec 13, 2022
06fd767
conditionally turn off goexperiment based on versioun
matloob Dec 13, 2022
1b41059
rename variable
matloob Dec 13, 2022
1c8389c
fix flag
matloob Dec 13, 2022
bb7eed4
add lcov fix
matloob Dec 13, 2022
6f3008a
address comments, and remove one config of builder
matloob Dec 14, 2022
06ee92d
decide to compile .a files based on whether they exist
matloob Dec 14, 2022
b3dfe4b
Merge branch 'master' of https://github.com/bazelbuild/rules_go into go
matloob Dec 14, 2022
de75424
Merge branch 'master' of https://github.com/bazelbuild/rules_go into go
matloob Dec 15, 2022
80235ea
fix add call
matloob Dec 15, 2022
929b55c
add nocoverageredesign to experiments earlier
matloob Dec 15, 2022
87f47ef
another small fix
matloob Dec 15, 2022
c12a10b
fix double
matloob Dec 15, 2022
108915e
Update go/private/sdk.bzl
matloob Dec 16, 2022
bc8eb25
add a simple test doing a build on 1.20rc1
matloob Dec 16, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion WORKSPACE
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_depe

go_rules_dependencies()

go_register_toolchains(version = "1.18.3")
go_register_toolchains(version = "1.20rc1")

http_archive(
name = "com_google_protobuf",
Expand Down
9 changes: 8 additions & 1 deletion go/private/BUILD.sdk.bazel
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
load("@{rules_go_repo_name}//go/private/rules:binary.bzl", "go_tool_binary")
load("@{rules_go_repo_name}//go/private/rules:sdk.bzl", "package_list")
load("@{rules_go_repo_name}//go/private/rules:transition.bzl", "non_go_reset_target")
load("@{rules_go_repo_name}//go/private:go_toolchain.bzl", "declare_go_toolchains")
load("@{rules_go_repo_name}//go:def.bzl", "go_sdk")

Expand Down Expand Up @@ -42,6 +43,7 @@ go_sdk(
package_list = ":package_list",
root_file = "ROOT",
tools = [":tools"],
version = "{version}",
)

go_tool_binary(
Expand All @@ -50,6 +52,11 @@ go_tool_binary(
sdk = ":go_sdk",
)

non_go_reset_target(
name = "builder_reset",
dep = ":builder"
)

# TODO(jayconrod): Gazelle depends on this file directly. This dependency
# should be broken, and this rule should be folded into go_sdk.
package_list(
Expand All @@ -60,7 +67,7 @@ package_list(
)

declare_go_toolchains(
builder = ":builder",
builder = ":builder_reset",
host_goos = "{goos}",
sdk = ":go_sdk",
)
Expand Down
18 changes: 17 additions & 1 deletion go/private/actions/stdlib.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -44,13 +44,29 @@ def _stdlib_library_to_source(go, attr, source, merge):
source["stdlib"] = _build_stdlib(go)

def _should_use_sdk_stdlib(go):
return (go.mode.goos == go.sdk.goos and
return (_has_precompiled_stdlib(go.sdk.version) and
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we instead check the existence of go.sdk.libs? So people can download a pre-compile Go SDK even after 1.20 to avoid building it on every machine

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done!

go.mode.goos == go.sdk.goos and
go.mode.goarch == go.sdk.goarch and
not go.mode.race and # TODO(jayconrod): use precompiled race
not go.mode.msan and
not go.mode.pure and
go.mode.link == LINKMODE_NORMAL)

def _has_precompiled_stdlib(version_string):
if version_string[:2] != "1.":
return False
minor = version_string[2:]
dot = minor.find(".")
if dot != -1:
minor = minor[:dot]
rc = minor.find("rc")
if rc != -1:
minor = minor[:rc]
beta = minor.find("beta")
if beta != -1:
minor = minor[:beta]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Assuming there could be other strings (e.g. "alpha"), and we are already at three (".", "rc", "beta"), could we instead cut off the first non-numeric character? You could use isdigit.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks good to me with one comment, we should be able to merge this right away.

@linzhp Could you test at Uber once the comment has been resolved?

Missed the coverage test failure, will also look into it.

I think the coverage test is the same issue on the boringcrypto cl: that we need to make the path to "Tool" relative on 1.19+.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I ran it locally with extra output and it complained about -test.coverprofile not being supported when built without coverage. Since the CI is passing for the other PR, I rather suspect something changed with respect to coverage instrumentation in Go 1.20.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, got it, that will require a closer look...

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok so that error was due to the flag-flip setting the GOEXPERIMENT coverageredesign by default. We'll need to eventually update the way coverage works in rules_go, but for now we can set GOEXPERIMENT=nocoverageredesign. I'll try to find a way to set that to get this change moving forward. It might be easiest to plumb something through the same way we do in the boringcrypto cl

return int(minor) < 20

def _build_stdlib_list_json(go):
out = go.declare_file(go, "stdlib.pkg.json")
args = go.builder_args(go, "stdliblist")
Expand Down
1 change: 1 addition & 0 deletions go/private/providers.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ GoSDK = provider(
"tools": ("List of executable files in the SDK built for " +
"the execution platform, excluding the go binary file"),
"go": "The go binary file",
"version": "The Go SDK version",
},
)

Expand Down
43 changes: 16 additions & 27 deletions go/private/rules/binary.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -415,11 +415,15 @@ def _go_tool_binary_impl(ctx):
if sdk.goos == "windows":
name += ".exe"

cout = ctx.actions.declare_file(name + ".a")
out = ctx.actions.declare_file(name)
if sdk.goos == "windows":
cmd = "@echo off\n {go} tool compile -o {cout} -trimpath=%cd% {srcs}".format(
gopath = ctx.actions.declare_directory("gopath")
gocache = ctx.actions.declare_directory("gocache")
cmd = "@echo off\nset GOCACHE=%cd%\\{gocache}\nset GOPATH=%cd%\\{gopath}\n{go} build -o {out} -trimpath {srcs}".format(
gopath = gopath.path,
gocache = gocache.path,
go = sdk.go.path.replace("/", "\\"),
cout = cout.path,
out = out.path,
srcs = " ".join([f.path for f in ctx.files.srcs]),
)
bat = ctx.actions.declare_file(name + ".bat")
Expand All @@ -429,39 +433,24 @@ def _go_tool_binary_impl(ctx):
)
ctx.actions.run(
executable = bat,
inputs = sdk.libs + sdk.headers + sdk.tools + ctx.files.srcs + [sdk.go],
outputs = [cout],
env = {"GOROOT": sdk.root_file.dirname}, # NOTE(#2005): avoid realpath in sandbox
mnemonic = "GoToolchainBinaryCompile",
inputs = sdk.headers + sdk.tools + sdk.srcs + ctx.files.srcs + [sdk.go],
outputs = [out, gopath, gocache],
mnemonic = "GoToolchainBinaryBuild",
)
else:
cmd = "{go} tool compile -o {cout} -trimpath=$PWD {srcs}".format(
# Note: GOPATH is needed for Go 1.16.
cmd = "GOCACHE=$(mktemp -d) GOPATH=$(mktemp -d) {go} build -o {out} -trimpath {srcs}".format(
go = sdk.go.path,
cout = cout.path,
out = out.path,
srcs = " ".join([f.path for f in ctx.files.srcs]),
)
ctx.actions.run_shell(
command = cmd,
inputs = sdk.libs + sdk.headers + sdk.tools + ctx.files.srcs + [sdk.go],
outputs = [cout],
env = {"GOROOT": sdk.root_file.dirname}, # NOTE(#2005): avoid realpath in sandbox
mnemonic = "GoToolchainBinaryCompile",
inputs = sdk.headers + sdk.tools + sdk.srcs + sdk.libs + ctx.files.srcs + [sdk.go],
outputs = [out],
mnemonic = "GoToolchainBinaryBuild",
)

out = ctx.actions.declare_file(name)
largs = ctx.actions.args()
largs.add_all(["tool", "link"])
largs.add("-o", out)
largs.add(cout)
ctx.actions.run(
executable = sdk.go,
arguments = [largs],
inputs = sdk.libs + sdk.headers + sdk.tools + [cout],
outputs = [out],
env = {"GOROOT_FINAL": "GOROOT"}, # Suppress go root paths to keep the output reproducible.
mnemonic = "GoToolchainBinary",
)

return [DefaultInfo(
files = depset([out]),
executable = out,
Expand Down
9 changes: 8 additions & 1 deletion go/private/rules/sdk.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ def _go_sdk_impl(ctx):
srcs = ctx.files.srcs,
tools = ctx.files.tools,
go = ctx.executable.go,
version = ctx.attr.version,
)]

go_sdk = rule(
Expand All @@ -56,7 +57,10 @@ go_sdk = rule(
"standard library that may be imported."),
),
"libs": attr.label_list(
allow_files = [".a"],
# allow_files is not set to [".a"] because that wouldn't allow
# for zero files to be present, as is the case in Go 1.20+.
# See also https://github.com/bazelbuild/bazel/issues/7516
allow_files = True,
doc = ("Pre-compiled .a files for the standard library, " +
"built for the execution platform"),
),
Expand All @@ -82,6 +86,9 @@ go_sdk = rule(
cfg = "exec",
doc = "The go binary",
),
"version": attr.string(
doc = "The version of the Go SDK.",
),
},
doc = ("Collects information about a Go SDK. The SDK must have a normal " +
"GOROOT directory structure."),
Expand Down
8 changes: 8 additions & 0 deletions go/private/rules/stdlib.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,10 @@ load(
"CgoContextInfo",
"GoConfigInfo",
)
load(
"//go/private/rules:transition.bzl",
"go_stdlib_transition",
)

def _stdlib_impl(ctx):
go = go_context(ctx)
Expand All @@ -33,12 +37,16 @@ def _stdlib_impl(ctx):

stdlib = rule(
implementation = _stdlib_impl,
cfg = go_stdlib_transition,
attrs = {
"cgo_context_data": attr.label(providers = [CgoContextInfo]),
"_go_config": attr.label(
default = "//:go_config",
providers = [GoConfigInfo],
),
"_allowlist_function_transition": attr.label(
default = "@bazel_tools//tools/allowlists/function_transition_allowlist",
),
},
doc = """stdlib builds the standard library for the target configuration
or uses the precompiled standard library from the SDK if it is suitable.""",
Expand Down
28 changes: 28 additions & 0 deletions go/private/rules/transition.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,13 @@ _reset_transition_dict = dict(_common_reset_transition_dict, **{

_reset_transition_keys = sorted([filter_transition_label(label) for label in _reset_transition_dict.keys()])

_stdlib_keep_keys = sorted([
"@io_bazel_rules_go//go/config:msan",
"@io_bazel_rules_go//go/config:race",
"@io_bazel_rules_go//go/config:pure",
"@io_bazel_rules_go//go/config:linkmode",
])

def _go_tool_transition_impl(settings, attr):
"""Sets most Go settings to default values (use for external Go tools).

Expand Down Expand Up @@ -266,6 +273,27 @@ non_go_tool_transition = transition(
outputs = _reset_transition_keys,
)

def _go_stdlib_transition_impl(settings, attr):
"""Sets all Go settings to their default values, except for those affecting the Go SDK.

This transition is similar to _non_go_tool_transition except that it keeps the
parts of the configuration that determine how to build the standard library.
It's used to consolidate the configurations used to build the standard library to limit
the number built.
"""
settings = dict(settings)
for label, value in _reset_transition_dict.items():
if label not in _stdlib_keep_keys:
settings[filter_transition_label(label)] = value
settings[filter_transition_label("@io_bazel_rules_go//go/private:bootstrap_nogo")] = False
return settings

go_stdlib_transition = transition(
implementation = _go_stdlib_transition_impl,
inputs = _reset_transition_keys,
outputs = _reset_transition_keys,
)

def _go_reset_target_impl(ctx):
t = ctx.attr.dep[0] # [0] seems to be necessary with the transition
providers = [t[p] for p in [GoLibrary, GoSource, GoArchive] if p in t]
Expand Down
1 change: 1 addition & 0 deletions go/private/sdk.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,7 @@ def _sdk_build_file(ctx, platform, version):
"{goarch}": goarch,
"{exe}": ".exe" if goos == "windows" else "",
"{rules_go_repo_name}": "io_bazel_rules_go",
"{version}": version,
},
)

Expand Down
1 change: 1 addition & 0 deletions go/tools/builders/stdlib.go
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`)
}
}
os.Setenv("CGO_LDFLAGS_ALLOW", b.String())
os.Setenv("GODEBUG", "installgoroot=all")

// Build the commands needed to build the std library in the right mode
// NOTE: the go command stamps compiled .a files with build ids, which are
Expand Down