From 4da516ce64e198e13b8fc21bba0395e0888961b5 Mon Sep 17 00:00:00 2001 From: Morgan Date: Wed, 19 Jun 2024 17:38:06 +0200 Subject: [PATCH] feat(transpiler): transpile gno standard libraries (#1695) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Merge order: 1. #1700 2. #1702 3. #1695 (this one!) -- review earlier ones first, if they're still open! This PR modifies the Gno transpiler (fka precompiler) to use Gno's standard libraries rather than Go's when performing transpilation. This creates the necessity to transpile Gno standard libraries, and as such support their native bindings. And it removes the necessity for a package like `stdshim`, and a mechanism like `stdlibWhitelist`. - Fixes #668. Fixes #1865. - Resolves #892. - Part of #814. - Makes #1475 / #1576 possible without using hacks like `stdshim`. cc/ @leohhhn @tbruyelle, as this relates to your work ## Why? - This PR enables us to perform Go type-checking across the board, and not use Go's standard libraries in transpiled code. This enables us to _properly support our own standard libraries_, such as `std` but any others we might want or need. - It also paves the way further to go full circle, and have Gno code be transpiled to Go, and then have "compilable" gno code ## Summary of changes - The transpiler has been thoroughly refactored. - The biggest change is described above: instead of maintaing the import paths like `"strconv"` and `"math"` the same (so using Gno's stdlibs in Gno, and Go's in Go), the import paths for standard libraries is now also updated to point to the Gno standard libraries. - Native functions are handled by removing their definitions when transpiling, and changing their call expressions where appropriate. This links the transpiled code directly to their native counterparts. - This removes the necessity for `stdlibWhitelist`. - As a consequence, `stdshim` is no longer needed and has been removed. - Test files are still not "strictly checked": they may reference stdlibs with no matching source, and will not be tested when running with `--gobuild`. This is because packages like `fmt` have no representation in Gno code; they only exist as injections in `tests/imports.go`. I'll fix this eventually :) - The CLI (`gno transpile`) has been changed to reflect the above changes. - Flag `--skip-fmt` has been removed (the result of transpile is always formatted, anyway), and `--gofmt-binary` too, obviously. `gno transpile` does not perform validation, but will gladly provide helpful validation with the `--gobuild` flag. - There is another PR that adds type checking in `gno lint`, without needing to run through the transpilation step first: https://github.com/gnolang/gno/pull/1730 - It now works by default by looking at "packages" rather than individual files. This is necessary so that when performing `transpile` on the `examples` directory, we can skip those where the gno.mod marks the module as draft. These modules make use of packages like "fmt", which because they don't have an underlying gno/go source, cannot be transpiled. - Running with `-gobuild` now handles more errors correctly; ie., all errors not previously captured by the `errorRe` which only matches those pertaining to a specific file/line. - `gnoFilesFromArgs` was unused and as such deleted - `gnomod`'s behaviour was slightly changed. - I am of the opinion that `gno mod download` should not precompile what it downloads; _especially_ to gather the dependencies it has. I've changed it so that it does a `OnlyImports` parse of the file it downloads to fetch additional dependencies Misc: - `Makefile` now contains a recipe to calculate the coverage for `gnovm/cmd/gno`, and also view it via the HTML interface. This is needed as it has a few extra steps (which @gfanton already previously added in the CI). - Realms r/demo/art/gnoface and r/x/manfred_outfmt have been marked as draft, as they depend on packages which are not actually present in the Gno standard libraries. - The transpiler now ignores draft packages by default. - `ReadMemPackage` now also considers Go files. This is meant to have on-chain the code for standard libraries like `std` which have native bindings. We still exclude Go code if it's not in a standard library. - `//go:build` constraints have been removed from standard libraries, as go files can only have one and we already add our own when transpiling ## Further improvements after this PR - Scope understanding in `transpiler` (so call expressions are not incorrectly rewritten) - Correctly transpile gno.mod --------- Co-authored-by: Antonio Navarro Perez Co-authored-by: Miloš Živković --- examples/gno.land/r/demo/art/gnoface/gno.mod | 2 + examples/gno.land/r/x/manfred_outfmt/gno.mod | 2 + gnovm/Makefile | 23 +- gnovm/cmd/gno/test.go | 3 +- .../gno_transpile/05_skip_fmt_flag.txtar | 36 -- .../gno_transpile/06_build_flag.txtar | 38 -- ...r.txtar => gobuild_flag_build_error.txtar} | 0 ...r.txtar => gobuild_flag_parse_error.txtar} | 0 ...elist_error.txtar => invalid_import.txtar} | 6 +- .../{01_no_args.txtar => no_args.txtar} | 0 ...es_parse_error.txtar => parse_error.txtar} | 0 ..._empty_dir.txtar => valid_empty_dir.txtar} | 0 .../gno_transpile/valid_gobuild_file.txtar | 31 ++ .../gno_transpile/valid_gobuild_flag.txtar | 72 ++++ .../gno_transpile/valid_output_flag.txtar | 41 ++ .../gno_transpile/valid_output_gobuild.txtar | 44 ++ .../gno_transpile/valid_transpile_file.txtar | 65 +++ ...es.txtar => valid_transpile_package.txtar} | 0 .../gno_transpile/valid_transpile_tree.txtar | 94 ++++ gnovm/cmd/gno/transpile.go | 272 +++++++++--- gnovm/cmd/gno/transpile_test.go | 80 ++++ gnovm/cmd/gno/util.go | 162 +++---- gnovm/cmd/gno/util_test.go | 47 ++ gnovm/pkg/gnolang/helpers.go | 28 ++ gnovm/pkg/gnolang/helpers_test.go | 55 +++ gnovm/pkg/gnolang/nodes.go | 12 +- gnovm/pkg/gnolang/realm.go | 15 - gnovm/pkg/gnomod/file.go | 9 +- gnovm/pkg/gnomod/gnomod.go | 41 +- gnovm/pkg/integration/gno.go | 2 + gnovm/pkg/transpiler/transpiler.go | 402 +++++++----------- gnovm/pkg/transpiler/transpiler_test.go | 232 +++++++--- gnovm/stdlibs/bytes/boundary_test.gno | 2 - .../crypto/chacha20/chacha/chacha_ref.gno | 2 - .../internal/bytealg/compare_generic.gno | 3 - .../internal/bytealg/count_generic.gno | 2 - .../internal/bytealg/index_generic.gno | 2 - .../internal/bytealg/indexbyte_generic.gno | 2 - gnovm/stdlibs/native.go | 58 ++- gnovm/stdlibs/stdlibs.go | 19 +- gnovm/stdlibs/stdshim/addr_set.gno | 47 -- gnovm/stdlibs/stdshim/banker.gno | 70 --- gnovm/stdlibs/stdshim/coins.gno | 174 -------- gnovm/stdlibs/stdshim/context.gno | 4 - gnovm/stdlibs/stdshim/crypto.gno | 17 - gnovm/stdlibs/stdshim/frame.gno | 18 - gnovm/stdlibs/time/time.gno | 3 - gnovm/tests/stdlibs/native.go | 36 +- misc/genstd/template.tmpl | 25 +- 49 files changed, 1327 insertions(+), 971 deletions(-) delete mode 100644 gnovm/cmd/gno/testdata/gno_transpile/05_skip_fmt_flag.txtar delete mode 100644 gnovm/cmd/gno/testdata/gno_transpile/06_build_flag.txtar rename gnovm/cmd/gno/testdata/gno_transpile/{07_build_flag_with_build_error.txtar => gobuild_flag_build_error.txtar} (100%) rename gnovm/cmd/gno/testdata/gno_transpile/{08_build_flag_with_parse_error.txtar => gobuild_flag_parse_error.txtar} (100%) rename gnovm/cmd/gno/testdata/gno_transpile/{09_gno_files_whitelist_error.txtar => invalid_import.txtar} (60%) rename gnovm/cmd/gno/testdata/gno_transpile/{01_no_args.txtar => no_args.txtar} (100%) rename gnovm/cmd/gno/testdata/gno_transpile/{03_gno_files_parse_error.txtar => parse_error.txtar} (100%) rename gnovm/cmd/gno/testdata/gno_transpile/{02_empty_dir.txtar => valid_empty_dir.txtar} (100%) create mode 100644 gnovm/cmd/gno/testdata/gno_transpile/valid_gobuild_file.txtar create mode 100644 gnovm/cmd/gno/testdata/gno_transpile/valid_gobuild_flag.txtar create mode 100644 gnovm/cmd/gno/testdata/gno_transpile/valid_output_flag.txtar create mode 100644 gnovm/cmd/gno/testdata/gno_transpile/valid_output_gobuild.txtar create mode 100644 gnovm/cmd/gno/testdata/gno_transpile/valid_transpile_file.txtar rename gnovm/cmd/gno/testdata/gno_transpile/{04_valid_gno_files.txtar => valid_transpile_package.txtar} (100%) create mode 100644 gnovm/cmd/gno/testdata/gno_transpile/valid_transpile_tree.txtar create mode 100644 gnovm/pkg/gnolang/helpers_test.go delete mode 100644 gnovm/stdlibs/stdshim/addr_set.gno delete mode 100644 gnovm/stdlibs/stdshim/banker.gno delete mode 100644 gnovm/stdlibs/stdshim/coins.gno delete mode 100644 gnovm/stdlibs/stdshim/context.gno delete mode 100644 gnovm/stdlibs/stdshim/crypto.gno delete mode 100644 gnovm/stdlibs/stdshim/frame.gno diff --git a/examples/gno.land/r/demo/art/gnoface/gno.mod b/examples/gno.land/r/demo/art/gnoface/gno.mod index bc17ee9df3b..6276629cba2 100644 --- a/examples/gno.land/r/demo/art/gnoface/gno.mod +++ b/examples/gno.land/r/demo/art/gnoface/gno.mod @@ -1,3 +1,5 @@ +// Draft + module gno.land/r/demo/art/gnoface require ( diff --git a/examples/gno.land/r/x/manfred_outfmt/gno.mod b/examples/gno.land/r/x/manfred_outfmt/gno.mod index e6f705c46b9..9804aecc7f1 100644 --- a/examples/gno.land/r/x/manfred_outfmt/gno.mod +++ b/examples/gno.land/r/x/manfred_outfmt/gno.mod @@ -1,3 +1,5 @@ +// Draft + module gno.land/r/x/manfred_outfmt require ( diff --git a/gnovm/Makefile b/gnovm/Makefile index 452b2dcc81a..aa80c61ac7d 100644 --- a/gnovm/Makefile +++ b/gnovm/Makefile @@ -27,6 +27,9 @@ GNOROOT_DIR ?= $(abspath $(lastword $(MAKEFILE_LIST))/../../) # We can't use '-trimpath' yet as amino use absolute path from call stack # to find some directory: see #1236 GOBUILD_FLAGS ?= -ldflags "-X github.com/gnolang/gno/gnovm/pkg/gnoenv._GNOROOT=$(GNOROOT_DIR)" +# file where to place cover profile; used for coverage commands which are +# more complex than adding -coverprofile, like test.cmd.coverage. +GOTEST_COVER_PROFILE ?= cmd-profile.out ######################################## # Dev tools @@ -66,6 +69,24 @@ test: _test.cmd _test.pkg _test.gnolang _test.cmd: go test ./cmd/... $(GOTEST_FLAGS) +# Run tests on ./cmd/, saving the result of the coverage in +# GOTEST_COVER_PROFILE. +.PHONY: test.cmd.coverage +test.cmd.coverage: + $(eval export TXTARCOVERDIR := $(shell mktemp -d --tmpdir gnovm-make.XXXXXXX)) + go test ./cmd/... -covermode atomic -test.gocoverdir='$(TXTARCOVERDIR)' $(GOTEST_FLAGS) + @echo "coverage results:" + go tool covdata percent -i="$(TXTARCOVERDIR)" + go tool covdata textfmt -v 1 -i="$(TXTARCOVERDIR)" -o '$(GOTEST_COVER_PROFILE)' + rm -rf "$(TXTARCOVERDIR)" + +# Run test.cmd.coverage, then view the result in the HTML browser render +# and delete the original file. +.PHONY: test.cmd.coverage_view +test.cmd.coverage_view: test.cmd.coverage + go tool cover -html='$(GOTEST_COVER_PROFILE)' + rm '$(GOTEST_COVER_PROFILE)' + .PHONY: _test.pkg _test.pkg: go test ./pkg/... $(GOTEST_FLAGS) @@ -74,7 +95,7 @@ _test.pkg: _test.gnolang: _test.gnolang.native _test.gnolang.stdlibs _test.gnolang.realm _test.gnolang.pkg0 _test.gnolang.pkg1 _test.gnolang.pkg2 _test.gnolang.other _test.gnolang.other:; go test tests/*.go -run "(TestFileStr|TestSelectors)" $(GOTEST_FLAGS) _test.gnolang.realm:; go test tests/*.go -run "TestFiles/^zrealm" $(GOTEST_FLAGS) -_test.gnolang.pkg0:; go test tests/*.go -run "TestStdlibs/(bufio|crypto|encoding|errors|internal|io|math|sort|std|stdshim|strconv|strings|testing|unicode)" $(GOTEST_FLAGS) +_test.gnolang.pkg0:; go test tests/*.go -run "TestStdlibs/(bufio|crypto|encoding|errors|internal|io|math|sort|std|strconv|strings|testing|unicode)" $(GOTEST_FLAGS) _test.gnolang.pkg1:; go test tests/*.go -run "TestStdlibs/regexp" $(GOTEST_FLAGS) _test.gnolang.pkg2:; go test tests/*.go -run "TestStdlibs/bytes" $(GOTEST_FLAGS) _test.gnolang.native:; go test tests/*.go -test.short -run "TestFilesNative/" $(GOTEST_FLAGS) diff --git a/gnovm/cmd/gno/test.go b/gnovm/cmd/gno/test.go index 2e966bd32a9..5884463a552 100644 --- a/gnovm/cmd/gno/test.go +++ b/gnovm/cmd/gno/test.go @@ -20,7 +20,6 @@ import ( "github.com/gnolang/gno/gnovm/pkg/gnoenv" gno "github.com/gnolang/gno/gnovm/pkg/gnolang" "github.com/gnolang/gno/gnovm/pkg/gnomod" - "github.com/gnolang/gno/gnovm/pkg/transpiler" "github.com/gnolang/gno/gnovm/tests" "github.com/gnolang/gno/tm2/pkg/commands" "github.com/gnolang/gno/tm2/pkg/errors" @@ -259,7 +258,7 @@ func gnoTestPkg( if gnoPkgPath == "" { // unable to read pkgPath from gno.mod, generate a random realm path io.ErrPrintfln("--- WARNING: unable to read package path from gno.mod or gno root directory; try creating a gno.mod file") - gnoPkgPath = transpiler.GnoRealmPkgsPrefixBefore + random.RandStr(8) + gnoPkgPath = gno.RealmPathPrefix + random.RandStr(8) } } memPkg := gno.ReadMemPackage(pkgPath, gnoPkgPath) diff --git a/gnovm/cmd/gno/testdata/gno_transpile/05_skip_fmt_flag.txtar b/gnovm/cmd/gno/testdata/gno_transpile/05_skip_fmt_flag.txtar deleted file mode 100644 index c07c670f721..00000000000 --- a/gnovm/cmd/gno/testdata/gno_transpile/05_skip_fmt_flag.txtar +++ /dev/null @@ -1,36 +0,0 @@ -# Run gno transpile with -skip-fmt flag -# NOTE(tb): this flag doesn't actually prevent the code format, because -# `gnolang.Transpile()` calls `format.Node()`. - -gno transpile -skip-fmt . - -! stdout .+ -! stderr .+ - -cmp main.gno.gen.go main.gno.gen.go.golden -cmp sub/sub.gno.gen.go sub/sub.gno.gen.go.golden - --- main.gno -- -package main - -func main(){} - --- sub/sub.gno -- -package sub - --- main.gno.gen.go.golden -- -// Code generated by github.com/gnolang/gno. DO NOT EDIT. - -//go:build gno - -//line main.gno:1:1 -package main - -func main() {} --- sub/sub.gno.gen.go.golden -- -// Code generated by github.com/gnolang/gno. DO NOT EDIT. - -//go:build gno - -//line sub.gno:1:1 -package sub diff --git a/gnovm/cmd/gno/testdata/gno_transpile/06_build_flag.txtar b/gnovm/cmd/gno/testdata/gno_transpile/06_build_flag.txtar deleted file mode 100644 index 110d04959c0..00000000000 --- a/gnovm/cmd/gno/testdata/gno_transpile/06_build_flag.txtar +++ /dev/null @@ -1,38 +0,0 @@ -# Run gno transpile with -gobuild flag - -gno transpile -gobuild . - -! stdout .+ -! stderr .+ - -cmp main.gno.gen.go main.gno.gen.go.golden -cmp sub/sub.gno.gen.go sub/sub.gno.gen.go.golden - --- main.gno -- -package main - -func main() { - var x = 1 - _=x -} --- sub/sub.gno -- -package sub --- main.gno.gen.go.golden -- -// Code generated by github.com/gnolang/gno. DO NOT EDIT. - -//go:build gno - -//line main.gno:1:1 -package main - -func main() { - var x = 1 - _ = x -} --- sub/sub.gno.gen.go.golden -- -// Code generated by github.com/gnolang/gno. DO NOT EDIT. - -//go:build gno - -//line sub.gno:1:1 -package sub diff --git a/gnovm/cmd/gno/testdata/gno_transpile/07_build_flag_with_build_error.txtar b/gnovm/cmd/gno/testdata/gno_transpile/gobuild_flag_build_error.txtar similarity index 100% rename from gnovm/cmd/gno/testdata/gno_transpile/07_build_flag_with_build_error.txtar rename to gnovm/cmd/gno/testdata/gno_transpile/gobuild_flag_build_error.txtar diff --git a/gnovm/cmd/gno/testdata/gno_transpile/08_build_flag_with_parse_error.txtar b/gnovm/cmd/gno/testdata/gno_transpile/gobuild_flag_parse_error.txtar similarity index 100% rename from gnovm/cmd/gno/testdata/gno_transpile/08_build_flag_with_parse_error.txtar rename to gnovm/cmd/gno/testdata/gno_transpile/gobuild_flag_parse_error.txtar diff --git a/gnovm/cmd/gno/testdata/gno_transpile/09_gno_files_whitelist_error.txtar b/gnovm/cmd/gno/testdata/gno_transpile/invalid_import.txtar similarity index 60% rename from gnovm/cmd/gno/testdata/gno_transpile/09_gno_files_whitelist_error.txtar rename to gnovm/cmd/gno/testdata/gno_transpile/invalid_import.txtar index 79d4d6a4a2c..0c51012feb7 100644 --- a/gnovm/cmd/gno/testdata/gno_transpile/09_gno_files_whitelist_error.txtar +++ b/gnovm/cmd/gno/testdata/gno_transpile/invalid_import.txtar @@ -1,10 +1,10 @@ -# Run gno transpile with gno files with whitelist errors +# Run gno transpile with gno files with an invalid import path ! gno transpile . ! stdout .+ -stderr '^main.gno:5:2: import "xxx" is not in the whitelist$' -stderr '^sub/sub.gno:3:8: import "xxx" is not in the whitelist$' +stderr '^main.gno:5:2: import "xxx" does not exist$' +stderr '^sub/sub.gno:3:8: import "xxx" does not exist$' stderr '^2 transpile error\(s\)$' # no *.gen.go files are created diff --git a/gnovm/cmd/gno/testdata/gno_transpile/01_no_args.txtar b/gnovm/cmd/gno/testdata/gno_transpile/no_args.txtar similarity index 100% rename from gnovm/cmd/gno/testdata/gno_transpile/01_no_args.txtar rename to gnovm/cmd/gno/testdata/gno_transpile/no_args.txtar diff --git a/gnovm/cmd/gno/testdata/gno_transpile/03_gno_files_parse_error.txtar b/gnovm/cmd/gno/testdata/gno_transpile/parse_error.txtar similarity index 100% rename from gnovm/cmd/gno/testdata/gno_transpile/03_gno_files_parse_error.txtar rename to gnovm/cmd/gno/testdata/gno_transpile/parse_error.txtar diff --git a/gnovm/cmd/gno/testdata/gno_transpile/02_empty_dir.txtar b/gnovm/cmd/gno/testdata/gno_transpile/valid_empty_dir.txtar similarity index 100% rename from gnovm/cmd/gno/testdata/gno_transpile/02_empty_dir.txtar rename to gnovm/cmd/gno/testdata/gno_transpile/valid_empty_dir.txtar diff --git a/gnovm/cmd/gno/testdata/gno_transpile/valid_gobuild_file.txtar b/gnovm/cmd/gno/testdata/gno_transpile/valid_gobuild_file.txtar new file mode 100644 index 00000000000..40bb1ecb98a --- /dev/null +++ b/gnovm/cmd/gno/testdata/gno_transpile/valid_gobuild_file.txtar @@ -0,0 +1,31 @@ +# Run gno transpile with -gobuild flag on an individual file + +gno transpile -gobuild -v main.gno + +! stdout .+ +cmp stderr stderr.golden + +cmp main.gno.gen.go main.gno.gen.go.golden + +-- stderr.golden -- +main.gno +main.gno [build] +-- main.gno -- +package main + +func main() { + var x = 1 + _=x +} +-- main.gno.gen.go.golden -- +// Code generated by github.com/gnolang/gno. DO NOT EDIT. + +//go:build gno + +//line main.gno:1:1 +package main + +func main() { + var x = 1 + _ = x +} diff --git a/gnovm/cmd/gno/testdata/gno_transpile/valid_gobuild_flag.txtar b/gnovm/cmd/gno/testdata/gno_transpile/valid_gobuild_flag.txtar new file mode 100644 index 00000000000..2eacfb9de60 --- /dev/null +++ b/gnovm/cmd/gno/testdata/gno_transpile/valid_gobuild_flag.txtar @@ -0,0 +1,72 @@ +# Run gno transpile with -gobuild flag + +gno transpile -gobuild -v . + +! stdout .+ +cmp stderr stderr.golden + +# The test file will be excluded from transpilation unless we pass it explicitly. +cmp main.gno.gen.go main.gno.gen.go.golden +! exists .main_test.gno.gen_test.go +cmp sub/sub.gno.gen.go sub/sub.gno.gen.go.golden +rm mai.gno.gen.gosub/sub.gno.gen.go + +# Re-try, but use an absolute path. +gno transpile -gobuild -v $WORK + +! stdout .+ +cmpenv stderr stderr2.golden + +cmp main.gno.gen.go main.gno.gen.go.golden +! exists .main_test.gno.gen_test.go +cmp sub/sub.gno.gen.go sub/sub.gno.gen.go.golden + +-- stderr.golden -- +. +sub +. [build] +sub [build] +-- stderr2.golden -- +$WORK +$WORK/sub +$WORK [build] +$WORK/sub [build] +-- main.gno -- +package main + +func main() { + var x = 1 + _=x +} +-- main.gno.gen.go.golden -- +// Code generated by github.com/gnolang/gno. DO NOT EDIT. + +//go:build gno + +//line main.gno:1:1 +package main + +func main() { + var x = 1 + _ = x +} +-- main_test.gno -- +package main + +import ( + "testing" + "badimport" +) + +func TestMain(t *testing.T) { + badimport.DoesNotExist() +} +-- sub/sub.gno -- +package sub +-- sub/sub.gno.gen.go.golden -- +// Code generated by github.com/gnolang/gno. DO NOT EDIT. + +//go:build gno + +//line sub.gno:1:1 +package sub diff --git a/gnovm/cmd/gno/testdata/gno_transpile/valid_output_flag.txtar b/gnovm/cmd/gno/testdata/gno_transpile/valid_output_flag.txtar new file mode 100644 index 00000000000..b1a63890f46 --- /dev/null +++ b/gnovm/cmd/gno/testdata/gno_transpile/valid_output_flag.txtar @@ -0,0 +1,41 @@ +# Run gno transpile with valid gno files, using the -output flag. + +gno transpile -v -output directory/hello/ . +! stdout .+ +cmp stderr stderr1.golden + +exists directory/hello/main.gno.gen.go +! exists main.gno.gen.go +rm directory + +# Try running using the absolute path to the directory. +gno transpile -v -output directory/hello $WORK +! stdout .+ +cmpenv stderr stderr2.golden + +exists directory/hello$WORK/main.gno.gen.go +! exists directory/hello/main.gno.gen.go +rm directory + +# Try running in subdirectory, using a "relative non-local path." (ie. has "../") +mkdir subdir +cd subdir +gno transpile -v -output hello .. +! stdout .+ +cmpenv stderr ../stderr3.golden + +exists hello$WORK/main.gno.gen.go +! exists main.gno.gen.go + +-- stderr1.golden -- +. +-- stderr2.golden -- +$WORK +-- stderr3.golden -- +.. +-- main.gno -- +package main + +func main() { + println("hello") +} diff --git a/gnovm/cmd/gno/testdata/gno_transpile/valid_output_gobuild.txtar b/gnovm/cmd/gno/testdata/gno_transpile/valid_output_gobuild.txtar new file mode 100644 index 00000000000..3540e865f3e --- /dev/null +++ b/gnovm/cmd/gno/testdata/gno_transpile/valid_output_gobuild.txtar @@ -0,0 +1,44 @@ +# Run gno transpile with valid gno files, using the -output and -gobuild flags together. + +gno transpile -v -output directory/hello/ -gobuild . +! stdout .+ +cmp stderr stderr1.golden + +exists directory/hello/main.gno.gen.go +! exists main.gno.gen.go +rm directory + +# Try running using the absolute path to the directory. +gno transpile -v -output directory/hello -gobuild $WORK +! stdout .+ +cmpenv stderr stderr2.golden + +exists directory/hello$WORK/main.gno.gen.go +! exists directory/hello/main.gno.gen.go +rm directory + +# Try running in subdirectory, using a "relative non-local path." (ie. has "../") +mkdir subdir +cd subdir +gno transpile -v -output hello -gobuild .. +! stdout .+ +cmpenv stderr ../stderr3.golden + +exists hello$WORK/main.gno.gen.go +! exists main.gno.gen.go + +-- stderr1.golden -- +. +directory/hello [build] +-- stderr2.golden -- +$WORK +directory/hello$WORK [build] +-- stderr3.golden -- +.. +hello$WORK [build] +-- main.gno -- +package main + +func main() { + println("hello") +} diff --git a/gnovm/cmd/gno/testdata/gno_transpile/valid_transpile_file.txtar b/gnovm/cmd/gno/testdata/gno_transpile/valid_transpile_file.txtar new file mode 100644 index 00000000000..86cc6f12f7a --- /dev/null +++ b/gnovm/cmd/gno/testdata/gno_transpile/valid_transpile_file.txtar @@ -0,0 +1,65 @@ +# Run gno transpile with an individual file. + +# Running transpile on the current directory should only precompile +# main.gno. +gno transpile -v . + +! stdout .+ +stderr ^\.$ + +exists main.gno.gen.go +! exists .hello_test.gno.gen_test.go +rm main.gno.gen.go + +# Running it using individual filenames should precompile hello_test.gno, as well. +gno transpile -v main.gno hello_test.gno + +! stdout .+ +cmp stderr transpile-files-stderr.golden + +cmp main.gno.gen.go main.gno.gen.go.golden +cmp .hello_test.gno.gen_test.go .hello_test.gno.gen_test.go.golden + +-- transpile-files-stderr.golden -- +main.gno +hello_test.gno +-- main.gno -- +package main + +func main() { + println("hello") +} + +-- hello_test.gno -- +package main + +import "std" + +func hello() { + std.AssertOriginCall() +} + +-- main.gno.gen.go.golden -- +// Code generated by github.com/gnolang/gno. DO NOT EDIT. + +//go:build gno + +//line main.gno:1:1 +package main + +func main() { + println("hello") +} +-- .hello_test.gno.gen_test.go.golden -- +// Code generated by github.com/gnolang/gno. DO NOT EDIT. + +//go:build gno && test + +//line hello_test.gno:1:1 +package main + +import "github.com/gnolang/gno/gnovm/stdlibs/std" + +func hello() { + std.AssertOriginCall(nil) +} diff --git a/gnovm/cmd/gno/testdata/gno_transpile/04_valid_gno_files.txtar b/gnovm/cmd/gno/testdata/gno_transpile/valid_transpile_package.txtar similarity index 100% rename from gnovm/cmd/gno/testdata/gno_transpile/04_valid_gno_files.txtar rename to gnovm/cmd/gno/testdata/gno_transpile/valid_transpile_package.txtar diff --git a/gnovm/cmd/gno/testdata/gno_transpile/valid_transpile_tree.txtar b/gnovm/cmd/gno/testdata/gno_transpile/valid_transpile_tree.txtar new file mode 100644 index 00000000000..a765ab5093b --- /dev/null +++ b/gnovm/cmd/gno/testdata/gno_transpile/valid_transpile_tree.txtar @@ -0,0 +1,94 @@ +# Run gno transpile with dependencies +env GNOROOT=$WORK + +gno transpile -v ./examples + +! stdout .+ +cmpenv stderr stderr.golden + +! exists examples/gno.land/r/question/question.gno.gen.go +cmp examples/gno.land/r/answer/answer.gno.gen.go examples/gno.land/r/answer/answer.gno.gen.go.golden +cmp examples/gno.land/r/answer/anti_answer.gno.gen.go examples/gno.land/r/answer/anti_answer.gno.gen.go.golden +cmp gnovm/stdlibs/math/math.gno.gen.go gnovm/stdlibs/math/math.gno.gen.go.golden + +-- stderr.golden -- +examples/gno.land/r/answer +$WORK/gnovm/stdlibs/math +examples/gno.land/r/question (skipped, gno.mod marks module as draft) +-- examples/gno.land/r/question/gno.mod -- +// Draft + +module gno.land/r/question + +-- examples/gno.land/r/question/question.gno -- +package question + +func Question() string { + return "What is the answer to Life, The Universe and Everything?" + invalid syntax +} + +-- examples/gno.land/r/answer/answer.gno -- +package answer + +import "math" + +func Answer() int { + return math.Sqrt(1764) +} + +-- examples/gno.land/r/answer/answer.gno.gen.go.golden -- +// Code generated by github.com/gnolang/gno. DO NOT EDIT. + +//go:build gno + +//line answer.gno:1:1 +package answer + +import "github.com/gnolang/gno/gnovm/stdlibs/math" + +func Answer() int { + return math.Sqrt(1764) +} +-- examples/gno.land/r/answer/anti_answer.gno -- +package answer + +import "math" + +func AntiAnswer() int { + return -math.Sqrt(1764) +} +-- examples/gno.land/r/answer/anti_answer.gno.gen.go.golden -- +// Code generated by github.com/gnolang/gno. DO NOT EDIT. + +//go:build gno + +//line anti_answer.gno:1:1 +package answer + +import "github.com/gnolang/gno/gnovm/stdlibs/math" + +func AntiAnswer() int { + return -math.Sqrt(1764) +} +-- examples/gno.land/r/answer/gno.mod -- +module gno.land/r/answer + +-- gnovm/stdlibs/math/math.gno -- +package math + +func Sqrt(i int) int { + return 42 +} + +-- gnovm/stdlibs/math/math.gno.gen.go.golden -- +// Code generated by github.com/gnolang/gno. DO NOT EDIT. + +//go:build gno + +//line math.gno:1:1 +package math + +func Sqrt(i int) int { + return 42 +} diff --git a/gnovm/cmd/gno/transpile.go b/gnovm/cmd/gno/transpile.go index 84744451f9d..2e12ee6f4b3 100644 --- a/gnovm/cmd/gno/transpile.go +++ b/gnovm/cmd/gno/transpile.go @@ -5,48 +5,61 @@ import ( "errors" "flag" "fmt" + "go/ast" "go/scanner" - "log" + "go/token" "os" + "os/exec" "path/filepath" + "regexp" + "slices" + "strconv" + "strings" + "github.com/gnolang/gno/gnovm/pkg/gnoenv" + "github.com/gnolang/gno/gnovm/pkg/gnomod" "github.com/gnolang/gno/gnovm/pkg/transpiler" "github.com/gnolang/gno/tm2/pkg/commands" ) -type importPath string - type transpileCfg struct { verbose bool - skipFmt bool + rootDir string skipImports bool gobuild bool goBinary string - gofmtBinary string output string } type transpileOptions struct { cfg *transpileCfg + // CLI output + io commands.IO // transpiled is the set of packages already // transpiled from .gno to .go. - transpiled map[importPath]struct{} + transpiled map[string]struct{} + // skipped packages (gno mod marks them as draft) + skipped []string } -func newTranspileOptions(cfg *transpileCfg) *transpileOptions { - return &transpileOptions{cfg, map[importPath]struct{}{}} +func newTranspileOptions(cfg *transpileCfg, io commands.IO) *transpileOptions { + return &transpileOptions{ + cfg: cfg, + io: io, + transpiled: map[string]struct{}{}, + } } func (p *transpileOptions) getFlags() *transpileCfg { return p.cfg } -func (p *transpileOptions) isTranspiled(pkg importPath) bool { +func (p *transpileOptions) isTranspiled(pkg string) bool { _, transpiled := p.transpiled[pkg] return transpiled } -func (p *transpileOptions) markAsTranspiled(pkg importPath) { +func (p *transpileOptions) markAsTranspiled(pkg string) { p.transpiled[pkg] = struct{}{} } @@ -74,11 +87,11 @@ func (c *transpileCfg) RegisterFlags(fs *flag.FlagSet) { "verbose output when running", ) - fs.BoolVar( - &c.skipFmt, - "skip-fmt", - false, - "do not check syntax of generated .go files", + fs.StringVar( + &c.rootDir, + "root-dir", + "", + "clone location of github.com/gnolang/gno (gno tries to guess it)", ) fs.BoolVar( @@ -102,13 +115,6 @@ func (c *transpileCfg) RegisterFlags(fs *flag.FlagSet) { "go binary to use for building", ) - fs.StringVar( - &c.gofmtBinary, - "go-fmt-binary", - "gofmt", - "gofmt binary to use for syntax checking", - ) - fs.StringVar( &c.output, "output", @@ -122,33 +128,54 @@ func execTranspile(cfg *transpileCfg, args []string, io commands.IO) error { return flag.ErrHelp } - // transpile .gno files. - paths, err := gnoFilesFromArgs(args) + // guess cfg.RootDir + if cfg.rootDir == "" { + cfg.rootDir = gnoenv.RootDir() + } + + // transpile .gno packages and files. + paths, err := gnoPackagesFromArgs(args) if err != nil { return fmt.Errorf("list paths: %w", err) } - opts := newTranspileOptions(cfg) + opts := newTranspileOptions(cfg, io) var errlist scanner.ErrorList - for _, filepath := range paths { - if err := transpileFile(filepath, opts); err != nil { + for _, path := range paths { + st, err := os.Stat(path) + if err != nil { + return err + } + if st.IsDir() { + err = transpilePkg(path, opts) + } else { + if opts.cfg.verbose { + io.ErrPrintln(filepath.Clean(path)) + } + + err = transpileFile(path, opts) + } + if err != nil { var fileErrlist scanner.ErrorList if !errors.As(err, &fileErrlist) { // Not an scanner.ErrorList: return immediately. - return fmt.Errorf("%s: transpile: %w", filepath, err) + return fmt.Errorf("%s: transpile: %w", path, err) } errlist = append(errlist, fileErrlist...) } } if errlist.Len() == 0 && cfg.gobuild { - paths, err := gnoPackagesFromArgs(args) - if err != nil { - return fmt.Errorf("list packages: %w", err) - } - for _, pkgPath := range paths { - err := goBuildFileOrPkg(pkgPath, cfg) + if slices.Contains(opts.skipped, pkgPath) { + continue + } + if cfg.output != "." { + if pkgPath, err = ResolvePath(cfg.output, pkgPath); err != nil { + return fmt.Errorf("resolve output path: %w", err) + } + } + err := goBuildFileOrPkg(io, pkgPath, cfg) if err != nil { var fileErrlist scanner.ErrorList if !errors.As(err, &fileErrlist) { @@ -169,19 +196,41 @@ func execTranspile(cfg *transpileCfg, args []string, io commands.IO) error { return nil } -func transpilePkg(pkgPath importPath, opts *transpileOptions) error { - if opts.isTranspiled(pkgPath) { +// transpilePkg transpiles all non-test files at the given location. +// Additionally, it checks the gno.mod in said location, and skips it if it is +// a draft module +func transpilePkg(dirPath string, opts *transpileOptions) error { + if opts.isTranspiled(dirPath) { + return nil + } + opts.markAsTranspiled(dirPath) + + gmod, err := gnomod.ParseAt(dirPath) + if err != nil && !errors.Is(err, gnomod.ErrGnoModNotFound) { + return err + } + if err == nil && gmod.Draft { + if opts.cfg.verbose { + opts.io.ErrPrintfln("%s (skipped, gno.mod marks module as draft)", filepath.Clean(dirPath)) + } + opts.skipped = append(opts.skipped, dirPath) return nil } - opts.markAsTranspiled(pkgPath) - files, err := filepath.Glob(filepath.Join(string(pkgPath), "*.gno")) + // XXX(morgan): Currently avoiding test files as they contain imports like "fmt". + // The transpiler doesn't currently support "test stdlibs", and even if it + // did all packages like "fmt" would have to exist as standard libraries to work. + // Easier to skip for now. + files, err := listNonTestFiles(dirPath) if err != nil { - log.Fatal(err) + return err } + if opts.cfg.verbose { + opts.io.ErrPrintln(filepath.Clean(dirPath)) + } for _, file := range files { - if err = transpileFile(file, opts); err != nil { + if err := transpileFile(file, opts); err != nil { return fmt.Errorf("%s: %w", file, err) } } @@ -191,14 +240,6 @@ func transpilePkg(pkgPath importPath, opts *transpileOptions) error { func transpileFile(srcPath string, opts *transpileOptions) error { flags := opts.getFlags() - gofmt := flags.gofmtBinary - if gofmt == "" { - gofmt = "gofmt" - } - - if flags.verbose { - fmt.Fprintf(os.Stderr, "%s\n", srcPath) - } // parse .gno. source, err := os.ReadFile(srcPath) @@ -207,7 +248,7 @@ func transpileFile(srcPath string, opts *transpileOptions) error { } // compute attributes based on filename. - targetFilename, tags := transpiler.GetTranspileFilenameAndTags(srcPath) + targetFilename, tags := transpiler.TranspiledFilenameAndTags(srcPath) // preprocess. transpileRes, err := transpiler.Transpile(string(source), tags, srcPath) @@ -218,7 +259,7 @@ func transpileFile(srcPath string, opts *transpileOptions) error { // resolve target path var targetPath string if flags.output != "." { - path, err := ResolvePath(flags.output, importPath(filepath.Dir(srcPath))) + path, err := ResolvePath(flags.output, filepath.Dir(srcPath)) if err != nil { return fmt.Errorf("resolve output path: %w", err) } @@ -233,18 +274,14 @@ func transpileFile(srcPath string, opts *transpileOptions) error { return fmt.Errorf("write .go file: %w", err) } - // check .go fmt, if `SkipFmt` sets to false. - if !flags.skipFmt { - err = transpiler.TranspileVerifyFile(targetPath, gofmt) + // transpile imported packages, if `SkipImports` sets to false + if !flags.skipImports && + !strings.HasSuffix(srcPath, "_filetest.gno") && !strings.HasSuffix(srcPath, "_test.gno") { + dirPaths, err := getPathsFromImportSpec(opts.cfg.rootDir, transpileRes.Imports) if err != nil { - return fmt.Errorf("check .go file: %w", err) + return err } - } - - // transpile imported packages, if `SkipImports` sets to false - if !flags.skipImports { - importPaths := getPathsFromImportSpec(transpileRes.Imports) - for _, path := range importPaths { + for _, path := range dirPaths { if err := transpilePkg(path, opts); err != nil { return err } @@ -254,13 +291,122 @@ func transpileFile(srcPath string, opts *transpileOptions) error { return nil } -func goBuildFileOrPkg(fileOrPkg string, cfg *transpileCfg) error { +func goBuildFileOrPkg(io commands.IO, fileOrPkg string, cfg *transpileCfg) error { verbose := cfg.verbose goBinary := cfg.goBinary if verbose { - fmt.Fprintf(os.Stderr, "%s\n", fileOrPkg) + io.ErrPrintfln("%s [build]", filepath.Clean(fileOrPkg)) + } + + return buildTranspiledPackage(fileOrPkg, goBinary) +} + +// getPathsFromImportSpec returns the directory paths where the code for each +// importSpec is stored (assuming they start with [transpiler.ImportPrefix]). +func getPathsFromImportSpec(rootDir string, importSpec []*ast.ImportSpec) (dirs []string, err error) { + for _, i := range importSpec { + path, err := strconv.Unquote(i.Path.Value) + if err != nil { + return nil, err + } + if strings.HasPrefix(path, transpiler.ImportPrefix) { + res := strings.TrimPrefix(path, transpiler.ImportPrefix) + + dirs = append(dirs, rootDir+filepath.FromSlash(res)) + } + } + return +} + +// buildTranspiledPackage tries to run `go build` against the transpiled .go files. +// +// This method is the most efficient to detect errors but requires that +// all the import are valid and available. +func buildTranspiledPackage(fileOrPkg, goBinary string) error { + // TODO: use cmd/compile instead of exec? + // TODO: find the nearest go.mod file, chdir in the same folder, trim prefix? + // TODO: temporarily create an in-memory go.mod or disable go modules for gno? + // TODO: ignore .go files that were not generated from gno? + + info, err := os.Stat(fileOrPkg) + if err != nil { + return fmt.Errorf("invalid file or package path %s: %w", fileOrPkg, err) + } + var ( + target string + chdir string + ) + if !info.IsDir() { + dstFilename, _ := transpiler.TranspiledFilenameAndTags(fileOrPkg) + // Makes clear to go compiler that this is a relative path, + // rather than a path to a package/module. + // can't use filepath.Join as it cleans its results. + target = filepath.Dir(fileOrPkg) + string(filepath.Separator) + dstFilename + } else { + // Go does not allow building packages using absolute paths, and requires + // relative paths to always be prefixed with `./` (because the argument + // go expects are import paths, not directories). + // To circumvent this, we use the -C flag to chdir into the right + // directory, then run `go build .` + chdir = fileOrPkg + target = "." + } + + // pre-alloc max 5 args + args := append(make([]string, 0, 5), "build") + if chdir != "" { + args = append(args, "-C", chdir) + } + args = append(args, "-tags=gno", target) + cmd := exec.Command(goBinary, args...) + out, err := cmd.CombinedOutput() + if errors.As(err, new(*exec.ExitError)) { + // there was a non-zero exit code; parse the go build errors + return parseGoBuildErrors(string(out)) + } + // other kinds of errors; return + return err +} + +var ( + reGoBuildError = regexp.MustCompile(`(?m)^(\S+):(\d+):(\d+): (.+)$`) + reGoBuildComment = regexp.MustCompile(`(?m)^#.*$`) +) + +// parseGoBuildErrors returns a scanner.ErrorList filled with all errors found +// in out, which is supposed to be the output of the `go build` command. +// +// TODO(tb): update when `go build -json` is released to replace regexp usage. +// See https://github.com/golang/go/issues/62067 +func parseGoBuildErrors(out string) error { + var errList scanner.ErrorList + matches := reGoBuildError.FindAllStringSubmatch(out, -1) + for _, match := range matches { + filename := match[1] + line, err := strconv.Atoi(match[2]) + if err != nil { + return fmt.Errorf("parse line go build error %s: %w", match, err) + } + + column, err := strconv.Atoi(match[3]) + if err != nil { + return fmt.Errorf("parse column go build error %s: %w", match, err) + } + msg := match[4] + errList.Add(token.Position{ + Filename: filename, + Line: line, + Column: column, + }, msg) + } + + replaced := reGoBuildError.ReplaceAllLiteralString(out, "") + replaced = reGoBuildComment.ReplaceAllString(replaced, "") + replaced = strings.TrimSpace(replaced) + if replaced != "" { + errList.Add(token.Position{}, "Additional go build errors:\n"+replaced) } - return transpiler.TranspileBuildPackage(fileOrPkg, goBinary) + return errList.Err() } diff --git a/gnovm/cmd/gno/transpile_test.go b/gnovm/cmd/gno/transpile_test.go index 2770026a01a..827c09e23f1 100644 --- a/gnovm/cmd/gno/transpile_test.go +++ b/gnovm/cmd/gno/transpile_test.go @@ -1,9 +1,13 @@ package main import ( + "go/scanner" + "go/token" + "strconv" "testing" "github.com/rogpeppe/go-internal/testscript" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/gnolang/gno/gnovm/pkg/integration" @@ -24,3 +28,79 @@ func Test_ScriptsTranspile(t *testing.T) { testscript.Run(t, p) } + +func Test_parseGoBuildErrors(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + output string + expectedError error + expectedErrorIs error + }{ + { + name: "empty output", + output: "", + expectedError: nil, + }, + { + name: "random output", + output: "xxx", + expectedError: scanner.ErrorList{ + &scanner.Error{ + Msg: "Additional go build errors:\nxxx", + }, + }, + }, + { + name: "some errors", + output: `xxx +main.gno:6:2: nasty error +pkg/file.gno:60:20: ugly error`, + expectedError: scanner.ErrorList{ + &scanner.Error{ + Pos: token.Position{ + Filename: "main.gno", + Line: 6, + Column: 2, + }, + Msg: "nasty error", + }, + &scanner.Error{ + Pos: token.Position{ + Filename: "pkg/file.gno", + Line: 60, + Column: 20, + }, + Msg: "ugly error", + }, + &scanner.Error{ + Msg: "Additional go build errors:\nxxx", + }, + }, + }, + { + name: "line parse error", + output: `main.gno:9000000000000000000000000000000000000000000000000000:11: error`, + expectedErrorIs: strconv.ErrRange, + }, + { + name: "column parse error", + output: `main.gno:1:9000000000000000000000000000000000000000000000000000: error`, + expectedErrorIs: strconv.ErrRange, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + err := parseGoBuildErrors(tt.output) + if eis := tt.expectedErrorIs; eis != nil { + assert.ErrorIs(t, err, eis) + } else { + assert.Equal(t, tt.expectedError, err) + } + }) + } +} diff --git a/gnovm/cmd/gno/util.go b/gnovm/cmd/gno/util.go index d9ec775dfca..480161c2b7e 100644 --- a/gnovm/cmd/gno/util.go +++ b/gnovm/cmd/gno/util.go @@ -2,7 +2,6 @@ package main import ( "fmt" - "go/ast" "io" "io/fs" "os" @@ -10,9 +9,6 @@ import ( "regexp" "strings" "time" - - "github.com/gnolang/gno/gnovm/pkg/gnoenv" - "github.com/gnolang/gno/gnovm/pkg/transpiler" ) func isGnoFile(f fs.DirEntry) bool { @@ -25,84 +21,69 @@ func isFileExist(path string) bool { return err == nil } -func gnoFilesFromArgs(args []string) ([]string, error) { - paths := []string{} - for _, arg := range args { - info, err := os.Stat(arg) +func gnoPackagesFromArgs(args []string) ([]string, error) { + var paths []string + + for _, argPath := range args { + info, err := os.Stat(argPath) if err != nil { return nil, fmt.Errorf("invalid file or package path: %w", err) } + if !info.IsDir() { - curpath := arg - paths = append(paths, curpath) - } else { - err = filepath.WalkDir(arg, func(curpath string, f fs.DirEntry, err error) error { - if err != nil { - return fmt.Errorf("%s: walk dir: %w", arg, err) - } - - if !isGnoFile(f) { - return nil // skip - } - paths = append(paths, curpath) - return nil - }) - if err != nil { - return nil, err - } + paths = append(paths, ensurePathPrefix(argPath)) + + continue + } + + // Gather package paths from the directory + err = walkDirForGnoFiles(argPath, func(path string) { + paths = append(paths, ensurePathPrefix(path)) + }) + if err != nil { + return nil, fmt.Errorf("unable to walk dir: %w", err) } } + return paths, nil } -func gnoPackagesFromArgs(args []string) ([]string, error) { - paths := []string{} - for _, arg := range args { - info, err := os.Stat(arg) +func ensurePathPrefix(path string) string { + if filepath.IsAbs(path) { + return path + } + + // cannot use path.Join or filepath.Join, because we need + // to ensure that ./ is the prefix to pass to go build. + // if not absolute. + return "." + string(filepath.Separator) + path +} + +func walkDirForGnoFiles(root string, addPath func(path string)) error { + visited := make(map[string]struct{}) + + walkFn := func(currPath string, f fs.DirEntry, err error) error { if err != nil { - return nil, fmt.Errorf("invalid file or package path: %w", err) + return fmt.Errorf("%s: walk dir: %w", root, err) } - if !info.IsDir() { - paths = append(paths, arg) - } else { - // if the passed arg is a dir, then we'll recursively walk the dir - // and look for directories containing at least one .gno file. - - visited := map[string]bool{} // used to run the builder only once per folder. - err = filepath.WalkDir(arg, func(curpath string, f fs.DirEntry, err error) error { - if err != nil { - return fmt.Errorf("%s: walk dir: %w", arg, err) - } - if f.IsDir() { - return nil // skip - } - if !isGnoFile(f) { - return nil // skip - } - - parentDir := filepath.Dir(curpath) - if _, found := visited[parentDir]; found { - return nil - } - visited[parentDir] = true - - pkg := parentDir - if !filepath.IsAbs(parentDir) { - // cannot use path.Join or filepath.Join, because we need - // to ensure that ./ is the prefix to pass to go build. - // if not absolute. - pkg = "./" + parentDir - } - - paths = append(paths, pkg) - return nil - }) - if err != nil { - return nil, err - } + + if f.IsDir() || !isGnoFile(f) { + return nil } + + parentDir := filepath.Dir(currPath) + if _, found := visited[parentDir]; found { + return nil + } + + visited[parentDir] = struct{}{} + + addPath(parentDir) + + return nil } - return paths, nil + + return filepath.WalkDir(root, walkFn) } // targetsFromPatterns returns a list of target paths that match the patterns. @@ -192,36 +173,27 @@ func fmtDuration(d time.Duration) string { return fmt.Sprintf("%.2fs", d.Seconds()) } -// getPathsFromImportSpec derive and returns ImportPaths -// without ImportPrefix from *ast.ImportSpec -func getPathsFromImportSpec(importSpec []*ast.ImportSpec) (importPaths []importPath) { - for _, i := range importSpec { - path := i.Path.Value[1 : len(i.Path.Value)-1] // trim leading and trailing `"` - if strings.HasPrefix(path, transpiler.ImportPrefix) { - res := strings.TrimPrefix(path, transpiler.ImportPrefix) - importPaths = append(importPaths, importPath("."+res)) - } +// ResolvePath determines the path where to place output files. +// output is the output directory provided by the user. +// dstPath is the desired output path by the gno program. +// +// If dstPath is relative non-local path (ie. contains ../), the dstPath will +// be made absolute and joined with output. +// +// Otherwise, the result is simply filepath.Join(output, dstPath). +// +// See related test for examples. +func ResolvePath(output, dstPath string) (string, error) { + if filepath.IsAbs(dstPath) || + filepath.IsLocal(dstPath) { + return filepath.Join(output, dstPath), nil } - return -} - -// ResolvePath joins the output dir with relative pkg path -// e.g -// Output Dir: Temp/gno-transpile -// Pkg Path: ../example/gno.land/p/pkg -// Returns -> Temp/gno-transpile/example/gno.land/p/pkg -func ResolvePath(output string, path importPath) (string, error) { - absOutput, err := filepath.Abs(output) + // Make dstPath absolute and join it with output. + absDst, err := filepath.Abs(dstPath) if err != nil { return "", err } - absPkgPath, err := filepath.Abs(string(path)) - if err != nil { - return "", err - } - pkgPath := strings.TrimPrefix(absPkgPath, gnoenv.RootDir()) - - return filepath.Join(absOutput, pkgPath), nil + return filepath.Join(output, absDst), nil } // WriteDirFile write file to the path and also create diff --git a/gnovm/cmd/gno/util_test.go b/gnovm/cmd/gno/util_test.go index 9e9659bfe4f..a92c924e272 100644 --- a/gnovm/cmd/gno/util_test.go +++ b/gnovm/cmd/gno/util_test.go @@ -295,3 +295,50 @@ func createGnoPackages(t *testing.T, tmpDir string) { } } } + +func TestResolvePath(t *testing.T) { + t.Parallel() + + if os.PathSeparator != '/' { + t.Skip("ResolvePath test is only written of UNIX-like filesystems") + } + wd, err := os.Getwd() + require.NoError(t, err) + tt := []struct { + output string + dstPath string + result string + }{ + { + "transpile-result", + "./examples/test/test1.gno.gen.go", + "transpile-result/examples/test/test1.gno.gen.go", + }, + { + "/transpile-result", + "./examples/test/test1.gno.gen.go", + "/transpile-result/examples/test/test1.gno.gen.go", + }, + { + "/transpile-result", + "/home/gno/examples/test/test1.gno.gen.go", + "/transpile-result/home/gno/examples/test/test1.gno.gen.go", + }, + { + "result", + "../hello", + filepath.Join("result", filepath.Join(wd, "../hello")), + }, + } + + for _, tc := range tt { + res, err := ResolvePath(tc.output, tc.dstPath) + // ResolvePath should error only in case we can't get the abs path; + // so never in normal conditions. + require.NoError(t, err) + assert.Equal(t, + tc.result, res, + "unexpected result of ResolvePath(%q, %q)", tc.output, tc.dstPath, + ) + } +} diff --git a/gnovm/pkg/gnolang/helpers.go b/gnovm/pkg/gnolang/helpers.go index 564ac0622c2..c6f7e696ea4 100644 --- a/gnovm/pkg/gnolang/helpers.go +++ b/gnovm/pkg/gnolang/helpers.go @@ -7,6 +7,34 @@ import ( "strings" ) +// ---------------------------------------- +// Functions centralizing definitions + +// RealmPathPrefix is the prefix used to identify pkgpaths which are meant to +// be realms and as such to have their state persisted. This is used by [IsRealmPath]. +const RealmPathPrefix = "gno.land/r/" + +// ReGnoRunPath is the path used for realms executed in maketx run. +// These are not considered realms, as an exception to the RealmPathPrefix rule. +var ReGnoRunPath = regexp.MustCompile(`^gno\.land/r/g[a-z0-9]+/run$`) + +// IsRealmPath determines whether the given pkgpath is for a realm, and as such +// should persist the global state. +func IsRealmPath(pkgPath string) bool { + return strings.HasPrefix(pkgPath, RealmPathPrefix) && + // MsgRun pkgPath aren't realms + !ReGnoRunPath.MatchString(pkgPath) +} + +// IsStdlib determines whether s is a pkgpath for a standard library. +func IsStdlib(s string) bool { + // NOTE(morgan): this is likely to change in the future as we add support for + // IBC/ICS and we allow import paths to other chains. It might be good to + // (eventually) follow the same rule as Go, which is: does the first + // element of the import path contain a dot? + return !strings.HasPrefix(s, "gno.land/") +} + // ---------------------------------------- // AST Construction (Expr) // These are copied over from go-amino-x, but produces Gno ASTs. diff --git a/gnovm/pkg/gnolang/helpers_test.go b/gnovm/pkg/gnolang/helpers_test.go new file mode 100644 index 00000000000..af8fa64ac79 --- /dev/null +++ b/gnovm/pkg/gnolang/helpers_test.go @@ -0,0 +1,55 @@ +package gnolang + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsRealmPath(t *testing.T) { + t.Parallel() + tt := []struct { + input string + result bool + }{ + {"gno.land/r/demo/users", true}, + {"gno.land/r/hello", true}, + {"gno.land/p/demo/users", false}, + {"gno.land/p/hello", false}, + {"gno.land/x", false}, + {"std", false}, + } + + for _, tc := range tt { + assert.Equal( + t, + tc.result, + IsRealmPath(tc.input), + "unexpected IsRealmPath(%q) result", tc.input, + ) + } +} + +func TestIsStdlib(t *testing.T) { + t.Parallel() + + tt := []struct { + s string + result bool + }{ + {"std", true}, + {"math", true}, + {"very/long/path/with_underscores", true}, + {"gno.land/r/demo/users", false}, + {"gno.land/hello", false}, + } + + for _, tc := range tt { + assert.Equal( + t, + tc.result, + IsStdlib(tc.s), + "IsStdlib(%q)", tc.s, + ) + } +} diff --git a/gnovm/pkg/gnolang/nodes.go b/gnovm/pkg/gnolang/nodes.go index 482f4850b6e..2897fdd5306 100644 --- a/gnovm/pkg/gnolang/nodes.go +++ b/gnovm/pkg/gnolang/nodes.go @@ -1115,11 +1115,21 @@ func ReadMemPackage(dir string, pkgPath string) *std.MemPackage { allowedFileExtensions := []string{ ".gno", } + // exceptions to allowedFileExtensions + var rejectedFileExtensions []string + + if IsStdlib(pkgPath) { + // Allows transpilation to work on stdlibs with native fns. + allowedFileExtensions = append(allowedFileExtensions, ".go") + rejectedFileExtensions = []string{".gen.go"} + } + list := make([]string, 0, len(files)) for _, file := range files { if file.IsDir() || strings.HasPrefix(file.Name(), ".") || - (!endsWith(file.Name(), allowedFileExtensions) && !contains(allowedFiles, file.Name())) { + (!endsWith(file.Name(), allowedFileExtensions) && !contains(allowedFiles, file.Name())) || + endsWith(file.Name(), rejectedFileExtensions) { continue } list = append(list, filepath.Join(dir, file.Name())) diff --git a/gnovm/pkg/gnolang/realm.go b/gnovm/pkg/gnolang/realm.go index 85f94d4fcbe..0036f9a54bf 100644 --- a/gnovm/pkg/gnolang/realm.go +++ b/gnovm/pkg/gnolang/realm.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "reflect" - "regexp" "strings" ) @@ -1514,20 +1513,6 @@ func isUnsaved(oo Object) bool { return oo.GetIsNewReal() || oo.GetIsDirty() } -// realmPathPrefix is the prefix used to identify pkgpaths which are meant to -// be realms and as such to have their state persisted. This is used by [IsRealmPath]. -const realmPathPrefix = "gno.land/r/" - -var ReGnoRunPath = regexp.MustCompile(`^gno\.land/r/g[a-z0-9]+/run$`) - -// IsRealmPath determines whether the given pkgpath is for a realm, and as such -// should persist the global state. -func IsRealmPath(pkgPath string) bool { - return strings.HasPrefix(pkgPath, realmPathPrefix) && - // MsgRun pkgPath aren't realms - !ReGnoRunPath.MatchString(pkgPath) -} - func prettyJSON(jstr []byte) []byte { var c interface{} err := json.Unmarshal(jstr, &c) diff --git a/gnovm/pkg/gnomod/file.go b/gnovm/pkg/gnomod/file.go index fda9263914e..b6ee95acac8 100644 --- a/gnovm/pkg/gnomod/file.go +++ b/gnovm/pkg/gnomod/file.go @@ -17,7 +17,7 @@ import ( "path/filepath" "strings" - "github.com/gnolang/gno/gnovm/pkg/transpiler" + gno "github.com/gnolang/gno/gnovm/pkg/gnolang" "golang.org/x/mod/modfile" "golang.org/x/mod/module" ) @@ -183,13 +183,8 @@ func (f *File) FetchDeps(path string, remote string, verbose bool) error { if strings.HasSuffix(path, modFile.Module.Mod.Path) { continue } - // skip if `std`, special case. - if path == transpiler.GnoStdPkgAfter { - continue - } - if strings.HasPrefix(path, transpiler.ImportPrefix) { - path = strings.TrimPrefix(path, transpiler.ImportPrefix+"/examples/") + if !gno.IsStdlib(path) { modFile.AddNewRequire(path, "v0.0.0-latest", true) } } diff --git a/gnovm/pkg/gnomod/gnomod.go b/gnovm/pkg/gnomod/gnomod.go index 014873b8faa..0effa532107 100644 --- a/gnovm/pkg/gnomod/gnomod.go +++ b/gnovm/pkg/gnomod/gnomod.go @@ -3,6 +3,8 @@ package gnomod import ( "errors" "fmt" + "go/parser" + gotoken "go/token" "os" "path/filepath" "strings" @@ -63,23 +65,12 @@ func writePackage(remote, basePath, pkgPath string) (requirements []string, err } else { // Is File // Transpile and write generated go file - if strings.HasSuffix(fileName, ".gno") { - filePath := filepath.Join(basePath, pkgPath) - targetFilename, _ := transpiler.GetTranspileFilenameAndTags(filePath) - transpileRes, err := transpiler.Transpile(string(res.Data), "", fileName) - if err != nil { - return nil, fmt.Errorf("transpile: %w", err) - } - - for _, i := range transpileRes.Imports { - requirements = append(requirements, i.Path.Value) - } - - targetFileNameWithPath := filepath.Join(basePath, dirPath, targetFilename) - err = os.WriteFile(targetFileNameWithPath, []byte(transpileRes.Translated), 0o644) - if err != nil { - return nil, fmt.Errorf("writefile %q: %w", targetFileNameWithPath, err) - } + file, err := parser.ParseFile(gotoken.NewFileSet(), fileName, res.Data, parser.ImportsOnly) + if err != nil { + return nil, fmt.Errorf("parse gno file: %w", err) + } + for _, i := range file.Imports { + requirements = append(requirements, i.Path.Value) } // Write file @@ -96,11 +87,12 @@ func writePackage(remote, basePath, pkgPath string) (requirements []string, err // GnoToGoMod make necessary modifications in the gno.mod // and return go.mod file. func GnoToGoMod(f File) (*File, error) { + // TODO(morgan): good candidate to move to pkg/transpiler. + gnoModPath := GetGnoModPath() - if strings.HasPrefix(f.Module.Mod.Path, transpiler.GnoRealmPkgsPrefixBefore) || - strings.HasPrefix(f.Module.Mod.Path, transpiler.GnoPurePkgsPrefixBefore) { - f.AddModuleStmt(transpiler.ImportPrefix + "/examples/" + f.Module.Mod.Path) + if !gnolang.IsStdlib(f.Module.Mod.Path) { + f.AddModuleStmt(transpiler.TranspileImportPath(f.Module.Mod.Path)) } for i := range f.Require { @@ -111,14 +103,13 @@ func GnoToGoMod(f File) (*File, error) { } } path := f.Require[i].Mod.Path - if strings.HasPrefix(f.Require[i].Mod.Path, transpiler.GnoRealmPkgsPrefixBefore) || - strings.HasPrefix(f.Require[i].Mod.Path, transpiler.GnoPurePkgsPrefixBefore) { + if !gnolang.IsStdlib(path) { // Add dependency with a modified import path - f.AddRequire(transpiler.ImportPrefix+"/examples/"+f.Require[i].Mod.Path, f.Require[i].Mod.Version) + f.AddRequire(transpiler.TranspileImportPath(path), f.Require[i].Mod.Version) } - f.AddReplace(f.Require[i].Mod.Path, f.Require[i].Mod.Version, filepath.Join(gnoModPath, path), "") + f.AddReplace(path, f.Require[i].Mod.Version, filepath.Join(gnoModPath, path), "") // Remove the old require since the new dependency was added above - f.DropRequire(f.Require[i].Mod.Path) + f.DropRequire(path) } // Remove replacements that are not replaced by directories. diff --git a/gnovm/pkg/integration/gno.go b/gnovm/pkg/integration/gno.go index ee0216fa9e8..a389b6a9b24 100644 --- a/gnovm/pkg/integration/gno.go +++ b/gnovm/pkg/integration/gno.go @@ -68,6 +68,8 @@ func SetupGno(p *testscript.Params, buildDir string) error { return fmt.Errorf("unable to create temporary home directory: %w", err) } env.Setenv("HOME", home) + // Avoids go command printing errors relating to lack of go.mod. + env.Setenv("GO111MODULE", "off") // Cleanup home folder env.Defer(func() { os.RemoveAll(home) }) diff --git a/gnovm/pkg/transpiler/transpiler.go b/gnovm/pkg/transpiler/transpiler.go index 8a91ae4a486..bd4bb1b1bc9 100644 --- a/gnovm/pkg/transpiler/transpiler.go +++ b/gnovm/pkg/transpiler/transpiler.go @@ -1,3 +1,5 @@ +// Package transpiler implements a source-to-source compiler for translating Gno +// code into Go code. package transpiler import ( @@ -9,102 +11,52 @@ import ( goscanner "go/scanner" "go/token" "os" - "os/exec" + "path" "path/filepath" - "regexp" - "sort" "strconv" "strings" + "github.com/gnolang/gno/gnovm/pkg/gnoenv" + gno "github.com/gnolang/gno/gnovm/pkg/gnolang" + "github.com/gnolang/gno/gnovm/stdlibs" "golang.org/x/tools/go/ast/astutil" ) -const ( - GnoRealmPkgsPrefixBefore = "gno.land/r/" - GnoRealmPkgsPrefixAfter = "github.com/gnolang/gno/examples/gno.land/r/" - GnoPurePkgsPrefixBefore = "gno.land/p/" - GnoPurePkgsPrefixAfter = "github.com/gnolang/gno/examples/gno.land/p/" - GnoStdPkgBefore = "std" - GnoStdPkgAfter = "github.com/gnolang/gno/gnovm/stdlibs/stdshim" -) - -var stdlibWhitelist = []string{ - // go - "bufio", - "bytes", - "compress/gzip", - "context", - "crypto/md5", - "crypto/sha1", - "crypto/chacha20", - "crypto/cipher", - "crypto/sha256", - "encoding/base64", - "encoding/binary", - "encoding/hex", - "encoding/json", - "encoding/xml", - "errors", - "hash", - "hash/adler32", - "internal/bytealg", - "internal/os", - "flag", - "fmt", - "io", - "io/util", - "math", - "math/big", - "math/bits", - "math/rand", - "net/url", - "path", - "regexp", - "sort", - "strconv", - "strings", - "text/template", - "time", - "unicode", - "unicode/utf8", - "unicode/utf16", +// ImportPrefix is the import path to the root of the gno repository, which should +// be used to create go import paths. +const ImportPrefix = "github.com/gnolang/gno" - // gno - "std", +// TranspileImportPath takes an import path s, and converts it into the full +// import path relative to the Gno repository. +func TranspileImportPath(s string) string { + return ImportPrefix + "/" + PackageDirLocation(s) } -var importPrefixWhitelist = []string{ - "github.com/gnolang/gno/_test", +// PackageDirLocation provides the supposed directory of the package, relative to the root dir. +// +// TODO(morgan): move out, this should go in a "resolver" package. +func PackageDirLocation(s string) string { + switch { + case !gno.IsStdlib(s): + return "examples/" + s + default: + return "gnovm/stdlibs/" + s + } } -const ImportPrefix = "github.com/gnolang/gno" - -type transpileResult struct { +// Result is returned by Transpile, returning the file's imports and output +// out the transpilation. +type Result struct { Imports []*ast.ImportSpec Translated string + File *ast.File } // TODO: func TranspileFile: supports caching. // TODO: func TranspilePkg: supports directories. -func guessRootDir(fileOrPkg string, goBinary string) (string, error) { - abs, err := filepath.Abs(fileOrPkg) - if err != nil { - return "", err - } - args := []string{"list", "-m", "-mod=mod", "-f", "{{.Dir}}", ImportPrefix} - cmd := exec.Command(goBinary, args...) - cmd.Dir = abs - out, err := cmd.CombinedOutput() - if err != nil { - return "", fmt.Errorf("can't guess --root-dir") - } - rootDir := strings.TrimSpace(string(out)) - return rootDir, nil -} - -// GetTranspileFilenameAndTags returns the filename and tags for transpiled files. -func GetTranspileFilenameAndTags(gnoFilePath string) (targetFilename, tags string) { +// TranspiledFilenameAndTags returns the filename and tags for transpiled files. +func TranspiledFilenameAndTags(gnoFilePath string) (targetFilename, tags string) { nameNoExtension := strings.TrimSuffix(filepath.Base(gnoFilePath), ".gno") switch { case strings.HasSuffix(gnoFilePath, "_filetest.gno"): @@ -120,17 +72,43 @@ func GetTranspileFilenameAndTags(gnoFilePath string) (targetFilename, tags strin return } -func Transpile(source string, tags string, filename string) (*transpileResult, error) { +// Transpile performs transpilation on the given source code. tags can be used +// to specify build tags; and filename helps generate useful error messages and +// discriminate between test and normal source files. +func Transpile(source, tags, filename string) (*Result, error) { fset := token.NewFileSet() - f, err := parser.ParseFile(fset, filename, source, parser.ParseComments) + f, err := parser.ParseFile(fset, filename, source, + // SkipObjectResolution -- unused here. + // ParseComments -- so that they show up when re-building the AST. + parser.SkipObjectResolution|parser.ParseComments) if err != nil { return nil, fmt.Errorf("parse: %w", err) } isTestFile := strings.HasSuffix(filename, "_test.gno") || strings.HasSuffix(filename, "_filetest.gno") - shouldCheckWhitelist := !isTestFile + ctx := &transpileCtx{ + rootDir: gnoenv.RootDir(), + } + stdlibPrefix := filepath.Join(ctx.rootDir, "gnovm", "stdlibs") + if isTestFile { + // XXX(morgan): this disables checking that a package exists (in examples or stdlibs) + // when transpiling a test file. After all Gno functions, including those in + // tests/imports.go are converted to native bindings, support should + // be added for transpiling stdlibs only available in tests/stdlibs, and + // enable as such "package checking" also on test files. + ctx.rootDir = "" + } + if strings.HasPrefix(filename, stdlibPrefix) { + // this is a standard library. Mark it in the options so the native + // bindings resolve correctly. + path := strings.TrimPrefix(filename, stdlibPrefix) + path = filepath.ToSlash(filepath.Dir(path)) + path = strings.TrimLeft(path, "/") + + ctx.stdlibPath = path + } - transformed, err := transpileAST(fset, f, shouldCheckWhitelist) + transformed, err := ctx.transformFile(fset, f) if err != nil { return nil, fmt.Errorf("transpileAST: %w", err) } @@ -152,190 +130,64 @@ func Transpile(source string, tags string, filename string) (*transpileResult, e return nil, fmt.Errorf("format.Node: %w", err) } - res := &transpileResult{ + res := &Result{ Imports: f.Imports, Translated: out.String(), + File: transformed, } return res, nil } -// TranspileVerifyFile tries to run `go fmt` against a transpiled .go file. -// -// This is fast and won't look the imports. -func TranspileVerifyFile(path string, gofmtBinary string) error { - // TODO: use cmd/parser instead of exec? +type transpileCtx struct { + // If rootDir is given, we will check that the directory of the import path + // exists (using rootDir/packageDirLocation()). + rootDir string + // This should be set if we're working with a file from a standard library. + // This allows us to easily check if a function has a native binding, and as + // such modify its call expressions appropriately. + stdlibPath string - args := strings.Split(gofmtBinary, " ") - args = append(args, []string{"-l", "-e", path}...) - cmd := exec.Command(args[0], args[1:]...) - out, err := cmd.CombinedOutput() - if err != nil { - fmt.Fprintln(os.Stderr, string(out)) - return fmt.Errorf("%s: %w", gofmtBinary, err) - } - return nil -} - -// TranspileBuildPackage tries to run `go build` against the transpiled .go files. -// -// This method is the most efficient to detect errors but requires that -// all the import are valid and available. -func TranspileBuildPackage(fileOrPkg, goBinary string) error { - // TODO: use cmd/compile instead of exec? - // TODO: find the nearest go.mod file, chdir in the same folder, rim prefix? - // TODO: temporarily create an in-memory go.mod or disable go modules for gno? - // TODO: ignore .go files that were not generated from gno? - // TODO: automatically transpile if not yet done. - - files := []string{} - - info, err := os.Stat(fileOrPkg) - if err != nil { - return fmt.Errorf("invalid file or package path %s: %w", fileOrPkg, err) - } - if !info.IsDir() { - file := fileOrPkg - files = append(files, file) - } else { - pkgDir := fileOrPkg - goGlob := filepath.Join(pkgDir, "*.go") - goMatches, err := filepath.Glob(goGlob) - if err != nil { - return fmt.Errorf("glob %s: %w", goGlob, err) - } - for _, goMatch := range goMatches { - switch { - case strings.HasPrefix(goMatch, "."): // skip - case strings.HasSuffix(goMatch, "_filetest.go"): // skip - case strings.HasSuffix(goMatch, "_filetest.gno.gen.go"): // skip - case strings.HasSuffix(goMatch, "_test.go"): // skip - case strings.HasSuffix(goMatch, "_test.gno.gen.go"): // skip - default: - files = append(files, goMatch) - } - } - } - - sort.Strings(files) - args := append([]string{"build", "-v", "-tags=gno"}, files...) - cmd := exec.Command(goBinary, args...) - rootDir, err := guessRootDir(fileOrPkg, goBinary) - if err == nil { - cmd.Dir = rootDir - } - out, err := cmd.CombinedOutput() - if _, ok := err.(*exec.ExitError); ok { - // exit error - return parseGoBuildErrors(string(out)) - } - return err -} - -var reGoBuildError = regexp.MustCompile(`(?m)^(\S+):(\d+):(\d+): (.+)$`) - -// parseGoBuildErrors returns a scanner.ErrorList filled with all errors found -// in out, which is supposed to be the output of the `go build` command. -// -// TODO(tb): update when `go build -json` is released to replace regexp usage. -// See https://github.com/golang/go/issues/62067 -func parseGoBuildErrors(out string) error { - var errList goscanner.ErrorList - matches := reGoBuildError.FindAllStringSubmatch(out, -1) - for _, match := range matches { - filename := match[1] - line, err := strconv.Atoi(match[2]) - if err != nil { - return fmt.Errorf("parse line go build error %s: %w", match, err) - } - - column, err := strconv.Atoi(match[3]) - if err != nil { - return fmt.Errorf("parse column go build error %s: %w", match, err) - } - msg := match[4] - errList.Add(token.Position{ - Filename: filename, - Line: line, - Column: column, - }, msg) - } - return errList.Err() + stdlibImports map[string]string // symbol -> import path } -func transpileAST(fset *token.FileSet, f *ast.File, checkWhitelist bool) (ast.Node, error) { +func (ctx *transpileCtx) transformFile(fset *token.FileSet, f *ast.File) (*ast.File, error) { var errs goscanner.ErrorList imports := astutil.Imports(fset, f) + ctx.stdlibImports = make(map[string]string) - // import whitelist - if checkWhitelist { - for _, paragraph := range imports { - for _, importSpec := range paragraph { - importPath := strings.TrimPrefix(strings.TrimSuffix(importSpec.Path.Value, `"`), `"`) - - if strings.HasPrefix(importPath, GnoRealmPkgsPrefixBefore) { - continue - } - - if strings.HasPrefix(importPath, GnoPurePkgsPrefixBefore) { - continue - } - - valid := false - for _, whitelisted := range stdlibWhitelist { - if importPath == whitelisted { - valid = true - break - } - } - if valid { - continue - } - - for _, whitelisted := range importPrefixWhitelist { - if strings.HasPrefix(importPath, whitelisted) { - valid = true - break - } - } - if valid { - continue - } - - errs.Add(fset.Position(importSpec.Pos()), fmt.Sprintf("import %q is not in the whitelist", importPath)) - } - } - } - - // rewrite imports + // rewrite imports to point to stdlibs/ or examples/ for _, paragraph := range imports { for _, importSpec := range paragraph { - importPath := strings.TrimPrefix(strings.TrimSuffix(importSpec.Path.Value, `"`), `"`) - - // std package - if importPath == GnoStdPkgBefore { - if !astutil.RewriteImport(fset, f, GnoStdPkgBefore, GnoStdPkgAfter) { - errs.Add(fset.Position(importSpec.Pos()), fmt.Sprintf("failed to replace the %q package with %q", GnoStdPkgBefore, GnoStdPkgAfter)) - } + importPath, err := strconv.Unquote(importSpec.Path.Value) + if err != nil { + errs.Add(fset.Position(importSpec.Pos()), fmt.Sprintf("can't unquote import path %s: %v", importSpec.Path.Value, err)) + continue } - // p/pkg packages - if strings.HasPrefix(importPath, GnoPurePkgsPrefixBefore) { - target := GnoPurePkgsPrefixAfter + strings.TrimPrefix(importPath, GnoPurePkgsPrefixBefore) - - if !astutil.RewriteImport(fset, f, importPath, target) { - errs.Add(fset.Position(importSpec.Pos()), fmt.Sprintf("failed to replace the %q package with %q", importPath, target)) + if ctx.rootDir != "" { + dirPath := filepath.Join(ctx.rootDir, PackageDirLocation(importPath)) + if _, err := os.Stat(dirPath); err != nil { + if !os.IsNotExist(err) { + return nil, err + } + errs.Add(fset.Position(importSpec.Pos()), fmt.Sprintf("import %q does not exist", importPath)) + continue } } - // r/realm packages - if strings.HasPrefix(importPath, GnoRealmPkgsPrefixBefore) { - target := GnoRealmPkgsPrefixAfter + strings.TrimPrefix(importPath, GnoRealmPkgsPrefixBefore) - - if !astutil.RewriteImport(fset, f, importPath, target) { - errs.Add(fset.Position(importSpec.Pos()), fmt.Sprintf("failed to replace the %q package with %q", importPath, target)) + // Create mapping + if gno.IsStdlib(importPath) { + if importSpec.Name != nil { + ctx.stdlibImports[importSpec.Name.Name] = importPath + } else { + // XXX: imperfect, see comment on transformCallExpr + ctx.stdlibImports[path.Base(importPath)] = importPath } } + + transp := TranspileImportPath(importPath) + importSpec.Path.Value = strconv.Quote(transp) } } @@ -343,14 +195,72 @@ func transpileAST(fset *token.FileSet, f *ast.File, checkWhitelist bool) (ast.No node := astutil.Apply(f, // pre func(c *astutil.Cursor) bool { - // do things here + node := c.Node() + // is function declaration without body? + // -> delete (native binding) + if fd, ok := node.(*ast.FuncDecl); ok && fd.Body == nil { + c.Delete() + return false // don't attempt to traverse children + } + + // is function call to a native function? + // -> rename if unexported, apply `nil,` for the first arg if necessary + if ce, ok := node.(*ast.CallExpr); ok { + return ctx.transformCallExpr(c, ce) + } + return true }, + // post func(c *astutil.Cursor) bool { - // and here return true }, ) - return node, errs.Err() + return node.(*ast.File), errs.Err() +} + +func (ctx *transpileCtx) transformCallExpr(_ *astutil.Cursor, ce *ast.CallExpr) bool { + switch fe := ce.Fun.(type) { + case *ast.SelectorExpr: + // XXX: This is not correct in 100% of cases. If I shadow the `std` symbol, and + // its replacement is a type with the method AssertOriginCall, this system + // will incorrectly add a `nil` as the first argument. + // A full fix requires understanding scope; the Go standard library recommends + // using go/types, which for proper functioning requires an importer + // which can work with Gno. This is deferred for a future PR. + id, ok := fe.X.(*ast.Ident) + if !ok { + break + } + ip, ok := ctx.stdlibImports[id.Name] + if !ok { + break + } + nat := stdlibs.FindNative(ip, gno.Name(fe.Sel.Name)) + if nat != nil && nat.HasMachineParam() { + // Because it's an import, the symbol is always exported, so no need for the + // X_ prefix we add below. + ce.Args = append([]ast.Expr{ast.NewIdent("nil")}, ce.Args...) + } + + case *ast.Ident: + // Is this a native binding? + // Note: this is only useful within packages like `std` and `math`. + // The logic here is not robust to be generic. It does not account for locally + // defined scope. However, because native bindings have a narrowly defined and + // controlled scope (standard libraries) this will work for our usecase. + nat := stdlibs.FindNative(ctx.stdlibPath, gno.Name(fe.Name)) + if ctx.stdlibPath != "" && nat != nil { + if nat.HasMachineParam() { + ce.Args = append([]ast.Expr{ast.NewIdent("nil")}, ce.Args...) + } + if !fe.IsExported() { + // Prefix unexported names with X_, per native binding convention + // (to export the symbol within Go). + fe.Name = "X_" + fe.Name + } + } + } + return true } diff --git a/gnovm/pkg/transpiler/transpiler_test.go b/gnovm/pkg/transpiler/transpiler_test.go index b9e9b218675..2a0707f7f79 100644 --- a/gnovm/pkg/transpiler/transpiler_test.go +++ b/gnovm/pkg/transpiler/transpiler_test.go @@ -2,21 +2,69 @@ package transpiler import ( "go/ast" - goscanner "go/scanner" - "go/token" + "path/filepath" "strings" "testing" + "github.com/gnolang/gno/gnovm/pkg/gnoenv" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +func TestTranspiledFilenameAndTags(t *testing.T) { + t.Parallel() + + tt := []struct { + name string + changed string + tags string + }{ + { + "hello.gno", + "hello.gno.gen.go", + "gno", + }, + { + "a/b/hello.gno", + "hello.gno.gen.go", + "gno", + }, + { + "hey_test.gno", + ".hey_test.gno.gen_test.go", + "gno && test", + }, + { + "hey_filetest.gno", + ".hey_filetest.gno.gen.go", + "gno && filetest", + }, + { + "badname.go", + "badname.go.gno.gen.go", + "gno", + }, + { + "badname_test.go", + "badname_test.go.gno.gen.go", + "gno", + }, + } + + for _, tc := range tt { + newName, tags := TranspiledFilenameAndTags(tc.name) + assert.Equal(t, tc.changed, newName, "name for %q", tc.name) + assert.Equal(t, tc.tags, tags, "tags for %q", tc.name) + } +} + func TestTranspile(t *testing.T) { t.Parallel() cases := []struct { name string tags string + filename string source string expectedOutput string expectedImports []*ast.ImportSpec @@ -75,7 +123,7 @@ func hello() string { //line foo.gno:1:1 package foo -import "github.com/gnolang/gno/gnovm/stdlibs/stdshim" +import "github.com/gnolang/gno/gnovm/stdlibs/std" func hello() string { _ = std.Foo @@ -87,9 +135,8 @@ func hello() string { Path: &ast.BasicLit{ ValuePos: 21, Kind: 9, - Value: `"github.com/gnolang/gno/gnovm/stdlibs/stdshim"`, + Value: `"github.com/gnolang/gno/gnovm/stdlibs/std"`, }, - EndPos: 26, }, }, }, @@ -119,7 +166,6 @@ func foo() { _ = users.Register } Kind: 9, Value: `"github.com/gnolang/gno/examples/gno.land/r/demo/users"`, }, - EndPos: 44, }, }, }, @@ -130,7 +176,7 @@ package foo import "gno.land/p/demo/avl" -func foo() { _ = avl.Tree } +func foo() { _ = avl.NewTree("hey", 1) } `, expectedOutput: ` // Code generated by github.com/gnolang/gno. DO NOT EDIT. @@ -140,7 +186,7 @@ package foo import "github.com/gnolang/gno/examples/gno.land/p/demo/avl" -func foo() { _ = avl.Tree } +func foo() { _ = avl.NewTree("hey", 1) } `, expectedImports: []*ast.ImportSpec{ { @@ -149,7 +195,6 @@ func foo() { _ = avl.Tree } Kind: 9, Value: `"github.com/gnolang/gno/examples/gno.land/p/demo/avl"`, }, - EndPos: 42, }, }, }, @@ -171,7 +216,7 @@ func hello() string { //line foo.gno:1:1 package foo -import bar "github.com/gnolang/gno/gnovm/stdlibs/stdshim" +import bar "github.com/gnolang/gno/gnovm/stdlibs/std" func hello() string { _ = bar.Foo @@ -187,14 +232,13 @@ func hello() string { Path: &ast.BasicLit{ ValuePos: 25, Kind: 9, - Value: `"github.com/gnolang/gno/gnovm/stdlibs/stdshim"`, + Value: `"github.com/gnolang/gno/gnovm/stdlibs/std"`, }, - EndPos: 30, }, }, }, { - name: "blacklisted-package", + name: "unknown-package", source: ` package foo @@ -202,7 +246,7 @@ import "reflect" func foo() { _ = reflect.ValueOf } `, - expectedError: `transpileAST: foo.gno:3:8: import "reflect" is not in the whitelist`, + expectedError: `transpileAST: foo.gno:3:8: import "reflect" does not exist`, }, { name: "syntax-error", @@ -226,6 +270,27 @@ import "gno.land/p/demo/unknownxyz" //line foo.gno:1:1 package foo +import "github.com/gnolang/gno/examples/gno.land/p/demo/unknownxyz" +`, + expectedError: `transpileAST: foo.gno:3:8: import "gno.land/p/demo/unknownxyz" does not exist`, + }, + { + // Test files should allow unknown imports while + // we still have "native" packages. + + name: "unknown-realm-test", + filename: "foo_test.gno", + source: ` +package foo + +import "gno.land/p/demo/unknownxyz" +`, + expectedOutput: ` +// Code generated by github.com/gnolang/gno. DO NOT EDIT. + +//line foo_test.gno:1:1 +package foo + import "github.com/gnolang/gno/examples/gno.land/p/demo/unknownxyz" `, expectedImports: []*ast.ImportSpec{ @@ -235,12 +300,11 @@ import "github.com/gnolang/gno/examples/gno.land/p/demo/unknownxyz" Kind: 9, Value: `"github.com/gnolang/gno/examples/gno.land/p/demo/unknownxyz"`, }, - EndPos: 49, }, }, }, { - name: "whitelisted-package", + name: "imported-package", source: ` package foo @@ -254,7 +318,7 @@ func foo() { _ = regexp.MatchString } //line foo.gno:1:1 package foo -import "regexp" +import "github.com/gnolang/gno/gnovm/stdlibs/regexp" func foo() { _ = regexp.MatchString } `, @@ -263,11 +327,87 @@ func foo() { _ = regexp.MatchString } Path: &ast.BasicLit{ ValuePos: 21, Kind: 9, - Value: `"regexp"`, + Value: `"github.com/gnolang/gno/gnovm/stdlibs/regexp"`, }, }, }, }, + { + name: "natbind-func", + filename: filepath.Join(gnoenv.RootDir(), "gnovm/stdlibs/math/math.gno"), + source: ` +package math + +import "std" + +func Float32bits(i float32) uint32 + +func testfunc() { + println(Float32bits(3.14159)) + std.AssertOriginCall() +} + +func otherFunc() { + std := 1 + // This is (incorrectly) changed for now. + std.AssertOriginCall() +} +`, + expectedOutput: ` +// Code generated by github.com/gnolang/gno. DO NOT EDIT. + +//line math.gno:1:1 +package math + +import "github.com/gnolang/gno/gnovm/stdlibs/std" + +func testfunc() { + println(Float32bits(3.14159)) + std.AssertOriginCall(nil) +} + +func otherFunc() { + std := 1 + // This is (incorrectly) changed for now. + std.AssertOriginCall(nil) +} +`, + expectedImports: []*ast.ImportSpec{ + { + Path: &ast.BasicLit{ + ValuePos: 22, + Kind: 9, + Value: `"github.com/gnolang/gno/gnovm/stdlibs/std"`, + }, + }, + }, + }, + { + name: "natbind-std", + filename: filepath.Join(gnoenv.RootDir(), "gnovm/stdlibs/std/std.gno"), + source: ` +package std + +func AssertOriginCall() +func origCaller() string + +func testfunc() { + AssertOriginCall() + println(origCaller()) +} +`, + expectedOutput: ` +// Code generated by github.com/gnolang/gno. DO NOT EDIT. + +//line std.gno:1:1 +package std + +func testfunc() { + AssertOriginCall(nil) + println(X_origCaller(nil)) +} +`, + }, } for _, c := range cases { c := c // scopelint @@ -276,7 +416,11 @@ func foo() { _ = regexp.MatchString } // "\n" is added for better test case readability, now trim it source := strings.TrimPrefix(c.source, "\n") - res, err := Transpile(source, c.tags, "foo.gno") + filename := c.filename + if filename == "" { + filename = "foo.gno" + } + res, err := Transpile(source, c.tags, filename) if c.expectedError != "" { require.EqualError(t, err, c.expectedError) @@ -289,53 +433,3 @@ func foo() { _ = regexp.MatchString } }) } } - -func TestParseGoBuildErrors(t *testing.T) { - tests := []struct { - name string - output string - expectedError error - }{ - { - name: "empty output", - output: "", - expectedError: nil, - }, - { - name: "random output", - output: "xxx", - expectedError: nil, - }, - { - name: "some errors", - output: `xxx -main.gno:6:2: nasty error -pkg/file.gno:60:20: ugly error`, - expectedError: goscanner.ErrorList{ - &goscanner.Error{ - Pos: token.Position{ - Filename: "main.gno", - Line: 6, - Column: 2, - }, - Msg: "nasty error", - }, - &goscanner.Error{ - Pos: token.Position{ - Filename: "pkg/file.gno", - Line: 60, - Column: 20, - }, - Msg: "ugly error", - }, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := parseGoBuildErrors(tt.output) - - assert.Equal(t, tt.expectedError, err) - }) - } -} diff --git a/gnovm/stdlibs/bytes/boundary_test.gno b/gnovm/stdlibs/bytes/boundary_test.gno index 9873b1db987..5d77c576ff8 100644 --- a/gnovm/stdlibs/bytes/boundary_test.gno +++ b/gnovm/stdlibs/bytes/boundary_test.gno @@ -1,8 +1,6 @@ // Copyright 2017 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// -//go:build linux package bytes_test diff --git a/gnovm/stdlibs/crypto/chacha20/chacha/chacha_ref.gno b/gnovm/stdlibs/crypto/chacha20/chacha/chacha_ref.gno index 903e2653572..91e5481fc42 100644 --- a/gnovm/stdlibs/crypto/chacha20/chacha/chacha_ref.gno +++ b/gnovm/stdlibs/crypto/chacha20/chacha/chacha_ref.gno @@ -2,8 +2,6 @@ // Use of this source code is governed by a license that can be // found in the LICENSE file. -//go:build (!amd64 && !386) || gccgo || appengine || nacl - package chacha import "encoding/binary" diff --git a/gnovm/stdlibs/internal/bytealg/compare_generic.gno b/gnovm/stdlibs/internal/bytealg/compare_generic.gno index e1795e47e9a..b56d0a67e02 100644 --- a/gnovm/stdlibs/internal/bytealg/compare_generic.gno +++ b/gnovm/stdlibs/internal/bytealg/compare_generic.gno @@ -2,8 +2,6 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !386 && !amd64 && !s390x && !arm && !arm64 && !ppc64 && !ppc64le && !mips && !mipsle && !wasm && !mips64 && !mips64le - package bytealg // import _ "unsafe" // for go:linkname @@ -35,7 +33,6 @@ samebytes: return 0 } -//go:linkname runtime_cmpstring runtime.cmpstring func runtime_cmpstring(a, b string) int { l := len(a) if len(b) < l { diff --git a/gnovm/stdlibs/internal/bytealg/count_generic.gno b/gnovm/stdlibs/internal/bytealg/count_generic.gno index 932a7c584c1..de08418fcaa 100644 --- a/gnovm/stdlibs/internal/bytealg/count_generic.gno +++ b/gnovm/stdlibs/internal/bytealg/count_generic.gno @@ -2,8 +2,6 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !amd64 && !arm && !arm64 && !ppc64le && !ppc64 && !riscv64 && !s390x - package bytealg func Count(b []byte, c byte) int { diff --git a/gnovm/stdlibs/internal/bytealg/index_generic.gno b/gnovm/stdlibs/internal/bytealg/index_generic.gno index a59e32938e7..d751b1bc940 100644 --- a/gnovm/stdlibs/internal/bytealg/index_generic.gno +++ b/gnovm/stdlibs/internal/bytealg/index_generic.gno @@ -2,8 +2,6 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !amd64 && !arm64 && !s390x && !ppc64le && !ppc64 - package bytealg const MaxBruteForce = 0 diff --git a/gnovm/stdlibs/internal/bytealg/indexbyte_generic.gno b/gnovm/stdlibs/internal/bytealg/indexbyte_generic.gno index 0a45f903843..47aee225df9 100644 --- a/gnovm/stdlibs/internal/bytealg/indexbyte_generic.gno +++ b/gnovm/stdlibs/internal/bytealg/indexbyte_generic.gno @@ -2,8 +2,6 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !386 && !amd64 && !s390x && !arm && !arm64 && !ppc64 && !ppc64le && !mips && !mipsle && !mips64 && !mips64le && !riscv64 && !wasm - package bytealg func IndexByte(b []byte, c byte) int { diff --git a/gnovm/stdlibs/native.go b/gnovm/stdlibs/native.go index 7319e393c35..3dd432c90c0 100644 --- a/gnovm/stdlibs/native.go +++ b/gnovm/stdlibs/native.go @@ -16,15 +16,25 @@ import ( libs_time "github.com/gnolang/gno/gnovm/stdlibs/time" ) -type nativeFunc struct { - gnoPkg string - gnoFunc gno.Name - params []gno.FieldTypeExpr - results []gno.FieldTypeExpr - f func(m *gno.Machine) +// NativeFunc represents a function in the standard library which has a native +// (go-based) implementation, commonly referred to as a "native binding". +type NativeFunc struct { + gnoPkg string + gnoFunc gno.Name + params []gno.FieldTypeExpr + results []gno.FieldTypeExpr + hasMachine bool + f func(m *gno.Machine) } -var nativeFuncs = [...]nativeFunc{ +// HasMachineParam returns whether the given native binding has a machine parameter. +// This means that the Go version of this function expects a *gno.Machine +// as its first parameter. +func (n *NativeFunc) HasMachineParam() bool { + return n.hasMachine +} + +var nativeFuncs = [...]NativeFunc{ { "crypto/ed25519", "verify", @@ -36,6 +46,7 @@ var nativeFuncs = [...]nativeFunc{ []gno.FieldTypeExpr{ {Name: gno.N("r0"), Type: gno.X("bool")}, }, + false, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -69,6 +80,7 @@ var nativeFuncs = [...]nativeFunc{ []gno.FieldTypeExpr{ {Name: gno.N("r0"), Type: gno.X("[32]byte")}, }, + false, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -96,6 +108,7 @@ var nativeFuncs = [...]nativeFunc{ []gno.FieldTypeExpr{ {Name: gno.N("r0"), Type: gno.X("uint32")}, }, + false, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -123,6 +136,7 @@ var nativeFuncs = [...]nativeFunc{ []gno.FieldTypeExpr{ {Name: gno.N("r0"), Type: gno.X("float32")}, }, + false, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -150,6 +164,7 @@ var nativeFuncs = [...]nativeFunc{ []gno.FieldTypeExpr{ {Name: gno.N("r0"), Type: gno.X("uint64")}, }, + false, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -177,6 +192,7 @@ var nativeFuncs = [...]nativeFunc{ []gno.FieldTypeExpr{ {Name: gno.N("r0"), Type: gno.X("float64")}, }, + false, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -206,6 +222,7 @@ var nativeFuncs = [...]nativeFunc{ {Name: gno.N("r0"), Type: gno.X("[]string")}, {Name: gno.N("r1"), Type: gno.X("[]int64")}, }, + true, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -245,6 +262,7 @@ var nativeFuncs = [...]nativeFunc{ {Name: gno.N("p4"), Type: gno.X("[]int64")}, }, []gno.FieldTypeExpr{}, + true, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -281,6 +299,7 @@ var nativeFuncs = [...]nativeFunc{ []gno.FieldTypeExpr{ {Name: gno.N("r0"), Type: gno.X("int64")}, }, + true, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -314,6 +333,7 @@ var nativeFuncs = [...]nativeFunc{ {Name: gno.N("p3"), Type: gno.X("int64")}, }, []gno.FieldTypeExpr{}, + true, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -347,6 +367,7 @@ var nativeFuncs = [...]nativeFunc{ {Name: gno.N("p3"), Type: gno.X("int64")}, }, []gno.FieldTypeExpr{}, + true, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -378,6 +399,7 @@ var nativeFuncs = [...]nativeFunc{ {Name: gno.N("p1"), Type: gno.X("[]string")}, }, []gno.FieldTypeExpr{}, + true, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -400,6 +422,7 @@ var nativeFuncs = [...]nativeFunc{ "AssertOriginCall", []gno.FieldTypeExpr{}, []gno.FieldTypeExpr{}, + true, func(m *gno.Machine) { libs_std.AssertOriginCall( m, @@ -413,6 +436,7 @@ var nativeFuncs = [...]nativeFunc{ []gno.FieldTypeExpr{ {Name: gno.N("r0"), Type: gno.X("bool")}, }, + true, func(m *gno.Machine) { r0 := libs_std.IsOriginCall( m, @@ -432,6 +456,7 @@ var nativeFuncs = [...]nativeFunc{ []gno.FieldTypeExpr{ {Name: gno.N("r0"), Type: gno.X("string")}, }, + true, func(m *gno.Machine) { r0 := libs_std.GetChainID( m, @@ -451,6 +476,7 @@ var nativeFuncs = [...]nativeFunc{ []gno.FieldTypeExpr{ {Name: gno.N("r0"), Type: gno.X("int64")}, }, + true, func(m *gno.Machine) { r0 := libs_std.GetHeight( m, @@ -471,6 +497,7 @@ var nativeFuncs = [...]nativeFunc{ {Name: gno.N("r0"), Type: gno.X("[]string")}, {Name: gno.N("r1"), Type: gno.X("[]int64")}, }, + true, func(m *gno.Machine) { r0, r1 := libs_std.X_origSend( m, @@ -495,6 +522,7 @@ var nativeFuncs = [...]nativeFunc{ []gno.FieldTypeExpr{ {Name: gno.N("r0"), Type: gno.X("string")}, }, + true, func(m *gno.Machine) { r0 := libs_std.X_origCaller( m, @@ -514,6 +542,7 @@ var nativeFuncs = [...]nativeFunc{ []gno.FieldTypeExpr{ {Name: gno.N("r0"), Type: gno.X("string")}, }, + true, func(m *gno.Machine) { r0 := libs_std.X_origPkgAddr( m, @@ -535,6 +564,7 @@ var nativeFuncs = [...]nativeFunc{ []gno.FieldTypeExpr{ {Name: gno.N("r0"), Type: gno.X("string")}, }, + true, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -565,6 +595,7 @@ var nativeFuncs = [...]nativeFunc{ {Name: gno.N("r0"), Type: gno.X("string")}, {Name: gno.N("r1"), Type: gno.X("string")}, }, + true, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -599,6 +630,7 @@ var nativeFuncs = [...]nativeFunc{ []gno.FieldTypeExpr{ {Name: gno.N("r0"), Type: gno.X("string")}, }, + false, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -627,6 +659,7 @@ var nativeFuncs = [...]nativeFunc{ []gno.FieldTypeExpr{ {Name: gno.N("r0"), Type: gno.X("string")}, }, + false, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -659,6 +692,7 @@ var nativeFuncs = [...]nativeFunc{ {Name: gno.N("r1"), Type: gno.X("[20]byte")}, {Name: gno.N("r2"), Type: gno.X("bool")}, }, + false, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -696,6 +730,7 @@ var nativeFuncs = [...]nativeFunc{ []gno.FieldTypeExpr{ {Name: gno.N("r0"), Type: gno.X("string")}, }, + false, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -725,6 +760,7 @@ var nativeFuncs = [...]nativeFunc{ []gno.FieldTypeExpr{ {Name: gno.N("r0"), Type: gno.X("[]byte")}, }, + false, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -759,6 +795,7 @@ var nativeFuncs = [...]nativeFunc{ {Name: gno.N("r0"), Type: gno.X("int")}, {Name: gno.N("r1"), Type: gno.X("error")}, }, + false, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -791,6 +828,7 @@ var nativeFuncs = [...]nativeFunc{ []gno.FieldTypeExpr{ {Name: gno.N("r0"), Type: gno.X("bool")}, }, + false, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -819,6 +857,7 @@ var nativeFuncs = [...]nativeFunc{ []gno.FieldTypeExpr{ {Name: gno.N("r0"), Type: gno.X("string")}, }, + false, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -850,6 +889,7 @@ var nativeFuncs = [...]nativeFunc{ []gno.FieldTypeExpr{ {Name: gno.N("r0"), Type: gno.X("string")}, }, + false, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -880,6 +920,7 @@ var nativeFuncs = [...]nativeFunc{ []gno.FieldTypeExpr{ {Name: gno.N("r0"), Type: gno.X("string")}, }, + false, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -907,6 +948,7 @@ var nativeFuncs = [...]nativeFunc{ []gno.FieldTypeExpr{ {Name: gno.N("r0"), Type: gno.X("string")}, }, + false, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -932,6 +974,7 @@ var nativeFuncs = [...]nativeFunc{ []gno.FieldTypeExpr{ {Name: gno.N("r0"), Type: gno.X("int64")}, }, + false, func(m *gno.Machine) { r0 := libs_testing.X_unixNano() @@ -951,6 +994,7 @@ var nativeFuncs = [...]nativeFunc{ {Name: gno.N("r1"), Type: gno.X("int32")}, {Name: gno.N("r2"), Type: gno.X("int64")}, }, + true, func(m *gno.Machine) { r0, r1, r2 := libs_time.X_now( m, diff --git a/gnovm/stdlibs/stdlibs.go b/gnovm/stdlibs/stdlibs.go index 48e69f78253..c9b16815ab5 100644 --- a/gnovm/stdlibs/stdlibs.go +++ b/gnovm/stdlibs/stdlibs.go @@ -13,11 +13,24 @@ func GetContext(m *gno.Machine) ExecContext { return libsstd.GetContext(m) } -func NativeStore(pkgPath string, name gno.Name) func(*gno.Machine) { - for _, nf := range nativeFuncs { +// FindNative returns the NativeFunc associated with the given pkgPath+name +// combination. If there is none, FindNative returns nil. +func FindNative(pkgPath string, name gno.Name) *NativeFunc { + for i, nf := range nativeFuncs { if nf.gnoPkg == pkgPath && name == nf.gnoFunc { - return nf.f + return &nativeFuncs[i] } } return nil } + +// NativeStore is used by the GnoVM to determine if the given function, +// specified by its pkgPath and name, has a native implementation; and if so +// retrieve it. +func NativeStore(pkgPath string, name gno.Name) func(*gno.Machine) { + nt := FindNative(pkgPath, name) + if nt == nil { + return nil + } + return nt.f +} diff --git a/gnovm/stdlibs/stdshim/addr_set.gno b/gnovm/stdlibs/stdshim/addr_set.gno deleted file mode 100644 index 3f46f48b8b5..00000000000 --- a/gnovm/stdlibs/stdshim/addr_set.gno +++ /dev/null @@ -1,47 +0,0 @@ -package std - -import "errors" - -//---------------------------------------- -// AddressSet - -type AddressSet interface { - Size() int - AddAddress(Address) error - HasAddress(Address) bool -} - -//---------------------------------------- -// AddressList implements AddressSet. -// TODO implement AddressTree with avl. - -type AddressList []Address - -func NewAddressList() *AddressList { - return &AddressList{} -} - -func (alist *AddressList) Size() int { - return len(*alist) -} - -func (alist *AddressList) AddAddress(newAddr Address) error { - // TODO optimize with binary algorithm - for _, addr := range *alist { - if addr == newAddr { - return errors.New("address already exists") - } - } - *alist = append(*alist, newAddr) - return nil -} - -func (alist *AddressList) HasAddress(newAddr Address) bool { - // TODO optimize with binary algorithm - for _, addr := range *alist { - if addr == newAddr { - return true - } - } - return false -} diff --git a/gnovm/stdlibs/stdshim/banker.gno b/gnovm/stdlibs/stdshim/banker.gno deleted file mode 100644 index 44e611b780f..00000000000 --- a/gnovm/stdlibs/stdshim/banker.gno +++ /dev/null @@ -1,70 +0,0 @@ -package std - -// Realm functions can call std.GetBanker(options) to get -// a banker instance. Banker objects cannot be persisted, -// but can be passed onto other functions to be transacted -// on. A banker instance can be passed onto other realm -// functions; this allows other realms to spend coins on -// behalf of the first realm. -// -// Banker panics on errors instead of returning errors. -// This also helps simplify the interface and prevent -// hidden bugs (e.g. ignoring errors) -// -// NOTE: this Gno interface is satisfied by a native go -// type, and those can't return non-primitive objects -// (without confusion). -type Banker interface { - GetCoins(addr Address) (dst Coins) - SendCoins(from, to Address, amt Coins) - TotalCoin(denom string) int64 - IssueCoin(addr Address, denom string, amount int64) - RemoveCoin(addr Address, denom string, amount int64) -} - -// Also available natively in stdlibs/context.go -type BankerType uint8 - -// Also available natively in stdlibs/context.go -const ( - // Can only read state. - BankerTypeReadonly BankerType = iota - // Can only send from tx send. - BankerTypeOrigSend - // Can send from all realm coins. - BankerTypeRealmSend - // Can issue and remove realm coins. - BankerTypeRealmIssue -) - -//---------------------------------------- -// adapter for native banker - -type bankAdapter struct { - nativeBanker Banker -} - -func (ba bankAdapter) GetCoins(addr Address) (dst Coins) { - // convert native -> gno - coins := ba.nativeBanker.GetCoins(addr) - for _, coin := range coins { - dst = append(dst, (Coin)(coin)) - } - return dst -} - -func (ba bankAdapter) SendCoins(from, to Address, amt Coins) { - ba.nativeBanker.SendCoins(from, to, amt) -} - -func (ba bankAdapter) TotalCoin(denom string) int64 { - return ba.nativeBanker.TotalCoin(denom) -} - -func (ba bankAdapter) IssueCoin(addr Address, denom string, amount int64) { - ba.nativeBanker.IssueCoin(addr, denom, amount) -} - -func (ba bankAdapter) RemoveCoin(addr Address, denom string, amount int64) { - ba.nativeBanker.RemoveCoin(addr, denom, amount) -} diff --git a/gnovm/stdlibs/stdshim/coins.gno b/gnovm/stdlibs/stdshim/coins.gno deleted file mode 100644 index 4589113bff4..00000000000 --- a/gnovm/stdlibs/stdshim/coins.gno +++ /dev/null @@ -1,174 +0,0 @@ -package std - -import "strconv" - -// NOTE: this is selectively copied over from tm2/pkgs/std/coin.go - -// Coin hold some amount of one currency. -// A negative amount is invalid. -type Coin struct { - Denom string `json:"denom"` - Amount int64 `json:"amount"` -} - -// NewCoin returns a new coin with a denomination and amount -func NewCoin(denom string, amount int64) Coin { - return Coin{ - Denom: denom, - Amount: amount, - } -} - -// String provides a human-readable representation of a coin -func (c Coin) String() string { - return strconv.Itoa(int(c.Amount)) + c.Denom -} - -// IsGTE returns true if they are the same type and the receiver is -// an equal or greater value -func (c Coin) IsGTE(other Coin) bool { - mustMatchDenominations(c.Denom, other.Denom) - - return c.Amount >= other.Amount -} - -// IsLT returns true if they are the same type and the receiver is -// a smaller value -func (c Coin) IsLT(other Coin) bool { - mustMatchDenominations(c.Denom, other.Denom) - - return c.Amount < other.Amount -} - -// IsEqual returns true if the two sets of Coins have the same value -func (c Coin) IsEqual(other Coin) bool { - mustMatchDenominations(c.Denom, other.Denom) - - return c.Amount == other.Amount -} - -// Add adds amounts of two coins with same denom. -// If the coins differ in denom then it panics. -// An overflow or underflow panics. -// An invalid result panics. -func (c Coin) Add(coinB Coin) Coin { - mustMatchDenominations(c.Denom, coinB.Denom) - - sum := c.Amount + coinB.Amount - - c.Amount = sum - return c -} - -// Sub subtracts amounts of two coins with same denom. -// If the coins differ in denom then it panics. -// An overflow or underflow panics. -// An invalid result panics. -func (c Coin) Sub(coinB Coin) Coin { - mustMatchDenominations(c.Denom, coinB.Denom) - - dff := c.Amount - coinB.Amount - c.Amount = dff - - return c -} - -// IsPositive returns true if coin amount is positive. -func (c Coin) IsPositive() bool { - return c.Amount > 0 -} - -// IsNegative returns true if the coin amount is negative and false otherwise. -func (c Coin) IsNegative() bool { - return c.Amount < 0 -} - -// IsZero returns if this represents no money -func (c Coin) IsZero() bool { - return c.Amount == 0 -} - -func mustMatchDenominations(denomA, denomB string) { - if denomA != denomB { - panic("incompatible coin denominations: " + denomA + ", " + denomB) - } -} - -// Coins is a set of Coin, one per currency -type Coins []Coin - -// NewCoins returns a new set of Coins given one or more Coins -// Consolidates any denom duplicates into one, keeping the properties of a mathematical set -func NewCoins(coins ...Coin) Coins { - coinMap := make(map[string]int64) - - for _, coin := range coins { - coinMap[coin.Denom] = coin.Amount - } - - var setCoins Coins - for denom, amount := range coinMap { - setCoins = append(setCoins, NewCoin(denom, amount)) - } - - return setCoins -} - -// String returns the string representation of Coins -func (cz Coins) String() string { - if len(cz) == 0 { - return "" - } - - res := "" - for i, c := range cz { - if i > 0 { - res += "," - } - res += c.String() - } - - return res -} - -// AmountOf returns the amount of a specific coin from the Coins set -func (cz Coins) AmountOf(denom string) int64 { - for _, c := range cz { - if c.Denom == denom { - return c.Amount - } - } - - return 0 -} - -// Add adds a Coin to the Coins set -func (cz Coins) Add(b Coins) Coins { - c := Coins{} - for _, ac := range cz { - bc := b.AmountOf(ac.Denom) - ac.Amount += bc - c = append(c, ac) - } - - for _, bc := range b { - cc := c.AmountOf(bc.Denom) - if cc == 0 { - c = append(c, bc) - } - } - - return c -} - -// expandNative expands for usage within natively bound functions. -func (cz Coins) expandNative() (denoms []string, amounts []int64) { - denoms = make([]string, len(cz)) - amounts = make([]int64, len(cz)) - for i, coin := range cz { - denoms[i] = coin.Denom - amounts[i] = coin.Amount - } - - return denoms, amounts -} diff --git a/gnovm/stdlibs/stdshim/context.gno b/gnovm/stdlibs/stdshim/context.gno deleted file mode 100644 index 878c963b22b..00000000000 --- a/gnovm/stdlibs/stdshim/context.gno +++ /dev/null @@ -1,4 +0,0 @@ -package std - -// ExecContext is not exposed, -// use native injections std.GetChainID(), std.GetHeight() etc instead. diff --git a/gnovm/stdlibs/stdshim/crypto.gno b/gnovm/stdlibs/stdshim/crypto.gno deleted file mode 100644 index 402a6af3e22..00000000000 --- a/gnovm/stdlibs/stdshim/crypto.gno +++ /dev/null @@ -1,17 +0,0 @@ -package std - -type Address string // NOTE: bech32 - -func (a Address) String() string { - return string(a) -} - -// IsValid checks if the address is valid bech32 encoded string -func (a Address) IsValid() bool { - _, _, ok := DecodeBech32(a) - return ok -} - -const RawAddressSize = 20 - -type RawAddress [RawAddressSize]byte diff --git a/gnovm/stdlibs/stdshim/frame.gno b/gnovm/stdlibs/stdshim/frame.gno deleted file mode 100644 index bc3a000f5a0..00000000000 --- a/gnovm/stdlibs/stdshim/frame.gno +++ /dev/null @@ -1,18 +0,0 @@ -package std - -type Realm struct { - addr Address - pkgPath string -} - -func (r Realm) Addr() Address { - return r.addr -} - -func (r Realm) PkgPath() string { - return r.pkgPath -} - -func (r Realm) IsUser() bool { - return r.pkgPath == "" -} diff --git a/gnovm/stdlibs/time/time.gno b/gnovm/stdlibs/time/time.gno index 521679e48d5..f3395142d1d 100644 --- a/gnovm/stdlibs/time/time.gno +++ b/gnovm/stdlibs/time/time.gno @@ -80,7 +80,6 @@ package time import ( "errors" - // XXX _ "unsafe" // for go:linkname ) // A Time represents an instant in time with nanosecond precision. @@ -1072,8 +1071,6 @@ func daysSinceEpoch(year int) uint64 { func now() (sec int64, nsec int32, mono int64) // injected // runtimeNano returns the current value of the runtime clock in nanoseconds. -// -//go:linkname runtimeNano runtime.nanotime func runtimeNano() int64 { _, _, mono := now() return mono diff --git a/gnovm/tests/stdlibs/native.go b/gnovm/tests/stdlibs/native.go index 0f33548054b..d2964a7958c 100644 --- a/gnovm/tests/stdlibs/native.go +++ b/gnovm/tests/stdlibs/native.go @@ -11,20 +11,31 @@ import ( testlibs_testing "github.com/gnolang/gno/gnovm/tests/stdlibs/testing" ) -type nativeFunc struct { - gnoPkg string - gnoFunc gno.Name - params []gno.FieldTypeExpr - results []gno.FieldTypeExpr - f func(m *gno.Machine) +// NativeFunc represents a function in the standard library which has a native +// (go-based) implementation, commonly referred to as a "native binding". +type NativeFunc struct { + gnoPkg string + gnoFunc gno.Name + params []gno.FieldTypeExpr + results []gno.FieldTypeExpr + hasMachine bool + f func(m *gno.Machine) } -var nativeFuncs = [...]nativeFunc{ +// HasMachineParam returns whether the given native binding has a machine parameter. +// This means that the Go version of this function expects a *gno.Machine +// as its first parameter. +func (n *NativeFunc) HasMachineParam() bool { + return n.hasMachine +} + +var nativeFuncs = [...]NativeFunc{ { "std", "AssertOriginCall", []gno.FieldTypeExpr{}, []gno.FieldTypeExpr{}, + true, func(m *gno.Machine) { testlibs_std.AssertOriginCall( m, @@ -38,6 +49,7 @@ var nativeFuncs = [...]nativeFunc{ []gno.FieldTypeExpr{ {Name: gno.N("r0"), Type: gno.X("bool")}, }, + true, func(m *gno.Machine) { r0 := testlibs_std.IsOriginCall( m, @@ -57,6 +69,7 @@ var nativeFuncs = [...]nativeFunc{ {Name: gno.N("p0"), Type: gno.X("int64")}, }, []gno.FieldTypeExpr{}, + true, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -76,6 +89,7 @@ var nativeFuncs = [...]nativeFunc{ "ClearStoreCache", []gno.FieldTypeExpr{}, []gno.FieldTypeExpr{}, + true, func(m *gno.Machine) { testlibs_std.ClearStoreCache( m, @@ -91,6 +105,7 @@ var nativeFuncs = [...]nativeFunc{ []gno.FieldTypeExpr{ {Name: gno.N("r0"), Type: gno.X("string")}, }, + true, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -118,6 +133,7 @@ var nativeFuncs = [...]nativeFunc{ {Name: gno.N("p0"), Type: gno.X("string")}, }, []gno.FieldTypeExpr{}, + true, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -139,6 +155,7 @@ var nativeFuncs = [...]nativeFunc{ {Name: gno.N("p0"), Type: gno.X("string")}, }, []gno.FieldTypeExpr{}, + true, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -161,6 +178,7 @@ var nativeFuncs = [...]nativeFunc{ {Name: gno.N("p1"), Type: gno.X("string")}, }, []gno.FieldTypeExpr{}, + true, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -188,6 +206,7 @@ var nativeFuncs = [...]nativeFunc{ {Name: gno.N("p3"), Type: gno.X("[]int64")}, }, []gno.FieldTypeExpr{}, + true, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -220,6 +239,7 @@ var nativeFuncs = [...]nativeFunc{ {Name: gno.N("p2"), Type: gno.X("[]int64")}, }, []gno.FieldTypeExpr{}, + true, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -250,6 +270,7 @@ var nativeFuncs = [...]nativeFunc{ {Name: gno.N("r0"), Type: gno.X("string")}, {Name: gno.N("r1"), Type: gno.X("string")}, }, + true, func(m *gno.Machine) { b := m.LastBlock() var ( @@ -282,6 +303,7 @@ var nativeFuncs = [...]nativeFunc{ []gno.FieldTypeExpr{ {Name: gno.N("r0"), Type: gno.X("int64")}, }, + false, func(m *gno.Machine) { r0 := testlibs_testing.X_unixNano() diff --git a/misc/genstd/template.tmpl b/misc/genstd/template.tmpl index f2cad0a851b..bfbe252a2d5 100644 --- a/misc/genstd/template.tmpl +++ b/misc/genstd/template.tmpl @@ -12,15 +12,25 @@ import ( {{- end }} ) -type nativeFunc struct { - gnoPkg string - gnoFunc gno.Name - params []gno.FieldTypeExpr - results []gno.FieldTypeExpr - f func(m *gno.Machine) +// NativeFunc represents a function in the standard library which has a native +// (go-based) implementation, commonly referred to as a "native binding". +type NativeFunc struct { + gnoPkg string + gnoFunc gno.Name + params []gno.FieldTypeExpr + results []gno.FieldTypeExpr + hasMachine bool + f func(m *gno.Machine) } -var nativeFuncs = [...]nativeFunc{ +// HasMachineParam returns whether the given native binding has a machine parameter. +// This means that the Go version of this function expects a *gno.Machine +// as its first parameter. +func (n *NativeFunc) HasMachineParam() bool { + return n.hasMachine +} + +var nativeFuncs = [...]NativeFunc{ {{- range $i, $m := .Mappings }} { {{ printf "%q" $m.GnoImportPath }}, @@ -36,6 +46,7 @@ var nativeFuncs = [...]nativeFunc{ {Name: gno.N("r{{ $i }}"), Type: gno.X({{ printf "%q" $r.GnoType }})}, {{- end }} }, + {{ if $m.MachineParam }}true{{ else }}false{{ end }}, func(m *gno.Machine) { {{ if $m.Params -}} b := m.LastBlock()