Merge pull request #3738 from grafana/feature/3703-typescript-support

TypeScript and ES6+ support using esbuild.

cmd/runtime_options.go

@@ -23,10 +23,11 @@ func runtimeOptionFlagSet(includeSysEnv bool) *pflag.FlagSet {
 	flags.SortFlags = false
 	flags.Bool("include-system-env-vars", includeSysEnv, "pass the real system environment variables to the runtime")
 	flags.String("compatibility-mode", "extended",
-		`JavaScript compiler compatibility mode, "extended" or "base"
+		`JavaScript compiler compatibility mode, "extended" or "base" or "experimental_enhanced"
 base: pure goja - Golang JS VM supporting ES5.1+
 extended: base + Babel with parts of ES2015 preset
 		  slower to compile in case the script uses syntax unsupported by base
+experimental_enhanced: esbuild-based transpiling for TypeScript and ES6+ support
 `)
 	flags.StringP("type", "t", "", "override test type, \"js\" or \"archive\"")
 	flags.StringArrayP("env", "e", nil, "add/override environment variable with `VAR=value`")

cmd/runtime_options_test.go

@@ -109,6 +109,7 @@ func TestRuntimeOptions(t *testing.T) {
 		defaultCompatMode   = null.NewString("extended", false)
 		baseCompatMode      = null.NewString("base", true)
 		extendedCompatMode  = null.NewString("extended", true)
+		enhancedCompatMode  = null.NewString("experimental_enhanced", true)
 		defaultTracesOutput = null.NewString("none", false)
 	)
 
@@ -143,6 +144,16 @@ func TestRuntimeOptions(t *testing.T) {
 				TracesOutput:         defaultTracesOutput,
 			},
 		},
+		"disabled sys env by default with experimental_enhanced compat mode": {
+			useSysEnv: false,
+			systemEnv: map[string]string{"test1": "val1", "K6_COMPATIBILITY_MODE": "experimental_enhanced"},
+			expRTOpts: lib.RuntimeOptions{
+				IncludeSystemEnvVars: null.NewBool(false, false),
+				CompatibilityMode:    enhancedCompatMode,
+				Env:                  map[string]string{},
+				TracesOutput:         defaultTracesOutput,
+			},
+		},
 		"disabled sys env by cli 1": {
 			useSysEnv: true,
 			systemEnv: map[string]string{"test1": "val1", "K6_COMPATIBILITY_MODE": "base"},

examples/enhanced/abort.ts

@@ -0,0 +1,5 @@
+import exec from "k6/execution";
+
+export default function () {
+	exec.test.abort("failed");
+}

examples/enhanced/script.ts

@@ -0,0 +1,6 @@
+import { User, newUser } from "./user.ts";
+
+export default () => {
+	const user: User = newUser("John");
+	console.log(user);
+};

examples/enhanced/user.ts

@@ -0,0 +1,20 @@
+interface User {
+	name: string;
+	id: number;
+}
+
+class UserAccount implements User {
+	name: string;
+	id: number;
+
+	constructor(name: string) {
+		this.name = name;
+		this.id = Math.floor(Math.random() * Number.MAX_SAFE_INTEGER);
+	}
+}
+
+function newUser(name: string): User {
+	return new UserAccount(name);
+}
+
+export { User, newUser };

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/Soontao/goHttpDigestClient v0.0.0-20170320082612-6d28bb1415c5
 	github.com/andybalholm/brotli v1.1.0
 	github.com/dop251/goja v0.0.0-20240220182346-e401ed450204
+	github.com/evanw/esbuild v0.21.2
 	github.com/fatih/color v1.16.0
 	github.com/go-sourcemap/sourcemap v2.1.4+incompatible
 	github.com/golang/protobuf v1.5.4

go.sum

@@ -57,6 +57,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/evanw/esbuild v0.21.2 h1:CLplcGi794CfHLVmUbvVfTMKkykm+nyIHU8SU60KUTA=
+github.com/evanw/esbuild v0.21.2/go.mod h1:D2vIQZqV/vIf/VRHtViaUtViZmG7o+kKmlBfVQuRi48=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=

js/bundle_test.go

@@ -159,7 +159,7 @@ func TestNewBundle(t *testing.T) {
 			}{
 				{
 					"InvalidCompat", "es1", `export default function() {};`,
-					`invalid compatibility mode "es1". Use: "extended", "base"`,
+					`invalid compatibility mode "es1". Use: "extended", "base", "experimental_enhanced"`,
 				},
 				// ES2015 modules are not supported
 				{

js/compiler/compiler.go

@@ -239,19 +239,33 @@ func (c *Compiler) compileImpl(
 		c.logger.WithError(state.srcMapError).Warnf("Couldn't load source map for %s", filename)
 		ast, err = parser.ParseFile(nil, filename, code, 0, parser.WithDisableSourceMaps)
 	}
-	if err != nil {
-		if compatibilityMode == lib.CompatibilityModeExtended {
-			code, state.srcMap, err = c.Transform(src, filename, state.srcMap)
-			if err != nil {
-				return nil, code, err
-			}
-			// the compatibility mode "decreases" here as we shouldn't transform twice
-			return c.compileImpl(code, filename, wrap, lib.CompatibilityModeBase, state.srcMap)
+
+	if err == nil {
+		pgm, err := goja.CompileAST(ast, c.Options.Strict)
+		return pgm, code, err
+	}
+
+	if compatibilityMode == lib.CompatibilityModeExtended {
+		code, state.srcMap, err = c.Transform(src, filename, state.srcMap)
+		if err != nil {
+			return nil, code, err
+		}
+		// the compatibility mode "decreases" here as we shouldn't transform twice
+		return c.compileImpl(code, filename, wrap, lib.CompatibilityModeBase, state.srcMap)
+	}
+
+	if compatibilityMode == lib.CompatibilityModeExperimentalEnhanced {
+		code, state.srcMap, err = esbuildTransform(src, filename)
+		if err != nil {
+			return nil, code, err
+		}
+		if c.Options.SourceMapLoader != nil {
+			// This hack is required for the source map to work
+			code += "\n//# sourceMappingURL=" + sourceMapURLFromBabel
 		}
-		return nil, code, err
+		return c.compileImpl(code, filename, wrap, lib.CompatibilityModeBase, state.srcMap)
 	}
-	pgm, err := goja.CompileAST(ast, c.Options.Strict)
-	return pgm, code, err
+	return nil, code, err
 }
 
 type babel struct {

js/compiler/enhanced.go

@@ -0,0 +1,55 @@
+package compiler
+
+import (
+	"path/filepath"
+
+	"github.com/dop251/goja/file"
+	"github.com/dop251/goja/parser"
+	"github.com/evanw/esbuild/pkg/api"
+)
+
+func esbuildTransform(src, filename string) (code string, srcMap []byte, err error) {
+	opts := api.TransformOptions{
+		Sourcefile:     filename,
+		Loader:         api.LoaderJS,
+		Target:         api.ESNext,
+		Format:         api.FormatCommonJS,
+		Sourcemap:      api.SourceMapExternal,
+		SourcesContent: api.SourcesContentInclude,
+		LegalComments:  api.LegalCommentsNone,
+		Platform:       api.PlatformNeutral,
+		LogLevel:       api.LogLevelSilent,
+		Charset:        api.CharsetUTF8,
+	}
+
+	if filepath.Ext(filename) == ".ts" {
+		opts.Loader = api.LoaderTS
+	}
+
+	result := api.Transform(src, opts)
+
+	if hasError, err := esbuildCheckError(&result); hasError {
+		return "", nil, err
+	}
+
+	return string(result.Code), result.Map, nil
+}
+
+func esbuildCheckError(result *api.TransformResult) (bool, error) {
+	if len(result.Errors) == 0 {
+		return false, nil
+	}
+
+	msg := result.Errors[0]
+	err := &parser.Error{Message: msg.Text}
+
+	if msg.Location != nil {
+		err.Position = file.Position{
+			Filename: msg.Location.File,
+			Line:     msg.Location.Line,
+			Column:   msg.Location.Column,
+		}
+	}
+
+	return true, err
+}

js/compiler/enhanced_test.go

@@ -0,0 +1,97 @@
+package compiler
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/dop251/goja"
+	"github.com/dop251/goja/parser"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.k6.io/k6/lib"
+	"go.k6.io/k6/lib/testutils"
+)
+
+func Test_esbuildTransform_js(t *testing.T) {
+	t.Parallel()
+
+	code, srcMap, err := esbuildTransform(`export default function(name) { return "Hello, " + name }`, "script.js")
+
+	require.NoError(t, err)
+	require.NotNil(t, srcMap)
+	require.NotEmpty(t, code)
+}
+
+func Test_esbuildTransform_ts(t *testing.T) {
+	t.Parallel()
+
+	script := `export function hello(name:string) : string { return "Hello, " + name}`
+
+	code, srcMap, err := esbuildTransform(script, "script.ts")
+
+	require.NoError(t, err)
+	require.NotNil(t, srcMap)
+	require.NotEmpty(t, code)
+}
+
+func Test_esbuildTransform_error(t *testing.T) {
+	t.Parallel()
+
+	script := `export function hello(name:string) : string { return "Hello, " + name}`
+
+	_, _, err := esbuildTransform(script, "script.js")
+
+	require.Error(t, err)
+
+	var perr *parser.Error
+
+	require.True(t, errors.As(err, &perr))
+	require.NotNil(t, perr.Position)
+	require.Equal(t, "script.js", perr.Position.Filename)
+	require.Equal(t, 1, perr.Position.Line)
+	require.Equal(t, 26, perr.Position.Column)
+	require.Equal(t, "Expected \")\" but found \":\"", perr.Message)
+}
+
+func TestCompile_experimental_enhanced(t *testing.T) {
+	t.Parallel()
+
+	t.Run("experimental_enhanced Invalid", func(t *testing.T) {
+		t.Parallel()
+		c := New(testutils.NewLogger(t))
+		src := `1+(function() { return 2; )()`
+		c.Options.CompatibilityMode = lib.CompatibilityModeExperimentalEnhanced
+		_, _, err := c.Compile(src, "script.js", false)
+		assert.IsType(t, &parser.Error{}, err)
+		assert.Contains(t, err.Error(), `script.js: Line 1:26 Unexpected ")"`)
+	})
+	t.Run("experimental_enhanced", func(t *testing.T) {
+		t.Parallel()
+		c := New(testutils.NewLogger(t))
+		c.Options.CompatibilityMode = lib.CompatibilityModeExperimentalEnhanced
+		pgm, code, err := c.Compile(`import "something"`, "script.js", true)
+		require.NoError(t, err)
+		assert.Equal(t, `var import_something = require("something");
+`, code)
+		rt := goja.New()
+		var requireCalled bool
+		require.NoError(t, rt.Set("require", func(s string) {
+			assert.Equal(t, "something", s)
+			requireCalled = true
+		}))
+		_, err = rt.RunProgram(pgm)
+		require.NoError(t, err)
+		require.True(t, requireCalled)
+	})
+	t.Run("experimental_enhanced sourcemap", func(t *testing.T) {
+		t.Parallel()
+		c := New(testutils.NewLogger(t))
+		c.Options.CompatibilityMode = lib.CompatibilityModeExperimentalEnhanced
+		c.Options.SourceMapLoader = func(_ string) ([]byte, error) { return nil, nil }
+		_, code, err := c.Compile(`import "something"`, "script.js", true)
+		require.NoError(t, err)
+		assert.Equal(t, `var import_something = require("something");
+
+//# sourceMappingURL=k6://internal-should-not-leak/file.map`, code)
+	})
+}

js/tc39/README.md

@@ -1,19 +1,17 @@
 # Introduction to a k6's TC39 testing
 
-The point of this module is to test k6 goja+babel combo against the tc39 test suite.
+The point of this module is to test k6 goja+babel and k6 goja+esbuild combo against the tc39 test suite.
 
 Ways to use it:
 1. run ./checkout.sh to checkout the last commit sha of [test262](https://github.com/tc39/test262)
    that was tested with this module
 2. Run `go test &> out.log`
 
-If there are failures there will be a JSON with what failed.
-The full list of failing tests, and the error, is in `breaking_test_errors.json`. All errors list there with the corresponding error will *not* be counted as errors - this is what the test expects, those specific errors.
+The full list of failing tests, and the error, is in `breaking_test_errors-*.json`. All errors list there with the corresponding error will *not* be counted as errors - this is what the test expects, those specific errors.
 Due to changes to goja it is not uncommon for the error to change, or there to be now a new error on previously passing test, or (hopefully) a test that was not passing but now is.
-In all of those cases `breaking_test_errors.json` needs to be updated. Currently, the output is the *difference* in errors, so we need to "null" the file. To that we set it to an empty JSON `echo '{}' > breaking_test_errors.json`.
-Run the test with output to a file `go test &> breaking_test_errors.json`. And then edit `out.log` so only the JSON is left. I personally search for `FAIL` and that should be the first thing just *after* the JSON, delete till the end of file. This is easiest done with sed(or vim) as in `sed -i '/FAIL/,$d' breaking_test_errors.json`.
+In all of those cases `breaking_test_errors-*.json` needs to be updated. Run the test with `-update` flag to update: `go test -update`
 
-NOTE: some text editors/IDEs will try to parse files ending in `json` as JSON, which given the size of `breaking_test_errors.json` might be a problem when it's not actually a JSON (before the edit). So it might be a better idea to name it something different if editing by hand and fix it later.
+NOTE: some text editors/IDEs will try to parse files ending in `json` as JSON, which given the size of `breaking_test_errors-*.json` might be a problem when it's not actually a JSON (before the edit). So it might be a better idea to name it something different if editing by hand and fix it later.
 
 This is a modified version of [the code in the goja
 repo](https://github.com/dop251/goja/blob/master/tc39_test.go)