Goal: Source mapping PC to TEAL

cmd/goal/clerk.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"encoding/base64"
+	"encoding/json"
 	"fmt"
 	"io"
 	"os"
@@ -49,6 +50,7 @@ var (
 	rejectsFilename string
 	closeToAddress  string
 	noProgramOutput bool
+	writeSourceMap  bool
 	signProgram     bool
 	programSource   string
 	argB64Strings   []string
@@ -123,6 +125,7 @@ func init() {
 
 	compileCmd.Flags().BoolVarP(&disassemble, "disassemble", "D", false, "disassemble a compiled program")
 	compileCmd.Flags().BoolVarP(&noProgramOutput, "no-out", "n", false, "don't write contract program binary")
+	compileCmd.Flags().BoolVarP(&writeSourceMap, "map", "m", false, "write out source map")
 	compileCmd.Flags().BoolVarP(&signProgram, "sign", "s", false, "sign program, output is a binary signed LogicSig record")
 	compileCmd.Flags().StringVarP(&outFilename, "outfile", "o", "", "Filename to write program bytes or signed LogicSig to")
 	compileCmd.Flags().StringVarP(&account, "account", "a", "", "Account address to sign the program (If not specified, uses default account)")
@@ -935,7 +938,7 @@ func mustReadFile(fname string) []byte {
 	return contents
 }
 
-func assembleFile(fname string, printWarnings bool) (program []byte) {
+func assembleFileImpl(fname string, printWarnings bool) *logic.OpStream {
 	text, err := readFile(fname)
 	if err != nil {
 		reportErrorf("%s: %s", fname, err)
@@ -967,9 +970,19 @@ func assembleFile(fname string, printWarnings bool) (program []byte) {
 		reportWarnRawf("%d warning%s", len(ops.Warnings), plural)
 	}
 
+	return ops
+}
+
+func assembleFile(fname string, printWarnings bool) (program []byte) {
+	ops := assembleFileImpl(fname, printWarnings)
 	return ops.Program
 }
 
+func assembleFileWithMap(fname string, printWarnings bool) ([]byte, logic.SourceMap) {
+	ops := assembleFileImpl(fname, printWarnings)
+	return ops.Program, logic.GetSourceMap([]string{fname}, ops.OffsetToLine)
+}
+
 func disassembleFile(fname, outname string) {
 	program, err := readFile(fname)
 	if err != nil {
@@ -1025,7 +1038,7 @@ var compileCmd = &cobra.Command{
 				}
 			}
 			shouldPrintAdditionalInfo := outname != stdoutFilenameValue
-			program := assembleFile(fname, true)
+			program, sourceMap := assembleFileWithMap(fname, true)
 			outblob := program
 			if signProgram {
 				dataDir := ensureSingleDataDir()
@@ -1056,6 +1069,17 @@ var compileCmd = &cobra.Command{
 					reportErrorf("%s: %s", outname, err)
 				}
 			}
+			if writeSourceMap {
+				mapname := fname + ".map"
+				pcblob, err := json.Marshal(sourceMap)
+				if err != nil {
+					reportErrorf("%s: %s", mapname, err)
+				}
+				err = writeFile(mapname, pcblob, 0666)
+				if err != nil {
+					reportErrorf("%s: %s", mapname, err)
+				}
+			}
 			if !signProgram && shouldPrintAdditionalInfo {
 				pd := logic.HashProgram(program)
 				addr := basics.Address(pd)

cmd/tealdbg/debugger.go

@@ -368,14 +368,14 @@ func (s *session) GetSourceMap() ([]byte, error) {
 	prevSourceLine := 0
 
 	// the very first entry is needed by CDT
-	lines[0] = MakeSourceMapLine(targetCol, sourceIdx, 0, sourceCol)
+	lines[0] = logic.MakeSourceMapLine(targetCol, sourceIdx, 0, sourceCol)
 	for targetLine := 1; targetLine < len(s.lines); targetLine++ {
 		if pc, ok := s.pcOffset[targetLine]; ok && pc != 0 {
 			sourceLine, ok = s.offsetToLine[pc]
 			if !ok {
 				lines[targetLine] = ""
 			} else {
-				lines[targetLine] = MakeSourceMapLine(targetCol, sourceIdx, sourceLine-prevSourceLine, sourceCol)
+				lines[targetLine] = logic.MakeSourceMapLine(targetCol, sourceIdx, sourceLine-prevSourceLine, sourceCol)
 				prevSourceLine = sourceLine
 			}
 		} else {
@@ -384,7 +384,7 @@ func (s *session) GetSourceMap() ([]byte, error) {
 			if targetLine == len(s.lines)-1 {
 				delta = 1
 			}
-			lines[targetLine] = MakeSourceMapLine(targetCol, sourceIdx, delta, sourceCol)
+			lines[targetLine] = logic.MakeSourceMapLine(targetCol, sourceIdx, delta, sourceCol)
 		}
 	}
 

cmd/tealdbg/util.go

@@ -17,7 +17,6 @@
 package main
 
 import (
-	"bytes"
 	"strconv"
 	"sync/atomic"
 )
@@ -100,29 +99,3 @@ func IsTextFile(data []byte) bool {
 	}
 	return printable
 }
-
-const b64table string = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
-
-// IntToVLQ writes out value to bytes.Buffer
-func IntToVLQ(v int, buf *bytes.Buffer) {
-	v <<= 1
-	if v < 0 {
-		v = -v
-		v |= 1
-	}
-	for v >= 32 {
-		buf.WriteByte(b64table[32|(v&31)])
-		v >>= 5
-	}
-	buf.WriteByte(b64table[v])
-}
-
-// MakeSourceMapLine creates source map mapping's line entry
-func MakeSourceMapLine(tcol, sindex, sline, scol int) string {
-	buf := bytes.NewBuffer(nil)
-	IntToVLQ(tcol, buf)
-	IntToVLQ(sindex, buf)
-	IntToVLQ(sline, buf)
-	IntToVLQ(scol, buf)
-	return buf.String()
-}

data/transactions/logic/sourcemap.go

@@ -0,0 +1,95 @@
+// Copyright (C) 2019-2022 Algorand, Inc.
+// This file is part of go-algorand
+//
+// go-algorand is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// go-algorand is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with go-algorand.  If not, see <https://www.gnu.org/licenses/>.
+
+package logic
+
+import (
+	"bytes"
+	"strings"
+)
+
+// sourceMapVersion is currently 3.
+// Refer to the full specs of sourcemap here: https://sourcemaps.info/spec.html
+const sourceMapVersion = 3
+const b64table string = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
+
+// SourceMap contains details from the source to assembly process.
+// Currently contains the map between TEAL source line to
+// the assembled bytecode position and details about
+// the template variables contained in the source file.
+type SourceMap struct {
+	Version    int      `json:"version"`
+	File       string   `json:"file,omitempty"`
+	SourceRoot string   `json:"sourceRoot,omitempty"`
+	Sources    []string `json:"sources"`
+	Names      []string `json:"names"`
+	Mapping    string   `json:"mapping"`
+}
+
+// GetSourceMap returns a struct containing details about
+// the assembled file and encoded mappings to the source file.
+func GetSourceMap(sourceNames []string, offsetToLine map[int]int) SourceMap {
+	maxPC := 0
+	for pc := range offsetToLine {
+		if pc > maxPC {
+			maxPC = pc
+		}
+	}
+
+	// Array where index is the PC and value is the line.
+	pcToLine := make([]string, maxPC+1)
+	for pc := range pcToLine {
+		if line, ok := offsetToLine[pc]; ok {
+			pcToLine[pc] = MakeSourceMapLine(0, 0, line, 0)
+		} else {
+			pcToLine[pc] = ""
+		}
+	}
+
+	// Encode the source map into a string
+	encodedMapping := strings.Join(pcToLine, ";")
+
+	return SourceMap{
+		Version: sourceMapVersion,
+		Sources: sourceNames,
+		Names:   []string{}, // TEAL code does not generate any names.
+		Mapping: encodedMapping,
+	}
+}
+
+// intToVLQ writes out value to bytes.Buffer
+func intToVLQ(v int, buf *bytes.Buffer) {
+	v <<= 1
+	if v < 0 {
+		v = -v
+		v |= 1
+	}
+	for v >= 32 {
+		buf.WriteByte(b64table[32|(v&31)])
+		v >>= 5
+	}
+	buf.WriteByte(b64table[v])
+}
+
+// MakeSourceMapLine creates source map mapping's line entry
+func MakeSourceMapLine(tcol, sindex, sline, scol int) string {
+	buf := bytes.NewBuffer(nil)
+	intToVLQ(tcol, buf)
+	intToVLQ(sindex, buf)
+	intToVLQ(sline, buf)
+	intToVLQ(scol, buf)
+	return buf.String()
+}

cmd/tealdbg/util_test.go → data/transactions/logic/sourcemap_test.go

@@ -14,15 +14,43 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with go-algorand.  If not, see <https://www.gnu.org/licenses/>.
 
-package main
+package logic
 
 import (
+	"strings"
 	"testing"
 
 	"github.com/algorand/go-algorand/test/partitiontest"
 	"github.com/stretchr/testify/require"
 )
 
+func TestGetSourceMap(t *testing.T) {
+	partitiontest.PartitionTest(t)
+	a := require.New(t)
+
+	sourceNames := []string{"test.teal"}
+	offsetToLine := map[int]int{
+		1: 1,
+		2: 2,
+		5: 3,
+	}
+	actualSourceMap := GetSourceMap(sourceNames, offsetToLine)
+
+	a.Equal(sourceMapVersion, actualSourceMap.Version)
+	a.Equal(sourceNames, actualSourceMap.Sources)
+	a.Equal([]string{}, actualSourceMap.Names)
+
+	// Check encoding for each line.
+	splitMapping := strings.Split(actualSourceMap.Mapping, ";")
+	for pc := range splitMapping {
+		if line, ok := offsetToLine[pc]; ok {
+			a.Equal(MakeSourceMapLine(0, 0, line, 0), splitMapping[pc])
+		} else {
+			a.Equal("", splitMapping[pc])
+		}
+	}
+}
+
 func TestVLQ(t *testing.T) {
 	partitiontest.PartitionTest(t)
 	a := require.New(t)

data/transactions/logic/tlhc.py

@@ -69,11 +69,11 @@ def main():
     out.write(code)
     try:
         out.close()
-    except Exeption as e:
+    except Exception as e:
         print(e)
     try:
         secretout.close()
-    except Exeption as e:
+    except Exception as e:
         print(e)
     return
 

test/scripts/e2e_subs/e2e-teal.sh

@@ -11,6 +11,8 @@ WALLET=$1
 
 gcmd="goal -w ${WALLET}"
 
+TEAL=test/scripts/e2e_subs/tealprogs
+
 ACCOUNT=$(${gcmd} account list|awk '{ print $3 }')
 
 # prints:
@@ -148,6 +150,16 @@ ${gcmd} clerk compile ${TEMPDIR}/true3.teal -o ${TEMPDIR}/true3.lsig
 cp ${TEMPDIR}/true3.lsig ${TEMPDIR}/true2.lsig
 printf '\x02' | dd of=${TEMPDIR}/true2.lsig bs=1 seek=0 count=1 conv=notrunc
 
+# Try to compile with source map, and check that map is correct.
+# Since the source map contains info about the file path,
+# we do this in place and clean up the file later.
+${gcmd} clerk compile ${TEAL}/quine.teal -m
+trap 'rm ${TEAL}/quine.teal.*' EXIT
+if ! diff ${TEAL}/quine.map ${TEAL}/quine.teal.map; then
+    echo "produced source maps do not match"
+    exit 1
+fi
+
 # compute the escrow account for the frankenstein program
 ACCOUNT_TRUE=$(python -c 'import algosdk, sys; print(algosdk.logic.address(sys.stdin.buffer.read()))' < ${TEMPDIR}/true2.lsig)
 # fund that escrow account