Op base64 decode (#3220)

b64 opcode, tests, and specs

.gitignore

@@ -65,3 +65,5 @@ assets
 
 index.html
 
+# test summary
+testresults.json

data/transactions/logic/README.md

@@ -165,6 +165,7 @@ various sizes.
 | `extract_uint16` | pop a byte-array A and integer B. Extract a range of bytes from A starting at B up to but not including B+2, convert bytes as big endian and push the uint64 result. If B+2 is larger than the array length, the program fails |
 | `extract_uint32` | pop a byte-array A and integer B. Extract a range of bytes from A starting at B up to but not including B+4, convert bytes as big endian and push the uint64 result. If B+4 is larger than the array length, the program fails |
 | `extract_uint64` | pop a byte-array A and integer B. Extract a range of bytes from A starting at B up to but not including B+8, convert bytes as big endian and push the uint64 result. If B+8 is larger than the array length, the program fails |
+| `base64_decode e` | decode X which was base64-encoded using _encoding alphabet_ E. Fail if X is not base64 encoded with alphabet E |
 
 These opcodes take byte-array values that are interpreted as
 big-endian unsigned integers.  For mathematical operators, the

data/transactions/logic/TEAL_opcodes.md

@@ -854,6 +854,17 @@ When A is a uint64, index 0 is the least significant bit. Setting bit 3 to 1 on
 - pop a byte-array A and integer B. Extract a range of bytes from A starting at B up to but not including B+8, convert bytes as big endian and push the uint64 result. If B+8 is larger than the array length, the program fails
 - LogicSigVersion >= 5
 
+## base64_decode e
+
+- Opcode: 0x5c {uint8 alphabet index}
+- Pops: *... stack*, []byte
+- Pushes: []byte
+- decode X which was base64-encoded using _encoding alphabet_ E. Fail if X is not base64 encoded with alphabet E
+- **Cost**: 25
+- LogicSigVersion >= 6
+
+decodes X using the base64 encoding alphabet E. Specify the alphabet with an immediate arg either as URL and Filename Safe (`URLAlph`) or Standard (`StdAlph`). See <a href="https://rfc-editor.org/rfc/rfc4648.html#section-4">RFC 4648</a> (sections 4 and 5)
+
 ## balance
 
 - Opcode: 0x60

data/transactions/logic/assembler.go

@@ -1244,6 +1244,28 @@ func assembleEcdsa(ops *OpStream, spec *OpSpec, args []string) error {
 	return nil
 }
 
+func assembleBase64Decode(ops *OpStream, spec *OpSpec, args []string) error {
+	if len(args) != 1 {
+		return ops.errorf("%s expects one argument", spec.Name)
+	}
+
+	alph, ok := base64AlphabetSpecByName[args[0]]
+	if !ok {
+		return ops.errorf("%s unknown alphabet: %#v", spec.Name, args[0])
+	}
+	if alph.version > ops.Version {
+		//nolint:errcheck // we continue to maintain typestack
+		ops.errorf("%s %s available in version %d. Missed #pragma version?", spec.Name, args[0], alph.version)
+	}
+
+	val := alph.field
+	ops.pending.WriteByte(spec.Opcode)
+	ops.pending.WriteByte(uint8(val))
+	ops.trace("%s (%s)", alph.field, alph.ftype)
+	ops.returns(alph.ftype)
+	return nil
+}
+
 type assembleFunc func(*OpStream, *OpSpec, []string) error
 
 // Basic assembly. Any extra bytes of opcode are encoded as byte immediates.
@@ -2668,6 +2690,20 @@ func disEcdsa(dis *disassembleState, spec *OpSpec) (string, error) {
 	return fmt.Sprintf("%s %s", spec.Name, EcdsaCurveNames[arg]), nil
 }
 
+func disBase64Decode(dis *disassembleState, spec *OpSpec) (string, error) {
+	lastIdx := dis.pc + 1
+	if len(dis.program) <= lastIdx {
+		missing := lastIdx - len(dis.program) + 1
+		return "", fmt.Errorf("unexpected %s opcode end: missing %d bytes", spec.Name, missing)
+	}
+	dis.nextpc = dis.pc + 2
+	b64dArg := dis.program[dis.pc+1]
+	if int(b64dArg) >= len(base64AlphabetNames) {
+		return "", fmt.Errorf("invalid base64_decode arg index %d at pc=%d", b64dArg, dis.pc)
+	}
+	return fmt.Sprintf("%s %s", spec.Name, base64AlphabetNames[b64dArg]), nil
+}
+
 type disInfo struct {
 	pcOffset       []PCOffset
 	hasStatefulOps bool

data/transactions/logic/assembler_test.go

@@ -344,6 +344,7 @@ itxna Logs 3
 
 const v6Nonsense = v5Nonsense + `
 itxn_next
+base64_decode URLAlph
 `
 
 var nonsense = map[uint64]string{
@@ -361,7 +362,7 @@ var compiled = map[uint64]string{
 	3: "032008b7a60cf8acd19181cf959a12f8acd19181cf951af8acd19181cf15f8acd191810f01020026050212340c68656c6c6f20776f726c6421208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d024242047465737400320032013202320328292929292a0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e0102222324252104082209240a220b230c240d250e230f23102311231223132314181b1c2b171615400003290349483403350222231d4a484848482a50512a63222352410003420000432105602105612105270463484821052b62482b642b65484821052b2106662b21056721072b682b692107210570004848210771004848361c0037001a0031183119311b311d311e311f3120210721051e312131223123312431253126312731283129312a312b312c312d312e312f4478222105531421055427042106552105082106564c4d4b02210538212106391c0081e80780046a6f686e",
 	4: "042004010200b7a60c26040242420c68656c6c6f20776f726c6421208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d047465737400320032013202320380021234292929292a0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e01022581f8acd19181cf959a1281f8acd19181cf951a81f8acd19181cf1581f8acd191810f082209240a220b230c240d250e230f23102311231223132314181b1c28171615400003290349483403350222231d4a484848482a50512a632223524100034200004322602261222b634848222862482864286548482228236628226724286828692422700048482471004848361c0037001a0031183119311b311d311e311f312024221e312131223123312431253126312731283129312a312b312c312d312e312f44782522531422542b2355220823564c4d4b0222382123391c0081e80780046a6f686e2281d00f24231f880003420001892223902291922394239593a0a1a2a3a4a5a6a7a8a9aaabacadae23af3a00003b003c003d8164",
 	5: "052004010002b7a60c26050242420c68656c6c6f20776f726c6421070123456789abcd208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d047465737400320032013202320380021234292929292b0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e01022581f8acd19181cf959a1281f8acd19181cf951a81f8acd19181cf1581f8acd191810f082209240a220b230c240d250e230f23102311231223132314181b1c28171615400003290349483403350222231d4a484848482b50512a632223524100034200004322602261222704634848222862482864286548482228246628226723286828692322700048482371004848361c0037001a0031183119311b311d311e311f312023221e312131223123312431253126312731283129312a312b312c312d312e312f447825225314225427042455220824564c4d4b0222382124391c0081e80780046a6f686e2281d00f23241f880003420001892224902291922494249593a0a1a2a3a4a5a6a7a8a9aaabacadae24af3a00003b003c003d816472064e014f012a57000823810858235b235a2359b03139330039b1b200b322c01a23c1001a2323c21a23c3233e233f8120af06002a494905002a49490700b53a03",
-	6: "062004010002b7a60c26050242420c68656c6c6f20776f726c6421070123456789abcd208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d047465737400320032013202320380021234292929292b0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e01022581f8acd19181cf959a1281f8acd19181cf951a81f8acd19181cf1581f8acd191810f082209240a220b230c240d250e230f23102311231223132314181b1c28171615400003290349483403350222231d4a484848482b50512a632223524100034200004322602261222704634848222862482864286548482228246628226723286828692322700048482371004848361c0037001a0031183119311b311d311e311f312023221e312131223123312431253126312731283129312a312b312c312d312e312f447825225314225427042455220824564c4d4b0222382124391c0081e80780046a6f686e2281d00f23241f880003420001892224902291922494249593a0a1a2a3a4a5a6a7a8a9aaabacadae24af3a00003b003c003d816472064e014f012a57000823810858235b235a2359b03139330039b1b200b322c01a23c1001a2323c21a23c3233e233f8120af06002a494905002a49490700b53a03b6",
+	6: "062004010002b7a60c26050242420c68656c6c6f20776f726c6421070123456789abcd208dae2087fbba51304eb02b91f656948397a7946390e8cb70fc9ea4d95f92251d047465737400320032013202320380021234292929292b0431003101310231043105310731083109310a310b310c310d310e310f3111311231133114311533000033000133000233000433000533000733000833000933000a33000b33000c33000d33000e33000f3300113300123300133300143300152d2e01022581f8acd19181cf959a1281f8acd19181cf951a81f8acd19181cf1581f8acd191810f082209240a220b230c240d250e230f23102311231223132314181b1c28171615400003290349483403350222231d4a484848482b50512a632223524100034200004322602261222704634848222862482864286548482228246628226723286828692322700048482371004848361c0037001a0031183119311b311d311e311f312023221e312131223123312431253126312731283129312a312b312c312d312e312f447825225314225427042455220824564c4d4b0222382124391c0081e80780046a6f686e2281d00f23241f880003420001892224902291922494249593a0a1a2a3a4a5a6a7a8a9aaabacadae24af3a00003b003c003d816472064e014f012a57000823810858235b235a2359b03139330039b1b200b322c01a23c1001a2323c21a23c3233e233f8120af06002a494905002a49490700b53a03b65c00",
 }
 
 func pseudoOp(opcode string) bool {

data/transactions/logic/doc.go

@@ -133,6 +133,7 @@ var opDocByName = map[string]string{
 	"extract_uint16": "pop a byte-array A and integer B. Extract a range of bytes from A starting at B up to but not including B+2, convert bytes as big endian and push the uint64 result. If B+2 is larger than the array length, the program fails",
 	"extract_uint32": "pop a byte-array A and integer B. Extract a range of bytes from A starting at B up to but not including B+4, convert bytes as big endian and push the uint64 result. If B+4 is larger than the array length, the program fails",
 	"extract_uint64": "pop a byte-array A and integer B. Extract a range of bytes from A starting at B up to but not including B+8, convert bytes as big endian and push the uint64 result. If B+8 is larger than the array length, the program fails",
+	"base64_decode":  "decode X which was base64-encoded using _encoding alphabet_ E. Fail if X is not base64 encoded with alphabet E",
 
 	"balance":           "get balance for account A, in microalgos. The balance is observed after the effects of previous transactions in the group, and after the fee for the current transaction is deducted.",
 	"min_balance":       "get minimum required balance for account A, in microalgos. Required balance is affected by [ASA](https://developer.algorand.org/docs/features/asa/#assets-overview) and [App](https://developer.algorand.org/docs/features/asc1/stateful/#minimum-balance-requirement-for-a-smart-contract) usage. When creating or opting into an app, the minimum balance grows before the app code runs, therefore the increase is visible there. When deleting or closing out, the minimum balance decreases after the app executes.",
@@ -229,6 +230,8 @@ var opcodeImmediateNotes = map[string]string{
 	"ecdsa_verify":        "{uint8 curve index}",
 	"ecdsa_pk_decompress": "{uint8 curve index}",
 	"ecdsa_pk_recover":    "{uint8 curve index}",
+
+	"base64_decode": "{uint8 alphabet index}",
 }
 
 // OpImmediateNote returns a short string about immediate data which follows the op byte
@@ -283,6 +286,7 @@ var opDocExtras = map[string]string{
 	"itxn_begin":          "`itxn_begin` initializes Sender to the application address; Fee to the minimum allowable, taking into account MinTxnFee and credit from overpaying in earlier transactions; FirstValid/LastValid to the values in the top-level transaction, and all other fields to zero values.",
 	"itxn_field":          "`itxn_field` fails if X is of the wrong type for F, including a byte array of the wrong size for use as an address when F is an address field. `itxn_field` also fails if X is an account or asset that does not appear in `txn.Accounts` or `txn.ForeignAssets` of the top-level transaction. (Setting addresses in asset creation are exempted from this requirement.)",
 	"itxn_submit":         "`itxn_submit` resets the current transaction so that it can not be resubmitted. A new `itxn_begin` is required to prepare another inner transaction.",
+	"base64_decode":       "decodes X using the base64 encoding alphabet E. Specify the alphabet with an immediate arg either as URL and Filename Safe (`URLAlph`) or Standard (`StdAlph`). See <a href=\"https://rfc-editor.org/rfc/rfc4648.html#section-4\">RFC 4648</a> (sections 4 and 5)",
 }
 
 // OpDocExtra returns extra documentation text about an op
@@ -295,7 +299,7 @@ func OpDocExtra(opName string) string {
 // opcodes consecutively, even if their opcode values are not.
 var OpGroups = map[string][]string{
 	"Arithmetic":            {"sha256", "keccak256", "sha512_256", "ed25519verify", "ecdsa_verify", "ecdsa_pk_recover", "ecdsa_pk_decompress", "+", "-", "/", "*", "<", ">", "<=", ">=", "&&", "||", "shl", "shr", "sqrt", "bitlen", "exp", "==", "!=", "!", "len", "itob", "btoi", "%", "|", "&", "^", "~", "mulw", "addw", "divmodw", "expw", "getbit", "setbit", "getbyte", "setbyte", "concat"},
-	"Byte Array Slicing":    {"substring", "substring3", "extract", "extract3", "extract_uint16", "extract_uint32", "extract_uint64"},
+	"Byte Array Slicing":    {"substring", "substring3", "extract", "extract3", "extract_uint16", "extract_uint32", "extract_uint64", "base64_decode"},
 	"Byte Array Arithmetic": {"b+", "b-", "b/", "b*", "b<", "b>", "b<=", "b>=", "b==", "b!=", "b%"},
 	"Byte Array Logic":      {"b|", "b&", "b^", "b~"},
 	"Loading Values":        {"intcblock", "intc", "intc_0", "intc_1", "intc_2", "intc_3", "pushint", "bytecblock", "bytec", "bytec_0", "bytec_1", "bytec_2", "bytec_3", "pushbytes", "bzero", "arg", "arg_0", "arg_1", "arg_2", "arg_3", "args", "txn", "gtxn", "txna", "txnas", "gtxna", "gtxnas", "gtxns", "gtxnsa", "gtxnsas", "global", "load", "loads", "store", "stores", "gload", "gloads", "gaid", "gaids"},

data/transactions/logic/eval.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"crypto/sha256"
 	"crypto/sha512"
+	"encoding/base64"
 	"encoding/binary"
 	"encoding/hex"
 	"errors"
@@ -4034,3 +4035,28 @@ func (cx *EvalContext) PcDetails() (pc int, dis string) {
 	}
 	return cx.pc, dis
 }
+
+func base64Decode(encoded []byte, encoding *base64.Encoding) ([]byte, error) {
+	decoded := make([]byte, encoding.DecodedLen(len(encoded)))
+	n, err := encoding.Strict().Decode(decoded, encoded)
+	if err != nil {
+		return decoded[:0], err
+	}
+	return decoded[:n], err
+}
+
+func opBase64Decode(cx *EvalContext) {
+	last := len(cx.stack) - 1
+	alphabetField := Base64Alphabet(cx.program[cx.pc+1])
+	fs, ok := base64AlphabetSpecByField[alphabetField]
+	if !ok || fs.version > cx.version {
+		cx.err = fmt.Errorf("invalid base64_decode alphabet %d", alphabetField)
+		return
+	}
+
+	encoding := base64.URLEncoding
+	if alphabetField == StdAlph {
+		encoding = base64.StdEncoding
+	}
+	cx.stack[last].Bytes, cx.err = base64Decode(cx.stack[last].Bytes, encoding)
+}

data/transactions/logic/evalStateful_test.go

@@ -2391,7 +2391,8 @@ func TestReturnTypes(t *testing.T) {
 		"args":              "args",
 		"itxn":              "itxn_begin; int pay; itxn_field TypeEnum; itxn_submit; itxn CreatedAssetID",
 		// This next one is a cop out.  Can't use itxna Logs until we have inner appl
-		"itxna": "itxn_begin; int pay; itxn_field TypeEnum; itxn_submit; itxn NumLogs",
+		"itxna":         "itxn_begin; int pay; itxn_field TypeEnum; itxn_submit; itxn NumLogs",
+		"base64_decode": `pushbytes "YWJjMTIzIT8kKiYoKSctPUB+"; base64_decode StdAlph; pushbytes "abc123!?$*&()'-=@~"; ==; pushbytes "YWJjMTIzIT8kKiYoKSctPUB-"; base64_decode URLAlph; pushbytes "abc123!?$*&()'-=@~"; ==; &&; assert`,
 	}
 
 	// these require special input data and tested separately

data/transactions/logic/eval_test.go

@@ -3719,6 +3719,54 @@ func BenchmarkBigMath(b *testing.B) {
 	}
 }
 
+func BenchmarkBase64Decode(b *testing.B) {
+	smallStd := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
+	smallURL := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
+	medStd := strings.Repeat(smallStd, 16)
+	medURL := strings.Repeat(smallURL, 16)
+	bigStd := strings.Repeat(medStd, 4)
+	bigURL := strings.Repeat(medURL, 4)
+
+	tags := []string{"small", "medium", "large"}
+	stds := []string{smallStd, medStd, bigStd}
+	urls := []string{smallURL, medURL, bigURL}
+	ops := []string{
+		"",
+		"len",
+		"b~",
+		"int 1; pop",
+		"keccak256",
+		"sha256",
+		"sha512_256",
+		"base64_decode StdAlph",
+		"base64_decode URLAlph",
+	}
+	benches := [][]string{}
+	for i, tag := range tags {
+		for _, op := range ops {
+			testName := op
+			encoded := stds[i]
+			if op == "base64_decode URLAlph" {
+				encoded = urls[i]
+			}
+			if len(op) > 0 {
+				op += "; "
+			}
+			op += "pop"
+			benches = append(benches, []string{
+				fmt.Sprintf("%s_%s", testName, tag),
+				"",
+				fmt.Sprintf(`byte "%s"; %s`, encoded, op),
+				"int 1",
+			})
+		}
+	}
+	for _, bench := range benches {
+		b.Run(bench[0], func(b *testing.B) {
+			benchmarkOperation(b, bench[1], bench[2], bench[3])
+		})
+	}
+}
 func BenchmarkAddx64(b *testing.B) {
 	progs := [][]string{
 		{"add long stack", addBenchmarkSource},
@@ -4909,3 +4957,115 @@ func TestPcDetails(t *testing.T) {
 		})
 	}
 }
+
+var minB64DecodeVersion uint64 = 6
+
+type b64DecodeTestCase struct {
+	Encoded string
+	IsURL   bool
+	Decoded string
+	Error   error
+}
+
+var testCases = []b64DecodeTestCase{
+	{"TU9CWS1ESUNLOwoKb3IsIFRIRSBXSEFMRS4KCgpCeSBIZXJtYW4gTWVsdmlsbGU=",
+		false,
+		`MOBY-DICK;
+
+or, THE WHALE.
+
+
+By Herman Melville`,
+		nil,
+	},
+	{"TU9CWS1ESUNLOwoKb3IsIFRIRSBXSEFMRS4KCgpCeSBIZXJtYW4gTWVsdmlsbGU=",
+		true,
+		`MOBY-DICK;
+
+or, THE WHALE.
+
+
+By Herman Melville`,
+		nil,
+	},
+	{"YWJjMTIzIT8kKiYoKSctPUB+", false, "abc123!?$*&()'-=@~", nil},
+	{"YWJjMTIzIT8kKiYoKSctPUB-", true, "abc123!?$*&()'-=@~", nil},
+	{"YWJjMTIzIT8kKiYoKSctPUB+", true, "", base64.CorruptInputError(23)},
+	{"YWJjMTIzIT8kKiYoKSctPUB-", false, "", base64.CorruptInputError(23)},
+}
+
+func TestBase64DecodeFunc(t *testing.T) {
+	partitiontest.PartitionTest(t)
+	t.Parallel()
+
+	for _, testCase := range testCases {
+		encoding := base64.StdEncoding
+		if testCase.IsURL {
+			encoding = base64.URLEncoding
+		}
+		encoding = encoding.Strict()
+		decoded, err := base64Decode([]byte(testCase.Encoded), encoding)
+		require.Equal(t, []byte(testCase.Decoded), decoded)
+		require.Equal(t, testCase.Error, err)
+	}
+}
+
+type b64DecodeTestArgs struct {
+	Raw     []byte
+	Encoded []byte
+	IsURL   bool
+	Program []byte
+}
+
+func b64TestDecodeAssembleWithArgs(t *testing.T) []b64DecodeTestArgs {
+	sourceTmpl := `#pragma version %d
+arg 0
+arg 1
+base64_decode %s
+==`
+	args := []b64DecodeTestArgs{}
+	for _, testCase := range testCases {
+		if testCase.Error == nil {
+			field := "StdAlph"
+			if testCase.IsURL {
+				field = "URLAlph"
+			}
+			source := fmt.Sprintf(sourceTmpl, minB64DecodeVersion, field)
+			ops, err := AssembleStringWithVersion(source, minB64DecodeVersion)
+			require.NoError(t, err)
+
+			arg := b64DecodeTestArgs{
+				Raw:     []byte(testCase.Decoded),
+				Encoded: []byte(testCase.Encoded),
+				IsURL:   testCase.IsURL,
+				Program: ops.Program,
+			}
+			args = append(args, arg)
+		}
+	}
+	return args
+}
+
+func b64TestDecodeEval(tb testing.TB, args []b64DecodeTestArgs) {
+	for _, data := range args {
+		var txn transactions.SignedTxn
+		txn.Lsig.Logic = data.Program
+		txn.Lsig.Args = [][]byte{data.Raw[:], data.Encoded[:]}
+		ep := defaultEvalParams(&strings.Builder{}, &txn)
+		pass, err := Eval(data.Program, ep)
+		if err != nil {
+			require.NoError(tb, err)
+		}
+		if !pass {
+			fmt.Printf("FAILING WITH data = %#v", data)
+			require.True(tb, pass)
+		}
+	}
+}
+
+func TestOpBase64Decode(t *testing.T) {
+	partitiontest.PartitionTest(t)
+	t.Parallel()
+	args := b64TestDecodeAssembleWithArgs(t)
+	b64TestDecodeEval(t, args)
+}