From 73c319c3b53a4d6e43d32ed967d563b40e90acda Mon Sep 17 00:00:00 2001 From: MAC Date: Fri, 9 Sep 2022 00:32:25 +0700 Subject: [PATCH 1/7] Add opcode json_ref --- packages/runtime/package.json | 1 + packages/runtime/src/errors/errors-list.ts | 18 +++ .../runtime/src/interpreter/opcode-list.ts | 113 ++++++++++++++++++ packages/runtime/src/lib/constants.ts | 6 + packages/runtime/src/lib/parsing.ts | 16 +++ packages/runtime/src/parser/parser.ts | 4 +- .../test/src/interpreter/opcode-list.ts | 48 ++++++++ 7 files changed, 205 insertions(+), 1 deletion(-) diff --git a/packages/runtime/package.json b/packages/runtime/package.json index 0bc6c1e65..5fb3f0824 100644 --- a/packages/runtime/package.json +++ b/packages/runtime/package.json @@ -34,6 +34,7 @@ "dependencies": { "@algo-builder/web": "workspace:*", "@nodelib/fs.walk": "^1.2.8", + "@types/json-bigint": "^1.0.1", "algosdk": "^1.19.0", "chalk": "^4.1.2", "debug": "^4.3.4", diff --git a/packages/runtime/src/errors/errors-list.ts b/packages/runtime/src/errors/errors-list.ts index 1d2ac7faf..eb60234b5 100644 --- a/packages/runtime/src/errors/errors-list.ts +++ b/packages/runtime/src/errors/errors-list.ts @@ -423,6 +423,24 @@ maximun uint128`, title: "replace opcode error", description: "Can not replace bytes due to invalid start or end index", }, + UNKNOWN_JSON_TYPE: { + number: 1062, + message: "JSON TYPE must be {JSONString, JSONUint64 and JSONObject}, got :%jsonType%, at line %line%", + title: "Unknown json type", + description: "Unknown json type", + }, + INVALID_JSON_PARSING: { + number: 1063, + message: "Invalid json parsing at line %line%", + title: "Invalid json parsing", + description: "Invalid json parsing", + }, + UNKNOWN_KEY_JSON: { + number: 1064, + message: "Unkown key %key% in json object at line %line%", + title: "Unknown key json", + description: "Unknown key json" + } }; const runtimeGeneralErrors = { diff --git a/packages/runtime/src/interpreter/opcode-list.ts b/packages/runtime/src/interpreter/opcode-list.ts index 7f220d71b..123989e34 100644 --- a/packages/runtime/src/interpreter/opcode-list.ts +++ b/packages/runtime/src/interpreter/opcode-list.ts @@ -1,5 +1,6 @@ /* eslint sonarjs/no-identical-functions: 0 */ import { parsing, tx as webTx, types } from "@algo-builder/web"; +import { stringToBytes } from "@algo-builder/web/build/lib/parsing"; import algosdk, { decodeAddress, decodeUint64, @@ -16,6 +17,7 @@ import chalk from "chalk"; import { ec as EC } from "elliptic"; import { Hasher, Message, sha256 } from "js-sha256"; import { sha512_256 } from "js-sha512"; +import JSONbig from "json-bigint"; import cloneDeep from "lodash.clonedeep"; import { Keccak, SHA3 } from "sha3"; @@ -30,6 +32,7 @@ import { AssetParamMap, GlobalFields, ITxArrFields, + json_refTypes, MathOp, MAX_APP_PROGRAM_COST, MAX_CONCAT_SIZE, @@ -49,6 +52,7 @@ import { bigintSqrt } from "../lib/math"; import { assertBase64, assertBase64Url, + assertJSON, assertLen, assertNumber, assertOnlyDigits, @@ -5087,3 +5091,112 @@ export class Replace3 extends Replace { return super.execute(stack); } } + +/** + * Opcode: json_ref r + * Stack: ..., A: []byte, B: []byte → ..., any + * return key B's value from a valid utf-8 encoded json object A + */ +export class Json_ref extends Op { + readonly line: number; + readonly jsonType: string; + length = 1; + + /** + * Asserts 1 argument is passed. + * @param args Expected arguments: [e], where e = {JSONString, JSONUint64 and JSONObject}. + * @param line line number in TEAL file + */ + constructor(args: string[], line: number) { + super(); + this.line = line; + assertLen(args.length, 1, line); + const argument = args[0]; + switch (argument) { + case json_refTypes.JSONString: { + this.jsonType = "byte"; + break; + } + case json_refTypes.JSONUint64: { + this.jsonType = "int"; + break; + } + case json_refTypes.JSONObject: { + this.jsonType = "object"; + break; + } + default: { + throw new RuntimeError(RUNTIME_ERRORS.TEAL.UNKNOWN_JSON_TYPE, { + jsonType: argument, + line: this.line, + }); + } + } + } + + computeCost(): number { + return 25 + 2 * Math.ceil(this.length / 7); // cost = 25 + ceil(bytes / 7) + } + + execute(stack: TEALStack): number { + this.assertMinStackLen(stack, 2, this.line); + const key = this.assertBytes(stack.pop(), this.line); + const object = this.assertBytes(stack.pop(), this.line); + this.length = object.length; + const enc = new TextDecoder("utf-8"); + const decoded = enc.decode(object); + const keyString = enc.decode(key); + assertJSON(decoded, this.line); + const nativeBigJSON = JSONbig({ useNativeBigInt: true }); + const jsonObject = nativeBigJSON.parse(decoded); + if (jsonObject[keyString] === undefined) { + throw new RuntimeError(RUNTIME_ERRORS.TEAL.UNKNOWN_KEY_JSON, { + key: keyString, + line: this.line, + }); + } + switch (this.jsonType) { + case "byte": { + if (typeof jsonObject[keyString] === 'string') { + stack.push(stringToBytes(jsonObject[keyString])); + } else { + throw new RuntimeError(RUNTIME_ERRORS.TEAL.INVALID_TYPE, { + expected: 'byte', + actual: typeof jsonObject[keyString], + line: this.line, + }); + } + break; + } + case "int": { + if (typeof jsonObject[keyString] === 'number' || + typeof jsonObject[keyString] === 'bigint') { + const result = BigInt(jsonObject[keyString]); + this.checkOverflow(result, this.line, MAX_UINT64); + stack.push(result); + } else { + throw new RuntimeError(RUNTIME_ERRORS.TEAL.INVALID_TYPE, { + expected: 'Uint64', + actual: typeof jsonObject[keyString], + line: this.line, + }); + } + break; + } + case "object": { + if (typeof jsonObject[keyString] === 'object') { + stack.push(stringToBytes(nativeBigJSON.stringify(jsonObject[keyString]))); + } else { + throw new RuntimeError(RUNTIME_ERRORS.TEAL.INVALID_TYPE, { + expected: 'object', + actual: typeof jsonObject[keyString], + line: this.line, + }); + } + break; + } + } + + return this.computeCost(); + } +} \ No newline at end of file diff --git a/packages/runtime/src/lib/constants.ts b/packages/runtime/src/lib/constants.ts index b3d3c2951..552d100d1 100644 --- a/packages/runtime/src/lib/constants.ts +++ b/packages/runtime/src/lib/constants.ts @@ -504,3 +504,9 @@ export enum TransactionTypeEnum { ASSET_FREEZE = "afrz", APPLICATION_CALL = "appl", } + +export const json_refTypes = { + JSONString: "JSONString", + JSONUint64: "JSONUint64", + JSONObject: "JSONObject", +} \ No newline at end of file diff --git a/packages/runtime/src/lib/parsing.ts b/packages/runtime/src/lib/parsing.ts index cdf7fec32..7b6297b73 100644 --- a/packages/runtime/src/lib/parsing.ts +++ b/packages/runtime/src/lib/parsing.ts @@ -1,5 +1,6 @@ import { parsing } from "@algo-builder/web"; import * as base32 from "hi-base32"; +import JSONbig from "json-bigint"; import { RUNTIME_ERRORS } from "../errors/errors-list"; import { RuntimeError } from "../errors/runtime-errors"; @@ -282,4 +283,19 @@ export function bigEndianBytesToBigInt(bytes: Uint8Array | Buffer): bigint { */ export function strHexToBytes(str: string): Uint8Array { return new Uint8Array(Buffer.from(str.slice(2), "hex")); +} + +/** + * assert if string given is a valid JSON object + * @param jsonString + */ + export function assertJSON(jsonString: string, line: number): void { + const strictBigJSON = JSONbig({ strict: true }); + try { + strictBigJSON.parse(jsonString); + } catch (e) { + throw new RuntimeError(RUNTIME_ERRORS.TEAL.INVALID_JSON_PARSING, { + line: line + }); + } } \ No newline at end of file diff --git a/packages/runtime/src/parser/parser.ts b/packages/runtime/src/parser/parser.ts index d3cacf9ae..9c8963fc0 100644 --- a/packages/runtime/src/parser/parser.ts +++ b/packages/runtime/src/parser/parser.ts @@ -110,6 +110,7 @@ import { ITxnField, ITxnNext, ITxnSubmit, + Json_ref, Keccak256, Label, Len, @@ -136,8 +137,8 @@ import { Select, SetBit, SetByte, - Sha256, Sha3_256, + Sha256, Sha512_256, Shl, Shr, @@ -404,6 +405,7 @@ opCodeMap[7] = { replace3: Replace3, sha3_256: Sha3_256, ed25519verify_bare: Ed25519verify_bare, + json_ref: Json_ref, }; // list of opcodes with exactly one parameter. diff --git a/packages/runtime/test/src/interpreter/opcode-list.ts b/packages/runtime/test/src/interpreter/opcode-list.ts index 08fd54f2e..34e588ca8 100644 --- a/packages/runtime/test/src/interpreter/opcode-list.ts +++ b/packages/runtime/test/src/interpreter/opcode-list.ts @@ -113,6 +113,7 @@ import { Itob, ITxn, ITxnas, + Json_ref, Keccak256, Label, Len, @@ -7283,4 +7284,51 @@ describe("Teal Opcodes", function () { assert.equal(1900, op.execute(stack)); }); }); + + describe("json_ref", function () { + const stack = new Stack(); + const jsonByte = "{\"key0\": 0,\"key1\": \"algo\",\"key2\":{\"key3\": \"teal\", \"key4\": {\"key40\": 10}}, \"key5\": 18446744073709551615 }"; + + it("should return correct JSONUint64", function () { + stack.push(parsing.stringToBytes("{\"maxUint64\": 18446744073709551615}")); + stack.push(parsing.stringToBytes("maxUint64")); + const op = new Json_ref(["JSONUint64"], 1); + op.execute(stack); + + const top = stack.pop(); + assert.deepEqual(18446744073709551615n, top); + }); + + it("should throw when get wrong type JSON(expect byte but got uint64)", function () { + stack.push(parsing.stringToBytes("{\"maxUint64\": 18446744073709551615}")); + stack.push(parsing.stringToBytes("maxUint64")); + const op = new Json_ref(["JSONString"], 1); + expectRuntimeError(() => op.execute(stack), RUNTIME_ERRORS.TEAL.INVALID_TYPE); + }); + + it("should return correct JSONString", function () { + stack.push(parsing.stringToBytes(jsonByte)); + stack.push(parsing.stringToBytes("key1")); + const op = new Json_ref(["JSONString"], 1); + op.execute(stack); + + const top = stack.pop(); + const expected = parsing.stringToBytes("algo"); + assert.deepEqual(expected, top); + }); + + it("should return correct JSONObject", function () { + stack.push(parsing.stringToBytes(jsonByte)); + stack.push(parsing.stringToBytes("key2")); + const op1 = new Json_ref(["JSONObject"], 1); + op1.execute(stack); + stack.push(parsing.stringToBytes("key4")); + const op2 = new Json_ref(["JSONObject"], 1); + op2.execute(stack); + + const top = stack.pop(); + const expected = parsing.stringToBytes("{\"key40\":10}"); + assert.deepEqual(top, expected); + }); + }); }); From f32503bcdf6eec6940c90c57e0d00e884907403d Mon Sep 17 00:00:00 2001 From: MAC Date: Fri, 9 Sep 2022 00:36:26 +0700 Subject: [PATCH 2/7] Update changelog --- CHANGELOG.md | 1 + yarn.lock | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66f32728a..369fa4b31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ from `algosdk` and sends it to the network. - Added replace2 and replace3 opcode to `runtime`. - Added sha3_256 opcode to `Runtime` - Added ed25519verify_bare opcode to `Runtime` +- Added json_ref opcode to `Runtime` #### @algo-builder/web - Added `appendSignMultisigTransaction` function to `WebMode` for appending signature to multisig transaction in the algosigner. diff --git a/yarn.lock b/yarn.lock index 92c75bac6..c79077a59 100644 --- a/yarn.lock +++ b/yarn.lock @@ -65,6 +65,7 @@ __metadata: "@types/chai": ^4.3.0 "@types/debug": ^4.1.7 "@types/elliptic": ^6.4.14 + "@types/json-bigint": ^1.0.1 "@types/lodash.clonedeep": ^4.5.6 "@types/mocha": ^9.1.0 "@types/node": ^17.0.21 @@ -78,6 +79,7 @@ __metadata: hi-base32: ^0.5.1 js-sha256: ^0.9.0 js-sha512: ^0.8.0 + json-bigint: ^1.0.0 lodash.clonedeep: ^4.5.0 mocha: ^9.2.1 murmurhash: ^2.0.0 @@ -725,6 +727,13 @@ __metadata: languageName: node linkType: hard +"@types/json-bigint@npm:^1.0.1": + version: 1.0.1 + resolution: "@types/json-bigint@npm:1.0.1" + checksum: b39e55a811f554bd25f1d991bc4fc70655216dff466f21f97160097573a4bc7b478c1907aa5194c79022c115f509f8e4712083c51f57665e7a2de7412ff7801f + languageName: node + linkType: hard + "@types/json-schema@npm:^7.0.9": version: 7.0.9 resolution: "@types/json-schema@npm:7.0.9" From 2ffc061abb8283e41c2df1ad773d31a1c5917868 Mon Sep 17 00:00:00 2001 From: MAC Date: Fri, 9 Sep 2022 08:59:03 +0700 Subject: [PATCH 3/7] Update yarnlock --- yarn.lock | 1 - 1 file changed, 1 deletion(-) diff --git a/yarn.lock b/yarn.lock index c79077a59..5255604fc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -79,7 +79,6 @@ __metadata: hi-base32: ^0.5.1 js-sha256: ^0.9.0 js-sha512: ^0.8.0 - json-bigint: ^1.0.0 lodash.clonedeep: ^4.5.0 mocha: ^9.2.1 murmurhash: ^2.0.0 From 997e0c5386212d0b25a4b2e1cc3e1f8341625a14 Mon Sep 17 00:00:00 2001 From: MAC Date: Fri, 16 Sep 2022 21:12:30 +0700 Subject: [PATCH 4/7] Fix error --- packages/runtime/src/interpreter/opcode-list.ts | 10 +++++----- packages/runtime/src/lib/parsing.ts | 2 +- packages/runtime/test/src/interpreter/opcode-list.ts | 10 +++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/runtime/src/interpreter/opcode-list.ts b/packages/runtime/src/interpreter/opcode-list.ts index 123989e34..9c0f192f5 100644 --- a/packages/runtime/src/interpreter/opcode-list.ts +++ b/packages/runtime/src/interpreter/opcode-list.ts @@ -5143,12 +5143,12 @@ export class Json_ref extends Op { const key = this.assertBytes(stack.pop(), this.line); const object = this.assertBytes(stack.pop(), this.line); this.length = object.length; - const enc = new TextDecoder("utf-8"); - const decoded = enc.decode(object); - const keyString = enc.decode(key); - assertJSON(decoded, this.line); + const utf8decoder = new TextDecoder("utf-8"); + const decodedObj = utf8decoder.decode(object); + const keyString = utf8decoder.decode(key); + assertJSON(decodedObj, this.line); const nativeBigJSON = JSONbig({ useNativeBigInt: true }); - const jsonObject = nativeBigJSON.parse(decoded); + const jsonObject = nativeBigJSON.parse(decodedObj); if (jsonObject[keyString] === undefined) { throw new RuntimeError(RUNTIME_ERRORS.TEAL.UNKNOWN_KEY_JSON, { key: keyString, diff --git a/packages/runtime/src/lib/parsing.ts b/packages/runtime/src/lib/parsing.ts index 7b6297b73..23dbb2ac8 100644 --- a/packages/runtime/src/lib/parsing.ts +++ b/packages/runtime/src/lib/parsing.ts @@ -286,7 +286,7 @@ export function strHexToBytes(str: string): Uint8Array { } /** - * assert if string given is a valid JSON object + * assert if given string is a valid JSON object * @param jsonString */ export function assertJSON(jsonString: string, line: number): void { diff --git a/packages/runtime/test/src/interpreter/opcode-list.ts b/packages/runtime/test/src/interpreter/opcode-list.ts index 34e588ca8..7c9985868 100644 --- a/packages/runtime/test/src/interpreter/opcode-list.ts +++ b/packages/runtime/test/src/interpreter/opcode-list.ts @@ -7285,11 +7285,11 @@ describe("Teal Opcodes", function () { }); }); - describe("json_ref", function () { + describe("json_ref", () => { const stack = new Stack(); const jsonByte = "{\"key0\": 0,\"key1\": \"algo\",\"key2\":{\"key3\": \"teal\", \"key4\": {\"key40\": 10}}, \"key5\": 18446744073709551615 }"; - it("should return correct JSONUint64", function () { + it("Should return correct JSONUint64", function () { stack.push(parsing.stringToBytes("{\"maxUint64\": 18446744073709551615}")); stack.push(parsing.stringToBytes("maxUint64")); const op = new Json_ref(["JSONUint64"], 1); @@ -7299,14 +7299,14 @@ describe("Teal Opcodes", function () { assert.deepEqual(18446744073709551615n, top); }); - it("should throw when get wrong type JSON(expect byte but got uint64)", function () { + it("Should throw when get wrong JSON type(expect byte but got uint64)", function () { stack.push(parsing.stringToBytes("{\"maxUint64\": 18446744073709551615}")); stack.push(parsing.stringToBytes("maxUint64")); const op = new Json_ref(["JSONString"], 1); expectRuntimeError(() => op.execute(stack), RUNTIME_ERRORS.TEAL.INVALID_TYPE); }); - it("should return correct JSONString", function () { + it("Should return correct JSONString", function () { stack.push(parsing.stringToBytes(jsonByte)); stack.push(parsing.stringToBytes("key1")); const op = new Json_ref(["JSONString"], 1); @@ -7317,7 +7317,7 @@ describe("Teal Opcodes", function () { assert.deepEqual(expected, top); }); - it("should return correct JSONObject", function () { + it("Should return correct JSONObject", function () { stack.push(parsing.stringToBytes(jsonByte)); stack.push(parsing.stringToBytes("key2")); const op1 = new Json_ref(["JSONObject"], 1); From c484ea98b1a7c31dd9d353ecc5044baa6e145c18 Mon Sep 17 00:00:00 2001 From: MAC Date: Fri, 16 Sep 2022 21:46:01 +0700 Subject: [PATCH 5/7] Erase lines --- packages/runtime/src/interpreter/opcode-list.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/runtime/src/interpreter/opcode-list.ts b/packages/runtime/src/interpreter/opcode-list.ts index 9c0f192f5..14839596f 100644 --- a/packages/runtime/src/interpreter/opcode-list.ts +++ b/packages/runtime/src/interpreter/opcode-list.ts @@ -5196,7 +5196,6 @@ export class Json_ref extends Op { break; } } - return this.computeCost(); } } \ No newline at end of file From 413c54cd579a42e8f742135ab7a53fb0315e333f Mon Sep 17 00:00:00 2001 From: MAC Date: Tue, 20 Sep 2022 23:36:15 +0700 Subject: [PATCH 6/7] Add more test and fix grammar --- packages/runtime/src/errors/errors-list.ts | 18 +++++++++--------- .../test/src/interpreter/opcode-list.ts | 16 ++++++++++++++++ 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/packages/runtime/src/errors/errors-list.ts b/packages/runtime/src/errors/errors-list.ts index eb60234b5..6c99cdcd9 100644 --- a/packages/runtime/src/errors/errors-list.ts +++ b/packages/runtime/src/errors/errors-list.ts @@ -425,21 +425,21 @@ maximun uint128`, }, UNKNOWN_JSON_TYPE: { number: 1062, - message: "JSON TYPE must be {JSONString, JSONUint64 and JSONObject}, got :%jsonType%, at line %line%", - title: "Unknown json type", - description: "Unknown json type", + message: "JSON TYPE must be {JSONString, JSONUint64 or JSONObject}, got :%jsonType%, at line %line%", + title: "Unknown JSON type", + description: "Unknown JSON type", }, INVALID_JSON_PARSING: { number: 1063, - message: "Invalid json parsing at line %line%", - title: "Invalid json parsing", - description: "Invalid json parsing", + message: "Invalid JSON parsing at line %line%", + title: "Invalid JSON parsing", + description: "Invalid JSON parsing", }, UNKNOWN_KEY_JSON: { number: 1064, - message: "Unkown key %key% in json object at line %line%", - title: "Unknown key json", - description: "Unknown key json" + message: "Unkown key %key% in JSON object at line %line%", + title: "Unknown key JSON", + description: "Unknown key JSON" } }; diff --git a/packages/runtime/test/src/interpreter/opcode-list.ts b/packages/runtime/test/src/interpreter/opcode-list.ts index 7c9985868..a38ed74c2 100644 --- a/packages/runtime/test/src/interpreter/opcode-list.ts +++ b/packages/runtime/test/src/interpreter/opcode-list.ts @@ -7330,5 +7330,21 @@ describe("Teal Opcodes", function () { const expected = parsing.stringToBytes("{\"key40\":10}"); assert.deepEqual(top, expected); }); + + it("Should throw error when parsing invalid JSON object(missing comma in JSON object)", function () { + const jsonByte = "{\"key0\": 0 \"key1\": 2}"; + stack.push(parsing.stringToBytes(jsonByte)); + stack.push(parsing.stringToBytes("key1")); + const op = new Json_ref(["JSONObject"], 1); + expectRuntimeError(() => op.execute(stack), RUNTIME_ERRORS.TEAL.INVALID_JSON_PARSING); + }); + + it("Should throw error when parsing invalid JSON object(duplicate key is not allowed in JSON object)", function () { + const jsonByte = "{\"key0\": 0,\"key1\": \"algo\",\"key2\":{\"key3\": \"teal\", \"key4\": {\"key40\": 10, \"key40\": \"should fail!\"}}}"; + stack.push(parsing.stringToBytes(jsonByte)); + stack.push(parsing.stringToBytes("key1")); + const op = new Json_ref(["JSONObject"], 1); + expectRuntimeError(() => op.execute(stack), RUNTIME_ERRORS.TEAL.INVALID_JSON_PARSING); + }); }); }); From 7c610fbd21a82061e2a901b34fd204a73c9db673 Mon Sep 17 00:00:00 2001 From: MAC Date: Wed, 21 Sep 2022 09:18:03 +0700 Subject: [PATCH 7/7] Fix error --- packages/runtime/src/lib/parsing.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/runtime/src/lib/parsing.ts b/packages/runtime/src/lib/parsing.ts index 97c47e70c..155b31fbd 100644 --- a/packages/runtime/src/lib/parsing.ts +++ b/packages/runtime/src/lib/parsing.ts @@ -284,7 +284,7 @@ export function bigEndianBytesToBigInt(bytes: Uint8Array | Buffer): bigint { export function strHexToBytes(str: string): Uint8Array { return new Uint8Array(Buffer.from(str.slice(2), "hex")); } - +/* * Function taken from algosdk.utils * ConcatArrays takes n number arrays and returns a joint Uint8Array * @param arrs - An arbitrary number of n array-like number list arguments