From c10a45624d095e902b8282949b7eec0d6d20ce83 Mon Sep 17 00:00:00 2001 From: luc-blaeser Date: Tue, 14 Jan 2025 14:12:13 +0100 Subject: [PATCH 01/11] Support low memory hook --- doc/md/canister-maintenance/memory.md | 26 ++++++++ nix/drun.nix | 15 +---- nix/sources.json | 8 +-- src/codegen/compile_classical.ml | 21 +++++++ src/codegen/compile_enhanced.ml | 21 +++++++ src/ir_def/arrange_ir.ml | 3 +- src/ir_def/check_ir.ml | 8 ++- src/ir_def/freevars.ml | 3 +- src/ir_def/ir.ml | 1 + src/ir_def/rename.ml | 6 +- src/ir_passes/async.ml | 6 +- src/ir_passes/await.ml | 6 +- src/ir_passes/const.ml | 6 +- src/ir_passes/eq.ml | 8 ++- src/ir_passes/erase_typ_field.ml | 6 +- src/ir_passes/show.ml | 8 ++- src/lowering/desugar.ml | 8 +++ src/mo_frontend/typing.ml | 4 ++ src/mo_frontend/typing.mli | 1 + test/run-drun/low-memory.mo | 63 ++++++++++++++++++++ test/run-drun/low-memory/low-memory-actor.mo | 27 +++++++++ test/run-drun/ok/low-memory.drun-run.ok | 7 +++ 22 files changed, 227 insertions(+), 35 deletions(-) create mode 100644 doc/md/canister-maintenance/memory.md create mode 100644 test/run-drun/low-memory.mo create mode 100644 test/run-drun/low-memory/low-memory-actor.mo create mode 100644 test/run-drun/ok/low-memory.drun-run.ok diff --git a/doc/md/canister-maintenance/memory.md b/doc/md/canister-maintenance/memory.md new file mode 100644 index 00000000000..cbddbada477 --- /dev/null +++ b/doc/md/canister-maintenance/memory.md @@ -0,0 +1,26 @@ +--- +sidebar_position: 5 +--- + +# Memory management + +## Low memory hook + +The IC allows to implement a low memory hook, which is a warning trigger when main memory is becoming scarce. + +For this purpose, a Motoko actor or actor class instance can implement the system function `onLowMemory()`. This system function is scheduled when canister's free main memory space has fallen below the defined threshold `wasm_memory_threshold`, that is is part of the canister settings. In Motoko, `onLowMemory()` implements the `canister_on_low_wasm_memory` hook defined in the IC specification. + +Example of using the low memory hook: +``` +actor { + system func onLowMemory() : async () { + Debug.print("Low memory!"); + } +} +``` + +A few aspects need to be considered with the low memory hook: +* The execution of `onLowMemory` happens with a certain delay, as it is scheduled as separate asynchronous message that runs after the message in which the threshold was crossed. +* Once executed, `onLowMemory` is only triggered again when the main memory free space went above the threshold (e.g. by lowering the threshold or shrinking the main memory through canister reinstallation) and when the free space again fell below the threshold. +* Traps or unhandled errors in `onLowMemory` are ignored. They only revert the changes done in `onLowMemory`. +* Due to its `async` return type, the `onLowMemory` function may send further messages and await results. diff --git a/nix/drun.nix b/nix/drun.nix index 2a0b84b25f8..d7251fe21bc 100644 --- a/nix/drun.nix +++ b/nix/drun.nix @@ -21,6 +21,8 @@ pkgs: "build-info-0.0.27" = "sha256-SkwWwDNrTsntkNiCv6rsyTFGazhpRDnKtVzPpYLKF9U="; "cloudflare-0.12.0" = "sha256-FxCAK7gUKp/63fdvzI5Ufsy4aur74fO4R/K3YFiUw0Y="; "icrc1-test-env-0.1.1" = "sha256-2PB7e64Owin/Eji3k8UoeWs+pfDfOOTaAyXjvjOZ/4g="; + "ic-bn-lib-0.1.0" = "sha256-wqWfF70B+YQWg63yiEvIxOq+LN1AasrNXcyPkDM4/jw="; + "ic-canister-sig-creation-1.1.0" = "sha256-c47Fh4kZbmezWCYVHMci2BMXJfESaOGsyNlWh8YR6oU="; "jsonrpc-0.12.1" = "sha256-3FtdZlt2PqVDkE5iKWYIp1eiIELsaYlUPRSP2Xp8ejM="; "lmdb-rkv-0.14.99" = "sha256-5WcUzapkrc/s3wCBNCuUDhtbp17n67rTbm2rx0qtITg="; }; @@ -42,19 +44,6 @@ pkgs: EOF cd - - # static linking of libunwind fails under nix Linux - patch rs/monitoring/backtrace/build.rs << EOF -@@ -1,8 +1,2 @@ - fn main() { -- if std::env::var("TARGET").unwrap() == "x86_64-unknown-linux-gnu" { -- println!("cargo:rustc-link-lib=static=unwind"); -- println!("cargo:rustc-link-lib=static=unwind-ptrace"); -- println!("cargo:rustc-link-lib=static=unwind-x86_64"); -- println!("cargo:rustc-link-lib=dylib=lzma"); -- } - } -EOF - mkdir -p .cargo cat > .cargo/config.toml << EOF [target.x86_64-apple-darwin] diff --git a/nix/sources.json b/nix/sources.json index bef8be1fa0a..46e1b3208b7 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -21,15 +21,15 @@ "version": "3.2.25" }, "ic": { - "branch": "luc/adjust-drun", + "branch": "luc/latest-drun", "description": "Internet Computer blockchain source: the client/replica software run by nodes", "homepage": "", "owner": "luc-blaeser", "repo": "ic", - "rev": "bebe89514a6abd26e940b295323823169911a965", - "sha256": "1g68fyi5acbcgs2kjribk97fj8ki5g6pd99nwl5azz1rw1b0xycx", + "rev": "de6b67ec757b960f5ba5a61efa8b5c35bf48fc39", + "sha256": "0ncp487frg455jg06887igh3lzav2zdy70wwvlnqpjisdlsh6b0k", "type": "tarball", - "url": "https://github.com/luc-blaeser/ic/archive/bebe89514a6abd26e940b295323823169911a965.tar.gz", + "url": "https://github.com/luc-blaeser/ic/archive/de6b67ec757b960f5ba5a61efa8b5c35bf48fc39.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "ic-hs": { diff --git a/src/codegen/compile_classical.ml b/src/codegen/compile_classical.ml index a16282ca4db..3b6a6105c48 100644 --- a/src/codegen/compile_classical.ml +++ b/src/codegen/compile_classical.ml @@ -5277,6 +5277,18 @@ module IC = struct edesc = nr (FuncExport (nr fi)) }) + let export_low_memory env = + assert (E.mode env = Flags.ICMode || E.mode env = Flags.RefMode); + let fi = E.add_fun env "canister_on_low_wasm_memory" + (Func.of_body env [] [] (fun env -> + G.i (Call (nr (E.built_in env "low_memory_exp"))) ^^ + GC.collect_garbage env)) + in + E.add_export env (nr { + name = Lib.Utf8.decode "canister_on_low_wasm_memory"; + edesc = nr (FuncExport (nr fi)) + }) + let export_wasi_start env = assert (E.mode env = Flags.WASIMode); let fi = E.add_fun env "_start" (Func.of_body env [] [] (fun env1 -> @@ -13014,6 +13026,15 @@ and main_actor as_opt mod_env ds fs up = IC.export_inspect env; end; + (* Export low memory hook (but only when required) *) + begin match up.low_memory.it with + | Ir.PrimE (Ir.TupPrim, []) -> () + | _ -> + Func.define_built_in env "low_memory_exp" [] [] (fun env -> + compile_exp_as env ae2 SR.unit up.low_memory); + IC.export_low_memory env; + end; + (* Helper function to build the stable actor wrapper *) Func.define_built_in mod_env IC.get_actor_to_persist_function_name [] [I32Type] (fun env -> compile_exp_as env ae2 SR.Vanilla build_stable_actor diff --git a/src/codegen/compile_enhanced.ml b/src/codegen/compile_enhanced.ml index 61d21e839f5..0f98f94a28b 100644 --- a/src/codegen/compile_enhanced.ml +++ b/src/codegen/compile_enhanced.ml @@ -4972,6 +4972,18 @@ module IC = struct edesc = nr (FuncExport (nr fi)) }) + let export_low_memory env = + assert (E.mode env = Flags.ICMode || E.mode env = Flags.RefMode); + let fi = E.add_fun env "canister_on_low_wasm_memory" + (Func.of_body env [] [] (fun env -> + G.i (Call (nr (E.built_in env "low_memory_exp"))) ^^ + GC.collect_garbage env)) + in + E.add_export env (nr { + name = Lib.Utf8.decode "canister_on_low_wasm_memory"; + edesc = nr (FuncExport (nr fi)) + }) + let initialize_main_actor_function_name = "@initialize_main_actor" let initialize_main_actor env = @@ -13106,6 +13118,15 @@ and main_actor as_opt mod_env ds fs up = IC.export_inspect env; end; + (* Export low memory hook (but only when required) *) + begin match up.low_memory.it with + | Ir.PrimE (Ir.TupPrim, []) -> () + | _ -> + Func.define_built_in env "low_memory_exp" [] [] (fun env -> + compile_exp_as env ae2 SR.unit up.low_memory); + IC.export_low_memory env; + end; + (* Helper function to build the stable actor wrapper *) Func.define_built_in mod_env IC.get_actor_to_persist_function_name [] [I64Type] (fun env -> compile_exp_as env ae2 SR.Vanilla build_stable_actor diff --git a/src/ir_def/arrange_ir.ml b/src/ir_def/arrange_ir.ml index b0f110029fb..25fe792e408 100644 --- a/src/ir_def/arrange_ir.ml +++ b/src/ir_def/arrange_ir.ml @@ -35,13 +35,14 @@ let rec exp e = match e.it with | TryE (e, cs, None) -> "TryE" $$ [exp e] @ List.map case cs | TryE (e, cs, Some (i, _)) -> "TryE" $$ [exp e] @ List.map case cs @ Atom ";" :: [id i] -and system { meta; preupgrade; postupgrade; heartbeat; timer; inspect; stable_record; stable_type} = (* TODO: show meta? *) +and system { meta; preupgrade; postupgrade; heartbeat; timer; inspect; low_memory; stable_record; stable_type} = (* TODO: show meta? *) "System" $$ [ "Pre" $$ [exp preupgrade]; "Post" $$ [exp postupgrade]; "Heartbeat" $$ [exp heartbeat]; "Timer" $$ [exp timer]; "Inspect" $$ [exp inspect]; + "OnLowMemory" $$ [exp low_memory]; "StableRecord" $$ [exp stable_record]; "StableType" $$ [typ stable_type] ] diff --git a/src/ir_def/check_ir.ml b/src/ir_def/check_ir.ml index 37e84da5623..e9eaf0aee49 100644 --- a/src/ir_def/check_ir.ml +++ b/src/ir_def/check_ir.ml @@ -817,7 +817,7 @@ let rec check_exp env (exp:Ir.exp) : unit = typ exp_r <: T.(Construct.err_contT unit); typ exp_c <: Construct.clean_contT; | ActorE (ds, fs, - { preupgrade; postupgrade; meta; heartbeat; timer; inspect; stable_record; stable_type }, t0) -> + { preupgrade; postupgrade; meta; heartbeat; timer; inspect; low_memory; stable_record; stable_type }, t0) -> (* TODO: check meta *) let env' = { env with async = None } in let scope1 = gather_block_decs env' ds in @@ -828,12 +828,14 @@ let rec check_exp env (exp:Ir.exp) : unit = check_exp env'' heartbeat; check_exp env'' timer; check_exp env'' inspect; + check_exp env'' low_memory; check_exp env'' stable_record; typ preupgrade <: T.unit; typ postupgrade <: T.unit; typ heartbeat <: T.unit; typ timer <: T.unit; typ inspect <: T.unit; + typ low_memory <: T.unit; typ stable_record <: stable_type; check (T.is_obj t0) "bad annotation (object type expected)"; let (s0, tfs0) = T.as_obj t0 in @@ -1160,7 +1162,7 @@ let check_comp_unit env = function let env' = adjoin env scope in check_decs env' ds | ActorU (as_opt, ds, fs, - { preupgrade; postupgrade; meta; heartbeat; timer; inspect; stable_type; stable_record }, t0) -> + { preupgrade; postupgrade; meta; heartbeat; timer; inspect; low_memory; stable_type; stable_record }, t0) -> let check p = check env no_region p in let (<:) t1 t2 = check_sub env no_region t1 t2 in let env' = match as_opt with @@ -1179,11 +1181,13 @@ let check_comp_unit env = function check_exp env'' timer; check_exp env'' inspect; check_exp env'' stable_record; + check_exp env'' low_memory; typ preupgrade <: T.unit; typ postupgrade <: T.unit; typ heartbeat <: T.unit; typ timer <: T.unit; typ inspect <: T.unit; + typ low_memory <: T.unit; typ stable_record <: stable_type; check (T.is_obj t0) "bad annotation (object type expected)"; let (s0, tfs0) = T.as_obj t0 in diff --git a/src/ir_def/freevars.ml b/src/ir_def/freevars.ml index ae8d8309cf5..374053af34d 100644 --- a/src/ir_def/freevars.ml +++ b/src/ir_def/freevars.ml @@ -123,12 +123,13 @@ let rec exp e : f = match e.it with and actor ds fs u = close (decs ds +++ fields fs +++ system u) -and system {meta; preupgrade; postupgrade; heartbeat; timer; inspect; stable_record; _} = +and system {meta; preupgrade; postupgrade; heartbeat; timer; inspect; low_memory; stable_record; _} = under_lambda (exp preupgrade) ++ under_lambda (exp postupgrade) ++ under_lambda (exp heartbeat) ++ under_lambda (exp timer) ++ under_lambda (exp inspect) ++ + under_lambda (exp low_memory) ++ under_lambda (exp stable_record) and exps es : f = unions exp es diff --git a/src/ir_def/ir.ml b/src/ir_def/ir.ml index 192a7453446..d2fa9a1f79a 100644 --- a/src/ir_def/ir.ml +++ b/src/ir_def/ir.ml @@ -85,6 +85,7 @@ and system = { heartbeat : exp; timer : exp; (* TODO: use an option type: (Default of exp | UserDefined of exp) option *) inspect : exp; + low_memory : exp; stable_record: exp; stable_type: Type.typ; } diff --git a/src/ir_def/rename.ml b/src/ir_def/rename.ml index f97594fba41..7cb2dd3431b 100644 --- a/src/ir_def/rename.ml +++ b/src/ir_def/rename.ml @@ -33,7 +33,7 @@ and exp' rho = function | VarE (m, i) -> VarE (m, id rho i) | LitE _ as e -> e | PrimE (p, es) -> PrimE (prim rho p, List.map (exp rho) es) - | ActorE (ds, fs, { meta; preupgrade; postupgrade; heartbeat; timer; inspect; stable_record; stable_type}, t) -> + | ActorE (ds, fs, { meta; preupgrade; postupgrade; heartbeat; timer; inspect; low_memory; stable_record; stable_type}, t) -> let ds', rho' = decs rho ds in ActorE (ds', @@ -44,6 +44,7 @@ and exp' rho = function heartbeat = exp rho' heartbeat; timer = exp rho' timer; inspect = exp rho' inspect; + low_memory = exp rho' low_memory; stable_type = stable_type; stable_record = exp rho' stable_record; }, @@ -200,7 +201,7 @@ let comp_unit rho cu = match cu with | LibU (ds, e) -> let ds', rho' = decs rho ds in LibU (ds', exp rho' e) - | ActorU (as_opt, ds, fs, { meta; preupgrade; postupgrade; heartbeat; timer; inspect; stable_record; stable_type }, t) -> + | ActorU (as_opt, ds, fs, { meta; preupgrade; postupgrade; heartbeat; timer; inspect; low_memory; stable_record; stable_type }, t) -> let as_opt', rho' = match as_opt with | None -> None, rho | Some as_ -> @@ -215,6 +216,7 @@ let comp_unit rho cu = match cu with heartbeat = exp rho'' heartbeat; timer = exp rho'' timer; inspect = exp rho'' inspect; + low_memory = exp rho'' low_memory; stable_record = exp rho'' stable_record; stable_type = stable_type; }, t) diff --git a/src/ir_passes/async.ml b/src/ir_passes/async.ml index b3b52989e7a..45ef1afea57 100644 --- a/src/ir_passes/async.ml +++ b/src/ir_passes/async.ml @@ -442,7 +442,7 @@ let transform prog = | (Returns | Replies), _ -> assert false end end - | ActorE (ds, fs, {meta; preupgrade; postupgrade; heartbeat; timer; inspect; stable_record; stable_type}, typ) -> + | ActorE (ds, fs, {meta; preupgrade; postupgrade; heartbeat; timer; inspect; low_memory; stable_record; stable_type}, typ) -> ActorE (t_decs ds, t_fields fs, {meta; preupgrade = t_exp preupgrade; @@ -450,6 +450,7 @@ let transform prog = heartbeat = t_exp heartbeat; timer = t_exp timer; inspect = t_exp inspect; + low_memory = t_exp low_memory; stable_record = t_exp stable_record; stable_type = t_typ stable_type; }, @@ -523,7 +524,7 @@ let transform prog = and t_comp_unit = function | LibU _ -> raise (Invalid_argument "cannot compile library") | ProgU ds -> ProgU (t_decs ds) - | ActorU (args_opt, ds, fs, {meta; preupgrade; postupgrade; heartbeat; timer; inspect; stable_record; stable_type}, t) -> + | ActorU (args_opt, ds, fs, {meta; preupgrade; postupgrade; heartbeat; timer; inspect; low_memory; stable_record; stable_type}, t) -> ActorU (Option.map t_args args_opt, t_decs ds, t_fields fs, { meta; preupgrade = t_exp preupgrade; @@ -531,6 +532,7 @@ let transform prog = heartbeat = t_exp heartbeat; timer = t_exp timer; inspect = t_exp inspect; + low_memory = t_exp low_memory; stable_record = t_exp stable_record; stable_type = t_typ stable_type; }, diff --git a/src/ir_passes/await.ml b/src/ir_passes/await.ml index 84a2170a8d2..291d5dbb85f 100644 --- a/src/ir_passes/await.ml +++ b/src/ir_passes/await.ml @@ -181,7 +181,7 @@ and t_exp' context exp = assert (not (T.is_shared_func (typ exp))); let context' = LabelEnv.singleton Return Label in FuncE (x, s, c, typbinds, pat, typs, t_exp context' exp1) - | ActorE (ds, ids, { meta; preupgrade; postupgrade; heartbeat; timer; inspect; stable_record; stable_type}, t) -> + | ActorE (ds, ids, { meta; preupgrade; postupgrade; heartbeat; timer; inspect; low_memory; stable_record; stable_type}, t) -> ActorE (t_decs context ds, ids, { meta; preupgrade = t_exp LabelEnv.empty preupgrade; @@ -189,6 +189,7 @@ and t_exp' context exp = heartbeat = t_ignore_throw LabelEnv.empty heartbeat; timer = t_ignore_throw LabelEnv.empty timer; inspect = t_exp LabelEnv.empty inspect; + low_memory = t_ignore_throw LabelEnv.empty low_memory; stable_record = t_exp LabelEnv.empty stable_record; stable_type; }, @@ -647,7 +648,7 @@ and t_comp_unit context = function expD (c_block context' ds (tupE []) (meta (T.unit) (fun v1 -> tupE []))) ] end - | ActorU (as_opt, ds, ids, { meta = m; preupgrade; postupgrade; heartbeat; timer; inspect; stable_record; stable_type}, t) -> + | ActorU (as_opt, ds, ids, { meta = m; preupgrade; postupgrade; heartbeat; timer; inspect; low_memory; stable_record; stable_type}, t) -> ActorU (as_opt, t_decs context ds, ids, { meta = m; preupgrade = t_exp LabelEnv.empty preupgrade; @@ -655,6 +656,7 @@ and t_comp_unit context = function heartbeat = t_ignore_throw LabelEnv.empty heartbeat; timer = t_ignore_throw LabelEnv.empty timer; inspect = t_exp LabelEnv.empty inspect; + low_memory = t_ignore_throw LabelEnv.empty low_memory; stable_record = t_exp LabelEnv.empty stable_record; stable_type; }, diff --git a/src/ir_passes/const.ml b/src/ir_passes/const.ml index 9bc4591cad1..e62e23ef575 100644 --- a/src/ir_passes/const.ml +++ b/src/ir_passes/const.ml @@ -164,7 +164,7 @@ let rec exp lvl (env : env) e : Lbool.t = surely_false | NewObjE _ -> (* mutable objects *) surely_false - | ActorE (ds, fs, {meta; preupgrade; postupgrade; heartbeat; timer; inspect; stable_record; stable_type}, _typ) -> + | ActorE (ds, fs, {meta; preupgrade; postupgrade; heartbeat; timer; inspect; low_memory; stable_record; stable_type}, _typ) -> (* this may well be “the” top-level actor, so don’t update lvl here *) let (env', _) = decs lvl env ds in exp_ lvl env' preupgrade; @@ -172,6 +172,7 @@ let rec exp lvl (env : env) e : Lbool.t = exp_ lvl env' heartbeat; exp_ lvl env' timer; exp_ lvl env' inspect; + exp_ lvl env' low_memory; exp_ lvl env' stable_record; surely_false in @@ -228,7 +229,7 @@ and block lvl env (ds, body) = and comp_unit = function | LibU _ -> raise (Invalid_argument "cannot compile library") | ProgU ds -> decs_ TopLvl M.empty ds - | ActorU (as_opt, ds, fs, {meta; preupgrade; postupgrade; heartbeat; timer; inspect; stable_record; stable_type}, typ) -> + | ActorU (as_opt, ds, fs, {meta; preupgrade; postupgrade; heartbeat; timer; inspect; low_memory; stable_record; stable_type}, typ) -> let env = match as_opt with | None -> M.empty | Some as_ -> args TopLvl M.empty as_ @@ -239,6 +240,7 @@ and comp_unit = function exp_ TopLvl env' heartbeat; exp_ TopLvl env' timer; exp_ TopLvl env' inspect; + exp_ TopLvl env' low_memory; exp_ TopLvl env' stable_record let analyze ((cu, _flavor) : prog) = diff --git a/src/ir_passes/eq.ml b/src/ir_passes/eq.ml index 52486abcf4b..5987e11a3fc 100644 --- a/src/ir_passes/eq.ml +++ b/src/ir_passes/eq.ml @@ -249,7 +249,7 @@ and t_exp' env = function NewObjE (sort, ids, t) | SelfCallE (ts, e1, e2, e3, e4) -> SelfCallE (ts, t_exp env e1, t_exp env e2, t_exp env e3, t_exp env e4) - | ActorE (ds, fields, {meta; preupgrade; postupgrade; heartbeat; timer; inspect; stable_record; stable_type}, typ) -> + | ActorE (ds, fields, {meta; preupgrade; postupgrade; heartbeat; timer; inspect; low_memory; stable_record; stable_type}, typ) -> (* Until Actor expressions become their own units, we repeat what we do in `comp_unit` below *) let env1 = empty_env () in @@ -259,6 +259,7 @@ and t_exp' env = function let heartbeat' = t_exp env1 heartbeat in let timer' = t_exp env1 timer in let inspect' = t_exp env1 inspect in + let low_memory' = t_exp env1 low_memory in let stable_record' = t_exp env1 stable_record in let decls = eq_decls !(env1.params) in ActorE (decls @ ds', fields, @@ -268,6 +269,7 @@ and t_exp' env = function heartbeat = heartbeat'; timer = timer'; inspect = inspect'; + low_memory = low_memory'; stable_record = stable_record'; stable_type; }, @@ -301,7 +303,7 @@ and t_comp_unit = function let ds' = t_decs env ds in let decls = eq_decls !(env.params) in ProgU (decls @ ds') - | ActorU (as_opt, ds, fields, {meta; preupgrade; postupgrade; heartbeat; timer; inspect; stable_record; stable_type}, typ) -> + | ActorU (as_opt, ds, fields, {meta; preupgrade; postupgrade; heartbeat; timer; inspect; low_memory; stable_record; stable_type}, typ) -> let env = empty_env () in let ds' = t_decs env ds in let preupgrade' = t_exp env preupgrade in @@ -309,6 +311,7 @@ and t_comp_unit = function let heartbeat' = t_exp env heartbeat in let timer' = t_exp env timer in let inspect' = t_exp env inspect in + let low_memory' = t_exp env low_memory in let stable_record' = t_exp env stable_record in let decls = eq_decls !(env.params) in ActorU (as_opt, decls @ ds', fields, @@ -318,6 +321,7 @@ and t_comp_unit = function heartbeat = heartbeat'; timer = timer'; inspect = inspect'; + low_memory = low_memory'; stable_record = stable_record'; stable_type; }, typ) diff --git a/src/ir_passes/erase_typ_field.ml b/src/ir_passes/erase_typ_field.ml index 2b5daf75d70..f569f0ee765 100644 --- a/src/ir_passes/erase_typ_field.ml +++ b/src/ir_passes/erase_typ_field.ml @@ -126,7 +126,7 @@ let transform prog = DefineE (id, mut, t_exp exp1) | FuncE (x, s, c, typbinds, args, ret_tys, exp) -> FuncE (x, s, c, t_typ_binds typbinds, t_args args, List.map t_typ ret_tys, t_exp exp) - | ActorE (ds, fs, {meta; preupgrade; postupgrade; heartbeat; timer; inspect; stable_record; stable_type}, typ) -> + | ActorE (ds, fs, {meta; preupgrade; postupgrade; heartbeat; timer; inspect; low_memory; stable_record; stable_type}, typ) -> ActorE (t_decs ds, t_fields fs, {meta; preupgrade = t_exp preupgrade; @@ -134,6 +134,7 @@ let transform prog = heartbeat = t_exp heartbeat; timer = t_exp timer; inspect = t_exp inspect; + low_memory = t_exp low_memory; stable_record = t_exp stable_record; stable_type = t_typ stable_type; }, @@ -211,7 +212,7 @@ let transform prog = and t_comp_unit = function | LibU _ -> raise (Invalid_argument "cannot compile library") | ProgU ds -> ProgU (t_decs ds) - | ActorU (args_opt, ds, fs, {meta; preupgrade; postupgrade; heartbeat; timer; inspect; stable_record; stable_type}, t) -> + | ActorU (args_opt, ds, fs, {meta; preupgrade; postupgrade; heartbeat; timer; inspect; low_memory; stable_record; stable_type}, t) -> ActorU (Option.map t_args args_opt, t_decs ds, t_fields fs, { meta; preupgrade = t_exp preupgrade; @@ -219,6 +220,7 @@ let transform prog = heartbeat = t_exp heartbeat; timer = t_exp timer; inspect = t_exp inspect; + low_memory = t_exp low_memory; stable_record = t_exp stable_record; stable_type = t_typ stable_type; }, diff --git a/src/ir_passes/show.ml b/src/ir_passes/show.ml index 23f195220d7..132d1e6d5bc 100644 --- a/src/ir_passes/show.ml +++ b/src/ir_passes/show.ml @@ -291,7 +291,7 @@ and t_exp' env = function NewObjE (sort, ids, t) | SelfCallE (ts, e1, e2, e3, e4) -> SelfCallE (ts, t_exp env e1, t_exp env e2, t_exp env e3, t_exp env e4) - | ActorE (ds, fields, {meta; preupgrade; postupgrade; heartbeat; timer; inspect; stable_record; stable_type}, typ) -> + | ActorE (ds, fields, {meta; preupgrade; postupgrade; heartbeat; timer; inspect; low_memory; stable_record; stable_type}, typ) -> (* Until Actor expressions become their own units, we repeat what we do in `comp_unit` below *) let env1 = empty_env () in @@ -301,6 +301,7 @@ and t_exp' env = function let heartbeat' = t_exp env1 heartbeat in let timer' = t_exp env1 timer in let inspect' = t_exp env1 inspect in + let low_memory' = t_exp env1 low_memory in let stable_record' = t_exp env1 stable_record in let decls = show_decls !(env1.params) in ActorE (decls @ ds', fields, @@ -310,6 +311,7 @@ and t_exp' env = function heartbeat = heartbeat'; timer = timer'; inspect = inspect'; + low_memory = low_memory'; stable_record = stable_record'; stable_type; }, @@ -342,7 +344,7 @@ and t_comp_unit = function let ds' = t_decs env ds in let decls = show_decls !(env.params) in ProgU (decls @ ds') - | ActorU (as_opt, ds, fields, {meta; preupgrade; postupgrade; heartbeat; timer; inspect; stable_record; stable_type}, typ) -> + | ActorU (as_opt, ds, fields, {meta; preupgrade; postupgrade; heartbeat; timer; inspect; low_memory; stable_record; stable_type}, typ) -> let env = empty_env () in let ds' = t_decs env ds in let preupgrade' = t_exp env preupgrade in @@ -350,6 +352,7 @@ and t_comp_unit = function let heartbeat' = t_exp env heartbeat in let timer' = t_exp env timer in let inspect' = t_exp env inspect in + let low_memory' = t_exp env low_memory in let stable_record' = t_exp env stable_record in let decls = show_decls !(env.params) in ActorU (as_opt, decls @ ds', fields, @@ -359,6 +362,7 @@ and t_comp_unit = function heartbeat = heartbeat'; timer = timer'; inspect = inspect'; + low_memory = low_memory'; stable_record = stable_record'; stable_type }, typ) diff --git a/src/lowering/desugar.ml b/src/lowering/desugar.ml index 7049a6d8a6c..91f1c3aa62a 100644 --- a/src/lowering/desugar.ml +++ b/src/lowering/desugar.ml @@ -424,6 +424,10 @@ and call_system_func_opt name es obj_typ = (unitE ()) (primE (Ir.OtherPrim "trap") [textE "canister_inspect_message explicitly refused message"])) + | "onLowMemory" -> + blockE + [ expD (callE (varE (var id.it note)) [T.Any] (unitE())) ] + (unitE ()) | name -> let inst = match name with | "preupgrade" | "postupgrade" -> [T.scope_bound] @@ -612,6 +616,10 @@ and build_actor at ts self_id es obj_typ = (match call_system_func_opt "inspect" es obj_typ with | Some call -> call | None -> tupE []); + low_memory = + (match call_system_func_opt "onLowMemory" es obj_typ with + | Some call -> call + | None -> tupE []); stable_record = with_stable_vars (fun e -> e); stable_type = ty; }, diff --git a/src/mo_frontend/typing.ml b/src/mo_frontend/typing.ml index eb0c2c36051..0d88381f456 100644 --- a/src/mo_frontend/typing.ml +++ b/src/mo_frontend/typing.ml @@ -394,12 +394,16 @@ let timer_type = [Func (Local, Returns, [], [Prim Nat64], [])], [Async (Fut, Var (default_scope_var, 0), unit)])) +let low_memory_type = + T.(Func (Local, Returns, [scope_bind], [], [Async (Fut, Var (default_scope_var, 0), unit)])) + let system_funcs tfs = [ ("heartbeat", heartbeat_type); ("timer", timer_type); T.("preupgrade", Func (Local, Returns, [scope_bind], [], [])); T.("postupgrade", Func (Local, Returns, [scope_bind], [], [])); + ("onLowMemory", low_memory_type); ("inspect", (let msg_typ = T.decode_msg_typ tfs in let record_typ = diff --git a/src/mo_frontend/typing.mli b/src/mo_frontend/typing.mli index be0d8b40b7e..c097df47e80 100644 --- a/src/mo_frontend/typing.mli +++ b/src/mo_frontend/typing.mli @@ -13,3 +13,4 @@ val check_actors : ?viper_mode:bool -> ?check_actors:bool -> scope -> Syntax.pro val check_stab_sig : scope -> Syntax.stab_sig -> (field list) Diag.result val heartbeat_type : typ +val low_memory_type : typ diff --git a/test/run-drun/low-memory.mo b/test/run-drun/low-memory.mo new file mode 100644 index 00000000000..29b04201acf --- /dev/null +++ b/test/run-drun/low-memory.mo @@ -0,0 +1,63 @@ +import Prim "mo:⛔"; +import LowMemoryActor "low-memory/low-memory-actor"; +import Cycles = "cycles/cycles"; + +actor Self { + type canister_settings = { + wasm_memory_threshold: ?Nat; + }; + + func setMemoryThreshold(a : actor {}, threshold : Nat) : async () { + let ic00 = actor "aaaaa-aa" : actor { + update_settings : shared { + canister_id : Principal; + settings : canister_settings; + } -> async (); + }; + let settings : canister_settings = { + wasm_memory_threshold = ?threshold; + }; + await ic00.update_settings({ + canister_id = Prim.principalOfActor(a); + settings }); + }; + + let kb = 1024; + let mb = 1024 * kb; + let gb = 1024 * mb; + let maxMemory = 4 * gb; // adjust when canister limits inspection is supported + + public shared func lowMemoryCallback(): async () { + Prim.debugPrint("Low memory callback"); + }; + + public shared func run() : async () { + Cycles.add(2_000_000_000_000); + let lowMemoryActor = await LowMemoryActor.LowMemoryActor(lowMemoryCallback); + + let threshold1 = maxMemory - (await lowMemoryActor.memorySize()) - mb : Nat; + await setMemoryThreshold(lowMemoryActor, threshold1); + Prim.debugPrint("Memory threshold set " # debug_show(threshold1)); + await lowMemoryActor.allocateMemory(); + + // Not yet implemented on IC: Should retrigger when lowering threshold. + // let threshold2 = maxMemory - (await lowMemoryActor.memorySize()) - mb : Nat; + // await lowMemoryActor.allocateMemory(); + // await setMemoryThreshold(lowMemoryActor, threshold2); + // Prim.debugPrint("Memory threshold set " # debug_show(threshold2)); + + // Not yet implemented on IC: Should retrigger when shrinking memory by reinstallation. + // let lowMemoryActor2 = await (system LowMemoryActor.LowMemoryActor)(#reinstall lowMemoryActor)(lowMemoryCallback); + // let threshold3 = maxMemory - (await lowMemoryActor2.memorySize()) - mb : Nat; + // await setMemoryThreshold(lowMemoryActor2, threshold3); + // Prim.debugPrint("Memory threshold set " # debug_show(threshold3)); + // await lowMemoryActor2.allocateMemory(); + }; +}; + +//SKIP run +//SKIP run-low +//SKIP run-ir +//SKIP ic-ref-run + +//CALL ingress run "DIDL\x00\x00" diff --git a/test/run-drun/low-memory/low-memory-actor.mo b/test/run-drun/low-memory/low-memory-actor.mo new file mode 100644 index 00000000000..0d58a62def9 --- /dev/null +++ b/test/run-drun/low-memory/low-memory-actor.mo @@ -0,0 +1,27 @@ +import Prim "mo:⛔"; + +actor class LowMemoryActor(callback: shared () -> async ()) { + system func onLowMemory() : async() { + Prim.debugPrint("Low memory!"); + await callback(); + Prim.debugPrint("Low memory callback done"); + }; + + type Node = { + array: [var Nat]; + next: ?Node; + }; + + var root : ?Node = null; + + public func allocateMemory() : async () { + let array = Prim.Array_init(8 * 1024 * 1024, 0); // 32 GB on 32-bit, 64 GB on 64-bit. + let node : Node = { array; next = root }; + root := ?node; + }; + + public func memorySize() : async Nat { + await async {}; // Allocate GC reserve (because of `--force-gc` flag during drun testing). + Prim.rts_memory_size(); + } +}; diff --git a/test/run-drun/ok/low-memory.drun-run.ok b/test/run-drun/ok/low-memory.drun-run.ok new file mode 100644 index 00000000000..61b81361a16 --- /dev/null +++ b/test/run-drun/ok/low-memory.drun-run.ok @@ -0,0 +1,7 @@ +ingress Completed: Reply: 0x4449444c016c01b3c4b1f204680100010a00000000000000000101 +ingress Completed: Reply: 0x4449444c0000 +debug.print: Memory threshold set 4_159_700_992 +debug.print: Low memory! +debug.print: Low memory callback +debug.print: Low memory callback done +ingress Completed: Reply: 0x4449444c0000 From f9cdc850322f048a8bace4034f4c29b4a66ad85f Mon Sep 17 00:00:00 2001 From: Nix hash updater <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 14 Jan 2025 22:09:34 +0000 Subject: [PATCH 02/11] Updating nix hashes --- nix/drun.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/drun.nix b/nix/drun.nix index d7251fe21bc..0f0076617e9 100644 --- a/nix/drun.nix +++ b/nix/drun.nix @@ -20,9 +20,9 @@ pkgs: outputHashes = { "build-info-0.0.27" = "sha256-SkwWwDNrTsntkNiCv6rsyTFGazhpRDnKtVzPpYLKF9U="; "cloudflare-0.12.0" = "sha256-FxCAK7gUKp/63fdvzI5Ufsy4aur74fO4R/K3YFiUw0Y="; - "icrc1-test-env-0.1.1" = "sha256-2PB7e64Owin/Eji3k8UoeWs+pfDfOOTaAyXjvjOZ/4g="; "ic-bn-lib-0.1.0" = "sha256-wqWfF70B+YQWg63yiEvIxOq+LN1AasrNXcyPkDM4/jw="; "ic-canister-sig-creation-1.1.0" = "sha256-c47Fh4kZbmezWCYVHMci2BMXJfESaOGsyNlWh8YR6oU="; + "icrc1-test-env-0.1.1" = "sha256-2PB7e64Owin/Eji3k8UoeWs+pfDfOOTaAyXjvjOZ/4g="; "jsonrpc-0.12.1" = "sha256-3FtdZlt2PqVDkE5iKWYIp1eiIELsaYlUPRSP2Xp8ejM="; "lmdb-rkv-0.14.99" = "sha256-5WcUzapkrc/s3wCBNCuUDhtbp17n67rTbm2rx0qtITg="; }; From 63733c989e4785d6390537d257413ca585ed05ce Mon Sep 17 00:00:00 2001 From: luc-blaeser Date: Wed, 15 Jan 2025 09:35:17 +0100 Subject: [PATCH 03/11] Adjust tests for all GC configurations --- doc/md/canister-maintenance/memory.md | 2 +- test/run-drun/low-memory.mo | 3 --- test/run-drun/ok/low-memory.drun-run.ok | 1 - 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/doc/md/canister-maintenance/memory.md b/doc/md/canister-maintenance/memory.md index cbddbada477..053efab8f3f 100644 --- a/doc/md/canister-maintenance/memory.md +++ b/doc/md/canister-maintenance/memory.md @@ -2,7 +2,7 @@ sidebar_position: 5 --- -# Memory management +# Memory diagnostics ## Low memory hook diff --git a/test/run-drun/low-memory.mo b/test/run-drun/low-memory.mo index 29b04201acf..372dfe165b9 100644 --- a/test/run-drun/low-memory.mo +++ b/test/run-drun/low-memory.mo @@ -37,20 +37,17 @@ actor Self { let threshold1 = maxMemory - (await lowMemoryActor.memorySize()) - mb : Nat; await setMemoryThreshold(lowMemoryActor, threshold1); - Prim.debugPrint("Memory threshold set " # debug_show(threshold1)); await lowMemoryActor.allocateMemory(); // Not yet implemented on IC: Should retrigger when lowering threshold. // let threshold2 = maxMemory - (await lowMemoryActor.memorySize()) - mb : Nat; // await lowMemoryActor.allocateMemory(); // await setMemoryThreshold(lowMemoryActor, threshold2); - // Prim.debugPrint("Memory threshold set " # debug_show(threshold2)); // Not yet implemented on IC: Should retrigger when shrinking memory by reinstallation. // let lowMemoryActor2 = await (system LowMemoryActor.LowMemoryActor)(#reinstall lowMemoryActor)(lowMemoryCallback); // let threshold3 = maxMemory - (await lowMemoryActor2.memorySize()) - mb : Nat; // await setMemoryThreshold(lowMemoryActor2, threshold3); - // Prim.debugPrint("Memory threshold set " # debug_show(threshold3)); // await lowMemoryActor2.allocateMemory(); }; }; diff --git a/test/run-drun/ok/low-memory.drun-run.ok b/test/run-drun/ok/low-memory.drun-run.ok index 61b81361a16..8727e74bd91 100644 --- a/test/run-drun/ok/low-memory.drun-run.ok +++ b/test/run-drun/ok/low-memory.drun-run.ok @@ -1,6 +1,5 @@ ingress Completed: Reply: 0x4449444c016c01b3c4b1f204680100010a00000000000000000101 ingress Completed: Reply: 0x4449444c0000 -debug.print: Memory threshold set 4_159_700_992 debug.print: Low memory! debug.print: Low memory callback debug.print: Low memory callback done From 11992252ae2ee5b8ba6b230c030ff13d4e5bb0fd Mon Sep 17 00:00:00 2001 From: luc-blaeser Date: Wed, 15 Jan 2025 09:53:48 +0100 Subject: [PATCH 04/11] Extend change log --- Changelog.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Changelog.md b/Changelog.md index 35686efe229..e6a438656c5 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,11 @@ # Motoko compiler changelog +## 0.13.6 (tbd) + +* motoko (`moc`) + + * Support low Wasm memory hook: `system func onLowMemory() : async () { ... }`. + ## 0.13.5 (2024-12-06) * motoko (`moc`) From c0706452a98768a62bb8d4e504256beec0674832 Mon Sep 17 00:00:00 2001 From: luc-blaeser Date: Wed, 15 Jan 2025 10:12:06 +0100 Subject: [PATCH 05/11] Adjust test and refine documentation --- doc/md/canister-maintenance/memory.md | 4 ++-- test/bench/ok/region-mem.drun-run.ok | 2 +- test/bench/ok/region0-mem.drun-run.ok | 2 +- test/bench/ok/stable-mem.drun-run.ok | 2 +- test/fail/ok/M0129.tc.ok | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/md/canister-maintenance/memory.md b/doc/md/canister-maintenance/memory.md index 053efab8f3f..383f77e6d2e 100644 --- a/doc/md/canister-maintenance/memory.md +++ b/doc/md/canister-maintenance/memory.md @@ -19,8 +19,8 @@ actor { } ``` -A few aspects need to be considered with the low memory hook: -* The execution of `onLowMemory` happens with a certain delay, as it is scheduled as separate asynchronous message that runs after the message in which the threshold was crossed. +The following properties apply to the low memory hook: +* The execution of `onLowMemory` happens with a certain delay, as it is scheduled as a separate asynchronous message that runs after the message in which the threshold was crossed. * Once executed, `onLowMemory` is only triggered again when the main memory free space went above the threshold (e.g. by lowering the threshold or shrinking the main memory through canister reinstallation) and when the free space again fell below the threshold. * Traps or unhandled errors in `onLowMemory` are ignored. They only revert the changes done in `onLowMemory`. * Due to its `async` return type, the `onLowMemory` function may send further messages and await results. diff --git a/test/bench/ok/region-mem.drun-run.ok b/test/bench/ok/region-mem.drun-run.ok index 28c1137fc32..3cd8271667f 100644 --- a/test/bench/ok/region-mem.drun-run.ok +++ b/test/bench/ok/region-mem.drun-run.ok @@ -1,4 +1,4 @@ ingress Completed: Reply: 0x4449444c016c01b3c4b1f204680100010a00000000000000000101 ingress Completed: Reply: 0x4449444c0000 -debug.print: {heap_diff = 0; instr_diff = 5_234_491_753} +debug.print: {heap_diff = 0; instr_diff = 5_240_635_753} ingress Completed: Reply: 0x4449444c0000 diff --git a/test/bench/ok/region0-mem.drun-run.ok b/test/bench/ok/region0-mem.drun-run.ok index 1d45643d325..1b1eccc80c7 100644 --- a/test/bench/ok/region0-mem.drun-run.ok +++ b/test/bench/ok/region0-mem.drun-run.ok @@ -1,4 +1,4 @@ ingress Completed: Reply: 0x4449444c016c01b3c4b1f204680100010a00000000000000000101 ingress Completed: Reply: 0x4449444c0000 -debug.print: {heap_diff = 0; instr_diff = 5_662_310_761} +debug.print: {heap_diff = 0; instr_diff = 5_668_454_761} ingress Completed: Reply: 0x4449444c0000 diff --git a/test/bench/ok/stable-mem.drun-run.ok b/test/bench/ok/stable-mem.drun-run.ok index 5e3eeec3bce..13861dd869a 100644 --- a/test/bench/ok/stable-mem.drun-run.ok +++ b/test/bench/ok/stable-mem.drun-run.ok @@ -1,4 +1,4 @@ ingress Completed: Reply: 0x4449444c016c01b3c4b1f204680100010a00000000000000000101 ingress Completed: Reply: 0x4449444c0000 -debug.print: {heap_diff = 0; instr_diff = 3_875_537_257} +debug.print: {heap_diff = 0; instr_diff = 3_881_681_257} ingress Completed: Reply: 0x4449444c0000 diff --git a/test/fail/ok/M0129.tc.ok b/test/fail/ok/M0129.tc.ok index 14f666c0262..5cfe1d802a1 100644 --- a/test/fail/ok/M0129.tc.ok +++ b/test/fail/ok/M0129.tc.ok @@ -1 +1 @@ -M0129.mo:2.15-2.21: type error [M0129], unexpected system method named foobar, expected heartbeat or timer or preupgrade or postupgrade or inspect +M0129.mo:2.15-2.21: type error [M0129], unexpected system method named foobar, expected heartbeat or timer or preupgrade or postupgrade or onLowMemory or inspect From 0ca1539711aef022ca5e3905b2a394e6e1c8e0b1 Mon Sep 17 00:00:00 2001 From: luc-blaeser Date: Wed, 15 Jan 2025 16:01:32 +0100 Subject: [PATCH 06/11] Update IC dependency --- nix/sources.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nix/sources.json b/nix/sources.json index 46e1b3208b7..e88e851b2f9 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -26,10 +26,10 @@ "homepage": "", "owner": "luc-blaeser", "repo": "ic", - "rev": "de6b67ec757b960f5ba5a61efa8b5c35bf48fc39", - "sha256": "0ncp487frg455jg06887igh3lzav2zdy70wwvlnqpjisdlsh6b0k", + "rev": "afc94db6269ec269ddb1eb5aded38e95fb019958", + "sha256": "1clc0183gsk824dg1qp8cqjfp44ywp9yll0a9a1frxgsbwllyk10", "type": "tarball", - "url": "https://github.com/luc-blaeser/ic/archive/de6b67ec757b960f5ba5a61efa8b5c35bf48fc39.tar.gz", + "url": "https://github.com/luc-blaeser/ic/archive/afc94db6269ec269ddb1eb5aded38e95fb019958.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "ic-hs": { From 3b264679aecf3cae7e48f19560f8c3f79df2a2a6 Mon Sep 17 00:00:00 2001 From: Luc Blaeser <112870813+luc-blaeser@users.noreply.github.com> Date: Fri, 17 Jan 2025 14:06:14 +0100 Subject: [PATCH 07/11] Apply suggestions from code review Co-authored-by: Claudio Russo --- doc/md/canister-maintenance/memory.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/md/canister-maintenance/memory.md b/doc/md/canister-maintenance/memory.md index 383f77e6d2e..0b5a4d81800 100644 --- a/doc/md/canister-maintenance/memory.md +++ b/doc/md/canister-maintenance/memory.md @@ -22,5 +22,5 @@ actor { The following properties apply to the low memory hook: * The execution of `onLowMemory` happens with a certain delay, as it is scheduled as a separate asynchronous message that runs after the message in which the threshold was crossed. * Once executed, `onLowMemory` is only triggered again when the main memory free space went above the threshold (e.g. by lowering the threshold or shrinking the main memory through canister reinstallation) and when the free space again fell below the threshold. -* Traps or unhandled errors in `onLowMemory` are ignored. They only revert the changes done in `onLowMemory`. +* Traps or unhandled errors in `onLowMemory` are ignored. Traps only revert the changes done in `onLowMemory`. * Due to its `async` return type, the `onLowMemory` function may send further messages and await results. From cc379c845f08eb14fd0036aa05b7ad04d40e2c25 Mon Sep 17 00:00:00 2001 From: luc-blaeser Date: Fri, 17 Jan 2025 14:11:18 +0100 Subject: [PATCH 08/11] Change to `lowmemory` instead of `onLowMemory` --- Changelog.md | 2 +- doc/md/canister-maintenance/memory.md | 12 ++++++------ src/ir_def/arrange_ir.ml | 2 +- src/lowering/desugar.ml | 4 ++-- src/mo_frontend/typing.ml | 2 +- test/fail/ok/M0129.tc.ok | 2 +- test/run-drun/low-memory/low-memory-actor.mo | 2 +- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Changelog.md b/Changelog.md index e6a438656c5..d051cbca21a 100644 --- a/Changelog.md +++ b/Changelog.md @@ -4,7 +4,7 @@ * motoko (`moc`) - * Support low Wasm memory hook: `system func onLowMemory() : async () { ... }`. + * Support low Wasm memory hook: `system func lowmemory() : async () { ... }`. ## 0.13.5 (2024-12-06) diff --git a/doc/md/canister-maintenance/memory.md b/doc/md/canister-maintenance/memory.md index 0b5a4d81800..99aaa29d282 100644 --- a/doc/md/canister-maintenance/memory.md +++ b/doc/md/canister-maintenance/memory.md @@ -8,19 +8,19 @@ sidebar_position: 5 The IC allows to implement a low memory hook, which is a warning trigger when main memory is becoming scarce. -For this purpose, a Motoko actor or actor class instance can implement the system function `onLowMemory()`. This system function is scheduled when canister's free main memory space has fallen below the defined threshold `wasm_memory_threshold`, that is is part of the canister settings. In Motoko, `onLowMemory()` implements the `canister_on_low_wasm_memory` hook defined in the IC specification. +For this purpose, a Motoko actor or actor class instance can implement the system function `lowmemory()`. This system function is scheduled when canister's free main memory space has fallen below the defined threshold `wasm_memory_threshold`, that is is part of the canister settings. In Motoko, `lowmemory()` implements the `canister_on_low_wasm_memory` hook defined in the IC specification. Example of using the low memory hook: ``` actor { - system func onLowMemory() : async () { + system func lowmemory() : async () { Debug.print("Low memory!"); } } ``` The following properties apply to the low memory hook: -* The execution of `onLowMemory` happens with a certain delay, as it is scheduled as a separate asynchronous message that runs after the message in which the threshold was crossed. -* Once executed, `onLowMemory` is only triggered again when the main memory free space went above the threshold (e.g. by lowering the threshold or shrinking the main memory through canister reinstallation) and when the free space again fell below the threshold. -* Traps or unhandled errors in `onLowMemory` are ignored. Traps only revert the changes done in `onLowMemory`. -* Due to its `async` return type, the `onLowMemory` function may send further messages and await results. +* The execution of `lowmemory` happens with a certain delay, as it is scheduled as a separate asynchronous message that runs after the message in which the threshold was crossed. +* Once executed, `lowmemory` is only triggered again when the main memory free space first exceeds and then falls below the threshold. +* Traps or unhandled errors in `lowmemory` are ignored. Traps only revert the changes done in `lowmemory`. +* Due to its `async` return type, the `lowmemory` function may send further messages and await results. diff --git a/src/ir_def/arrange_ir.ml b/src/ir_def/arrange_ir.ml index 25fe792e408..55a42d9cec9 100644 --- a/src/ir_def/arrange_ir.ml +++ b/src/ir_def/arrange_ir.ml @@ -42,7 +42,7 @@ and system { meta; preupgrade; postupgrade; heartbeat; timer; inspect; low_memor "Heartbeat" $$ [exp heartbeat]; "Timer" $$ [exp timer]; "Inspect" $$ [exp inspect]; - "OnLowMemory" $$ [exp low_memory]; + "LowMemory" $$ [exp low_memory]; "StableRecord" $$ [exp stable_record]; "StableType" $$ [typ stable_type] ] diff --git a/src/lowering/desugar.ml b/src/lowering/desugar.ml index 91f1c3aa62a..ceffa3ca186 100644 --- a/src/lowering/desugar.ml +++ b/src/lowering/desugar.ml @@ -424,7 +424,7 @@ and call_system_func_opt name es obj_typ = (unitE ()) (primE (Ir.OtherPrim "trap") [textE "canister_inspect_message explicitly refused message"])) - | "onLowMemory" -> + | "lowmemory" -> blockE [ expD (callE (varE (var id.it note)) [T.Any] (unitE())) ] (unitE ()) @@ -617,7 +617,7 @@ and build_actor at ts self_id es obj_typ = | Some call -> call | None -> tupE []); low_memory = - (match call_system_func_opt "onLowMemory" es obj_typ with + (match call_system_func_opt "lowmemory" es obj_typ with | Some call -> call | None -> tupE []); stable_record = with_stable_vars (fun e -> e); diff --git a/src/mo_frontend/typing.ml b/src/mo_frontend/typing.ml index 0d88381f456..acf547246ea 100644 --- a/src/mo_frontend/typing.ml +++ b/src/mo_frontend/typing.ml @@ -403,7 +403,7 @@ let system_funcs tfs = ("timer", timer_type); T.("preupgrade", Func (Local, Returns, [scope_bind], [], [])); T.("postupgrade", Func (Local, Returns, [scope_bind], [], [])); - ("onLowMemory", low_memory_type); + ("lowmemory", low_memory_type); ("inspect", (let msg_typ = T.decode_msg_typ tfs in let record_typ = diff --git a/test/fail/ok/M0129.tc.ok b/test/fail/ok/M0129.tc.ok index 5cfe1d802a1..aafbcc41faf 100644 --- a/test/fail/ok/M0129.tc.ok +++ b/test/fail/ok/M0129.tc.ok @@ -1 +1 @@ -M0129.mo:2.15-2.21: type error [M0129], unexpected system method named foobar, expected heartbeat or timer or preupgrade or postupgrade or onLowMemory or inspect +M0129.mo:2.15-2.21: type error [M0129], unexpected system method named foobar, expected heartbeat or timer or preupgrade or postupgrade or lowmemory or inspect diff --git a/test/run-drun/low-memory/low-memory-actor.mo b/test/run-drun/low-memory/low-memory-actor.mo index 0d58a62def9..ec7c1d654a3 100644 --- a/test/run-drun/low-memory/low-memory-actor.mo +++ b/test/run-drun/low-memory/low-memory-actor.mo @@ -1,7 +1,7 @@ import Prim "mo:⛔"; actor class LowMemoryActor(callback: shared () -> async ()) { - system func onLowMemory() : async() { + system func lowmemory() : async() { Prim.debugPrint("Low memory!"); await callback(); Prim.debugPrint("Low memory callback done"); From 7d6c23ada3a3b46ea8b4d55d6e3c8a05cfe7bbc1 Mon Sep 17 00:00:00 2001 From: luc-blaeser Date: Fri, 17 Jan 2025 15:29:23 +0100 Subject: [PATCH 09/11] Use `async*` for `lowmemory` hook --- Changelog.md | 2 +- doc/md/canister-maintenance/memory.md | 2 +- src/ir_def/check_ir.ml | 6 ++-- src/lowering/desugar.ml | 5 ++- src/mo_frontend/typing.ml | 2 +- test/run-drun/low-memory.mo | 37 ++++++++++---------- test/run-drun/low-memory/low-memory-actor.mo | 10 +++--- 7 files changed, 33 insertions(+), 31 deletions(-) diff --git a/Changelog.md b/Changelog.md index d051cbca21a..e6b4f992353 100644 --- a/Changelog.md +++ b/Changelog.md @@ -4,7 +4,7 @@ * motoko (`moc`) - * Support low Wasm memory hook: `system func lowmemory() : async () { ... }`. + * Support low Wasm memory hook: `system func lowmemory() : async* () { ... }`. ## 0.13.5 (2024-12-06) diff --git a/doc/md/canister-maintenance/memory.md b/doc/md/canister-maintenance/memory.md index 99aaa29d282..2239b9295ab 100644 --- a/doc/md/canister-maintenance/memory.md +++ b/doc/md/canister-maintenance/memory.md @@ -13,7 +13,7 @@ For this purpose, a Motoko actor or actor class instance can implement the syste Example of using the low memory hook: ``` actor { - system func lowmemory() : async () { + system func lowmemory() : async* () { Debug.print("Low memory!"); } } diff --git a/src/ir_def/check_ir.ml b/src/ir_def/check_ir.ml index e9eaf0aee49..8d8d23c56d9 100644 --- a/src/ir_def/check_ir.ml +++ b/src/ir_def/check_ir.ml @@ -828,7 +828,8 @@ let rec check_exp env (exp:Ir.exp) : unit = check_exp env'' heartbeat; check_exp env'' timer; check_exp env'' inspect; - check_exp env'' low_memory; + let async_cap = Some Async_cap.top_cap in + check_exp { env'' with async = async_cap } low_memory; check_exp env'' stable_record; typ preupgrade <: T.unit; typ postupgrade <: T.unit; @@ -1181,7 +1182,8 @@ let check_comp_unit env = function check_exp env'' timer; check_exp env'' inspect; check_exp env'' stable_record; - check_exp env'' low_memory; + let async_cap = Some Async_cap.top_cap in + check_exp { env'' with async = async_cap } low_memory; typ preupgrade <: T.unit; typ postupgrade <: T.unit; typ heartbeat <: T.unit; diff --git a/src/lowering/desugar.ml b/src/lowering/desugar.ml index ceffa3ca186..33630d0792a 100644 --- a/src/lowering/desugar.ml +++ b/src/lowering/desugar.ml @@ -425,9 +425,8 @@ and call_system_func_opt name es obj_typ = (primE (Ir.OtherPrim "trap") [textE "canister_inspect_message explicitly refused message"])) | "lowmemory" -> - blockE - [ expD (callE (varE (var id.it note)) [T.Any] (unitE())) ] - (unitE ()) + awaitE T.Cmp + (callE (varE (var id.it note)) [T.scope_bound] (unitE())) | name -> let inst = match name with | "preupgrade" | "postupgrade" -> [T.scope_bound] diff --git a/src/mo_frontend/typing.ml b/src/mo_frontend/typing.ml index acf547246ea..c40dcdb9976 100644 --- a/src/mo_frontend/typing.ml +++ b/src/mo_frontend/typing.ml @@ -395,7 +395,7 @@ let timer_type = [Async (Fut, Var (default_scope_var, 0), unit)])) let low_memory_type = - T.(Func (Local, Returns, [scope_bind], [], [Async (Fut, Var (default_scope_var, 0), unit)])) + T.(Func (Local, Returns, [scope_bind], [], [Async (Cmp, Var (default_scope_var, 0), unit)])) let system_funcs tfs = [ diff --git a/test/run-drun/low-memory.mo b/test/run-drun/low-memory.mo index 372dfe165b9..6b76c80ee8d 100644 --- a/test/run-drun/low-memory.mo +++ b/test/run-drun/low-memory.mo @@ -4,7 +4,7 @@ import Cycles = "cycles/cycles"; actor Self { type canister_settings = { - wasm_memory_threshold: ?Nat; + wasm_memory_threshold : ?Nat; }; func setMemoryThreshold(a : actor {}, threshold : Nat) : async () { @@ -15,11 +15,12 @@ actor Self { } -> async (); }; let settings : canister_settings = { - wasm_memory_threshold = ?threshold; + wasm_memory_threshold = ?threshold; }; await ic00.update_settings({ canister_id = Prim.principalOfActor(a); - settings }); + settings; + }); }; let kb = 1024; @@ -27,27 +28,27 @@ actor Self { let gb = 1024 * mb; let maxMemory = 4 * gb; // adjust when canister limits inspection is supported - public shared func lowMemoryCallback(): async () { - Prim.debugPrint("Low memory callback"); + public shared func lowMemoryCallback() : async () { + Prim.debugPrint("Low memory callback"); }; public shared func run() : async () { Cycles.add(2_000_000_000_000); - let lowMemoryActor = await LowMemoryActor.LowMemoryActor(lowMemoryCallback); - - let threshold1 = maxMemory - (await lowMemoryActor.memorySize()) - mb : Nat; - await setMemoryThreshold(lowMemoryActor, threshold1); - await lowMemoryActor.allocateMemory(); + let lowMemoryActor1 = await LowMemoryActor.LowMemoryActor(lowMemoryCallback); + // await lowMemoryActor1.allocateMemory(); + + let threshold1 = maxMemory - (await lowMemoryActor1.memorySize()) - mb : Nat; + await setMemoryThreshold(lowMemoryActor1, threshold1); + await lowMemoryActor1.allocateMemory(); - // Not yet implemented on IC: Should retrigger when lowering threshold. - // let threshold2 = maxMemory - (await lowMemoryActor.memorySize()) - mb : Nat; - // await lowMemoryActor.allocateMemory(); - // await setMemoryThreshold(lowMemoryActor, threshold2); - // Not yet implemented on IC: Should retrigger when shrinking memory by reinstallation. - // let lowMemoryActor2 = await (system LowMemoryActor.LowMemoryActor)(#reinstall lowMemoryActor)(lowMemoryCallback); - // let threshold3 = maxMemory - (await lowMemoryActor2.memorySize()) - mb : Nat; - // await setMemoryThreshold(lowMemoryActor2, threshold3); + // let lowMemoryActor2 = await (system LowMemoryActor.LowMemoryActor)(#reinstall lowMemoryActor1)(lowMemoryCallback); + // await lowMemoryActor2.allocateMemory(); + // await lowMemoryActor2.allocateMemory(); + + // Not yet implemented on IC: Should retrigger when lowering threshold. + // let threshold2 = maxMemory - (await lowMemoryActor2.memorySize()) - mb : Nat; + // await setMemoryThreshold(lowMemoryActor2, threshold2); // await lowMemoryActor2.allocateMemory(); }; }; diff --git a/test/run-drun/low-memory/low-memory-actor.mo b/test/run-drun/low-memory/low-memory-actor.mo index ec7c1d654a3..b059518bd4b 100644 --- a/test/run-drun/low-memory/low-memory-actor.mo +++ b/test/run-drun/low-memory/low-memory-actor.mo @@ -1,15 +1,15 @@ import Prim "mo:⛔"; -actor class LowMemoryActor(callback: shared () -> async ()) { - system func lowmemory() : async() { +actor class LowMemoryActor(callback : shared () -> async ()) { + system func lowmemory() : async* () { Prim.debugPrint("Low memory!"); await callback(); Prim.debugPrint("Low memory callback done"); }; type Node = { - array: [var Nat]; - next: ?Node; + array : [var Nat]; + next : ?Node; }; var root : ?Node = null; @@ -23,5 +23,5 @@ actor class LowMemoryActor(callback: shared () -> async ()) { public func memorySize() : async Nat { await async {}; // Allocate GC reserve (because of `--force-gc` flag during drun testing). Prim.rts_memory_size(); - } + }; }; From 21940abe0db03955d9b2198da38b33aae41f6432 Mon Sep 17 00:00:00 2001 From: Luc Blaeser <112870813+luc-blaeser@users.noreply.github.com> Date: Fri, 17 Jan 2025 16:55:34 +0100 Subject: [PATCH 10/11] Adjust documentation Co-authored-by: Gabor Greif --- doc/md/canister-maintenance/memory.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/md/canister-maintenance/memory.md b/doc/md/canister-maintenance/memory.md index 2239b9295ab..5294c5b2d44 100644 --- a/doc/md/canister-maintenance/memory.md +++ b/doc/md/canister-maintenance/memory.md @@ -23,4 +23,4 @@ The following properties apply to the low memory hook: * The execution of `lowmemory` happens with a certain delay, as it is scheduled as a separate asynchronous message that runs after the message in which the threshold was crossed. * Once executed, `lowmemory` is only triggered again when the main memory free space first exceeds and then falls below the threshold. * Traps or unhandled errors in `lowmemory` are ignored. Traps only revert the changes done in `lowmemory`. -* Due to its `async` return type, the `lowmemory` function may send further messages and await results. +* Due to its `async*` return type, the `lowmemory` function may send further messages and `await` results. From 3dbb2dd13d2230cd8ddbf7a56ed041e983295309 Mon Sep 17 00:00:00 2001 From: luc-blaeser Date: Mon, 20 Jan 2025 15:15:02 +0100 Subject: [PATCH 11/11] Update IC dependency --- nix/sources.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nix/sources.json b/nix/sources.json index 94a130213db..86e6de79a04 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -26,10 +26,10 @@ "homepage": "", "owner": "luc-blaeser", "repo": "ic", - "rev": "afc94db6269ec269ddb1eb5aded38e95fb019958", - "sha256": "1clc0183gsk824dg1qp8cqjfp44ywp9yll0a9a1frxgsbwllyk10", + "rev": "01c8dfda47126bb5f688302e0161f17c2c0cfe6f", + "sha256": "16h037xagxwf00k13y3h0gvfs7f3c45z2ml0j1pgvzg2fx5pzahy", "type": "tarball", - "url": "https://github.com/luc-blaeser/ic/archive/afc94db6269ec269ddb1eb5aded38e95fb019958.tar.gz", + "url": "https://github.com/luc-blaeser/ic/archive/01c8dfda47126bb5f688302e0161f17c2c0cfe6f.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "ic-hs": {