optimize memory consumption for verifier subprocess

src/app/archive/lib/test.ml

@@ -20,10 +20,8 @@ let%test_module "Archive node unit tests" =
 
     let verifier =
       Async.Thread_safe.block_on_async_exn (fun () ->
-          Verifier.create ~logger ~proof_level ~constraint_constants
-            ~conf_dir:None
-            ~pids:(Child_processes.Termination.create_pid_table ())
-            ~commit_id:"not specified for unit tests" () )
+          Verifier.For_tests.default ~constraint_constants ~logger ~proof_level
+            () )
 
     module Genesis_ledger = (val Genesis_ledger.for_unit_tests)
 

src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml

@@ -1736,10 +1736,11 @@ let internal_commands ~itn_features logger =
                     | Error err ->
                         failwithf "Could not parse JSON: %s" err () ) )
           in
+
           let%bind verifier =
-            Verifier.create ~commit_id:Mina_version.commit_id ~logger
-              ~proof_level ~constraint_constants ~pids:(Pid.Table.create ())
-              ~conf_dir:(Some conf_dir) ()
+            Verifier.For_tests.default ~constraint_constants ~proof_level
+              ~commit_id:Mina_version.commit_id ~logger
+              ~pids:(Pid.Table.create ()) ~conf_dir:(Some conf_dir) ()
           in
           let%bind result =
             let cap lst =

src/app/cli/src/init/transaction_snark_profiler.ml

@@ -17,10 +17,8 @@ let run ~genesis_constants ~constraint_constants ~proof_level
     Parallel.init_master () ;
     let verifier =
       Async.Thread_safe.block_on_async_exn (fun () ->
-          Verifier.create ~commit_id:Mina_version.commit_id ~logger ~proof_level
-            ~constraint_constants ~conf_dir:None
-            ~pids:(Child_processes.Termination.create_pid_table ())
-            () )
+          Verifier.For_tests.default ~commit_id:Mina_version.commit_id ~logger
+            ~proof_level ~constraint_constants () )
     in
     let rec go n =
       if n <= 0 then ()

src/lib/bootstrap_controller/bootstrap_controller.ml

@@ -745,10 +745,8 @@ let%test_module "Bootstrap_controller tests" =
 
     let verifier =
       Async.Thread_safe.block_on_async_exn (fun () ->
-          Verifier.create ~logger ~proof_level ~constraint_constants
-            ~conf_dir:None
-            ~pids:(Child_processes.Termination.create_pid_table ())
-            ~commit_id:"not specified for unit tests" () )
+          Verifier.For_tests.default ~constraint_constants ~logger ~proof_level
+            () )
 
     module Genesis_ledger = (val precomputed_values.genesis_ledger)
 

src/lib/ledger_catchup/normal_catchup.ml

@@ -908,10 +908,8 @@ let%test_module "Ledger_catchup tests" =
 
     let verifier =
       Async.Thread_safe.block_on_async_exn (fun () ->
-          Verifier.create ~logger ~proof_level ~constraint_constants
-            ~conf_dir:None
-            ~pids:(Child_processes.Termination.create_pid_table ())
-            ~commit_id:"not specified for unit tests" () )
+          Verifier.For_tests.default ~constraint_constants ~logger ~proof_level
+            () )
 
     module Context = struct
       let logger = logger

src/lib/ledger_catchup/super_catchup.ml

@@ -1453,10 +1453,8 @@ let%test_module "Ledger_catchup tests" =
 
     let verifier =
       Async.Thread_safe.block_on_async_exn (fun () ->
-          Verifier.create ~logger ~proof_level ~constraint_constants
-            ~conf_dir:None
-            ~pids:(Child_processes.Termination.create_pid_table ())
-            ~commit_id:"not specified for unit tests" () )
+          Verifier.For_tests.default ~constraint_constants ~logger ~proof_level
+            () )
 
     module Context = struct
       let logger = logger

src/lib/mina_graphql/mina_graphql.ml

@@ -2663,7 +2663,7 @@ module Queries = struct
       ~args:Arg.[]
       ~resolve:(fun { ctx = mina; _ } () ->
         let open Deferred.Result.Let_syntax in
-        Mina_lib.verifier mina |> Verifier.get_blockchain_verification_key
+        Mina_lib.prover mina |> Prover.get_blockchain_verification_key
         |> Deferred.Result.map_error ~f:Error.to_string_hum
         >>| Pickles.Verification_key.to_yojson >>| Yojson.Safe.to_basic )
 

src/lib/mina_lib/mina_lib.ml

@@ -1564,15 +1564,23 @@ let create ~commit_id ?wallets (config : Config.t) =
                       ~metadata:[ ("exn", Error_json.error_to_yojson err) ] ) )
               (fun () ->
                 O1trace.thread "manage_verifier_subprocess" (fun () ->
+                    let%bind blockchain_verification_key =
+                      Prover.get_blockchain_verification_key prover
+                      >>| Or_error.ok_exn
+                    in
+                    let%bind transaction_verification_key =
+                      Prover.get_transaction_verification_key prover
+                      >>| Or_error.ok_exn
+                    in
                     let%bind verifier =
                       Verifier.create ~commit_id ~logger:config.logger
                         ~enable_internal_tracing:
                           (Internal_tracing.is_enabled ())
                         ~internal_trace_filename:"verifier-internal-trace.jsonl"
                         ~proof_level:config.precomputed_values.proof_level
-                        ~constraint_constants:
-                          config.precomputed_values.constraint_constants
-                        ~pids:config.pids ~conf_dir:(Some config.conf_dir) ()
+                        ~pids:config.pids ~conf_dir:(Some config.conf_dir)
+                        ~blockchain_verification_key
+                        ~transaction_verification_key ()
                     in
                     let%map () = set_itn_data (module Verifier) verifier in
                     verifier ) )
@@ -2250,7 +2258,7 @@ let get_filtered_log_entries
   in
   (get_from_idx curr_idx messages [], is_started)
 
-let verifier { processes = { verifier; _ }; _ } = verifier
+let prover { processes = { prover; _ }; _ } = prover
 
 let vrf_evaluator { processes = { vrf_evaluator; _ }; _ } = vrf_evaluator
 

src/lib/mina_lib/mina_lib.mli

@@ -233,7 +233,7 @@ val start_filtered_log : t -> string list -> unit Or_error.t
 
 val get_filtered_log_entries : t -> int -> string list * bool
 
-val verifier : t -> Verifier.t
+val prover : t -> Prover.t
 
 val vrf_evaluator : t -> Vrf_evaluator.t
 

src/lib/mina_lib/tests/tests.ml

@@ -140,10 +140,10 @@ let%test_module "Epoch ledger sync tests" =
 
     let make_verifier (module Context : CONTEXT) =
       let open Context in
-      Verifier.create ~logger ~proof_level:precomputed_values.proof_level
-        ~constraint_constants:precomputed_values.constraint_constants ~pids
+      Verifier.For_tests.default ~constraint_constants ~logger
+        ~proof_level:precomputed_values.proof_level ~pids
         ~conf_dir:(Some (make_dirname "verifier"))
-        ~commit_id:"not specified for unit tests" ()
+        ()
 
     let make_empty_ledger (module Context : CONTEXT) =
       Mina_ledger.Ledger.create

src/lib/network_pool/batcher.ml

@@ -498,10 +498,8 @@ module Snark_pool = struct
 
       let verifier =
         Async.Thread_safe.block_on_async_exn (fun () ->
-            Verifier.create ~logger ~proof_level ~constraint_constants
-              ~conf_dir:None
-              ~pids:(Child_processes.Termination.create_pid_table ())
-              ~commit_id:"not specified for unit tests" () )
+            Verifier.For_tests.default ~constraint_constants ~logger
+              ~proof_level () )
 
       let gen_proofs =
         let open Quickcheck.Generator.Let_syntax in

src/lib/network_pool/snark_pool.ml

@@ -591,10 +591,8 @@ let%test_module "random set test" =
 
     let verifier =
       Async.Thread_safe.block_on_async_exn (fun () ->
-          Verifier.create ~logger ~proof_level ~constraint_constants
-            ~conf_dir:None
-            ~pids:(Child_processes.Termination.create_pid_table ())
-            ~commit_id:"not specified for unit tests" () )
+          Verifier.For_tests.default ~constraint_constants ~logger ~proof_level
+            () )
 
     module Mock_snark_pool =
       Make (Mocks.Base_ledger) (Mocks.Staged_ledger) (Mocks.Transition_frontier)

src/lib/network_pool/test.ml

@@ -27,10 +27,8 @@ let%test_module "network pool test" =
 
     let verifier =
       Async.Thread_safe.block_on_async_exn (fun () ->
-          Verifier.create ~logger ~proof_level ~constraint_constants
-            ~conf_dir:None
-            ~pids:(Child_processes.Termination.create_pid_table ())
-            ~commit_id:"not specified for unit tests" () )
+          Verifier.For_tests.default ~constraint_constants ~logger ~proof_level
+            () )
 
     module Mock_snark_pool =
       Snark_pool.Make (Mocks.Base_ledger) (Mocks.Staged_ledger)

src/lib/network_pool/transaction_pool.ml

@@ -1672,10 +1672,8 @@ let%test_module _ =
 
     let verifier =
       Async.Thread_safe.block_on_async_exn (fun () ->
-          Verifier.create ~logger ~proof_level ~constraint_constants
-            ~conf_dir:None
-            ~pids:(Child_processes.Termination.create_pid_table ())
-            ~commit_id:"not specified for unit tests" () )
+          Verifier.For_tests.default ~constraint_constants ~logger ~proof_level
+            () )
 
     let `VK vk, `Prover prover =
       Transaction_snark.For_tests.create_trivial_snapp ~constraint_constants ()
@@ -3090,10 +3088,8 @@ let%test_module _ =
               authorization would be rejected" =
       Thread_safe.block_on_async_exn (fun () ->
           let%bind verifier_full =
-            Verifier.create ~logger ~proof_level:Full ~constraint_constants
-              ~conf_dir:None
-              ~pids:(Child_processes.Termination.create_pid_table ())
-              ~commit_id:"not specified for unit tests" ()
+            Verifier.For_tests.default ~constraint_constants ~logger
+              ~proof_level:Full ()
           in
           let%bind test =
             setup_test ~verifier:verifier_full

src/lib/prover/intf.ml

@@ -57,4 +57,10 @@ module type S = sig
      sets the process kind for the Itn logger to "prover"
   *)
   val set_itn_logger_data : t -> daemon_port:int -> unit Deferred.Or_error.t
+
+  val get_blockchain_verification_key :
+    t -> Pickles.Verification_key.t Deferred.Or_error.t
+
+  val get_transaction_verification_key :
+    t -> Pickles.Verification_key.t Deferred.Or_error.t
 end

src/lib/prover/prover.ml

@@ -56,6 +56,12 @@ module Worker_state = struct
     val toggle_internal_tracing : bool -> unit
 
     val set_itn_logger_data : daemon_port:int -> unit
+
+    val get_blockchain_verification_key :
+      unit -> Pickles.Verification_key.t Deferred.t
+
+    val get_transaction_verification_key :
+      unit -> Pickles.Verification_key.t Deferred.t
   end
 
   (* bin_io required by rpc_parallel *)
@@ -103,7 +109,6 @@ module Worker_state = struct
           Pickles.Cache_handle.generate_or_load B.cache_handle
           |> Promise.to_deferred
         in
-
         ( module struct
           module T = T
           module B = B
@@ -161,6 +166,12 @@ module Worker_state = struct
 
           let set_itn_logger_data ~daemon_port =
             Itn_logger.set_data ~process_kind:"prover" ~daemon_port
+
+          let get_blockchain_verification_key () =
+            Lazy.force B.Proof.verification_key
+
+          let get_transaction_verification_key () =
+            Lazy.force T.verification_key
         end : S )
     | Check ->
         Deferred.return
@@ -202,6 +213,12 @@ module Worker_state = struct
             let toggle_internal_tracing _ = ()
 
             let set_itn_logger_data ~daemon_port:_ = ()
+
+            let get_blockchain_verification_key () =
+              Deferred.return (Lazy.force Pickles.Verification_key.dummy)
+
+            let get_transaction_verification_key () =
+              Deferred.return (Lazy.force Pickles.Verification_key.dummy)
           end : S )
     | No_check ->
         Deferred.return
@@ -221,6 +238,12 @@ module Worker_state = struct
             let toggle_internal_tracing _ = ()
 
             let set_itn_logger_data ~daemon_port:_ = ()
+
+            let get_blockchain_verification_key () =
+              Deferred.return (Lazy.force Pickles.Verification_key.dummy)
+
+            let get_transaction_verification_key () =
+              Deferred.return (Lazy.force Pickles.Verification_key.dummy)
           end : S )
 
   let get = Fn.id
@@ -275,6 +298,18 @@ module Functions = struct
         let (module M) = Worker_state.get w in
         M.set_itn_logger_data ~daemon_port ;
         Deferred.unit )
+
+  let get_blockchain_verification_key =
+    create bin_unit [%bin_type_class: Pickles.Verification_key.Stable.Latest.t]
+      (fun w () ->
+        let (module M) = Worker_state.get w in
+        M.get_blockchain_verification_key () )
+
+  let get_transaction_verification_key =
+    create bin_unit [%bin_type_class: Pickles.Verification_key.Stable.Latest.t]
+      (fun w () ->
+        let (module M) = Worker_state.get w in
+        M.get_transaction_verification_key () )
 end
 
 module Worker = struct
@@ -288,6 +323,10 @@ module Worker = struct
       ; verify_blockchain : ('w, Blockchain.t, unit Or_error.t) F.t
       ; toggle_internal_tracing : ('w, bool, unit) F.t
       ; set_itn_logger_data : ('w, int, unit) F.t
+      ; get_blockchain_verification_key :
+          ('w, unit, Pickles.Verification_key.t) F.t
+      ; get_transaction_verification_key :
+          ('w, unit, Pickles.Verification_key.t) F.t
       }
 
     module Worker_state = Worker_state
@@ -316,6 +355,8 @@ module Worker = struct
         ; verify_blockchain = f verify_blockchain
         ; toggle_internal_tracing = f toggle_internal_tracing
         ; set_itn_logger_data = f set_itn_logger_data
+        ; get_blockchain_verification_key = f get_blockchain_verification_key
+        ; get_transaction_verification_key = f get_transaction_verification_key
         }
 
       let init_worker_state
@@ -566,3 +607,11 @@ let toggle_internal_tracing { connection; _ } enabled =
 let set_itn_logger_data { connection; _ } ~daemon_port =
   Worker.Connection.run connection ~f:Worker.functions.set_itn_logger_data
     ~arg:daemon_port
+
+let get_blockchain_verification_key { connection; _ } =
+  Worker.Connection.run connection
+    ~f:Worker.functions.get_blockchain_verification_key ~arg:()
+
+let get_transaction_verification_key { connection; _ } =
+  Worker.Connection.run connection
+    ~f:Worker.functions.get_transaction_verification_key ~arg:()

src/lib/staged_ledger/staged_ledger.ml

@@ -2389,10 +2389,8 @@ let%test_module "staged ledger tests" =
 
     let verifier =
       Async.Thread_safe.block_on_async_exn (fun () ->
-          Verifier.create ~logger ~proof_level ~constraint_constants
-            ~conf_dir:None
-            ~pids:(Child_processes.Termination.create_pid_table ())
-            ~commit_id:"not specified for unit tests" () )
+          Verifier.For_tests.default ~constraint_constants ~logger ~proof_level
+            () )
 
     let find_vk ledger =
       Zkapp_command.Verifiable.load_vk_from_ledger ~get:(Ledger.get ledger)
@@ -5187,12 +5185,10 @@ let%test_module "staged ledger tests" =
                           (Staged_ledger_diff.With_valid_signatures_and_proofs
                            .commands diff )
                         = 1 ) ;
+
                       let%bind verifier_full =
-                        Verifier.create ~logger ~proof_level:Full
-                          ~constraint_constants ~conf_dir:None
-                          ~pids:
-                            (Child_processes.Termination.create_pid_table ())
-                          ~commit_id:"not specified for unit tests" ()
+                        Verifier.For_tests.default ~constraint_constants ~logger
+                          ~proof_level:Full ()
                       in
                       match%map
                         Sl.apply ~constraint_constants ~global_slot !sl

src/lib/transaction_inclusion_status/transaction_inclusion_status.ml

@@ -92,10 +92,8 @@ let%test_module "transaction_status" =
 
     let verifier =
       Async.Thread_safe.block_on_async_exn (fun () ->
-          Verifier.create ~logger ~proof_level ~constraint_constants
-            ~conf_dir:None
-            ~pids:(Child_processes.Termination.create_pid_table ())
-            ~commit_id:"not specified for unit tests" () )
+          Verifier.For_tests.default ~constraint_constants ~logger ~proof_level
+            () )
 
     let key_gen =
       let open Quickcheck.Generator in

src/lib/transition_frontier/full_frontier/full_frontier.ml

@@ -962,10 +962,7 @@ module For_tests = struct
 
   let verifier () =
     Async.Thread_safe.block_on_async_exn (fun () ->
-        Verifier.create ~logger ~proof_level ~constraint_constants
-          ~conf_dir:None
-          ~pids:(Child_processes.Termination.create_pid_table ())
-          ~commit_id:"not specified for unit tests" () )
+        Verifier.For_tests.default ~constraint_constants ~logger ~proof_level () )
 
   module Genesis_ledger = (val precomputed_values.genesis_ledger)
 

src/lib/transition_handler/catchup_scheduler.ml

@@ -366,8 +366,8 @@ let%test_module "Transition_handler.Catchup_scheduler tests" =
 
     let verifier =
       Async.Thread_safe.block_on_async_exn (fun () ->
-          Verifier.create ~logger ~proof_level ~constraint_constants
-            ~conf_dir:None ~pids ~commit_id:"not specified for unit tests" () )
+          Verifier.For_tests.default ~constraint_constants ~logger ~proof_level
+            ~pids () )
 
     (* cast a breadcrumb into a cached, enveloped, partially validated transition *)
     let downcast_breadcrumb breadcrumb =