Merge pull request #8961 from MinaProtocol/fix/verifier-async

Make verifier async

.gitignore

@@ -76,3 +76,4 @@ automation/services/watchdog/check_libp2p/check_libp2p
 *.backup
 
 *.terraform.lock.hcl
+_opam

dev/Makefile

@@ -0,0 +1,5 @@
+all:
+	docker-compose up
+
+ssh:
+	docker exec -it -w "/mina" mina /bin/bash

dev/README.md

@@ -0,0 +1,9 @@
+# Dev Environment
+
+To build your changes using docker, you can use the following flow:
+
+- in one window run `make`
+- in another one run `make ssh` to have a shell into the container
+    + don't forget to run `./scripts/setup-ocaml.sh` to set up opam/ocaml/etc.
+    + don't forget to run `eval $(opam config env)` as well
+    + `make build` should work after that :)

dev/docker-compose.yml

@@ -0,0 +1,19 @@
+version: "3.9"
+
+services:
+  mina:
+    container_name: mina
+    build:
+        context: ../dockerfiles
+        dockerfile: Dockerfile-toolchain
+    volumes:
+      - "..:/mina" # mount the mina folder so that changes are reflected in the container
+      - "_opam:/mina/_opam" # exclude any existing local switch
+      - "opam:/home/opam/.opam" # cache opam stuff
+    stdin_open: true # docker run -i
+    tty: true # docker run -t
+
+# volumes
+volumes:
+  opam:
+  _opam:

src/app/reformat/reformat.ml

@@ -16,7 +16,8 @@ let dirs_trustlist =
   ; "tablecloth"
   ; "zexe"
   ; "marlin"
-  ; "snarky" ]
+  ; "snarky"
+  ; "_opam" ]
 
 let rec fold_over_files ~path ~process_path ~init ~f =
   let%bind all = Sys.ls_dir path in

src/lib/blockchain_snark/blockchain_snark_state.mli

@@ -14,7 +14,7 @@ type tag =
 val verify :
      (Protocol_state.Value.t * Proof.t) list
   -> key:Pickles.Verification_key.t
-  -> bool
+  -> bool Async.Deferred.t
 
 val check :
      Witness.t

src/lib/pickles/common.ml

@@ -175,9 +175,10 @@ module Ipa = struct
         Array.of_list_map comm_chals ~f:(fun (comm, _) ->
             Or_infinity.Finite comm )
       in
-      Marlin_plonk_bindings.Pasta_fp_urs.batch_accumulator_check
-        (Backend.Tick.Keypair.load_urs ())
-        comms chals
+      let urs = Backend.Tick.Keypair.load_urs () in
+      Async.In_thread.run (fun () ->
+          Marlin_plonk_bindings.Pasta_fp_urs.batch_accumulator_check urs comms
+            chals )
   end
 end
 

src/lib/pickles/pickles.ml

@@ -244,7 +244,7 @@ module type Proof_intf = sig
 
   val id : Verification_key.Id.t Lazy.t
 
-  val verify : (statement * t) list -> bool
+  val verify : (statement * t) list -> bool Async.Deferred.t
 end
 
 module Prover = struct
@@ -853,7 +853,11 @@ module Side_loaded = struct
                           Pow_2_roots_of_unity (Int.ceil_log2 input_size)
                       ; h= d.h } )
               ; index=
-                  (match vk.wrap_vk with None -> return false | Some x -> x)
+                  ( match vk.wrap_vk with
+                  | None ->
+                      return (Async.return false)
+                  | Some x ->
+                      x )
               ; data=
                   (* This isn't used in verify_heterogeneous, so we can leave this dummy *)
                   {constraints= 0} }
@@ -1033,7 +1037,10 @@ let%test_module "test no side-loaded" =
       in
       [(Field.Constant.zero, b0); (Field.Constant.one, b1)]
 
-    let%test_unit "verify" = assert (Blockchain_snark.Proof.verify xs)
+    let%test_unit "verify" =
+      assert (
+        Async.Thread_safe.block_on_async_exn (fun () ->
+            Blockchain_snark.Proof.verify xs ) )
   end )
 
 (*

src/lib/pickles/pickles.mli

@@ -58,7 +58,7 @@ module type Proof_intf = sig
 
   val id : Verification_key.Id.t Lazy.t
 
-  val verify : (statement * t) list -> bool
+  val verify : (statement * t) list -> bool Async.Deferred.t
 end
 
 module Proof : sig
@@ -90,7 +90,7 @@ val verify :
   -> (module Statement_value_intf with type t = 'a)
   -> Verification_key.t
   -> ('a * ('n, 'n) Proof.t) list
-  -> bool
+  -> bool Async.Deferred.t
 
 module Prover : sig
   type ('prev_values, 'local_widths, 'local_heights, 'a_value, 'proof) t =
@@ -169,7 +169,7 @@ module Side_loaded : sig
   val verify :
        value_to_field_elements:('value -> Impls.Step.Field.Constant.t array)
     -> (Verification_key.t * 'value * Proof.t) list
-    -> bool
+    -> bool Async.Deferred.t
 
   (* Must be called in the inductive rule snarky function defining a
    rule for which this tag is used as a predecessor. *)

src/lib/pickles/verify.ml

@@ -166,66 +166,66 @@ let verify_heterogenous (ts : Instance.t list) =
         plonk )
   in
   let open Backend.Tock.Proof in
+  let open Async in
+  let%bind accumulator_check =
+    Ipa.Step.accumulator_check
+      (List.map ts ~f:(fun (T (_, _, _, _, T t)) ->
+           ( t.statement.proof_state.me_only.sg
+           , Ipa.Step.compute_challenges
+               t.statement.proof_state.deferred_values.bulletproof_challenges
+           ) ))
+  in
   Common.time "batch_step_dlog_check" (fun () ->
-      check
-        ( lazy "batch_step_dlog_check"
-        , Ipa.Step.accumulator_check
-            (List.map ts ~f:(fun (T (_, _, _, _, T t)) ->
-                 ( t.statement.proof_state.me_only.sg
-                 , Ipa.Step.compute_challenges
-                     t.statement.proof_state.deferred_values
-                       .bulletproof_challenges ) )) ) ) ;
-  Common.time "dlog_check" (fun () ->
-      check
-        ( lazy "dlog_check"
-        , batch_verify
-            (List.map2_exn ts in_circuit_plonks
-               ~f:(fun (T
-                         ( ( module
-                         Max_branching )
-                         , ( module
-                         A_value )
-                         , key
-                         , app_state
-                         , T t ))
-                  plonk
-                  ->
-                 let prepared_statement :
-                     _ Types.Dlog_based.Statement.In_circuit.t =
-                   { pass_through=
-                       Common.hash_pairing_me_only
-                         ~app_state:A_value.to_field_elements
-                         (Reduced_me_only.Pairing_based.prepare
-                            ~dlog_plonk_index:key.commitments
-                            {t.statement.pass_through with app_state})
-                   ; proof_state=
-                       { t.statement.proof_state with
-                         deferred_values=
-                           {t.statement.proof_state.deferred_values with plonk}
-                       ; me_only=
-                           Common.hash_dlog_me_only Max_branching.n
-                             (Reduced_me_only.Dlog_based.prepare
-                                t.statement.proof_state.me_only) } }
-                 in
-                 let input =
-                   tock_unpadded_public_input_of_statement prepared_statement
-                 in
-                 ( key.index
-                 , t.proof
-                 , input
-                 , Some
-                     (Vector.to_list
-                        (Vector.map2
-                           ~f:(fun g cs ->
-                             { Challenge_polynomial.challenges=
-                                 Vector.to_array
-                                   (Ipa.Wrap.compute_challenges cs)
-                             ; commitment= g } )
-                           (Vector.extend_exn t.statement.pass_through.sg
-                              Max_branching.n
-                              (Lazy.force Dummy.Ipa.Wrap.sg))
-                           t.statement.proof_state.me_only
-                             .old_bulletproof_challenges)) ) )) ) ) ;
+      check (lazy "batch_step_dlog_check", accumulator_check) ) ;
+  let%map dlog_check =
+    batch_verify
+      (List.map2_exn ts in_circuit_plonks
+         ~f:(fun (T
+                   ( ( module
+                   Max_branching )
+                   , ( module
+                   A_value )
+                   , key
+                   , app_state
+                   , T t ))
+            plonk
+            ->
+           let prepared_statement : _ Types.Dlog_based.Statement.In_circuit.t =
+             { pass_through=
+                 Common.hash_pairing_me_only
+                   ~app_state:A_value.to_field_elements
+                   (Reduced_me_only.Pairing_based.prepare
+                      ~dlog_plonk_index:key.commitments
+                      {t.statement.pass_through with app_state})
+             ; proof_state=
+                 { t.statement.proof_state with
+                   deferred_values=
+                     {t.statement.proof_state.deferred_values with plonk}
+                 ; me_only=
+                     Common.hash_dlog_me_only Max_branching.n
+                       (Reduced_me_only.Dlog_based.prepare
+                          t.statement.proof_state.me_only) } }
+           in
+           let input =
+             tock_unpadded_public_input_of_statement prepared_statement
+           in
+           ( key.index
+           , t.proof
+           , input
+           , Some
+               (Vector.to_list
+                  (Vector.map2
+                     ~f:(fun g cs ->
+                       { Challenge_polynomial.challenges=
+                           Vector.to_array (Ipa.Wrap.compute_challenges cs)
+                       ; commitment= g } )
+                     (Vector.extend_exn t.statement.pass_through.sg
+                        Max_branching.n
+                        (Lazy.force Dummy.Ipa.Wrap.sg))
+                     t.statement.proof_state.me_only.old_bulletproof_challenges))
+           ) ))
+  in
+  Common.time "dlog_check" (fun () -> check (lazy "dlog_check", dlog_check)) ;
   match result () with
   | Ok () ->
       true

src/lib/prover/prover.ml

@@ -49,7 +49,7 @@ module Worker_state = struct
       -> Pending_coinbase_witness.t
       -> Blockchain.t Async.Deferred.Or_error.t
 
-    val verify : Protocol_state.Value.t -> Proof.t -> bool
+    val verify : Protocol_state.Value.t -> Proof.t -> bool Deferred.t
   end
 
   (* bin_io required by rpc_parallel *)
@@ -168,7 +168,7 @@ module Worker_state = struct
                        "Prover threw an error while extending block: $error" ) ;
                  Async.Deferred.return res
 
-               let verify _state _proof = true
+               let verify _state _proof = Deferred.return true
              end
              : S )
          | None ->
@@ -183,7 +183,7 @@ module Worker_state = struct
                          ~proof:Mina_base.Proof.blockchain_dummy
                          ~state:next_state)
 
-               let verify _ _ = true
+               let verify _ _ = Deferred.return true
              end
              : S )
        in
@@ -221,8 +221,7 @@ module Functions = struct
         let (module W) = Worker_state.get w in
         W.verify
           (Blockchain_snark.Blockchain.state chain)
-          (Blockchain_snark.Blockchain.proof chain)
-        |> Deferred.return )
+          (Blockchain_snark.Blockchain.proof chain) )
 end
 
 module Worker = struct

src/lib/transaction_snark/transaction_snark.ml

@@ -2984,13 +2984,13 @@ module Verification = struct
   module type S = sig
     val tag : tag
 
-    val verify : (t * Sok_message.t) list -> bool
+    val verify : (t * Sok_message.t) list -> bool Async.Deferred.t
 
     val id : Pickles.Verification_key.Id.t Lazy.t
 
     val verification_key : Pickles.Verification_key.t Lazy.t
 
-    val verify_against_digest : t -> bool
+    val verify_against_digest : t -> bool Async.Deferred.t
 
     val constraint_system_digests : (string * Md5_lib.t) list Lazy.t
   end
@@ -3314,15 +3314,18 @@ let generate_transaction_witness ?preeval ~constraint_constants ~sok_message
         pending_coinbase_stack_state handler
 
 let verify (ts : (t * _) list) ~key =
-  List.for_all ts ~f:(fun ({statement; _}, message) ->
-      Sok_message.Digest.equal
-        (Sok_message.digest message)
-        statement.sok_digest )
-  && Pickles.verify
-       (module Nat.N2)
-       (module Statement.With_sok)
-       key
-       (List.map ts ~f:(fun ({statement; proof}, _) -> (statement, proof)))
+  if
+    List.for_all ts ~f:(fun ({statement; _}, message) ->
+        Sok_message.Digest.equal
+          (Sok_message.digest message)
+          statement.sok_digest )
+  then
+    Pickles.verify
+      (module Nat.N2)
+      (module Statement.With_sok)
+      key
+      (List.map ts ~f:(fun ({statement; proof}, _) -> (statement, proof)))
+  else Async.return false
 
 let constraint_system_digests ~constraint_constants () =
   let digest = Tick.R1CS_constraint_system.digest in
@@ -3361,11 +3364,14 @@ struct
     Proof.verify [(statement, proof)]
 
   let verify ts =
-    List.for_all ts ~f:(fun (p, m) ->
-        Sok_message.Digest.equal (Sok_message.digest m) p.statement.sok_digest
-    )
-    && Proof.verify
-         (List.map ts ~f:(fun ({statement; proof}, _) -> (statement, proof)))
+    if
+      List.for_all ts ~f:(fun (p, m) ->
+          Sok_message.Digest.equal (Sok_message.digest m)
+            p.statement.sok_digest )
+    then
+      Proof.verify
+        (List.map ts ~f:(fun ({statement; proof}, _) -> (statement, proof)))
+    else Async.return false
 
   let of_transaction_union sok_digest source target ~init_stack
       ~pending_coinbase_stack_state ~next_available_token_before
@@ -4303,7 +4309,9 @@ let%test_module "transaction_snark" =
                     merge ~sok_digest proof12 proof23 )
                 |> Or_error.ok_exn
               in
-              Proof.verify [(proof13.statement, proof13.proof)] ) )
+              Async.Thread_safe.block_on_async (fun () ->
+                  Proof.verify [(proof13.statement, proof13.proof)] )
+              |> Result.ok_exn ) )
 
     let%test "base_and_merge: transactions in one block (t1,t2 in b1), \
               carryforward the state from a previous transaction t0 in b1" =

src/lib/transaction_snark/transaction_snark.mli

@@ -232,19 +232,22 @@ type tag =
   , Nat.N2.n )
   Pickles.Tag.t
 
-val verify : (t * Sok_message.t) list -> key:Pickles.Verification_key.t -> bool
+val verify :
+     (t * Sok_message.t) list
+  -> key:Pickles.Verification_key.t
+  -> bool Async.Deferred.t
 
 module Verification : sig
   module type S = sig
     val tag : tag
 
-    val verify : (t * Sok_message.t) list -> bool
+    val verify : (t * Sok_message.t) list -> bool Async.Deferred.t
 
     val id : Pickles.Verification_key.Id.t Lazy.t
 
     val verification_key : Pickles.Verification_key.t Lazy.t
 
-    val verify_against_digest : t -> bool
+    val verify_against_digest : t -> bool Async.Deferred.t
 
     val constraint_system_digests : (string * Md5_lib.t) list Lazy.t
   end