Merge pull request #11185 from MinaProtocol/dynamic-lagrange-commitme…

…nt-selection

Dynamic lagrange commitment selection

src/app/replayer/test/archive_db.sql

@@ -1860,11 +1860,11 @@ COPY public.accounts_created (block_id, account_identifier_id, creation_fee) FRO
 --
 
 COPY public.blocks (id, state_hash, parent_id, parent_hash, creator_id, block_winner_id, snarked_ledger_hash_id, staking_epoch_data_id, next_epoch_data_id, min_window_density, total_currency, ledger_hash, height, global_slot_since_hard_fork, global_slot_since_genesis, "timestamp", chain_status) FROM stdin;
-1	3NLDRvvByy2GMYivoFg7FADtNzjqZZ71jX2AZDVdxqtYLusb7AcV	\N	3NKBXHShSYqxwWuxRiFXCSCNKmDVJaEgTHh4FRSYWzMTJ9MRseTC	5	5	1	1	2	77	11616000000065089	jxgSP5L3y7YLoG37xJ5XQmCGb7676TDuek4j3qZVs9LjUF6Gbd4	1	0	0	1651708749987	canonical
-2	3NKHwN7Hg65eGsM2fY3kn3m5KvVsK6GVFgAgVNoXDRnCRhr1UVWJ	1	3NLDRvvByy2GMYivoFg7FADtNzjqZZ71jX2AZDVdxqtYLusb7AcV	4	3	1	1	3	77	11616000000065089	jxHbatt2mrbNoNwFJJbZdWVqSHGejMcwbRkS85yJBy91eHZixxt	2	1	1	1651708944856	pending
-3	3NLejSrDnJMo1BRVSq6fqzZMAupQRiA1eB98yEbh6SNxyiGXWYRJ	2	3NKHwN7Hg65eGsM2fY3kn3m5KvVsK6GVFgAgVNoXDRnCRhr1UVWJ	4	3	1	1	4	77	11616000000065089	jxiLr7L1xNDGtqRAh4UpA3bJC1C1Tvk8UoVboR3hkhNUppHP9Z1	3	2	2	1651709109987	pending
-4	3NKRDvPbuYVUX23tMVCMA33U5Qrua6A4EFpTfyu4UJzsa27x97aY	3	3NLejSrDnJMo1BRVSq6fqzZMAupQRiA1eB98yEbh6SNxyiGXWYRJ	4	3	1	1	5	77	11616000000065089	jw7bVbZrKcaxZKEvm1xj2rqrmb42FNVuLQ6yyhJE9CmycZ9wzVY	4	3	3	1651709289987	pending
-5	3NLuycS3WqnYZBxgLTYBXv8a1qYaS6CSt2ZQ4cG3gi4rvD64yDsg	4	3NKRDvPbuYVUX23tMVCMA33U5Qrua6A4EFpTfyu4UJzsa27x97aY	4	3	1	1	6	77	11616000000065089	jx5XboJuVNJ4u8m7KpQi3SwtG4gE9cxE5toj8xDWYBBBY4urWh3	5	4	4	1651709469987	pending
+1	3NLDRvvByy2GMYivoFg7FADtNzjqZZ71jX2AZDVdxqtYLusb7AcV	\N	3NKBXHShSYqxwWuxRiFXCSCNKmDVJaEgTHh4FRSYWzMTJ9MRseTC	5	5	1	1	2	77	11616000000065089	jxjJQUU4K3u2yXpLPCFqcYbqA3ZfTrEgn3Gqw96V5rwWRgkGZtr	1	0	0	1651708749987	canonical
+2	3NKHwN7Hg65eGsM2fY3kn3m5KvVsK6GVFgAgVNoXDRnCRhr1UVWJ	1	3NLDRvvByy2GMYivoFg7FADtNzjqZZ71jX2AZDVdxqtYLusb7AcV	4	3	1	1	3	77	11616000000065089	jwtggYGqA1qVbLqrpuXoEksBi3ddT7GM1qVAbUMvDg2jz3nMj64	2	1	1	1651708944856	pending
+3	3NLejSrDnJMo1BRVSq6fqzZMAupQRiA1eB98yEbh6SNxyiGXWYRJ	2	3NKHwN7Hg65eGsM2fY3kn3m5KvVsK6GVFgAgVNoXDRnCRhr1UVWJ	4	3	1	1	4	77	11616000000065089	jxcKVgdor95eAatRgWe3TZWrweSjxvAwvcZJ1g6TMHb4dcDVU7n	3	2	2	1651709109987	pending
+4	3NKRDvPbuYVUX23tMVCMA33U5Qrua6A4EFpTfyu4UJzsa27x97aY	3	3NLejSrDnJMo1BRVSq6fqzZMAupQRiA1eB98yEbh6SNxyiGXWYRJ	4	3	1	1	5	77	11616000000065089	jwztmpmEC61N5QYkaMb1tbiNCDeNVhoyhj8bU7eKV65syUDEqT6	4	3	3	1651709289987	pending
+5	3NLuycS3WqnYZBxgLTYBXv8a1qYaS6CSt2ZQ4cG3gi4rvD64yDsg	4	3NKRDvPbuYVUX23tMVCMA33U5Qrua6A4EFpTfyu4UJzsa27x97aY	4	3	1	1	6	77	11616000000065089	jwexhTzbjxHmy7wrbEPLzt3aQKsszzwoq9vYqkgdV43dCh6iq6A	5	4	4	1651709469987	pending
 \.
 
 
@@ -2099,7 +2099,7 @@ COPY public.public_keys (id, value) FROM stdin;
 --
 
 COPY public.snarked_ledger_hashes (id, value) FROM stdin;
-1	jxgSP5L3y7YLoG37xJ5XQmCGb7676TDuek4j3qZVs9LjUF6Gbd4
+1	jxjJQUU4K3u2yXpLPCFqcYbqA3ZfTrEgn3Gqw96V5rwWRgkGZtr
 \.
 
 

src/lib/crypto/kimchi_bindings/stubs/binding_generation/src/main.rs

@@ -335,6 +335,7 @@ fn generate_kimchi_bindings(mut w: impl std::io::Write, env: &mut Env) {
                 decl_func!(w, env, caml_fp_srs_write => "write");
                 decl_func!(w, env, caml_fp_srs_read => "read");
                 decl_func!(w, env, caml_fp_srs_lagrange_commitment => "lagrange_commitment");
+                decl_func!(w, env, caml_fp_srs_add_lagrange_basis=> "add_lagrange_basis");
                 decl_func!(w, env, caml_fp_srs_commit_evaluations => "commit_evaluations");
                 decl_func!(w, env, caml_fp_srs_b_poly_commitment => "b_poly_commitment");
                 decl_func!(w, env, caml_fp_srs_batch_accumulator_check => "batch_accumulator_check");
@@ -348,6 +349,7 @@ fn generate_kimchi_bindings(mut w: impl std::io::Write, env: &mut Env) {
                 decl_func!(w, env, caml_fq_srs_write => "write");
                 decl_func!(w, env, caml_fq_srs_read => "read");
                 decl_func!(w, env, caml_fq_srs_lagrange_commitment => "lagrange_commitment");
+                decl_func!(w, env, caml_fq_srs_add_lagrange_basis=> "add_lagrange_basis");
                 decl_func!(w, env, caml_fq_srs_commit_evaluations => "commit_evaluations");
                 decl_func!(w, env, caml_fq_srs_b_poly_commitment => "b_poly_commitment");
                 decl_func!(w, env, caml_fq_srs_batch_accumulator_check => "batch_accumulator_check");

src/lib/crypto/kimchi_bindings/stubs/kimchi_bindings.ml

@@ -105,6 +105,9 @@ module Protocol = struct
         -> Pasta_bindings.Fq.t Kimchi_types.or_infinity Kimchi_types.poly_comm
         = "caml_fp_srs_lagrange_commitment"
 
+      external add_lagrange_basis : t -> int -> unit
+        = "caml_fp_srs_add_lagrange_basis"
+
       external commit_evaluations :
            t
         -> int
@@ -144,6 +147,9 @@ module Protocol = struct
         -> Pasta_bindings.Fp.t Kimchi_types.or_infinity Kimchi_types.poly_comm
         = "caml_fq_srs_lagrange_commitment"
 
+      external add_lagrange_basis : t -> int -> unit
+        = "caml_fq_srs_add_lagrange_basis"
+
       external commit_evaluations :
            t
         -> int

src/lib/crypto/kimchi_bindings/stubs/src/srs.rs

@@ -94,6 +94,18 @@ macro_rules! impl_srs {
                 Ok(srs.commit_non_hiding(&p, None).into())
             }
 
+            #[ocaml_gen::func]
+            #[ocaml::func]
+            pub fn [<$name:snake _add_lagrange_basis>](
+                srs: $name,
+                log2_size: ocaml::Int,
+            ) {
+                let ptr: &mut commitment_dlog::srs::SRS<GAffine> =
+                    unsafe { &mut *(std::sync::Arc::as_ptr(&srs) as *mut _) };
+                let domain = EvaluationDomain::<$F>::new(1 << (log2_size as usize)).expect("invalid domain size");
+                ptr.add_lagrange_basis(domain);
+            }
+
             #[ocaml_gen::func]
             #[ocaml::func]
             pub fn [<$name:snake _commit_evaluations>](

src/lib/crypto/kimchi_bindings/wasm/src/srs.rs

@@ -73,6 +73,17 @@ macro_rules! impl_srs {
                 Arc::new(SRS::create(depth as usize)).into()
             }
 
+            #[wasm_bindgen]
+            pub fn [<$name:snake _add_lagrange_basis>](
+                srs: &[<Wasm $field_name:camel Srs>],
+                log2_size: i32,
+            ) {
+                let ptr: &mut commitment_dlog::srs::SRS<$G> =
+                    unsafe { &mut *(std::sync::Arc::as_ptr(&srs) as *mut _) };
+                let domain = EvaluationDomain::<$F>::new(1 << (log2_size as usize)).expect("invalid domain size");
+                ptr.add_lagrange_basis(domain);
+            }
+
             #[wasm_bindgen]
             pub fn [<$name:snake _write>](
                 append: Option<bool>,

src/lib/pickles/composition_types/branch_data.ml

@@ -4,46 +4,7 @@ open Pickles_types
 (** Data specific to a branch of a proof-system that's necessary for
     finalizing the deferred-values in a wrap proof of that branch. *)
 
-(** Inside the circuit, we will represent a value of this type
-    as a sequence of 2 bits:
-    00: N0
-    10: N1
-    11: N2 *)
-module Proofs_verified = struct
-  [%%versioned
-  module Stable = struct
-    module V1 = struct
-      type t = N0 | N1 | N2
-      [@@deriving sexp, sexp, compare, yojson, hash, equal]
-
-      let to_latest = Fn.id
-    end
-  end]
-
-  module Checked = struct
-    type 'f boolean = 'f Snarky_backendless.Cvar.t Snarky_backendless.Boolean.t
-
-    type 'f t = ('f boolean, Nat.N2.n) Vector.t
-  end
-
-  let to_mask : t -> (bool, Nat.N2.n) Vector.t = function
-    | N0 ->
-        [ false; false ]
-    | N1 ->
-        [ true; false ]
-    | N2 ->
-        [ true; true ]
-
-  let of_mask_exn : (bool, Nat.N2.n) Vector.t -> t = function
-    | [ false; false ] ->
-        N0
-    | [ true; false ] ->
-        N1
-    | [ true; true ] ->
-        N2
-    | [ false; true ] ->
-        failwith "Invalid mask"
-end
+module Proofs_verified = Pickles_base.Proofs_verified
 
 module Domain_log2 = struct
   [%%versioned
@@ -98,22 +59,23 @@ let pack (type f)
   (* shift domain_log2 over by 2 bits (multiply by 4) *)
   times4 domain_log2
   + project
-      (Pickles_types.Vector.to_list (Proofs_verified.to_mask proofs_verified))
+      (Pickles_types.Vector.to_list
+         (Proofs_verified.Prefix_mask.there proofs_verified) )
 
 let unpack (type f)
     (module Impl : Snarky_backendless.Snark_intf.Run with type field = f) (x : f)
     : t =
   match Impl.Field.Constant.unpack x with
   | x0 :: x1 :: y0 :: y1 :: y2 :: y3 :: y4 :: y5 :: y6 :: y7 :: _ ->
-      { proofs_verified = Proofs_verified.of_mask_exn [ x0; x1 ]
+      { proofs_verified = Proofs_verified.Prefix_mask.back [ x0; x1 ]
       ; domain_log2 = Domain_log2.of_bits_msb [ y7; y6; y5; y4; y3; y2; y1; y0 ]
       }
   | _ ->
       assert false
 
 module Checked = struct
   type 'f t =
-    { proofs_verified_mask : 'f Proofs_verified.Checked.t
+    { proofs_verified_mask : 'f Proofs_verified.Prefix_mask.Checked.t
     ; domain_log2 : 'f Snarky_backendless.Cvar.t
     }
   [@@deriving hlist]
@@ -141,10 +103,8 @@ let typ (type f)
     (assert_16_bits : Impl.Field.t -> unit) : (f Checked.t, t) Impl.Typ.t =
   let open Impl in
   let proofs_verified_mask :
-      (f Proofs_verified.Checked.t, Proofs_verified.t) Typ.t =
-    Typ.transport
-      (Vector.typ Boolean.typ Nat.N2.n)
-      ~there:Proofs_verified.to_mask ~back:Proofs_verified.of_mask_exn
+      (f Proofs_verified.Prefix_mask.Checked.t, Proofs_verified.t) Typ.t =
+    Proofs_verified.Prefix_mask.typ (module Impl)
   in
   let domain_log2 : (Field.t, Domain_log2.t) Typ.t =
     let (Typ t) =

src/lib/pickles/composition_types/branch_data.mli

@@ -1,22 +1,5 @@
 open Core_kernel
-
-module Proofs_verified : sig
-  [%%versioned:
-  module Stable : sig
-    module V1 : sig
-      type t = N0 | N1 | N2
-      [@@deriving sexp, sexp, compare, yojson, hash, equal]
-    end
-  end]
-
-  module Checked : sig
-    type 'f boolean = 'f Snarky_backendless.Cvar.t Snarky_backendless.Boolean.t
-
-    open Pickles_types
-
-    type 'f t = ('f boolean, Nat.N2.n) Vector.t
-  end
-end
+module Proofs_verified = Pickles_base.Proofs_verified
 
 module Domain_log2 : sig
   [%%versioned:
@@ -42,7 +25,7 @@ end]
 
 module Checked : sig
   type 'f t =
-    { proofs_verified_mask : 'f Proofs_verified.Checked.t
+    { proofs_verified_mask : 'f Proofs_verified.Prefix_mask.Checked.t
     ; domain_log2 : 'f Snarky_backendless.Cvar.t
     }
 

src/lib/pickles/one_hot_vector/one_hot_vector.ml

@@ -5,10 +5,13 @@ module Constant = struct
   type t = int
 end
 
+(* TODO: Optimization(?) Have this have length n - 1 since the last one is
+    determined by the remaining ones. *)
+type ('f, 'n) t =
+  ('f Snarky_backendless.Cvar.t Snarky_backendless.Boolean.t, 'n) Vector.t
+
 module T (Impl : Snarky_backendless.Snark_intf.Run) = struct
-  (* TODO: Optimization. Have this have length n - 1 since the last one is
-     determined by the remaining ones. *)
-  type 'n t = (Impl.Boolean.var, 'n) Vector.t
+  type nonrec 'n t = (Impl.field, 'n) t
 end
 
 module Make (Impl : Snarky_backendless.Snark_intf.Run) = struct

src/lib/pickles/one_hot_vector/one_hot_vector.mli

@@ -4,8 +4,12 @@ module Constant : sig
   type t = int
 end
 
+type ('f, 'n) t =
+  private
+  ('f Snarky_backendless.Cvar.t Snarky_backendless.Boolean.t, 'n) Vector.t
+
 module T (Impl : Snarky_backendless.Snark_intf.Run) : sig
-  type 'n t = private (Impl.Boolean.var, 'n) Vector.t
+  type nonrec 'n t = (Impl.field, 'n) t
 end
 
 module Make (Impl : Snarky_backendless.Snark_intf.Run) : sig

src/lib/pickles/pickles.mli

@@ -223,6 +223,8 @@ module Side_loaded : sig
   (* Must be called immediately before calling the prover for the inductive rule
      for which this tag is used as a predecessor. *)
   val in_prover : ('var, 'value, 'n1, 'n2) Tag.t -> Verification_key.t -> unit
+
+  val srs_precomputation : unit -> unit
 end
 
 (** This compiles a series of inductive rules defining a set into a proof

src/lib/pickles/side_loaded_verification_key.ml

@@ -178,7 +178,11 @@ end
 module Stable = struct
   module V2 = struct
     module T = struct
-      type t = (Backend.Tock.Curve.Affine.t, Vk.t) Poly.Stable.V2.t
+      type t =
+        ( Backend.Tock.Curve.Affine.t
+        , Pickles_base.Proofs_verified.Stable.V1.t
+        , Vk.t )
+        Poly.Stable.V2.t
       [@@deriving hash]
 
       let to_latest = Fn.id
@@ -187,13 +191,17 @@ module Stable = struct
 
       let version_byte = Base58_check.Version_bytes.verification_key
 
-      let to_repr { Poly.max_width; wrap_index; wrap_vk = _ } =
-        { Repr.Stable.V2.max_width; wrap_index }
+      let to_repr { Poly.max_proofs_verified; wrap_index; wrap_vk = _ } =
+        { Repr.Stable.V2.max_proofs_verified; wrap_index }
 
-      let of_repr ({ Repr.Stable.V2.max_width; wrap_index = c } : R.Stable.V2.t)
-          : t =
+      let of_repr
+          ({ Repr.Stable.V2.max_proofs_verified; wrap_index = c } :
+            R.Stable.V2.t ) : t =
         let d =
-          (Common.wrap_domains ~proofs_verified:(Width.to_int max_width)).h
+          (Common.wrap_domains
+             ~proofs_verified:
+               (Pickles_base.Proofs_verified.to_int max_proofs_verified) )
+            .h
         in
         let log2_size = Import.Domain.log2_size d in
         let max_quot_size = Common.max_quot_size_int (Import.Domain.size d) in
@@ -231,7 +239,7 @@ module Stable = struct
               ; lookup_index = None
               } )
         in
-        { Poly.max_width; wrap_index = c; wrap_vk }
+        { Poly.max_proofs_verified; wrap_index = c; wrap_vk }
 
       (* Proxy derivers to [R.t]'s, ignoring [wrap_vk] *)
 
@@ -277,7 +285,7 @@ Stable.Latest.
   , compare )]
 
 let dummy : t =
-  { max_width = Width.zero
+  { max_proofs_verified = N2
   ; wrap_index =
       (let g = Backend.Tock.Curve.(to_affine_exn one) in
        { sigma_comm = Vector.init Plonk_types.Permuts.n ~f:(fun _ -> g)
@@ -297,8 +305,8 @@ module Checked = struct
   open Impl
 
   type t =
-    { (* TODO: Not sure this is used *)
-      max_width : Width.Checked.t
+    { max_proofs_verified :
+        Impl.field Pickles_base.Proofs_verified.One_hot.Checked.t
           (** The maximum of all of the [step_widths]. *)
     ; wrap_index : Inner_curve.t Plonk_verification_key_evals.t
           (** The plonk verification key for the 'wrapping' proof that this key
@@ -312,10 +320,13 @@ module Checked = struct
 
   let to_input =
     let open Random_oracle_input.Chunked in
-    fun { max_width; wrap_index } : _ Random_oracle_input.Chunked.t ->
-      let width w = (Width.Checked.to_field w, width_size) in
+    fun { max_proofs_verified; wrap_index } : _ Random_oracle_input.Chunked.t ->
+      let max_proofs_verified =
+        Pickles_base.Proofs_verified.One_hot.Checked.to_input
+          max_proofs_verified
+      in
       List.reduce_exn ~f:append
-        [ packed (width max_width)
+        [ max_proofs_verified
         ; wrap_index_to_input
             (Fn.compose Array.of_list Inner_curve.to_field_elements)
             wrap_index
@@ -339,9 +350,11 @@ let typ : (Checked.t, t) Impls.Step.Typ.t =
   let open Step_main_inputs in
   let open Impl in
   Typ.of_hlistable
-    [ Width.typ; Plonk_verification_key_evals.typ Inner_curve.typ ]
+    [ Pickles_base.Proofs_verified.One_hot.typ (module Impls.Step)
+    ; Plonk_verification_key_evals.typ Inner_curve.typ
+    ]
     ~var_to_hlist:Checked.to_hlist ~var_of_hlist:Checked.of_hlist
     ~value_of_hlist:(fun _ ->
       failwith "Side_loaded_verification_key: value_of_hlist" )
-    ~value_to_hlist:(fun { Poly.wrap_index; max_width; _ } ->
-      [ max_width; wrap_index ] )
+    ~value_to_hlist:(fun { Poly.wrap_index; max_proofs_verified; _ } ->
+      [ max_proofs_verified; wrap_index ] )

src/lib/pickles/step_main.ml

@@ -45,9 +45,8 @@ let verify_one
           sponge
         in
         (* TODO: Refactor args into an "unfinalized proof" struct *)
-        finalize_other_proof d.max_proofs_verified ~max_width:d.max_width
-          ~step_domains:d.step_domains ~sponge ~prev_challenges
-          proof_state.deferred_values prev_proof_evals )
+        finalize_other_proof d.max_proofs_verified ~step_domains:d.step_domains
+          ~sponge ~prev_challenges proof_state.deferred_values prev_proof_evals )
   in
   let branch_data = proof_state.deferred_values.branch_data in
   let statement =
@@ -78,8 +77,7 @@ let verify_one
   in
   let verified =
     with_label __LOC__ (fun () ->
-        verify ~proofs_verified:d.max_proofs_verified
-          ~wrap_domain:d.wrap_domains.h
+        verify ~proofs_verified:d.max_proofs_verified ~wrap_domain:d.wrap_domain
           ~is_base_case:(Boolean.not should_verify)
           ~sg_old:prev_challenge_polynomial_commitments ~proof:wrap_proof
           ~wrap_verification_key:d.wrap_key statement unfinalized )
@@ -341,11 +339,10 @@ let step_main :
                         `Known
                           (Vector.map basic.proofs_verifieds ~f:Field.of_int)
                     ; max_proofs_verified = (module Max_proofs_verified)
-                    ; max_width = None
                     ; typ = basic.typ
                     ; var_to_field_elements = basic.var_to_field_elements
                     ; value_to_field_elements = basic.value_to_field_elements
-                    ; wrap_domains = basic.wrap_domains
+                    ; wrap_domain = `Known basic.wrap_domains.h
                     ; step_domains = `Known basic.step_domains
                     ; wrap_key = dlog_plonk_index
                     }