feat: selective disclosure tooling (#335)

* tlsn-formats minimal

* initial work on verifiers

* decouple proof builder from SessionData

* add commit remaining fn

* unit tests passing for commit and prove

* remove wip verify modules

* remove unused

* serde derives, fix test

* move tls prover into module and implement http prover

* docs

* rebase fixes

* remove dead code

* update integration test

* remove unwrap

* handle invalid json path commitment

* fix body proof build, and add commit all methods

* fix http comments

* add extra check to range

* rename GarbleRole to DEAPRole

* add gf2 comment

* comment unused body_built field

* fix notary-server integration test

notary-server/tests/integration_test.rs

@@ -15,7 +15,7 @@ use std::{
     time::Duration,
 };
 use tls_server_fixture::{bind_test_server_hyper, CA_CERT_DER, SERVER_DOMAIN};
-use tlsn_prover::{Prover, ProverConfig};
+use tlsn_prover::tls::{Prover, ProverConfig};
 use tokio_rustls::TlsConnector;
 use tokio_util::compat::{FuturesAsyncReadCompatExt, TokioAsyncReadCompatExt};
 use tracing::debug;

tlsn/Cargo.toml

@@ -3,6 +3,7 @@ members = [
     "tlsn-core",
     "tlsn-notary",
     "tlsn-prover",
+    "tlsn-formats",
     "tlsn-server-fixture",
     "tests-integration",
     "examples",
@@ -15,6 +16,8 @@ tlsn-prover = { path = "tlsn-prover" }
 tlsn-notary = { path = "tlsn-notary" }
 tlsn-server-fixture = { path = "tlsn-server-fixture" }
 
+tlsn-formats = { path = "tlsn-formats" }
+
 tlsn-tls-core = { path = "../components/tls/tls-core" }
 tlsn-tls-mpc = { path = "../components/tls/tls-mpc" }
 tlsn-tls-client = { path = "../components/tls/tls-client" }
@@ -53,6 +56,7 @@ bincode = "1"
 hex = "0.4"
 bytes = "1.4"
 opaque-debug = "0.3"
+spansy = { git = "https://github.com/sinui0/spansy", rev = "becb33d" }
 
 tracing = "0.1"
 tracing-subscriber = "0.3"

tlsn/examples/discord/discord_dm.rs

@@ -13,7 +13,7 @@ use tokio_rustls::TlsConnector;
 use tokio_util::compat::{FuturesAsyncReadCompatExt, TokioAsyncReadCompatExt};
 use tracing::debug;
 
-use tlsn_prover::{Prover, ProverConfig};
+use tlsn_prover::tls::{Prover, ProverConfig};
 
 // Setting of the application server
 const SERVER_DOMAIN: &str = "discord.com";

tlsn/examples/simple/simple_prover.rs

@@ -9,7 +9,7 @@ use tlsn_core::proof::TlsProof;
 use tokio::io::AsyncWriteExt as _;
 use tokio_util::compat::{FuturesAsyncReadCompatExt, TokioAsyncReadCompatExt};
 
-use tlsn_prover::{Prover, ProverConfig};
+use tlsn_prover::tls::{Prover, ProverConfig};
 
 // Setting of the application server
 const SERVER_DOMAIN: &str = "example.com";

tlsn/examples/twitter/twitter_dm.rs

@@ -12,7 +12,7 @@ use tokio_rustls::TlsConnector;
 use tokio_util::compat::{FuturesAsyncReadCompatExt, TokioAsyncReadCompatExt};
 use tracing::debug;
 
-use tlsn_prover::{Prover, ProverConfig};
+use tlsn_prover::tls::{Prover, ProverConfig};
 
 // Setting of the application server
 const SERVER_DOMAIN: &str = "twitter.com";

tlsn/tests-integration/Cargo.toml

@@ -5,8 +5,9 @@ edition = "2021"
 publish = false
 
 [dev-dependencies]
+tlsn-core.workspace = true
 tlsn-tls-core.workspace = true
-tlsn-prover.workspace = true
+tlsn-prover = { workspace = true, features = ["formats"] }
 tlsn-notary.workspace = true
 tlsn-server-fixture.workspace = true
 
@@ -19,3 +20,6 @@ tokio-util.workspace = true
 
 tracing.workspace = true
 tracing-subscriber.workspace = true
+
+serde_json = "1.0"
+bincode = "*"

tlsn/tests-integration/tests/test.rs

@@ -1,7 +1,7 @@
 use futures::AsyncWriteExt;
 use hyper::{body::to_bytes, Body, Request, StatusCode};
 use tlsn_notary::{bind_notary, NotaryConfig};
-use tlsn_prover::{Prover, ProverConfig};
+use tlsn_prover::tls::{Prover, ProverConfig};
 use tlsn_server_fixture::{CA_CERT_DER, SERVER_DOMAIN};
 use tokio::io::{AsyncRead, AsyncWrite};
 use tokio_util::compat::{FuturesAsyncReadCompatExt, TokioAsyncReadCompatExt};
@@ -73,17 +73,19 @@ async fn prover<T: AsyncWrite + AsyncRead + Send + Unpin + 'static>(notary_socke
 
     client_socket.close().await.unwrap();
 
-    let mut prover = prover_task.await.unwrap().unwrap().start_notarize();
-
-    let sent_len = prover.sent_transcript().data().len();
-    let recv_len = prover.recv_transcript().data().len();
+    let mut prover = prover_task
+        .await
+        .unwrap()
+        .unwrap()
+        .to_http()
+        .unwrap()
+        .start_notarize();
 
-    let builder = prover.commitment_builder();
+    prover.commit().unwrap();
 
-    builder.commit_sent(0..sent_len).unwrap();
-    builder.commit_recv(0..recv_len).unwrap();
+    let notarized_session = prover.finalize().await.unwrap();
 
-    _ = prover.finalize().await.unwrap();
+    _ = notarized_session.proof_builder().build().unwrap();
 }
 
 #[instrument(skip(socket))]

tlsn/tlsn-core/src/commitment/builder.rs

@@ -6,7 +6,8 @@ use utils::range::RangeSet;
 
 use crate::{
     commitment::{
-        blake3::Blake3Commitment, Commitment, CommitmentId, CommitmentInfo, TranscriptCommitments,
+        blake3::Blake3Commitment, Commitment, CommitmentId, CommitmentInfo, CommitmentKind,
+        TranscriptCommitments,
     },
     merkle::MerkleTree,
     transcript::get_value_ids,
@@ -16,6 +17,12 @@ use crate::{
 /// An error for [`TranscriptCommitmentBuilder`]
 #[derive(Debug, thiserror::Error)]
 pub enum TranscriptCommitmentBuilderError {
+    /// Empty range
+    #[error("can not commit to an empty range")]
+    EmptyRange,
+    /// Range out of bounds
+    #[error("range out of bounds")]
+    RangeOutOfBounds,
     /// Failed to retrieve encodings for the provided transcript ranges.
     #[error("failed to retrieve encodings for the provided transcript ranges")]
     MissingEncodings,
@@ -35,6 +42,8 @@ pub struct TranscriptCommitmentBuilder {
     merkle_leaves: Vec<Hash>,
     /// A function that returns the encodings for the provided transcript byte ids.
     encoding_provider: EncodingProvider,
+    sent_len: usize,
+    recv_len: usize,
 }
 
 opaque_debug::implement!(TranscriptCommitmentBuilder);
@@ -46,12 +55,14 @@ impl TranscriptCommitmentBuilder {
     ///
     /// * `encoding_provider` - A function that returns the encodings for the provided transcript byte ids.
     #[doc(hidden)]
-    pub fn new(encoding_provider: EncodingProvider) -> Self {
+    pub fn new(encoding_provider: EncodingProvider, sent_len: usize, recv_len: usize) -> Self {
         Self {
             commitments: HashMap::default(),
             commitment_info: BiMap::default(),
             merkle_leaves: Vec::default(),
             encoding_provider,
+            sent_len,
+            recv_len,
         }
     }
 
@@ -71,12 +82,40 @@ impl TranscriptCommitmentBuilder {
         self.add_substrings_commitment(ranges.into(), Direction::Received)
     }
 
+    /// Gets the commitment id for the provided commitment info.
+    pub fn get_id(
+        &self,
+        kind: CommitmentKind,
+        ranges: impl Into<RangeSet<usize>>,
+        direction: Direction,
+    ) -> Option<CommitmentId> {
+        self.commitment_info
+            .get_by_right(&CommitmentInfo {
+                kind,
+                ranges: ranges.into(),
+                direction,
+            })
+            .copied()
+    }
+
     /// Add a commitment to substrings of the transcript
     fn add_substrings_commitment(
         &mut self,
         ranges: RangeSet<usize>,
         direction: Direction,
     ) -> Result<CommitmentId, TranscriptCommitmentBuilderError> {
+        let max = ranges
+            .max()
+            .ok_or(TranscriptCommitmentBuilderError::EmptyRange)?;
+        let len = match direction {
+            Direction::Sent => self.sent_len,
+            Direction::Received => self.recv_len,
+        };
+
+        if max > len {
+            return Err(TranscriptCommitmentBuilderError::RangeOutOfBounds);
+        }
+
         let ids: Vec<_> = get_value_ids(&ranges, direction).collect();
 
         let id_refs = ids.iter().map(|id| id.as_ref()).collect::<Vec<_>>();

tlsn/tlsn-core/src/proof/substrings.rs

@@ -7,10 +7,12 @@ use serde::{Deserialize, Serialize};
 use utils::range::{RangeDisjoint, RangeSet, RangeUnion};
 
 use crate::{
-    commitment::{Commitment, CommitmentId, CommitmentInfo, CommitmentOpening},
+    commitment::{
+        Commitment, CommitmentId, CommitmentInfo, CommitmentOpening, TranscriptCommitments,
+    },
     merkle::MerkleProof,
     transcript::get_value_ids,
-    Direction, EncodingId, RedactedTranscript, SessionData, SessionHeader, TranscriptSlice,
+    Direction, EncodingId, RedactedTranscript, SessionHeader, Transcript, TranscriptSlice,
     MAX_TOTAL_COMMITTED_DATA,
 };
 
@@ -33,32 +35,38 @@ pub enum SubstringsProofBuilderError {
 
 /// A builder for [`SubstringsProof`]
 pub struct SubstringsProofBuilder<'a> {
-    data: &'a SessionData,
+    commitments: &'a TranscriptCommitments,
+    transcript_tx: &'a Transcript,
+    transcript_rx: &'a Transcript,
     openings: HashMap<CommitmentId, (CommitmentInfo, CommitmentOpening)>,
 }
 
 opaque_debug::implement!(SubstringsProofBuilder<'_>);
 
 impl<'a> SubstringsProofBuilder<'a> {
     /// Creates a new builder.
-    pub(crate) fn new(data: &'a SessionData) -> Self {
+    pub fn new(
+        commitments: &'a TranscriptCommitments,
+        transcript_tx: &'a Transcript,
+        transcript_rx: &'a Transcript,
+    ) -> Self {
         Self {
-            data,
+            commitments,
+            transcript_tx,
+            transcript_rx,
             openings: HashMap::default(),
         }
     }
 
     /// Reveals data corresponding to the provided commitment id
     pub fn reveal(&mut self, id: CommitmentId) -> Result<&mut Self, SubstringsProofBuilderError> {
         let commitment = self
-            .data
-            .commitments()
+            .commitments
             .get(&id)
             .ok_or(SubstringsProofBuilderError::InvalidCommitmentId(id))?;
 
         let info = self
-            .data
-            .commitments()
+            .commitments
             .get_info(&id)
             .expect("info exists if commitment exists");
 
@@ -69,8 +77,8 @@ impl<'a> SubstringsProofBuilder<'a> {
         };
 
         let transcript = match info.direction() {
-            Direction::Sent => self.data.sent_transcript(),
-            Direction::Received => self.data.recv_transcript(),
+            Direction::Sent => self.transcript_tx,
+            Direction::Received => self.transcript_rx,
         };
 
         let data = transcript.get_bytes_in_ranges(info.ranges());
@@ -89,15 +97,19 @@ impl<'a> SubstringsProofBuilder<'a> {
 
     /// Builds the [`SubstringsProof`]
     pub fn build(self) -> Result<SubstringsProof, SubstringsProofBuilderError> {
-        let Self { data, openings } = self;
+        let Self {
+            commitments,
+            openings,
+            ..
+        } = self;
 
         let mut indices = openings
             .keys()
             .map(|id| id.to_inner() as usize)
             .collect::<Vec<_>>();
         indices.sort();
 
-        let inclusion_proof = data.commitments().merkle_tree().proof(&indices);
+        let inclusion_proof = commitments.merkle_tree().proof(&indices);
 
         Ok(SubstringsProof {
             openings,
@@ -118,7 +130,7 @@ pub enum SubstringsProofError {
     MaxDataExceeded(usize),
     /// The proof contains duplicate transcript data.
     #[error("proof contains duplicate transcript data")]
-    DuplicateData,
+    DuplicateData(Direction, RangeSet<usize>),
     /// Range of the opening is out of bounds.
     #[error("range of opening {0:?} is out of bounds: {1}")]
     RangeOutOfBounds(CommitmentId, usize),
@@ -184,13 +196,13 @@ impl SubstringsProof {
             match direction {
                 Direction::Sent => {
                     if !sent_ranges.is_disjoint(&ranges) {
-                        return Err(SubstringsProofError::DuplicateData);
+                        return Err(SubstringsProofError::DuplicateData(direction, ranges));
                     }
                     sent_ranges = sent_ranges.union(&ranges);
                 }
                 Direction::Received => {
                     if !recv_ranges.is_disjoint(&ranges) {
-                        return Err(SubstringsProofError::DuplicateData);
+                        return Err(SubstringsProofError::DuplicateData(direction, ranges));
                     }
                     recv_ranges = recv_ranges.union(&ranges);
                 }

tlsn/tlsn-core/src/session/data.rs

@@ -71,6 +71,6 @@ impl SessionData {
 
     /// Returns a substrings proof builder.
     pub fn build_substrings_proof(&self) -> SubstringsProofBuilder {
-        SubstringsProofBuilder::new(self)
+        SubstringsProofBuilder::new(&self.commitments, &self.transcript_tx, &self.transcript_rx)
     }
 }

tlsn/tlsn-core/src/session/mod.rs

@@ -20,6 +20,8 @@ pub struct NotarizedSession {
     data: SessionData,
 }
 
+opaque_debug::implement!(NotarizedSession);
+
 impl NotarizedSession {
     /// Create a new notarized session.
     pub fn new(header: SessionHeader, signature: Option<Signature>, data: SessionData) -> Self {

tlsn/tlsn-core/src/transcript.rs

@@ -108,6 +108,22 @@ impl RedactedTranscript {
             self.data[range].fill(value);
         }
     }
+
+    /// Sets all bytes in the transcript which were redacted in the given range.
+    ///
+    /// # Arguments
+    ///
+    /// * `value` - The value to set the redacted bytes to
+    /// * `range` - The range of redacted bytes to set
+    pub fn set_redacted_range(&mut self, value: u8, range: Range<usize>) {
+        for range in self
+            .redacted
+            .difference(&(0..self.data.len()).difference(&range))
+            .iter_ranges()
+        {
+            self.data[range].fill(value);
+        }
+    }
 }
 
 /// Slice of a transcript.

tlsn/tlsn-core/tests/api.rs

@@ -75,7 +75,8 @@ fn test_api() {
     // Commitment to the handshake which the Prover sent at the start of the TLS handshake
     let (hs_decommitment, hs_commitment) = handshake_data.hash_commit();
 
-    let mut commitment_builder = TranscriptCommitmentBuilder::new(encodings_provider);
+    let mut commitment_builder =
+        TranscriptCommitmentBuilder::new(encodings_provider, data_sent.len(), data_recv.len());
 
     let commitment_id_1 = commitment_builder.commit_sent(range1.clone()).unwrap();
     let commitment_id_2 = commitment_builder.commit_recv(range2.clone()).unwrap();

tlsn/tlsn-formats/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "tlsn-formats"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+tlsn-core.workspace = true
+tlsn-utils.workspace = true
+
+bytes.workspace = true
+spansy = { workspace = true, features = ["serde"] }
+serde.workspace = true
+thiserror.workspace = true
+
+[dev-dependencies]
+tlsn-core = { workspace = true, features = ["fixtures"] }
+rstest.workspace = true