From 841ff78a7ab3996991443d577b27daf444f156b4 Mon Sep 17 00:00:00 2001 From: laser Date: Mon, 4 May 2020 12:18:55 -0700 Subject: [PATCH 1/8] alternative verify post --- generated/generated.go | 31 ++++++++++++++++--------- proofs.go | 20 +++++++++++++--- rust/src/proofs/api.rs | 47 ++++++++++++++++++++++++++++++++------ rust/src/proofs/helpers.rs | 45 ++++++++++++++++++++++++++++++++++++ 4 files changed, 122 insertions(+), 21 deletions(-) diff --git a/generated/generated.go b/generated/generated.go index d1efb870..15dc78a8 100644 --- a/generated/generated.go +++ b/generated/generated.go @@ -750,27 +750,36 @@ func FilVerifyWindowPost(randomness Fil32ByteArray, replicasPtr []FilPublicRepli } // FilVerifyWinningPost function as declared in filecoin-ffi/filcrypto.h:675 -func FilVerifyWinningPost(randomness Fil32ByteArray, replicasPtr []FilPublicReplicaInfo, replicasLen uint, proofsPtr []FilPoStProof, proofsLen uint, proverId Fil32ByteArray) *FilVerifyWinningPoStResponse { +func FilVerifyWinningPost(randomness Fil32ByteArray, proverId Fil32ByteArray, replicasPtr []FilPublicReplicaInfo, replicasLen uint, proofTypesPtr []FilRegisteredPoStProof, proofTypesLen uint, proofChunkSizesPtr []uint16, proofChunkSizesLen uint, flattenedProofsPtr string, flattenedProofsLen uint) *FilVerifyWinningPoStResponse { crandomness, crandomnessAllocMap := randomness.PassValue() + cproverId, cproverIdAllocMap := proverId.PassValue() creplicasPtr, creplicasPtrAllocMap := unpackArgSFilPublicReplicaInfo(replicasPtr) creplicasLen, creplicasLenAllocMap := (C.size_t)(replicasLen), cgoAllocsUnknown - cproofsPtr, cproofsPtrAllocMap := unpackArgSFilPoStProof(proofsPtr) - cproofsLen, cproofsLenAllocMap := (C.size_t)(proofsLen), cgoAllocsUnknown - cproverId, cproverIdAllocMap := proverId.PassValue() - __ret := C.fil_verify_winning_post(crandomness, creplicasPtr, creplicasLen, cproofsPtr, cproofsLen, cproverId) - runtime.KeepAlive(cproverIdAllocMap) - runtime.KeepAlive(cproofsLenAllocMap) - packSFilPoStProof(proofsPtr, cproofsPtr) - runtime.KeepAlive(cproofsPtrAllocMap) + cproofTypesPtr, cproofTypesPtrAllocMap := (*C.fil_RegisteredPoStProof)(unsafe.Pointer((*sliceHeader)(unsafe.Pointer(&proofTypesPtr)).Data)), cgoAllocsUnknown + cproofTypesLen, cproofTypesLenAllocMap := (C.size_t)(proofTypesLen), cgoAllocsUnknown + cproofChunkSizesPtr, cproofChunkSizesPtrAllocMap := (*C.uint16_t)(unsafe.Pointer((*sliceHeader)(unsafe.Pointer(&proofChunkSizesPtr)).Data)), cgoAllocsUnknown + cproofChunkSizesLen, cproofChunkSizesLenAllocMap := (C.size_t)(proofChunkSizesLen), cgoAllocsUnknown + flattenedProofsPtr = safeString(flattenedProofsPtr) + cflattenedProofsPtr, cflattenedProofsPtrAllocMap := unpackPUint8TString(flattenedProofsPtr) + cflattenedProofsLen, cflattenedProofsLenAllocMap := (C.size_t)(flattenedProofsLen), cgoAllocsUnknown + __ret := C.fil_verify_winning_post(crandomness, cproverId, creplicasPtr, creplicasLen, cproofTypesPtr, cproofTypesLen, cproofChunkSizesPtr, cproofChunkSizesLen, cflattenedProofsPtr, cflattenedProofsLen) + runtime.KeepAlive(cflattenedProofsLenAllocMap) + runtime.KeepAlive(flattenedProofsPtr) + runtime.KeepAlive(cflattenedProofsPtrAllocMap) + runtime.KeepAlive(cproofChunkSizesLenAllocMap) + runtime.KeepAlive(cproofChunkSizesPtrAllocMap) + runtime.KeepAlive(cproofTypesLenAllocMap) + runtime.KeepAlive(cproofTypesPtrAllocMap) runtime.KeepAlive(creplicasLenAllocMap) packSFilPublicReplicaInfo(replicasPtr, creplicasPtr) runtime.KeepAlive(creplicasPtrAllocMap) + runtime.KeepAlive(cproverIdAllocMap) runtime.KeepAlive(crandomnessAllocMap) __v := NewFilVerifyWinningPoStResponseRef(unsafe.Pointer(__ret)) return __v } -// FilWriteWithAlignment function as declared in filecoin-ffi/filcrypto.h:686 +// FilWriteWithAlignment function as declared in filecoin-ffi/filcrypto.h:690 func FilWriteWithAlignment(registeredProof FilRegisteredSealProof, srcFd int32, srcSize uint64, dstFd int32, existingPieceSizesPtr []uint64, existingPieceSizesLen uint) *FilWriteWithAlignmentResponse { cregisteredProof, cregisteredProofAllocMap := (C.fil_RegisteredSealProof)(registeredProof), cgoAllocsUnknown csrcFd, csrcFdAllocMap := (C.int)(srcFd), cgoAllocsUnknown @@ -789,7 +798,7 @@ func FilWriteWithAlignment(registeredProof FilRegisteredSealProof, srcFd int32, return __v } -// FilWriteWithoutAlignment function as declared in filecoin-ffi/filcrypto.h:697 +// FilWriteWithoutAlignment function as declared in filecoin-ffi/filcrypto.h:701 func FilWriteWithoutAlignment(registeredProof FilRegisteredSealProof, srcFd int32, srcSize uint64, dstFd int32) *FilWriteWithoutAlignmentResponse { cregisteredProof, cregisteredProofAllocMap := (C.fil_RegisteredSealProof)(registeredProof), cgoAllocsUnknown csrcFd, csrcFdAllocMap := (C.int)(srcFd), cgoAllocsUnknown diff --git a/proofs.go b/proofs.go index 38ccc164..fe85aa92 100644 --- a/proofs.go +++ b/proofs.go @@ -73,13 +73,27 @@ func VerifyWinningPoSt(info abi.WinningPoStVerifyInfo) (bool, error) { return false, err } + var flattenedProofs string + proofTypes := make([]generated.FilRegisteredPoStProof, filPoStProofsLen) + proofChunkSizes := make([]uint16, filPoStProofsLen) + + for idx := range filPoStProofs { + flattenedProofs = flattenedProofs + filPoStProofs[idx].ProofPtr + proofTypes[idx] = filPoStProofs[idx].RegisteredProof + proofChunkSizes[idx] = uint16(filPoStProofs[idx].ProofLen) + } + resp := generated.FilVerifyWinningPost( to32ByteArray(info.Randomness), + proverID, filPublicReplicaInfos, filPublicReplicaInfosLen, - filPoStProofs, - filPoStProofsLen, - proverID, + proofTypes, + uint(len(proofTypes)), + proofChunkSizes, + uint(len(proofChunkSizes)), + flattenedProofs, + uint(len(flattenedProofs)), ) resp.Deref() diff --git a/rust/src/proofs/api.rs b/rust/src/proofs/api.rs index 618b6f54..d5fb3c63 100644 --- a/rust/src/proofs/api.rs +++ b/rust/src/proofs/api.rs @@ -13,6 +13,7 @@ use std::slice::from_raw_parts; use super::helpers::{c_to_rust_post_proofs, to_private_replica_info_map}; use super::types::*; +use crate::proofs::helpers::c_to_rust_post_proofs_meow; use crate::util::api::init_log; /// TODO: document @@ -484,11 +485,15 @@ pub unsafe extern "C" fn fil_verify_seal( #[no_mangle] pub unsafe extern "C" fn fil_verify_winning_post( randomness: fil_32ByteArray, + prover_id: fil_32ByteArray, replicas_ptr: *const fil_PublicReplicaInfo, replicas_len: libc::size_t, - proofs_ptr: *const fil_PoStProof, - proofs_len: libc::size_t, - prover_id: fil_32ByteArray, + proof_types_ptr: *const fil_RegisteredPoStProof, + proof_types_len: libc::size_t, + proof_chunk_sizes_ptr: *const u16, + proof_chunk_sizes_len: libc::size_t, + flattened_proofs_ptr: *const u8, + flattened_proofs_len: libc::size_t, ) -> *mut fil_VerifyWinningPoStResponse { catch_panic_response(|| { init_log(); @@ -500,7 +505,15 @@ pub unsafe extern "C" fn fil_verify_winning_post( let convert = super::helpers::to_public_replica_info_map(replicas_ptr, replicas_len); let result = convert.and_then(|replicas| { - let post_proofs = c_to_rust_post_proofs(proofs_ptr, proofs_len)?; + let post_proofs = c_to_rust_post_proofs_meow( + proof_types_ptr, + proof_types_len, + proof_chunk_sizes_ptr, + proof_chunk_sizes_len, + flattened_proofs_ptr, + flattened_proofs_len, + )?; + let proofs: Vec = post_proofs.iter().flat_map(|pp| pp.clone().proof).collect(); filecoin_proofs_api::post::verify_winning_post( @@ -1594,13 +1607,33 @@ pub mod tests { comm_r: (*resp_b2).comm_r, }]; + let mut ptypes: Vec = Default::default(); + let mut pchunks: Vec = Default::default(); + let mut flattened: Vec = Default::default(); + + for pp in std::slice::from_raw_parts((*resp_h).proofs_ptr, (*resp_h).proofs_len).iter() + { + ptypes.push(pp.registered_proof.clone()); + + let mut proof: Vec = Default::default(); + proof.extend_from_slice(std::slice::from_raw_parts(pp.proof_ptr, pp.proof_len)); + + flattened.append(&mut proof); + + pchunks.push(pp.proof_len as u16); + } + let resp_i = fil_verify_winning_post( randomness, + prover_id, public_replicas.as_ptr(), public_replicas.len(), - (*resp_h).proofs_ptr, - (*resp_h).proofs_len, - prover_id, + ptypes.as_ptr(), + ptypes.len(), + pchunks.as_ptr(), + pchunks.len(), + flattened.as_ptr(), + flattened.len(), ); if (*resp_i).status_code != FCPResponseStatus::FCPNoError { diff --git a/rust/src/proofs/helpers.rs b/rust/src/proofs/helpers.rs index 132666a4..c8dff08e 100644 --- a/rust/src/proofs/helpers.rs +++ b/rust/src/proofs/helpers.rs @@ -132,3 +132,48 @@ pub unsafe fn c_to_rust_post_proofs( Ok(out) } + +pub unsafe fn c_to_rust_post_proofs_meow( + proof_types_ptr: *const fil_RegisteredPoStProof, + proof_types_len: libc::size_t, + proof_chunk_sizes_ptr: *const u16, + proof_chunk_sizes_len: libc::size_t, + flattened_proofs_ptr: *const u8, + flattened_proofs_len: libc::size_t, +) -> Result> { + ensure!( + !flattened_proofs_ptr.is_null(), + "flattened_proofs_ptr must not be null" + ); + + ensure!( + !proof_chunk_sizes_ptr.is_null(), + "proof_chunk_sizes_ptr must not be null" + ); + + ensure!( + !proof_types_ptr.is_null(), + "proof_types_ptr must not be null" + ); + + let flattened_proofs = from_raw_parts(flattened_proofs_ptr, flattened_proofs_len); + let chunk_sizes = from_raw_parts(proof_chunk_sizes_ptr, proof_chunk_sizes_len); + let proof_types = from_raw_parts(proof_types_ptr, proof_types_len); + + let mut out: Vec = Vec::with_capacity(chunk_sizes.len()); + + let mut offset: usize = 0; + for (idx, chunk_size) in chunk_sizes.iter().enumerate() { + let mut proof: Vec = Default::default(); + proof.extend_from_slice(&flattened_proofs[offset..offset + *chunk_size as usize]); + + out.push(PoStProof { + registered_proof: proof_types[idx].into(), + proof, + }); + + offset += *chunk_size as usize + } + + Ok(out) +} From 7302e635658993af566971ae476387e65b47c63f Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 5 May 2020 09:06:39 -0700 Subject: [PATCH 2/8] manually allocate proxy object to prevent sharing Go heap with C --- .circleci/config.yml | 1 + generated/customallocs.go | 31 +++++++++++++++++++++++++++++++ proofs.go | 22 ++++++++++++++++------ 3 files changed, 48 insertions(+), 6 deletions(-) create mode 100644 generated/customallocs.go diff --git a/.circleci/config.yml b/.circleci/config.yml index d8715638..1453221b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -158,6 +158,7 @@ workflows: branches: only: - master + - feat/cgocheck-2-panic run_leak_detector: true - build_and_test_linux_cgo_bindings: filters: diff --git a/generated/customallocs.go b/generated/customallocs.go new file mode 100644 index 00000000..38f671dd --- /dev/null +++ b/generated/customallocs.go @@ -0,0 +1,31 @@ +package generated + +/* +#cgo LDFLAGS: -L${SRCDIR}/.. -lfilcrypto +#cgo pkg-config: ${SRCDIR}/../filcrypto.pc +#include "../filcrypto.h" +#include +#include "cgo_helpers.h" +*/ +import "C" +import ( + "unsafe" +) + +func (x *FilPrivateReplicaInfo) AllocateProxy() func() { + mem81a31e9b := allocFilPrivateReplicaInfoMemory(1) + ref81a31e9b := (*C.fil_PrivateReplicaInfo)(mem81a31e9b) + ref81a31e9b.cache_dir_path = C.CString(x.CacheDirPath) + ref81a31e9b.comm_r = *(*[32]C.uint8_t)(unsafe.Pointer(&x.CommR)) + ref81a31e9b.registered_proof = (C.fil_RegisteredPoStProof)(x.RegisteredProof) + ref81a31e9b.replica_path = C.CString(x.ReplicaPath) + ref81a31e9b.sector_id = (C.uint64_t)(x.SectorId) + + x.ref81a31e9b = ref81a31e9b + + return func() { + C.free(unsafe.Pointer(ref81a31e9b.cache_dir_path)) + C.free(unsafe.Pointer(ref81a31e9b.replica_path)) + C.free(unsafe.Pointer(ref81a31e9b)) + } +} diff --git a/proofs.go b/proofs.go index fe85aa92..3a00b198 100644 --- a/proofs.go +++ b/proofs.go @@ -516,10 +516,11 @@ func GenerateWinningPoSt( privateSectorInfo SortedPrivateSectorInfo, randomness abi.PoStRandomness, ) ([]abi.PoStProof, error) { - filReplicas, filReplicasLen, err := toFilPrivateReplicaInfos(privateSectorInfo.Values(), "winning") + filReplicas, filReplicasLen, free, err := toFilPrivateReplicaInfos(privateSectorInfo.Values(), "winning") if err != nil { return nil, errors.Wrap(err, "failed to create private replica info array for FFI") } + defer free() proverID, err := toProverID(minerID) if err != nil { @@ -555,10 +556,11 @@ func GenerateWindowPoSt( privateSectorInfo SortedPrivateSectorInfo, randomness abi.PoStRandomness, ) ([]abi.PoStProof, error) { - filReplicas, filReplicasLen, err := toFilPrivateReplicaInfos(privateSectorInfo.Values(), "window") + filReplicas, filReplicasLen, free, err := toFilPrivateReplicaInfos(privateSectorInfo.Values(), "window") if err != nil { return nil, errors.Wrap(err, "failed to create private replica info array for FFI") } + defer free() proverID, err := toProverID(minerID) if err != nil { @@ -706,18 +708,20 @@ func toFilPublicReplicaInfos(src []abi.SectorInfo, typ string) ([]generated.FilP return out, uint(len(out)), nil } -func toFilPrivateReplicaInfos(src []PrivateSectorInfo, typ string) ([]generated.FilPrivateReplicaInfo, uint, error) { +func toFilPrivateReplicaInfos(src []PrivateSectorInfo, typ string) ([]generated.FilPrivateReplicaInfo, uint, func(), error) { + frees := make([]func(), len(src)) + out := make([]generated.FilPrivateReplicaInfo, len(src)) for idx := range out { commR, err := to32ByteCommR(src[idx].SealedCID) if err != nil { - return nil, 0, err + return nil, 0, func() {}, err } pp, err := toFilRegisteredPoStProof(src[idx].PoStProofType, typ) if err != nil { - return nil, 0, err + return nil, 0, func() {}, err } out[idx] = generated.FilPrivateReplicaInfo{ @@ -727,9 +731,15 @@ func toFilPrivateReplicaInfos(src []PrivateSectorInfo, typ string) ([]generated. ReplicaPath: src[idx].SealedSectorPath, SectorId: uint64(src[idx].SectorNumber), } + + frees[idx] = out[idx].AllocateProxy() } - return out, uint(len(out)), nil + return out, uint(len(out)), func() { + for idx := range frees { + frees[idx]() + } + }, nil } func fromFilPoStProofs(src []generated.FilPoStProof) ([]abi.PoStProof, error) { From 03ce5339c122b12f06d5eef3e74b3a34d345b53e Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 5 May 2020 09:06:58 -0700 Subject: [PATCH 3/8] Revert "alternative verify post" This reverts commit 841ff78a7ab3996991443d577b27daf444f156b4. --- generated/generated.go | 31 +++++++++---------------- proofs.go | 20 +++------------- rust/src/proofs/api.rs | 47 ++++++-------------------------------- rust/src/proofs/helpers.rs | 45 ------------------------------------ 4 files changed, 21 insertions(+), 122 deletions(-) diff --git a/generated/generated.go b/generated/generated.go index 15dc78a8..d1efb870 100644 --- a/generated/generated.go +++ b/generated/generated.go @@ -750,36 +750,27 @@ func FilVerifyWindowPost(randomness Fil32ByteArray, replicasPtr []FilPublicRepli } // FilVerifyWinningPost function as declared in filecoin-ffi/filcrypto.h:675 -func FilVerifyWinningPost(randomness Fil32ByteArray, proverId Fil32ByteArray, replicasPtr []FilPublicReplicaInfo, replicasLen uint, proofTypesPtr []FilRegisteredPoStProof, proofTypesLen uint, proofChunkSizesPtr []uint16, proofChunkSizesLen uint, flattenedProofsPtr string, flattenedProofsLen uint) *FilVerifyWinningPoStResponse { +func FilVerifyWinningPost(randomness Fil32ByteArray, replicasPtr []FilPublicReplicaInfo, replicasLen uint, proofsPtr []FilPoStProof, proofsLen uint, proverId Fil32ByteArray) *FilVerifyWinningPoStResponse { crandomness, crandomnessAllocMap := randomness.PassValue() - cproverId, cproverIdAllocMap := proverId.PassValue() creplicasPtr, creplicasPtrAllocMap := unpackArgSFilPublicReplicaInfo(replicasPtr) creplicasLen, creplicasLenAllocMap := (C.size_t)(replicasLen), cgoAllocsUnknown - cproofTypesPtr, cproofTypesPtrAllocMap := (*C.fil_RegisteredPoStProof)(unsafe.Pointer((*sliceHeader)(unsafe.Pointer(&proofTypesPtr)).Data)), cgoAllocsUnknown - cproofTypesLen, cproofTypesLenAllocMap := (C.size_t)(proofTypesLen), cgoAllocsUnknown - cproofChunkSizesPtr, cproofChunkSizesPtrAllocMap := (*C.uint16_t)(unsafe.Pointer((*sliceHeader)(unsafe.Pointer(&proofChunkSizesPtr)).Data)), cgoAllocsUnknown - cproofChunkSizesLen, cproofChunkSizesLenAllocMap := (C.size_t)(proofChunkSizesLen), cgoAllocsUnknown - flattenedProofsPtr = safeString(flattenedProofsPtr) - cflattenedProofsPtr, cflattenedProofsPtrAllocMap := unpackPUint8TString(flattenedProofsPtr) - cflattenedProofsLen, cflattenedProofsLenAllocMap := (C.size_t)(flattenedProofsLen), cgoAllocsUnknown - __ret := C.fil_verify_winning_post(crandomness, cproverId, creplicasPtr, creplicasLen, cproofTypesPtr, cproofTypesLen, cproofChunkSizesPtr, cproofChunkSizesLen, cflattenedProofsPtr, cflattenedProofsLen) - runtime.KeepAlive(cflattenedProofsLenAllocMap) - runtime.KeepAlive(flattenedProofsPtr) - runtime.KeepAlive(cflattenedProofsPtrAllocMap) - runtime.KeepAlive(cproofChunkSizesLenAllocMap) - runtime.KeepAlive(cproofChunkSizesPtrAllocMap) - runtime.KeepAlive(cproofTypesLenAllocMap) - runtime.KeepAlive(cproofTypesPtrAllocMap) + cproofsPtr, cproofsPtrAllocMap := unpackArgSFilPoStProof(proofsPtr) + cproofsLen, cproofsLenAllocMap := (C.size_t)(proofsLen), cgoAllocsUnknown + cproverId, cproverIdAllocMap := proverId.PassValue() + __ret := C.fil_verify_winning_post(crandomness, creplicasPtr, creplicasLen, cproofsPtr, cproofsLen, cproverId) + runtime.KeepAlive(cproverIdAllocMap) + runtime.KeepAlive(cproofsLenAllocMap) + packSFilPoStProof(proofsPtr, cproofsPtr) + runtime.KeepAlive(cproofsPtrAllocMap) runtime.KeepAlive(creplicasLenAllocMap) packSFilPublicReplicaInfo(replicasPtr, creplicasPtr) runtime.KeepAlive(creplicasPtrAllocMap) - runtime.KeepAlive(cproverIdAllocMap) runtime.KeepAlive(crandomnessAllocMap) __v := NewFilVerifyWinningPoStResponseRef(unsafe.Pointer(__ret)) return __v } -// FilWriteWithAlignment function as declared in filecoin-ffi/filcrypto.h:690 +// FilWriteWithAlignment function as declared in filecoin-ffi/filcrypto.h:686 func FilWriteWithAlignment(registeredProof FilRegisteredSealProof, srcFd int32, srcSize uint64, dstFd int32, existingPieceSizesPtr []uint64, existingPieceSizesLen uint) *FilWriteWithAlignmentResponse { cregisteredProof, cregisteredProofAllocMap := (C.fil_RegisteredSealProof)(registeredProof), cgoAllocsUnknown csrcFd, csrcFdAllocMap := (C.int)(srcFd), cgoAllocsUnknown @@ -798,7 +789,7 @@ func FilWriteWithAlignment(registeredProof FilRegisteredSealProof, srcFd int32, return __v } -// FilWriteWithoutAlignment function as declared in filecoin-ffi/filcrypto.h:701 +// FilWriteWithoutAlignment function as declared in filecoin-ffi/filcrypto.h:697 func FilWriteWithoutAlignment(registeredProof FilRegisteredSealProof, srcFd int32, srcSize uint64, dstFd int32) *FilWriteWithoutAlignmentResponse { cregisteredProof, cregisteredProofAllocMap := (C.fil_RegisteredSealProof)(registeredProof), cgoAllocsUnknown csrcFd, csrcFdAllocMap := (C.int)(srcFd), cgoAllocsUnknown diff --git a/proofs.go b/proofs.go index 3a00b198..039eab45 100644 --- a/proofs.go +++ b/proofs.go @@ -73,27 +73,13 @@ func VerifyWinningPoSt(info abi.WinningPoStVerifyInfo) (bool, error) { return false, err } - var flattenedProofs string - proofTypes := make([]generated.FilRegisteredPoStProof, filPoStProofsLen) - proofChunkSizes := make([]uint16, filPoStProofsLen) - - for idx := range filPoStProofs { - flattenedProofs = flattenedProofs + filPoStProofs[idx].ProofPtr - proofTypes[idx] = filPoStProofs[idx].RegisteredProof - proofChunkSizes[idx] = uint16(filPoStProofs[idx].ProofLen) - } - resp := generated.FilVerifyWinningPost( to32ByteArray(info.Randomness), - proverID, filPublicReplicaInfos, filPublicReplicaInfosLen, - proofTypes, - uint(len(proofTypes)), - proofChunkSizes, - uint(len(proofChunkSizes)), - flattenedProofs, - uint(len(flattenedProofs)), + filPoStProofs, + filPoStProofsLen, + proverID, ) resp.Deref() diff --git a/rust/src/proofs/api.rs b/rust/src/proofs/api.rs index d5fb3c63..618b6f54 100644 --- a/rust/src/proofs/api.rs +++ b/rust/src/proofs/api.rs @@ -13,7 +13,6 @@ use std::slice::from_raw_parts; use super::helpers::{c_to_rust_post_proofs, to_private_replica_info_map}; use super::types::*; -use crate::proofs::helpers::c_to_rust_post_proofs_meow; use crate::util::api::init_log; /// TODO: document @@ -485,15 +484,11 @@ pub unsafe extern "C" fn fil_verify_seal( #[no_mangle] pub unsafe extern "C" fn fil_verify_winning_post( randomness: fil_32ByteArray, - prover_id: fil_32ByteArray, replicas_ptr: *const fil_PublicReplicaInfo, replicas_len: libc::size_t, - proof_types_ptr: *const fil_RegisteredPoStProof, - proof_types_len: libc::size_t, - proof_chunk_sizes_ptr: *const u16, - proof_chunk_sizes_len: libc::size_t, - flattened_proofs_ptr: *const u8, - flattened_proofs_len: libc::size_t, + proofs_ptr: *const fil_PoStProof, + proofs_len: libc::size_t, + prover_id: fil_32ByteArray, ) -> *mut fil_VerifyWinningPoStResponse { catch_panic_response(|| { init_log(); @@ -505,15 +500,7 @@ pub unsafe extern "C" fn fil_verify_winning_post( let convert = super::helpers::to_public_replica_info_map(replicas_ptr, replicas_len); let result = convert.and_then(|replicas| { - let post_proofs = c_to_rust_post_proofs_meow( - proof_types_ptr, - proof_types_len, - proof_chunk_sizes_ptr, - proof_chunk_sizes_len, - flattened_proofs_ptr, - flattened_proofs_len, - )?; - + let post_proofs = c_to_rust_post_proofs(proofs_ptr, proofs_len)?; let proofs: Vec = post_proofs.iter().flat_map(|pp| pp.clone().proof).collect(); filecoin_proofs_api::post::verify_winning_post( @@ -1607,33 +1594,13 @@ pub mod tests { comm_r: (*resp_b2).comm_r, }]; - let mut ptypes: Vec = Default::default(); - let mut pchunks: Vec = Default::default(); - let mut flattened: Vec = Default::default(); - - for pp in std::slice::from_raw_parts((*resp_h).proofs_ptr, (*resp_h).proofs_len).iter() - { - ptypes.push(pp.registered_proof.clone()); - - let mut proof: Vec = Default::default(); - proof.extend_from_slice(std::slice::from_raw_parts(pp.proof_ptr, pp.proof_len)); - - flattened.append(&mut proof); - - pchunks.push(pp.proof_len as u16); - } - let resp_i = fil_verify_winning_post( randomness, - prover_id, public_replicas.as_ptr(), public_replicas.len(), - ptypes.as_ptr(), - ptypes.len(), - pchunks.as_ptr(), - pchunks.len(), - flattened.as_ptr(), - flattened.len(), + (*resp_h).proofs_ptr, + (*resp_h).proofs_len, + prover_id, ); if (*resp_i).status_code != FCPResponseStatus::FCPNoError { diff --git a/rust/src/proofs/helpers.rs b/rust/src/proofs/helpers.rs index c8dff08e..132666a4 100644 --- a/rust/src/proofs/helpers.rs +++ b/rust/src/proofs/helpers.rs @@ -132,48 +132,3 @@ pub unsafe fn c_to_rust_post_proofs( Ok(out) } - -pub unsafe fn c_to_rust_post_proofs_meow( - proof_types_ptr: *const fil_RegisteredPoStProof, - proof_types_len: libc::size_t, - proof_chunk_sizes_ptr: *const u16, - proof_chunk_sizes_len: libc::size_t, - flattened_proofs_ptr: *const u8, - flattened_proofs_len: libc::size_t, -) -> Result> { - ensure!( - !flattened_proofs_ptr.is_null(), - "flattened_proofs_ptr must not be null" - ); - - ensure!( - !proof_chunk_sizes_ptr.is_null(), - "proof_chunk_sizes_ptr must not be null" - ); - - ensure!( - !proof_types_ptr.is_null(), - "proof_types_ptr must not be null" - ); - - let flattened_proofs = from_raw_parts(flattened_proofs_ptr, flattened_proofs_len); - let chunk_sizes = from_raw_parts(proof_chunk_sizes_ptr, proof_chunk_sizes_len); - let proof_types = from_raw_parts(proof_types_ptr, proof_types_len); - - let mut out: Vec = Vec::with_capacity(chunk_sizes.len()); - - let mut offset: usize = 0; - for (idx, chunk_size) in chunk_sizes.iter().enumerate() { - let mut proof: Vec = Default::default(); - proof.extend_from_slice(&flattened_proofs[offset..offset + *chunk_size as usize]); - - out.push(PoStProof { - registered_proof: proof_types[idx].into(), - proof, - }); - - offset += *chunk_size as usize - } - - Ok(out) -} From 479060e986971e4df7d27cfd5f14f41924c2737a Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 5 May 2020 09:22:01 -0700 Subject: [PATCH 4/8] manually allocate proxy object to prevent sharing Go heap with C --- generated/customallocs.go | 16 ++++++++++++++++ proofs.go | 20 +++++++++++++++----- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/generated/customallocs.go b/generated/customallocs.go index 38f671dd..6b09581e 100644 --- a/generated/customallocs.go +++ b/generated/customallocs.go @@ -29,3 +29,19 @@ func (x *FilPrivateReplicaInfo) AllocateProxy() func() { C.free(unsafe.Pointer(ref81a31e9b)) } } + +func (x *FilPoStProof) AllocateProxy() func() { + mem3451bfa := allocFilPoStProofMemory(1) + ref3451bfa := (*C.fil_PoStProof)(mem3451bfa) + + ref3451bfa.registered_proof = (C.fil_RegisteredPoStProof)(x.RegisteredProof) + ref3451bfa.proof_len = (C.size_t)(x.ProofLen) + ref3451bfa.proof_ptr = (*C.uchar)(unsafe.Pointer(C.CString(x.ProofPtr))) + + x.ref3451bfa = ref3451bfa + + return func() { + C.free(unsafe.Pointer(ref3451bfa.proof_ptr)) + C.free(unsafe.Pointer(ref3451bfa)) + } +} diff --git a/proofs.go b/proofs.go index 039eab45..b921c509 100644 --- a/proofs.go +++ b/proofs.go @@ -63,10 +63,11 @@ func VerifyWinningPoSt(info abi.WinningPoStVerifyInfo) (bool, error) { return false, errors.Wrap(err, "failed to create public replica info array for FFI") } - filPoStProofs, filPoStProofsLen, err := toFilPoStProofs(info.Proofs, "winning") + filPoStProofs, filPoStProofsLen, free, err := toFilPoStProofs(info.Proofs, "winning") if err != nil { return false, errors.Wrap(err, "failed to create PoSt proofs array for FFI") } + defer free() proverID, err := toProverID(info.Prover) if err != nil { @@ -100,10 +101,11 @@ func VerifyWindowPoSt(info abi.WindowPoStVerifyInfo) (bool, error) { return false, errors.Wrap(err, "failed to create public replica info array for FFI") } - filPoStProofs, filPoStProofsLen, err := toFilPoStProofs(info.Proofs, "window") + filPoStProofs, filPoStProofsLen, free, err := toFilPoStProofs(info.Proofs, "window") if err != nil { return false, errors.Wrap(err, "failed to create PoSt proofs array for FFI") } + defer free() proverID, err := toProverID(info.Prover) if err != nil { @@ -748,12 +750,14 @@ func fromFilPoStProofs(src []generated.FilPoStProof) ([]abi.PoStProof, error) { return out, nil } -func toFilPoStProofs(src []abi.PoStProof, typ string) ([]generated.FilPoStProof, uint, error) { +func toFilPoStProofs(src []abi.PoStProof, typ string) ([]generated.FilPoStProof, uint, func(), error) { + frees := make([]func(), len(src)) + out := make([]generated.FilPoStProof, len(src)) for idx := range out { pp, err := toFilRegisteredPoStProof(src[idx].RegisteredProof, typ) if err != nil { - return nil, 0, err + return nil, 0, func() {}, err } out[idx] = generated.FilPoStProof{ @@ -761,9 +765,15 @@ func toFilPoStProofs(src []abi.PoStProof, typ string) ([]generated.FilPoStProof, ProofLen: uint(len(src[idx].ProofBytes)), ProofPtr: string(src[idx].ProofBytes), } + + frees[idx] = out[idx].AllocateProxy() } - return out, uint(len(out)), nil + return out, uint(len(out)), func() { + for idx := range frees { + frees[idx]() + } + }, nil } func to32ByteArray(in []byte) generated.Fil32ByteArray { From e3dc65c89275fe8282c595be4bbbe1505f9828d6 Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 5 May 2020 10:06:30 -0700 Subject: [PATCH 5/8] pass flattened messages to fil_hash_verify to prevent pointer-to-pointer issues with CGO --- bls.go | 10 ++++---- generated/cgo_helpers.go | 48 ----------------------------------- generated/generated.go | 55 +++++++++++++++++++++------------------- rust/src/bls/api.rs | 36 +++++++++++++++----------- 4 files changed, 55 insertions(+), 94 deletions(-) diff --git a/bls.go b/bls.go index ae630ceb..5afdf977 100644 --- a/bls.go +++ b/bls.go @@ -44,11 +44,11 @@ func Verify(signature *Signature, digests []Digest, publicKeys []PublicKey) bool // HashVerify verifies that a signature is the aggregated signature of hashed messages. func HashVerify(signature *Signature, messages []Message, publicKeys []PublicKey) bool { - messagesPtrs := make([]string, len(messages)) + var flattenedMessages string messagesSizes := make([]uint, len(messages)) - for idx, msg := range messages { - messagesPtrs[idx] = string(msg) - messagesSizes[idx] = uint(len(msg)) + for idx := range messages { + flattenedMessages = flattenedMessages + string(messages[idx]) + messagesSizes[idx] = uint(len(messages[idx])) } flattenedPublicKeys := make([]byte, PublicKeyBytes*len(publicKeys)) @@ -56,7 +56,7 @@ func HashVerify(signature *Signature, messages []Message, publicKeys []PublicKey copy(flattenedPublicKeys[(PublicKeyBytes*idx):(PublicKeyBytes*(1+idx))], publicKey[:]) } - isValid := generated.FilHashVerify(string(signature[:]), messagesPtrs, messagesSizes, uint(len(messages)), string(flattenedPublicKeys), uint(len(flattenedPublicKeys))) + isValid := generated.FilHashVerify(string(signature[:]), flattenedMessages, uint(len(flattenedMessages)), messagesSizes, uint(len(messagesSizes)), string(flattenedPublicKeys), uint(len(flattenedPublicKeys))) return isValid > 0 } diff --git a/generated/cgo_helpers.go b/generated/cgo_helpers.go index 39b5a318..2797e128 100644 --- a/generated/cgo_helpers.go +++ b/generated/cgo_helpers.go @@ -3444,54 +3444,6 @@ func packSFilPrivateReplicaInfo(v []FilPrivateReplicaInfo, ptr0 *C.fil_PrivateRe } } -// allocPUint8TMemory allocates memory for type *C.uint8_t in C. -// The caller is responsible for freeing the this memory via C.free. -func allocPUint8TMemory(n int) unsafe.Pointer { - mem, err := C.calloc(C.size_t(n), (C.size_t)(sizeOfPUint8TValue)) - if mem == nil { - panic(fmt.Sprintln("memory alloc error: ", err)) - } - return mem -} - -const sizeOfPUint8TValue = unsafe.Sizeof([1]*C.uint8_t{}) - -// unpackArgSUString transforms a sliced Go data structure into plain C format. -func unpackArgSUString(x []string) (unpacked **C.uint8_t, allocs *cgoAllocMap) { - if x == nil { - return nil, nil - } - allocs = new(cgoAllocMap) - defer runtime.SetFinalizer(allocs, func(a *cgoAllocMap) { - go a.Free() - }) - - len0 := len(x) - mem0 := allocPUint8TMemory(len0) - allocs.Add(mem0) - h0 := &sliceHeader{ - Data: mem0, - Cap: len0, - Len: len0, - } - v0 := *(*[]*C.uint8_t)(unsafe.Pointer(h0)) - for i0 := range x { - v0[i0], _ = unpackPUint8TString(x[i0]) - } - h := (*sliceHeader)(unsafe.Pointer(&v0)) - unpacked = (**C.uint8_t)(h.Data) - return -} - -// packSUString reads sliced Go data structure out from plain C format. -func packSUString(v []string, ptr0 **C.uint8_t) { - const m = 0x7fffffff - for i0 := range v { - ptr1 := (*(*[m / sizeOfPtr]*C.uint8_t)(unsafe.Pointer(ptr0)))[i0] - v[i0] = packPUint8TString(ptr1) - } -} - // unpackArgSFilPublicReplicaInfo transforms a sliced Go data structure into plain C format. func unpackArgSFilPublicReplicaInfo(x []FilPublicReplicaInfo) (unpacked *C.fil_PublicReplicaInfo, allocs *cgoAllocMap) { if x == nil { diff --git a/generated/generated.go b/generated/generated.go index d1efb870..f79585a7 100644 --- a/generated/generated.go +++ b/generated/generated.go @@ -436,30 +436,33 @@ func FilHash(messagePtr string, messageLen uint) *FilHashResponse { } // FilHashVerify function as declared in filecoin-ffi/filcrypto.h:499 -func FilHashVerify(signaturePtr string, messagesPtr []string, messagesSizesPtr []uint, messagesLen uint, flattenedPublicKeysPtr string, flattenedPublicKeysLen uint) int32 { +func FilHashVerify(signaturePtr string, flattenedMessagesPtr string, flattenedMessagesLen uint, messageSizesPtr []uint, messageSizesLen uint, flattenedPublicKeysPtr string, flattenedPublicKeysLen uint) int32 { signaturePtr = safeString(signaturePtr) csignaturePtr, csignaturePtrAllocMap := unpackPUint8TString(signaturePtr) - cmessagesPtr, cmessagesPtrAllocMap := unpackArgSUString(messagesPtr) - cmessagesSizesPtr, cmessagesSizesPtrAllocMap := (*C.size_t)(unsafe.Pointer((*sliceHeader)(unsafe.Pointer(&messagesSizesPtr)).Data)), cgoAllocsUnknown - cmessagesLen, cmessagesLenAllocMap := (C.size_t)(messagesLen), cgoAllocsUnknown + flattenedMessagesPtr = safeString(flattenedMessagesPtr) + cflattenedMessagesPtr, cflattenedMessagesPtrAllocMap := unpackPUint8TString(flattenedMessagesPtr) + cflattenedMessagesLen, cflattenedMessagesLenAllocMap := (C.size_t)(flattenedMessagesLen), cgoAllocsUnknown + cmessageSizesPtr, cmessageSizesPtrAllocMap := (*C.size_t)(unsafe.Pointer((*sliceHeader)(unsafe.Pointer(&messageSizesPtr)).Data)), cgoAllocsUnknown + cmessageSizesLen, cmessageSizesLenAllocMap := (C.size_t)(messageSizesLen), cgoAllocsUnknown flattenedPublicKeysPtr = safeString(flattenedPublicKeysPtr) cflattenedPublicKeysPtr, cflattenedPublicKeysPtrAllocMap := unpackPUint8TString(flattenedPublicKeysPtr) cflattenedPublicKeysLen, cflattenedPublicKeysLenAllocMap := (C.size_t)(flattenedPublicKeysLen), cgoAllocsUnknown - __ret := C.fil_hash_verify(csignaturePtr, cmessagesPtr, cmessagesSizesPtr, cmessagesLen, cflattenedPublicKeysPtr, cflattenedPublicKeysLen) + __ret := C.fil_hash_verify(csignaturePtr, cflattenedMessagesPtr, cflattenedMessagesLen, cmessageSizesPtr, cmessageSizesLen, cflattenedPublicKeysPtr, cflattenedPublicKeysLen) runtime.KeepAlive(cflattenedPublicKeysLenAllocMap) runtime.KeepAlive(flattenedPublicKeysPtr) runtime.KeepAlive(cflattenedPublicKeysPtrAllocMap) - runtime.KeepAlive(cmessagesLenAllocMap) - runtime.KeepAlive(cmessagesSizesPtrAllocMap) - packSUString(messagesPtr, cmessagesPtr) - runtime.KeepAlive(cmessagesPtrAllocMap) + runtime.KeepAlive(cmessageSizesLenAllocMap) + runtime.KeepAlive(cmessageSizesPtrAllocMap) + runtime.KeepAlive(cflattenedMessagesLenAllocMap) + runtime.KeepAlive(flattenedMessagesPtr) + runtime.KeepAlive(cflattenedMessagesPtrAllocMap) runtime.KeepAlive(signaturePtr) runtime.KeepAlive(csignaturePtrAllocMap) __v := (int32)(__ret) return __v } -// FilInitLogFd function as declared in filecoin-ffi/filcrypto.h:515 +// FilInitLogFd function as declared in filecoin-ffi/filcrypto.h:516 func FilInitLogFd(logFd int32) *FilInitLogFdResponse { clogFd, clogFdAllocMap := (C.int)(logFd), cgoAllocsUnknown __ret := C.fil_init_log_fd(clogFd) @@ -468,14 +471,14 @@ func FilInitLogFd(logFd int32) *FilInitLogFdResponse { return __v } -// FilPrivateKeyGenerate function as declared in filecoin-ffi/filcrypto.h:520 +// FilPrivateKeyGenerate function as declared in filecoin-ffi/filcrypto.h:521 func FilPrivateKeyGenerate() *FilPrivateKeyGenerateResponse { __ret := C.fil_private_key_generate() __v := NewFilPrivateKeyGenerateResponseRef(unsafe.Pointer(__ret)) return __v } -// FilPrivateKeyGenerateWithSeed function as declared in filecoin-ffi/filcrypto.h:533 +// FilPrivateKeyGenerateWithSeed function as declared in filecoin-ffi/filcrypto.h:534 func FilPrivateKeyGenerateWithSeed(rawSeed Fil32ByteArray) *FilPrivateKeyGenerateResponse { crawSeed, crawSeedAllocMap := rawSeed.PassValue() __ret := C.fil_private_key_generate_with_seed(crawSeed) @@ -484,7 +487,7 @@ func FilPrivateKeyGenerateWithSeed(rawSeed Fil32ByteArray) *FilPrivateKeyGenerat return __v } -// FilPrivateKeyPublicKey function as declared in filecoin-ffi/filcrypto.h:544 +// FilPrivateKeyPublicKey function as declared in filecoin-ffi/filcrypto.h:545 func FilPrivateKeyPublicKey(rawPrivateKeyPtr string) *FilPrivateKeyPublicKeyResponse { rawPrivateKeyPtr = safeString(rawPrivateKeyPtr) crawPrivateKeyPtr, crawPrivateKeyPtrAllocMap := unpackPUint8TString(rawPrivateKeyPtr) @@ -495,7 +498,7 @@ func FilPrivateKeyPublicKey(rawPrivateKeyPtr string) *FilPrivateKeyPublicKeyResp return __v } -// FilPrivateKeySign function as declared in filecoin-ffi/filcrypto.h:557 +// FilPrivateKeySign function as declared in filecoin-ffi/filcrypto.h:558 func FilPrivateKeySign(rawPrivateKeyPtr string, messagePtr string, messageLen uint) *FilPrivateKeySignResponse { rawPrivateKeyPtr = safeString(rawPrivateKeyPtr) crawPrivateKeyPtr, crawPrivateKeyPtrAllocMap := unpackPUint8TString(rawPrivateKeyPtr) @@ -512,7 +515,7 @@ func FilPrivateKeySign(rawPrivateKeyPtr string, messagePtr string, messageLen ui return __v } -// FilSealCommitPhase1 function as declared in filecoin-ffi/filcrypto.h:565 +// FilSealCommitPhase1 function as declared in filecoin-ffi/filcrypto.h:566 func FilSealCommitPhase1(registeredProof FilRegisteredSealProof, commR Fil32ByteArray, commD Fil32ByteArray, cacheDirPath string, replicaPath string, sectorId uint64, proverId Fil32ByteArray, ticket Fil32ByteArray, seed Fil32ByteArray, piecesPtr []FilPublicPieceInfo, piecesLen uint) *FilSealCommitPhase1Response { cregisteredProof, cregisteredProofAllocMap := (C.fil_RegisteredSealProof)(registeredProof), cgoAllocsUnknown ccommR, ccommRAllocMap := commR.PassValue() @@ -546,7 +549,7 @@ func FilSealCommitPhase1(registeredProof FilRegisteredSealProof, commR Fil32Byte return __v } -// FilSealCommitPhase2 function as declared in filecoin-ffi/filcrypto.h:577 +// FilSealCommitPhase2 function as declared in filecoin-ffi/filcrypto.h:578 func FilSealCommitPhase2(sealCommitPhase1OutputPtr string, sealCommitPhase1OutputLen uint, sectorId uint64, proverId Fil32ByteArray) *FilSealCommitPhase2Response { sealCommitPhase1OutputPtr = safeString(sealCommitPhase1OutputPtr) csealCommitPhase1OutputPtr, csealCommitPhase1OutputPtrAllocMap := unpackPUint8TString(sealCommitPhase1OutputPtr) @@ -563,7 +566,7 @@ func FilSealCommitPhase2(sealCommitPhase1OutputPtr string, sealCommitPhase1Outpu return __v } -// FilSealPreCommitPhase1 function as declared in filecoin-ffi/filcrypto.h:586 +// FilSealPreCommitPhase1 function as declared in filecoin-ffi/filcrypto.h:587 func FilSealPreCommitPhase1(registeredProof FilRegisteredSealProof, cacheDirPath string, stagedSectorPath string, sealedSectorPath string, sectorId uint64, proverId Fil32ByteArray, ticket Fil32ByteArray, piecesPtr []FilPublicPieceInfo, piecesLen uint) *FilSealPreCommitPhase1Response { cregisteredProof, cregisteredProofAllocMap := (C.fil_RegisteredSealProof)(registeredProof), cgoAllocsUnknown cacheDirPath = safeString(cacheDirPath) @@ -595,7 +598,7 @@ func FilSealPreCommitPhase1(registeredProof FilRegisteredSealProof, cacheDirPath return __v } -// FilSealPreCommitPhase2 function as declared in filecoin-ffi/filcrypto.h:600 +// FilSealPreCommitPhase2 function as declared in filecoin-ffi/filcrypto.h:601 func FilSealPreCommitPhase2(sealPreCommitPhase1OutputPtr string, sealPreCommitPhase1OutputLen uint, cacheDirPath string, sealedSectorPath string) *FilSealPreCommitPhase2Response { sealPreCommitPhase1OutputPtr = safeString(sealPreCommitPhase1OutputPtr) csealPreCommitPhase1OutputPtr, csealPreCommitPhase1OutputPtrAllocMap := unpackPUint8TString(sealPreCommitPhase1OutputPtr) @@ -616,7 +619,7 @@ func FilSealPreCommitPhase2(sealPreCommitPhase1OutputPtr string, sealPreCommitPh return __v } -// FilUnseal function as declared in filecoin-ffi/filcrypto.h:608 +// FilUnseal function as declared in filecoin-ffi/filcrypto.h:609 func FilUnseal(registeredProof FilRegisteredSealProof, cacheDirPath string, sealedSectorPath string, unsealOutputPath string, sectorId uint64, proverId Fil32ByteArray, ticket Fil32ByteArray, commD Fil32ByteArray) *FilUnsealResponse { cregisteredProof, cregisteredProofAllocMap := (C.fil_RegisteredSealProof)(registeredProof), cgoAllocsUnknown cacheDirPath = safeString(cacheDirPath) @@ -645,7 +648,7 @@ func FilUnseal(registeredProof FilRegisteredSealProof, cacheDirPath string, seal return __v } -// FilUnsealRange function as declared in filecoin-ffi/filcrypto.h:620 +// FilUnsealRange function as declared in filecoin-ffi/filcrypto.h:621 func FilUnsealRange(registeredProof FilRegisteredSealProof, cacheDirPath string, sealedSectorPath string, unsealOutputPath string, sectorId uint64, proverId Fil32ByteArray, ticket Fil32ByteArray, commD Fil32ByteArray, offset uint64, length uint64) *FilUnsealRangeResponse { cregisteredProof, cregisteredProofAllocMap := (C.fil_RegisteredSealProof)(registeredProof), cgoAllocsUnknown cacheDirPath = safeString(cacheDirPath) @@ -678,7 +681,7 @@ func FilUnsealRange(registeredProof FilRegisteredSealProof, cacheDirPath string, return __v } -// FilVerify function as declared in filecoin-ffi/filcrypto.h:642 +// FilVerify function as declared in filecoin-ffi/filcrypto.h:643 func FilVerify(signaturePtr string, flattenedDigestsPtr string, flattenedDigestsLen uint, flattenedPublicKeysPtr string, flattenedPublicKeysLen uint) int32 { signaturePtr = safeString(signaturePtr) csignaturePtr, csignaturePtrAllocMap := unpackPUint8TString(signaturePtr) @@ -701,7 +704,7 @@ func FilVerify(signaturePtr string, flattenedDigestsPtr string, flattenedDigests return __v } -// FilVerifySeal function as declared in filecoin-ffi/filcrypto.h:652 +// FilVerifySeal function as declared in filecoin-ffi/filcrypto.h:653 func FilVerifySeal(registeredProof FilRegisteredSealProof, commR Fil32ByteArray, commD Fil32ByteArray, proverId Fil32ByteArray, ticket Fil32ByteArray, seed Fil32ByteArray, sectorId uint64, proofPtr string, proofLen uint) *FilVerifySealResponse { cregisteredProof, cregisteredProofAllocMap := (C.fil_RegisteredSealProof)(registeredProof), cgoAllocsUnknown ccommR, ccommRAllocMap := commR.PassValue() @@ -728,7 +731,7 @@ func FilVerifySeal(registeredProof FilRegisteredSealProof, commR Fil32ByteArray, return __v } -// FilVerifyWindowPost function as declared in filecoin-ffi/filcrypto.h:665 +// FilVerifyWindowPost function as declared in filecoin-ffi/filcrypto.h:666 func FilVerifyWindowPost(randomness Fil32ByteArray, replicasPtr []FilPublicReplicaInfo, replicasLen uint, proofsPtr []FilPoStProof, proofsLen uint, proverId Fil32ByteArray) *FilVerifyWindowPoStResponse { crandomness, crandomnessAllocMap := randomness.PassValue() creplicasPtr, creplicasPtrAllocMap := unpackArgSFilPublicReplicaInfo(replicasPtr) @@ -749,7 +752,7 @@ func FilVerifyWindowPost(randomness Fil32ByteArray, replicasPtr []FilPublicRepli return __v } -// FilVerifyWinningPost function as declared in filecoin-ffi/filcrypto.h:675 +// FilVerifyWinningPost function as declared in filecoin-ffi/filcrypto.h:676 func FilVerifyWinningPost(randomness Fil32ByteArray, replicasPtr []FilPublicReplicaInfo, replicasLen uint, proofsPtr []FilPoStProof, proofsLen uint, proverId Fil32ByteArray) *FilVerifyWinningPoStResponse { crandomness, crandomnessAllocMap := randomness.PassValue() creplicasPtr, creplicasPtrAllocMap := unpackArgSFilPublicReplicaInfo(replicasPtr) @@ -770,7 +773,7 @@ func FilVerifyWinningPost(randomness Fil32ByteArray, replicasPtr []FilPublicRepl return __v } -// FilWriteWithAlignment function as declared in filecoin-ffi/filcrypto.h:686 +// FilWriteWithAlignment function as declared in filecoin-ffi/filcrypto.h:687 func FilWriteWithAlignment(registeredProof FilRegisteredSealProof, srcFd int32, srcSize uint64, dstFd int32, existingPieceSizesPtr []uint64, existingPieceSizesLen uint) *FilWriteWithAlignmentResponse { cregisteredProof, cregisteredProofAllocMap := (C.fil_RegisteredSealProof)(registeredProof), cgoAllocsUnknown csrcFd, csrcFdAllocMap := (C.int)(srcFd), cgoAllocsUnknown @@ -789,7 +792,7 @@ func FilWriteWithAlignment(registeredProof FilRegisteredSealProof, srcFd int32, return __v } -// FilWriteWithoutAlignment function as declared in filecoin-ffi/filcrypto.h:697 +// FilWriteWithoutAlignment function as declared in filecoin-ffi/filcrypto.h:698 func FilWriteWithoutAlignment(registeredProof FilRegisteredSealProof, srcFd int32, srcSize uint64, dstFd int32) *FilWriteWithoutAlignmentResponse { cregisteredProof, cregisteredProofAllocMap := (C.fil_RegisteredSealProof)(registeredProof), cgoAllocsUnknown csrcFd, csrcFdAllocMap := (C.int)(srcFd), cgoAllocsUnknown diff --git a/rust/src/bls/api.rs b/rust/src/bls/api.rs index a7746a85..87d09f34 100644 --- a/rust/src/bls/api.rs +++ b/rust/src/bls/api.rs @@ -188,9 +188,10 @@ pub unsafe extern "C" fn fil_verify( #[no_mangle] pub unsafe extern "C" fn fil_hash_verify( signature_ptr: *const u8, - messages_ptr: *const *const u8, - messages_sizes_ptr: *const libc::size_t, - messages_len: libc::size_t, + flattened_messages_ptr: *const u8, + flattened_messages_len: libc::size_t, + message_sizes_ptr: *const libc::size_t, + message_sizes_len: libc::size_t, flattened_public_keys_ptr: *const u8, flattened_public_keys_len: libc::size_t, ) -> libc::c_int { @@ -198,13 +199,17 @@ pub unsafe extern "C" fn fil_hash_verify( let raw_signature = from_raw_parts(signature_ptr, SIGNATURE_BYTES); let signature = try_ffi!(Signature::from_bytes(raw_signature), 0); - let raw_messages_sizes: &[usize] = from_raw_parts(messages_sizes_ptr, messages_len); - let raw_messages: &[*const u8] = from_raw_parts(messages_ptr, messages_len); - let messages: Vec<&[u8]> = raw_messages - .iter() - .zip(raw_messages_sizes.iter()) - .map(|(ptr, size)| from_raw_parts(*ptr, *size)) - .collect(); + let flattened = from_raw_parts(flattened_messages_ptr, flattened_messages_len); + let chunk_sizes = from_raw_parts(message_sizes_ptr, message_sizes_len); + + // split the flattened message array into slices of individual messages to + // be hashed + let mut messages: Vec<&[u8]> = Vec::with_capacity(message_sizes_len); + let mut offset = 0; + for chunk_size in chunk_sizes.iter() { + messages.push(&flattened[offset..offset + *chunk_size]); + offset += *chunk_size + } let raw_public_keys = from_raw_parts(flattened_public_keys_ptr, flattened_public_keys_len); @@ -369,13 +374,14 @@ mod tests { assert_eq!(1, verified); - let messages = [message.as_ptr()]; - let messages_sizes = [message.len()]; + let flattened_messages = message; + let message_sizes = [message.len()]; let verified = fil_hash_verify( signature.as_ptr(), - messages.as_ptr(), - messages_sizes.as_ptr(), - 1, + flattened_messages.as_ptr(), + flattened_messages.len(), + message_sizes.as_ptr(), + message_sizes.len(), public_key.as_ptr(), public_key.len(), ); From a5b9e249a3ac502dc93efc9ce49884253f7a8cb0 Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 5 May 2020 10:16:03 -0700 Subject: [PATCH 6/8] rename symbols within custom allocate methods --- generated/customallocs.go | 45 ++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/generated/customallocs.go b/generated/customallocs.go index 6b09581e..4c8d10dd 100644 --- a/generated/customallocs.go +++ b/generated/customallocs.go @@ -12,36 +12,43 @@ import ( "unsafe" ) +// AllocateProxy allocates a FilPrivateReplicaInfo proxy object in the C heap, +// returning a function which, when called, frees the allocated memory. This +// method exists because the default c-for-go allocation strategy allocates a +// C struct with a field whose values is a pointer into the Go heap, which is +// not permitted by the most strict CGO check (cgocheck=2). func (x *FilPrivateReplicaInfo) AllocateProxy() func() { - mem81a31e9b := allocFilPrivateReplicaInfoMemory(1) - ref81a31e9b := (*C.fil_PrivateReplicaInfo)(mem81a31e9b) - ref81a31e9b.cache_dir_path = C.CString(x.CacheDirPath) - ref81a31e9b.comm_r = *(*[32]C.uint8_t)(unsafe.Pointer(&x.CommR)) - ref81a31e9b.registered_proof = (C.fil_RegisteredPoStProof)(x.RegisteredProof) - ref81a31e9b.replica_path = C.CString(x.ReplicaPath) - ref81a31e9b.sector_id = (C.uint64_t)(x.SectorId) + mem := allocFilPrivateReplicaInfoMemory(1) + proxy := (*C.fil_PrivateReplicaInfo)(mem) + proxy.cache_dir_path = C.CString(x.CacheDirPath) + proxy.comm_r = *(*[32]C.uint8_t)(unsafe.Pointer(&x.CommR)) + proxy.registered_proof = (C.fil_RegisteredPoStProof)(x.RegisteredProof) + proxy.replica_path = C.CString(x.ReplicaPath) + proxy.sector_id = (C.uint64_t)(x.SectorId) - x.ref81a31e9b = ref81a31e9b + x.ref81a31e9b = proxy return func() { - C.free(unsafe.Pointer(ref81a31e9b.cache_dir_path)) - C.free(unsafe.Pointer(ref81a31e9b.replica_path)) - C.free(unsafe.Pointer(ref81a31e9b)) + C.free(unsafe.Pointer(proxy.cache_dir_path)) + C.free(unsafe.Pointer(proxy.replica_path)) + C.free(unsafe.Pointer(proxy)) } } +// AllocateProxy allocates a FilPoStProof proxy object in the C heap, +// returning a function which, when called, frees the allocated memory. func (x *FilPoStProof) AllocateProxy() func() { - mem3451bfa := allocFilPoStProofMemory(1) - ref3451bfa := (*C.fil_PoStProof)(mem3451bfa) + mem := allocFilPoStProofMemory(1) + proxy := (*C.fil_PoStProof)(mem) - ref3451bfa.registered_proof = (C.fil_RegisteredPoStProof)(x.RegisteredProof) - ref3451bfa.proof_len = (C.size_t)(x.ProofLen) - ref3451bfa.proof_ptr = (*C.uchar)(unsafe.Pointer(C.CString(x.ProofPtr))) + proxy.registered_proof = (C.fil_RegisteredPoStProof)(x.RegisteredProof) + proxy.proof_len = (C.size_t)(x.ProofLen) + proxy.proof_ptr = (*C.uchar)(unsafe.Pointer(C.CString(x.ProofPtr))) - x.ref3451bfa = ref3451bfa + x.ref3451bfa = proxy return func() { - C.free(unsafe.Pointer(ref3451bfa.proof_ptr)) - C.free(unsafe.Pointer(ref3451bfa)) + C.free(unsafe.Pointer(proxy.proof_ptr)) + C.free(unsafe.Pointer(proxy)) } } From 257cf48e1a5089f07486f1e1c5e57ea1fb9fa02c Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 5 May 2020 10:19:15 -0700 Subject: [PATCH 7/8] run go test with cgocheck=2 --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1453221b..0a04cb9f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -321,13 +321,13 @@ commands: no_output_timeout: 60m - run: name: Test project - command: RUST_LOG=info go test -p 1 -timeout 60m + command: GODEBUG=cgocheck=2 RUST_LOG=info go test -p 1 -timeout 60m no_output_timeout: 60m compile_tests: steps: - run: name: Build project and tests, but don't actually run the tests (used to verify that build/link works with Darwin) - command: RUST_LOG=info go test -run=^$ + command: GODEBUG=cgocheck=2 RUST_LOG=info go test -run=^$ restore_parameter_cache: steps: - restore_cache: From ce595c4cb7a455ede88892a6addf390aba2e674b Mon Sep 17 00:00:00 2001 From: laser Date: Tue, 5 May 2020 12:29:35 -0700 Subject: [PATCH 8/8] delete temporary branches from circleci config --- .circleci/config.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0a04cb9f..27e44e4e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -158,7 +158,6 @@ workflows: branches: only: - master - - feat/cgocheck-2-panic run_leak_detector: true - build_and_test_linux_cgo_bindings: filters: @@ -171,7 +170,6 @@ workflows: branches: only: - master - - feat/lint-during-build - build_darwin_cgo_bindings - publish_darwin_staticlib: filters: