From a8c880c71874655ab9808d74662395b398b9d43a Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 1 Feb 2018 17:21:49 +0200 Subject: [PATCH 001/157] sendrawtransaction message when conflicting with memppol --- src/core/main.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/main.cpp b/src/core/main.cpp index 292bd9e2..9e27ef1f 100644 --- a/src/core/main.cpp +++ b/src/core/main.cpp @@ -1446,7 +1446,8 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa { if(fDebug)LogPrint("mchn","Conflicting with in-memory %s\n",tx.vin[i].ToString().c_str()); // Disable replacement feature for now - return false; + return state.Invalid(error("AcceptToMemoryPool : Conflicting with in-memory tx"), + REJECT_DUPLICATE, "bad-txns-inputs-spent"); } } } From 76a01b46e821797d1efc57c9489fc68635f75b3f Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 6 Feb 2018 18:53:40 +0200 Subject: [PATCH 002/157] Refactored AcceptMultiChainTransaction --- src/protocol/multichaintx.cpp | 2080 +++++++++++++++++++++++++++++++-- src/script/standard.cpp | 20 +- 2 files changed, 1979 insertions(+), 121 deletions(-) diff --git a/src/protocol/multichaintx.cpp b/src/protocol/multichaintx.cpp index 23b24c74..b2363258 100644 --- a/src/protocol/multichaintx.cpp +++ b/src/protocol/multichaintx.cpp @@ -14,177 +14,2020 @@ using namespace std; string EncodeHexTx(const CTransaction& tx); +bool ExtractDestinations10008(const CScript& scriptPubKey, txnouttype& typeRet, vector& addressRet, int& nRequiredRet, bool no_clear, bool *not_cleared); + +#define MC_MTX_OUTPUT_DETAIL_FLAG_NONE 0x00000000 +#define MC_MTX_OUTPUT_DETAIL_FLAG_OP_RETURN_ENTITY_ITEM 0x00000001 +#define MC_MTX_OUTPUT_DETAIL_FLAG_PERMISSION_CREATE 0x00000002 +#define MC_MTX_OUTPUT_DETAIL_FLAG_PERMISSION_ADMIN 0x00000004 +#define MC_MTX_OUTPUT_DETAIL_FLAG_NOT_PURE_PERMISSION 0x00000008 +#define MC_MTX_OUTPUT_DETAIL_FLAG_NOT_OP_RETURN 0x00000010 +#define MC_MTX_OUTPUT_DETAIL_FLAG_FOLLOWON_DETAILS 0x00000020 +#define MC_MTX_OUTPUT_DETAIL_FLAG_NO_DESTINATION 0x00000040 + +typedef struct CMultiChainTxDetails +{ + bool fCheckCachedScript; // If tx contains admin/miner grants, it should have cached input script + bool fScriptHashAllFound; // Input script with SIGHASH_ALL found, OP_RETURN metadata data can be transferred in this tx + bool fRejectIfOpDropOpReturn; // Tx should be rejected of OP_DROP+OP_RETURN script is found + bool fSeedNodeInvolved; // Connect permission of seed node changed + bool fFullReplayCheckRequired; // Tx should be rechecked when mempool is replayed + bool fAdminMinerGrant; // Admin/miner grant in this transaction + + vector vInputScriptTypes; // Input script types + vector vInputDestinations; // Addresses used in input scripts + vector vInputHashTypes; // Input hash types + vector vInputCanGrantAdminMine; // Flags - input can be used as signer for admin/mine grants + vector vInputHadAdminPermissionBeforeThisTx; // Admin permissions before this tx - for approval + + set vAllowedAdmins; // Admin permissions before this tx - for grants + set vAllowedActivators; // Activate permissions before this tx - for grants + + vector vOutputScriptFlags; // Output script flags, filled when script is processed for the first time + vector vOutputPermissionRequired; // Number of required receive permissions + vector < vector > vOutputDestinations; // Output destinations + vector < CTxDestination > vOutputSingleDestination; // Single destination for setting permission + + + unsigned char details_script[MC_ENT_MAX_SCRIPT_SIZE]; // Entity details script + int details_script_size; // Entity details script size + int details_script_type; // Entity details script type - new/update + uint32_t new_entity_type; // New entity type + int new_entity_output; // Output where new entity is defined + + CMultiChainTxDetails() + { + Zero(); + } + + ~CMultiChainTxDetails() + { + + } + + void Zero(); + bool IsRelevantInput(int vin,int vout); + +} CMultiChainTxDetails; + +void CMultiChainTxDetails::Zero() +{ + vInputScriptTypes.clear(); + vInputDestinations.clear(); + vInputHashTypes.clear(); + vInputCanGrantAdminMine.clear(); + vInputHadAdminPermissionBeforeThisTx.clear(); + + vAllowedAdmins.clear(); + vAllowedActivators.clear(); + + vOutputScriptFlags.clear(); + vOutputPermissionRequired.clear(); + vOutputDestinations.clear(); + vOutputSingleDestination.clear(); + + fCheckCachedScript=false; + fScriptHashAllFound=false; + fRejectIfOpDropOpReturn=false; + fSeedNodeInvolved=false; + fFullReplayCheckRequired=false; + fAdminMinerGrant=false; + + details_script_size=0; + details_script_type=-1; + new_entity_type=MC_ENT_TYPE_NONE; + new_entity_output=-1; + +} + +bool CMultiChainTxDetails::IsRelevantInput(int vin, int vout) +{ + if( (vInputHashTypes[vin] == SIGHASH_ALL) || ( (vInputHashTypes[vin] == SIGHASH_SINGLE) && ((vin == vout) ) ) ) + { + return true; + } + return false; +} uint160 mc_GenesisAdmin(const CTransaction& tx) { uint32_t type,from,to,timestamp; for (unsigned int j = 0; j < tx.vout.size(); j++) { - mc_gState->m_TmpScript->Clear(); - const CScript& script1 = tx.vout[j].scriptPubKey; - CScript::const_iterator pc1 = script1.begin(); - CTxDestination addressRet; - - mc_gState->m_TmpScript->SetScript((unsigned char*)(&pc1[0]),(size_t)(script1.end()-pc1),MC_SCR_TYPE_SCRIPTPUBKEY); - for (int e = 0; e < mc_gState->m_TmpScript->GetNumElements(); e++) + mc_gState->m_TmpScript->Clear(); + const CScript& script1 = tx.vout[j].scriptPubKey; + CScript::const_iterator pc1 = script1.begin(); + CTxDestination addressRet; + + mc_gState->m_TmpScript->SetScript((unsigned char*)(&pc1[0]),(size_t)(script1.end()-pc1),MC_SCR_TYPE_SCRIPTPUBKEY); + for (int e = 0; e < mc_gState->m_TmpScript->GetNumElements(); e++) + { + mc_gState->m_TmpScript->SetElement(e); + if(mc_gState->m_TmpScript->GetPermission(&type,&from,&to,×tamp) == 0) + { + if(type == MC_PTP_GLOBAL_ALL) + { + CTxDestination addressRet; + if(ExtractDestination(script1, addressRet)) + { + CKeyID *lpKeyID=boost::get (&addressRet); + if(lpKeyID) + { + return *(uint160*)lpKeyID; + } + } + } + } + } + } + return 0; +} + +bool mc_ExtractInputAssetQuantities(mc_Buffer *assets, const CScript& script1, uint256 hash, string& reason) +{ + int err; + int64_t quantity; + CScript::const_iterator pc1 = script1.begin(); + + mc_gState->m_TmpScript->Clear(); + mc_gState->m_TmpScript->SetScript((unsigned char*)(&pc1[0]),(size_t)(script1.end()-pc1),MC_SCR_TYPE_SCRIPTPUBKEY); + + for (int e = 0; e < mc_gState->m_TmpScript->GetNumElements(); e++) + { + mc_gState->m_TmpScript->SetElement(e); + err=mc_gState->m_TmpScript->GetAssetQuantities(assets,MC_SCR_ASSET_SCRIPT_TYPE_TRANSFER | MC_SCR_ASSET_SCRIPT_TYPE_FOLLOWON); + if((err != MC_ERR_NOERROR) && (err != MC_ERR_WRONG_SCRIPT)) + { + reason="Asset transfer script rejected - error in script"; + return false; + } + err=mc_gState->m_TmpScript->GetAssetGenesis(&quantity); + if(err == 0) + { + mc_EntityDetails entity; + unsigned char buf_amounts[MC_AST_ASSET_FULLREF_BUF_SIZE]; + memset(buf_amounts,0,MC_AST_ASSET_FULLREF_BUF_SIZE); + if(mc_gState->m_Assets->FindEntityByTxID(&entity,(unsigned char*)&hash)) + { + memcpy(buf_amounts,entity.GetFullRef(),MC_AST_ASSET_FULLREF_SIZE); + int row=assets->Seek(buf_amounts); + if(row>=0) + { + int64_t last=mc_GetABQuantity(assets->GetRow(row)); + quantity+=last; + mc_SetABQuantity(assets->GetRow(row),quantity); + } + else + { + mc_SetABQuantity(buf_amounts,quantity); + assets->Add(buf_amounts); + } + } + else + { + reason="Asset transfer script rejected - issue tx not found"; + return false; + } + } + else + { + if(err != MC_ERR_WRONG_SCRIPT) + { + reason="Asset transfer script rejected - error in input issue script"; + return false; + } + } + } + + return true; +} + +bool mc_CompareAssetQuantities(string& reason) +{ + unsigned char *ptrIn; + unsigned char *ptrOut; + int64_t quantity; + mc_EntityDetails entity; + + for(int i=0;im_TmpAssetsIn->GetCount();i++) + { + ptrIn=mc_gState->m_TmpAssetsIn->GetRow(i); + int row=mc_gState->m_TmpAssetsOut->Seek(ptrIn); + quantity=mc_GetABQuantity(ptrIn); + if(quantity>0) + { + if(row>=0) + { + ptrOut=mc_gState->m_TmpAssetsOut->GetRow(row); + if(memcmp(ptrIn,ptrOut,MC_AST_ASSET_QUANTITY_OFFSET+MC_AST_ASSET_QUANTITY_SIZE)) + { + reason="Asset transfer script rejected - mismatch in input/output quantities"; + return false; + } + } + else + { + reason="Asset transfer script rejected - mismatch in input/output quantities"; + return false; + } + } + } + + for(int i=0;im_TmpAssetsOut->GetCount();i++) + { + ptrOut=mc_gState->m_TmpAssetsOut->GetRow(i); + int row=mc_gState->m_TmpAssetsIn->Seek(ptrOut); + quantity=mc_GetABQuantity(ptrOut); + + if(mc_gState->m_Assets->FindEntityByFullRef(&entity,ptrOut) == 0) + { + reason="Asset transfer script rejected - asset not found"; + return false; + } + + if(quantity>0) + { + if(row>=0) + { + ptrIn=mc_gState->m_TmpAssetsIn->GetRow(row); + if(memcmp(ptrIn,ptrOut,MC_AST_ASSET_QUANTITY_OFFSET+MC_AST_ASSET_QUANTITY_SIZE)) + { + reason="Asset transfer script rejected - mismatch in input/output quantities"; + return false; + } + } + else + { + reason="Asset transfer script rejected - mismatch in input/output quantities"; + return false; + } + } + } + + return true; +} + +void MultiChainTransaction_SetTmpOutputScript(const CScript& script1) +{ + CScript::const_iterator pc1 = script1.begin(); + mc_gState->m_TmpScript->Clear(); + mc_gState->m_TmpScript->SetScript((unsigned char*)(&pc1[0]),(size_t)(script1.end()-pc1),MC_SCR_TYPE_SCRIPTPUBKEY); +} + +bool MultiChainTransaction_CheckCachedScriptFlag(const CTransaction& tx) +{ + bool flag=false; + if(mc_gState->m_NetworkParams->GetInt64Param("supportminerprecheck")) + { + flag=true; + } + if(tx.IsCoinBase()) + { + flag=false; + } + return flag; +} + +bool MultiChainTransaction_CheckCoinbaseInputs(const CTransaction& tx, + CMultiChainTxDetails *details) +{ + if(!tx.IsCoinBase()) + { + return false; + } + + if(mc_gState->m_Permissions->m_Block == -1) + { + details->vInputScriptTypes.push_back(TX_PUBKEYHASH); + details->vInputDestinations.push_back(mc_GenesisAdmin(tx)); // Genesis admin is considered to be admin/opener of everything in genesis coinbase + } + else + { + details->vInputScriptTypes.push_back(TX_NONSTANDARD); + details->vInputDestinations.push_back(0); // Invalid signer, but we have to fill arrays for the input + } + + details->fScriptHashAllFound=true; + details->vInputHashTypes.push_back(SIGHASH_ALL); + details->vInputCanGrantAdminMine.push_back(!details->fCheckCachedScript); + + return true; +} + + +bool MultiChainTransaction_CheckInputs(const CTransaction& tx, // Tx to check + const CCoinsViewCache &inputs, // Tx inputs from UTXO database + CMultiChainTxDetails *details, // Tx details object + string& reason) // Error message + +{ + for (unsigned int i = 0; i < tx.vin.size(); i++) + { + const CScript& script2 = tx.vin[i].scriptSig; + CScript::const_iterator pc2 = script2.begin(); + if (!script2.IsPushOnly()) + { + reason="sigScript should be push-only"; + return false; + } + + const COutPoint &prevout = tx.vin[i].prevout; + const CCoins *coins = inputs.AccessCoins(prevout.hash); + assert(coins); + + const CScript& script1 = coins->vout[prevout.n].scriptPubKey; + CScript::const_iterator pc1 = script1.begin(); + + txnouttype typeRet; + int nRequiredRet; + vector addressRets; + int op_addr_offset,op_addr_size,is_redeem_script,sighash_type,check_last; + + sighash_type=SIGHASH_NONE; + if(ExtractDestinations(script1,typeRet,addressRets,nRequiredRet)) // Standard script + { + if (typeRet != TX_NULL_DATA) // Regular script + { + CKeyID *lpKeyID=boost::get (&addressRets[0]); + CScriptID *lpScriptID=boost::get (&addressRets[0]); + if( (lpKeyID == NULL) && (lpScriptID == NULL) ) + { + reason="Internal error: cannot extract address from input scriptPubKey"; + return false; + } + else + { + if(typeRet != TX_MULTISIG) + { + if(lpKeyID) + { + details->vInputDestinations.push_back(*(uint160*)lpKeyID); + } + if(lpScriptID) + { + details->vInputDestinations.push_back(*(uint160*)lpScriptID); + } + } + else + { + details->vInputDestinations.push_back(0); // Multisig scripts cannot be used for signing tx objects - issues, stream items, etc. + } + } + + check_last=0; + if( (typeRet == TX_PUBKEY) || (typeRet == TX_MULTISIG) ) + { + check_last=1; + details->fRejectIfOpDropOpReturn=true; // pay-to-pubkey and bare multisig scripts cannot be considered "publisher" for the stream, + // because we cannot extract publisher address from the input script itself. + // Though we can still accept this transaction, it is rejected for consistency with principle + // "each input which signs the stream item must have permitted writer" + } + + // Find sighash_type + mc_ExtractAddressFromInputScript((unsigned char*)(&pc2[0]),(int)(script2.end()-pc2),&op_addr_offset,&op_addr_size,&is_redeem_script,&sighash_type,check_last); + if(sighash_type == SIGHASH_ALL) + { + details->fScriptHashAllFound=true; + } + if(sighash_type == SIGHASH_SINGLE) + { + if(i >= tx.vout.size()) + { + reason="SIGHASH_SINGLE input without matching output"; + return false; + } + } + } + else // Null-data script + { + details->fRejectIfOpDropOpReturn=true; // Null data scripts cannot be used in txs with OP_DROP+OP_RETURN + // We should not be there at all + details->vInputDestinations.push_back(0); + } + details->vInputScriptTypes.push_back(typeRet); + } + else // Non-standard script + { + details->fRejectIfOpDropOpReturn=true; // Non-standard inputs cannot be used in txs with OP_DROP+OP_RETURN + // We cannot be sure where are the signatures in input script + details->vInputScriptTypes.push_back(TX_NONSTANDARD); + details->vInputDestinations.push_back(0); + } + + details->vInputHashTypes.push_back(sighash_type); + + if(mc_gState->m_Features->PerAssetPermissions()) // Checking per-asset send permissions + { + mc_gState->m_TmpAssetsTmp->Clear(); + if(!mc_ExtractInputAssetQuantities(mc_gState->m_TmpAssetsTmp,script1,prevout.hash,reason)) + { + return false; + } + if(!mc_VerifyAssetPermissions(mc_gState->m_TmpAssetsTmp,addressRets,1,MC_PTP_SEND,reason)) + { + return false; + } + } + + // Filling input asset quantity list + if(!mc_ExtractInputAssetQuantities(mc_gState->m_TmpAssetsIn,script1,prevout.hash,reason)) + { + return false; + } + + // Initialization, input cannot be used as signer for admin/miner grant + details->vInputCanGrantAdminMine.push_back(!details->fCheckCachedScript); + } + + return true; +} + +bool MultiChainTransaction_CheckNewEntity(int vout, + bool& fScriptParsed, + CMultiChainTxDetails *details, + string& reason) +{ + int err; + int entity_update; + + mc_gState->m_TmpScript->SetElement(0); + err=mc_gState->m_TmpScript->GetNewEntityType(&(details->new_entity_type),&entity_update,details->details_script,&(details->details_script_size)); + if(err == 0) + { + fScriptParsed=true; + if(details->new_entity_output >= 0) + { + reason="Metadata script rejected - too many new entities"; + return false; + } + if(entity_update) + { + reason="Metadata script rejected - entity update script should be preceded by entityref"; + return false; + } + if(details->new_entity_type <= mc_gState->m_Assets->MaxEntityType()) + { + details->new_entity_output=vout; + details->details_script_type=entity_update; + } + else + { + reason="Metadata script rejected - unsupported new entity type"; + return false; + } + } + else + { + if(err != MC_ERR_WRONG_SCRIPT) + { + reason="Entity details script rejected - error in script"; + return false; + } + } + + return true; +} + +bool MultiChainTransaction_CheckCachedScript(const CTransaction& tx, + const CCoinsViewCache &inputs, + bool& fScriptParsed, + CMultiChainTxDetails *details, + string& reason) +{ + int err; + int cs_offset,cs_new_offset,cs_size,cs_vin; + unsigned char *cs_script; + + mc_gState->m_TmpScript->SetElement(0); + cs_offset=0; + while( (err=mc_gState->m_TmpScript->GetCachedScript(cs_offset,&cs_new_offset,&cs_vin,&cs_script,&cs_size)) != MC_ERR_WRONG_SCRIPT ) + { + fScriptParsed=true; + if(err != MC_ERR_NOERROR) + { + reason="Metadata script rejected - error in cached script"; + return false; + } + if(cs_offset) + { + if( cs_vin >= (int)tx.vin.size() ) + { + reason="Metadata script rejected - invalid input in cached script"; + return false; + } + + const COutPoint &prevout = tx.vin[cs_vin].prevout; + const CCoins *coins = inputs.AccessCoins(prevout.hash); + + const CScript& script3 = coins->vout[prevout.n].scriptPubKey; + CScript::const_iterator pc3 = script3.begin(); + + if(cs_size != (int)script3.size()) + { + reason="Metadata script rejected - cached script mismatch"; + return false; + } + if(memcmp(cs_script,(unsigned char*)&pc3[0],cs_size)) + { + reason="Metadata script rejected - cached script mismatch"; + return false; + } + if(details->fCheckCachedScript) + { + if(details->vInputHashTypes[cs_vin] == SIGHASH_ALL) + { + details->vInputCanGrantAdminMine[cs_vin]=true; + } + } + } + cs_offset=cs_new_offset; + } + + return true; +} + +void MultiChainTransaction_FillAdminPermissionsBeforeTx(const CTransaction& tx, + CMultiChainTxDetails *details) +{ + if(details->vInputHadAdminPermissionBeforeThisTx.size() == 0) + { + for (unsigned int i = 0; i < tx.vin.size(); i++) + { + details->vInputHadAdminPermissionBeforeThisTx.push_back(mc_gState->m_Permissions->CanAdmin(NULL,(unsigned char*)&(details->vInputDestinations[i])) != 0); + } + } +} + +bool MultiChainTransaction_CheckOpReturnScript(const CTransaction& tx, + const CCoinsViewCache &inputs, + int vout, + CMultiChainTxDetails *details, + string& reason) +{ + bool fScriptParsed=false; + uint32_t timestamp,approval; + vector addressRets; + CTxDestination single_address; + + mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(NULL); // Format scripts are eliminated for protocol checks + + if(!details->fScriptHashAllFound) + { + // SIGHASH_NONE or not signed at all + if( ((int)details->vInputHashTypes.size() <= vout) || (details->vInputHashTypes[vout] != SIGHASH_SINGLE) ) + { + reason="Output with metadata should be properly signed"; + return false; + } + } + + if( mc_gState->m_TmpScript->GetNumElements() > 1 ) // OP_DROP+OP_RETURN script + { + if(details->fRejectIfOpDropOpReturn) // We cannot extract address sighash_type properly from the input script + // as we don't know where are the signatures + { + reason="Non-standard, P2PK or bare multisig inputs cannot be used in this tx"; + return false; + } + + if(mc_gState->m_TmpScript->IsDirtyOpReturnScript()) + { + reason="Non-standard, Only OP_DROP elements are allowed in metadata outputs with OP_DROP"; + return false; + } + } + + if( mc_gState->m_TmpScript->GetNumElements() == 2 ) // Cached input script or new entity + { + if(!fScriptParsed) + { + // Cached scripts + if(!MultiChainTransaction_CheckCachedScript(tx, inputs, fScriptParsed, details, reason)) + { + return false; + } + } + + if(!fScriptParsed) + { + // New entities + if(!MultiChainTransaction_CheckNewEntity(vout, fScriptParsed, details, reason)) + { + return false; + } + } + + if(!fScriptParsed) + { + reason="Metadata script rejected - Unrecognized script, should be new entity or input script cache"; + return false; + } + } + + if( (mc_gState->m_TmpScript->GetNumElements() > 3) && + (mc_gState->m_Features->MultipleStreamKeys() == 0) ) // More than 2 OP_DROPs + { + reason="Metadata script rejected - too many elements"; + return false; + } + + if(mc_gState->m_TmpScript->GetNumElements() == 3 ) // 2 OP_DROPs + OP_RETURN - possible upgrade approval + // Admin permissions before tx should be used + // Performed only if it is indeed needed + { + mc_gState->m_TmpScript->SetElement(1); + + if(mc_gState->m_TmpScript->GetApproval(&approval,×tamp) == 0) + { + MultiChainTransaction_FillAdminPermissionsBeforeTx(tx,details); + } + } + + if(mc_gState->m_TmpScript->GetNumElements() >= 3 ) // This output should be processed later - after all permissions + { + details->vOutputScriptFlags[vout] |= MC_MTX_OUTPUT_DETAIL_FLAG_OP_RETURN_ENTITY_ITEM; + } + + details->vOutputPermissionRequired.push_back(0); // Stubs for these arrays elements, not used later + details->vOutputDestinations.push_back(addressRets); + details->vOutputSingleDestination.push_back(single_address); + + return true; +} + +bool MultiChainTransaction_CheckAssetUpdateDetails(mc_EntityDetails *entity, + int vout, + CMultiChainTxDetails *details, + string& reason) +{ + int err; + int entity_update; + + if(mc_gState->m_TmpScript->GetNumElements() > 3) + { + reason="Metadata script rejected - too many elements in asset update script"; + return false; + } + + mc_gState->m_TmpScript->SetElement(1); + + err=mc_gState->m_TmpScript->GetNewEntityType(&(details->new_entity_type),&entity_update,details->details_script,&(details->details_script_size)); + + if(err == 0) + { + if(details->details_script_type >= 0) + { + reason="Metadata script rejected - too many new entities/entity updates"; + return false; + } + + if(entity_update == 0) + { + reason="Metadata script rejected - wrong element, should be entity update"; + return false; + } + if(details->new_entity_type != MC_ENT_TYPE_ASSET) + { + reason="Metadata script rejected - entity type mismatch in update script"; + return false; + } + details->details_script_type=entity_update; + } + else + { + reason="Metadata script rejected - wrong element, should be entity update"; + return false; + } + // The script itself will be processed later + details->vOutputScriptFlags[vout] |= MC_MTX_OUTPUT_DETAIL_FLAG_FOLLOWON_DETAILS; + return true; +} + +bool MultiChainTransaction_CheckUpgradeApproval(const CTransaction& tx, + mc_EntityDetails *entity, + int offset, + int vout, + CMultiChainTxDetails *details, + string& reason) +{ + uint32_t timestamp,approval; + uint256 upgrade_hash; + bool fAdminFound; + + if(mc_gState->m_TmpScript->GetNumElements() > 3) + { + reason="Metadata script rejected - too many elements in upgrade approval script"; + return false; + } + + mc_gState->m_TmpScript->SetElement(1); + + if(mc_gState->m_TmpScript->GetApproval(&approval,×tamp)) + { + reason="Metadata script rejected - wrong element, should be upgrade approval"; + return false; + } + + upgrade_hash=*(uint256*)(entity->GetTxID()); + if(approval) + { + LogPrintf("Found approval script in tx %s for %s\n", + tx.GetHash().GetHex().c_str(), + upgrade_hash.ToString().c_str()); + } + else + { + LogPrintf("Found disapproval script in tx %s for %s\n", + tx.GetHash().GetHex().c_str(), + upgrade_hash.ToString().c_str()); + } + + fAdminFound=false; + for (unsigned int i = 0; i < tx.vin.size(); i++) + { + if(details->vInputHadAdminPermissionBeforeThisTx[i]) + { + if(details->IsRelevantInput(i,vout)) + { + if(mc_gState->m_Permissions->SetApproval(entity->GetTxID()+MC_AST_SHORT_TXID_OFFSET,approval, + (unsigned char*)&(details->vInputDestinations[i]),entity->UpgradeStartBlock(),timestamp,MC_PFL_NONE,1,offset) == 0) + { + fAdminFound=true; + } + } + } + } + + if(!fAdminFound) + { + reason="Inputs don't belong to valid admin for approval script"; + return false; + } + + return true; +} + +bool MultiChainTransaction_CheckStreamItem(mc_EntityDetails *entity, + int vout, + CMultiChainTxDetails *details, + string& reason) +{ + bool fAllValidPublishers; + unsigned char item_key[MC_ENT_MAX_ITEM_KEY_SIZE]; + int item_key_size; + + // Multiple keys, if not allowed, check for count is made in different place + for (int e = 1; e < mc_gState->m_TmpScript->GetNumElements()-1; e++) + { + mc_gState->m_TmpScript->SetElement(e); + + if(mc_gState->m_TmpScript->GetItemKey(item_key,&item_key_size)) + { + reason="Metadata script rejected - wrong element, should be item key"; + return false; + } + } + + fAllValidPublishers=true; + if(entity->AnyoneCanWrite() == 0) + { + for (unsigned int i = 0; i < details->vInputDestinations.size(); i++) + { + if(details->IsRelevantInput(i,vout)) + { + if(fAllValidPublishers) + { + if(mc_gState->m_Permissions->CanWrite(entity->GetTxID(),(unsigned char*)&(details->vInputDestinations[i])) == 0) + { + fAllValidPublishers=false; + } + } + } + } + } + + if(!fAllValidPublishers) + { + reason="Metadata script rejected - Inputs don't belong to valid publisher"; + return false; + } + + return true; +} + +bool MultiChainTransaction_CheckEntityItem(const CTransaction& tx, + int offset, + int vout, + CMultiChainTxDetails *details, + string& reason) +{ + unsigned char short_txid[MC_AST_SHORT_TXID_SIZE]; + mc_EntityDetails entity; + + mc_gState->m_TmpScript->SetElement(0); + // Should be spke + if(mc_gState->m_TmpScript->GetEntity(short_txid)) // Entity element + { + reason="Metadata script rejected - wrong element, should be entityref"; + return false; + } + + if(mc_gState->m_Assets->FindEntityByShortTxID(&entity,short_txid) == 0) + { + reason="Metadata script rejected - entity not found"; + return false; + } + + if(entity.GetEntityType() == MC_ENT_TYPE_ASSET) // Asset update + { + if(!MultiChainTransaction_CheckAssetUpdateDetails(&entity,vout,details,reason)) + { + return false; + } + } + else + { + if(entity.GetEntityType() == MC_ENT_TYPE_UPGRADE) // Upgrade approval + { + if(!MultiChainTransaction_CheckUpgradeApproval(tx,&entity,offset,vout,details,reason)) + { + return false; + } + } + else // (Pseudo)stream item + { + if(!MultiChainTransaction_CheckStreamItem(&entity,vout,details,reason)) + { + return false; + } + } + } + + return true; +} + +bool MultiChainTransaction_CheckDestinations(const CScript& script1, + int vout, + CMultiChainTxDetails *details, + string& reason) +{ + bool fNoDestinationInOutput; + bool not_cleared; + + txnouttype typeRet; + int nRequiredRet,receive_required; + vector addressRets; + CTxDestination single_destination; + + not_cleared=false; + fNoDestinationInOutput=false; + + if(mc_gState->m_Features->FixedDestinationExtraction()) + { + fNoDestinationInOutput=!ExtractDestinations(script1,typeRet,addressRets,nRequiredRet); + } + else // Bug in 10008, permission was set to address in non-standard output + { + fNoDestinationInOutput=!ExtractDestinations10008(script1,typeRet,addressRets,nRequiredRet,true,¬_cleared); + } + if(fNoDestinationInOutput) + { + details->vOutputScriptFlags[vout] |= MC_MTX_OUTPUT_DETAIL_FLAG_NO_DESTINATION; + } + + if(fNoDestinationInOutput && + ( (MCP_ANYONE_CAN_RECEIVE == 0) || (MCP_ALLOW_ARBITRARY_OUTPUTS == 0) ) ) + { + reason="Script rejected - destination required "; + return false; + } + + if((MCP_ALLOW_ARBITRARY_OUTPUTS == 0) || (mc_gState->m_Features->FixedDestinationExtraction() == 0) ) + { + if((typeRet == TX_MULTISIG) && (MCP_ALLOW_MULTISIG_OUTPUTS == 0)) + { + reason="Script rejected - multisig is not allowed"; + return false; + } + + if((typeRet == TX_SCRIPTHASH) && (MCP_ALLOW_P2SH_OUTPUTS == 0)) + { + reason="Script rejected - P2SH is not allowed"; + return false; + } + } + + receive_required=addressRets.size(); + if(typeRet == TX_MULTISIG) + { + receive_required-=nRequiredRet; + receive_required+=1; + if(receive_required>(int)addressRets.size()) + { + receive_required=addressRets.size(); + } + } + + if(addressRets.size()) + { + single_destination=addressRets[0]; + } + if(not_cleared) + { + receive_required=0; + addressRets.clear(); + } + details->vOutputPermissionRequired.push_back(receive_required); + details->vOutputDestinations.push_back(addressRets); + details->vOutputSingleDestination.push_back(single_destination); + + return true; +} + +bool MultiChainTransaction_ProcessPermissions(const CTransaction& tx, + int offset, + int vout, + uint32_t permission_type, + bool fFirstPass, + CMultiChainTxDetails *details, + string& reason) +{ + bool fIsPurePermission; + bool fNoDestinationInOutput; + unsigned char short_txid[MC_AST_SHORT_TXID_SIZE]; + mc_EntityDetails entity; + uint32_t type,from,to,timestamp,flags; + + fIsPurePermission=false; + if(mc_gState->m_TmpScript->GetNumElements()) + { + fIsPurePermission=true; + } + + fNoDestinationInOutput=( (details->vOutputScriptFlags[vout] & MC_MTX_OUTPUT_DETAIL_FLAG_NO_DESTINATION) != 0); + + entity.Zero(); + for (int e = 0; e < mc_gState->m_TmpScript->GetNumElements(); e++) + { + mc_gState->m_TmpScript->SetElement(e); + if(mc_gState->m_TmpScript->GetEntity(short_txid) == 0) // Entity element + { + if(entity.GetEntityType()) + { + reason="Script rejected - duplicate entity script"; + return false; + } + if(mc_gState->m_Assets->FindEntityByShortTxID(&entity,short_txid) == 0) + { + reason="Script rejected - entity not found"; + return false; + } + } + else // Not entity element + { + if(mc_gState->m_TmpScript->GetPermission(&type,&from,&to,×tamp) == 0) // Grant script + { + if(fNoDestinationInOutput) + { + reason="Script rejected - wrong destination type in output with permission script"; + return false; + } + + if(fFirstPass) + { + if( type & ( MC_PTP_CREATE | MC_PTP_ISSUE | MC_PTP_ACTIVATE ) ) + { + details->vOutputScriptFlags[vout] |= MC_MTX_OUTPUT_DETAIL_FLAG_PERMISSION_CREATE; + } + if( type & ( MC_PTP_MINE | MC_PTP_ADMIN ) ) + { + details->fAdminMinerGrant=true; + details->vOutputScriptFlags[vout] |= MC_MTX_OUTPUT_DETAIL_FLAG_PERMISSION_ADMIN; + } + } + + type &= permission_type; // Processing only relevant permissions + + // Admins and activators are those having permissions before thix tx + for (unsigned int i = 0; i < details->vInputDestinations.size(); i++) + { + if(details->IsRelevantInput(i,vout)) + { + if(mc_gState->m_Permissions->CanAdmin(entity.GetTxID(),(unsigned char*)&(details->vInputDestinations[i]))) + { + details->vAllowedAdmins.insert(strprintf("%d-%d-%d",i,vout,e)); + } + if(mc_gState->m_Permissions->CanActivate(entity.GetTxID(),(unsigned char*)&(details->vInputDestinations[i]))) + { + details->vAllowedActivators.insert(strprintf("%d-%d-%d",i,vout,e)); + } + } + } + + if(type) + { + CKeyID *lpKeyID=boost::get (&(details->vOutputSingleDestination[vout])); + CScriptID *lpScriptID=boost::get (&(details->vOutputSingleDestination[vout])); + + // Permissions cannot be granted to nonstandard outputs or bare multisigs + if(((lpKeyID == NULL) && (lpScriptID == NULL)) || (details->vOutputDestinations[vout].size() > 1)) + { + reason="Permission script rejected - wrong destination type"; + return false; + } + + CBitcoinAddress address; + unsigned char* ptr=NULL; + flags=MC_PFL_NONE; + if(lpKeyID != NULL) + { + address=CBitcoinAddress(*lpKeyID); + ptr=(unsigned char*)(lpKeyID); + if(type & MC_PTP_CONNECT) + { + if(mc_gState->m_pSeedNode) + { + CNode* seed_node; + seed_node=(CNode*)(mc_gState->m_pSeedNode); + + // If connect permission of seed node was involved, we may want to disconnect from it + if(memcmp(ptr,seed_node->kAddrRemote.begin(),20) == 0) + { + details->fSeedNodeInvolved=true; + } + } + } + } + else + { + flags=MC_PFL_IS_SCRIPTHASH; + address=CBitcoinAddress(*lpScriptID); + ptr=(unsigned char*)(lpScriptID); + } + + if(fDebug)LogPrint("mchn","Found permission script in tx %s for %s - (%08x: %d - %d)\n", + tx.GetHash().GetHex().c_str(), + address.ToString().c_str(), + type, from, to); + + bool fAdminFound=false; + bool fAdminFoundWithoutCachedScript=false; + bool fActivateIsEnough=mc_gState->m_Permissions->IsActivateEnough(type); + + for (unsigned int i = 0; i < tx.vin.size(); i++) + { + if( ( !fActivateIsEnough && (details->vAllowedAdmins.count(strprintf("%d-%d-%d",i,vout,e)) > 0)) || + ( fActivateIsEnough && (details->vAllowedActivators.count(strprintf("%d-%d-%d",i,vout,e)) > 0)) ) + { + if(details->vInputDestinations[i] != 0) + { + if( ( (type & (MC_PTP_ADMIN | MC_PTP_MINE)) == 0) || details->vInputCanGrantAdminMine[i] || (entity.GetEntityType() > 0) ) + { + details->fFullReplayCheckRequired=true; + if(mc_gState->m_Permissions->SetPermission(entity.GetTxID(),ptr,type,(unsigned char*)&(details->vInputDestinations[i]), + from,to,timestamp,flags,1,offset) == 0) + { + fAdminFound=true; + } + } + else + { + fAdminFoundWithoutCachedScript=true; + } + } + } + } + + if(!fAdminFound) + { + reason="Inputs don't belong to valid admin"; + if(fAdminFoundWithoutCachedScript) + { + reason="Inputs require scriptPubKey cache to support miner precheck"; + } + return false; + } + } + entity.Zero(); // Entity element in the non-op-return output should be followed by permission element + // So only permission can reset it + } + else // Not permission script + { + if(entity.GetEntityType()) + { + reason="Script rejected - entity script should be followed by permission"; + return false; + } + fIsPurePermission=false; + } + } + } + + if(entity.GetEntityType()) + { + reason="Script rejected - incomplete entity script"; + return false; + } + + if(fFirstPass) + { + if(tx.vout[vout].nValue > 0) + { + fIsPurePermission=false; + } + + if(!fIsPurePermission) + { + details->vOutputScriptFlags[vout] |= MC_MTX_OUTPUT_DETAIL_FLAG_NOT_PURE_PERMISSION; + } + details->vOutputScriptFlags[vout] |= MC_MTX_OUTPUT_DETAIL_FLAG_NOT_OP_RETURN; + } + + return true; +} + +bool MultiChainTransaction_CheckAssetTransfers(const CTransaction& tx, + int offset, + int vout, + CMultiChainTxDetails *details, + string& reason) +{ + int receive_required=details->vOutputPermissionRequired[vout]; + + if(mc_gState->m_Features->PerAssetPermissions()) // Checking per-asset receive permissions + { + mc_gState->m_TmpAssetsTmp->Clear(); + if(!mc_ExtractOutputAssetQuantities(mc_gState->m_TmpAssetsTmp,reason,true)) + { + return false; + } + if(!mc_VerifyAssetPermissions(mc_gState->m_TmpAssetsTmp,details->vOutputDestinations[vout],receive_required,MC_PTP_RECEIVE,reason)) + { + return false; + } + } + + if(!mc_ExtractOutputAssetQuantities(mc_gState->m_TmpAssetsOut,reason,false))// Filling output asset quantity list + { + return false; + } + // Check for dust and receive permissions + // Not required for pure grants + if(details->vOutputScriptFlags[vout] & MC_MTX_OUTPUT_DETAIL_FLAG_NOT_PURE_PERMISSION) + { + if((offset < 0) && Params().RequireStandard()) // If not in block - part of IsStandard check + { + if (tx.vout[vout].IsDust(::minRelayTxFee)) + { + if(!tx.IsCoinBase()) + { + reason="Transaction amount too small"; + return false; + } + } + } + for(int a=0;a<(int)details->vOutputDestinations[vout].size();a++) + { + CKeyID *lpKeyID=boost::get (&(details->vOutputDestinations[vout][a])); + CScriptID *lpScriptID=boost::get (&(details->vOutputDestinations[vout][a])); + if((lpKeyID == NULL) && (lpScriptID == NULL)) + { + reason="Script rejected - wrong destination type"; + return false; + } + unsigned char* ptr=NULL; + if(lpKeyID != NULL) + { + ptr=(unsigned char*)(lpKeyID); + } + else + { + ptr=(unsigned char*)(lpScriptID); + } + + bool fCanReceive=mc_gState->m_Permissions->CanReceive(NULL,ptr); + + // Miner can send funds to himself in coinbase even without receive permission + // It is relevant only for native currency as other assets will be unbalanced + if(tx.IsCoinBase()) + { + fCanReceive |= mc_gState->m_Permissions->CanMine(NULL,ptr); + } + if(fCanReceive) + { + receive_required--; + } + } + if(receive_required>0) + { + if( (tx.vout[vout].nValue > 0) || + (mc_gState->m_TmpScript->GetNumElements() > 0) || + (mc_gState->m_Features->AnyoneCanReceiveEmpty() == 0) ) + { + reason="One of the outputs doesn't have receive permission"; + return false; + } + } + } + + return true; +} + +bool MultiChainTransaction_CheckOutputs(const CTransaction& tx, // Tx to check + const CCoinsViewCache &inputs, // Tx inputs from UTXO database + int offset, // Tx offset in block, -1 if in memppol + CMultiChainTxDetails *details, // Tx details object + string& reason) // Error message +{ + uint32_t permission_type; + for (unsigned int vout = 0; vout < tx.vout.size(); vout++) + { + details->vOutputScriptFlags.push_back(MC_MTX_OUTPUT_DETAIL_FLAG_NONE); + } + + for (unsigned int vout = 0; vout < tx.vout.size(); vout++) // Basic checks, destinations and simple grants + { + MultiChainTransaction_SetTmpOutputScript(tx.vout[vout].scriptPubKey); + + if(mc_gState->m_TmpScript->IsOpReturnScript()) + { + if(!MultiChainTransaction_CheckOpReturnScript(tx,inputs,vout,details,reason)) + { + return false; + } + } + else + { + if(!MultiChainTransaction_CheckDestinations(tx.vout[vout].scriptPubKey,vout,details,reason)) + { + return false; + } + + permission_type=MC_PTP_CONNECT | MC_PTP_SEND | MC_PTP_RECEIVE | MC_PTP_WRITE; + if(!MultiChainTransaction_ProcessPermissions(tx,offset,vout,permission_type,true,details,reason)) + { + return false; + } + } + } + + for (unsigned int vout = 0; vout < tx.vout.size(); vout++) // create, issue, activate grants (requiring pre-tx permissions) + { + if(details->vOutputScriptFlags[vout] & MC_MTX_OUTPUT_DETAIL_FLAG_PERMISSION_CREATE) + { + MultiChainTransaction_SetTmpOutputScript(tx.vout[vout].scriptPubKey); + + permission_type=MC_PTP_CREATE | MC_PTP_ISSUE | MC_PTP_ACTIVATE; + if(!MultiChainTransaction_ProcessPermissions(tx,offset,vout,permission_type,false,details,reason)) + { + return false; + } + + } + } + + for (unsigned int vout = 0; vout < tx.vout.size(); vout++) // mine, admin grants (requiring cached script) + { + if(details->vOutputScriptFlags[vout] & MC_MTX_OUTPUT_DETAIL_FLAG_PERMISSION_ADMIN) + { + MultiChainTransaction_SetTmpOutputScript(tx.vout[vout].scriptPubKey); + + permission_type=MC_PTP_MINE | MC_PTP_ADMIN; + if(!MultiChainTransaction_ProcessPermissions(tx,offset,vout,permission_type,false,details,reason)) + { + return false; + } + } + } + + mc_gState->m_TmpAssetsOut->Clear(); + + for (unsigned int vout = 0; vout < tx.vout.size(); vout++) + { + // Entity items (stream items, upgrade approvals, updates) + if(details->vOutputScriptFlags[vout] & MC_MTX_OUTPUT_DETAIL_FLAG_OP_RETURN_ENTITY_ITEM) + { + MultiChainTransaction_SetTmpOutputScript(tx.vout[vout].scriptPubKey); + + if(!MultiChainTransaction_CheckEntityItem(tx,offset,vout,details,reason)) + { + return false; + } + } + // Assets quantities and permission checks + if(details->vOutputScriptFlags[vout] & MC_MTX_OUTPUT_DETAIL_FLAG_NOT_OP_RETURN) + { + MultiChainTransaction_SetTmpOutputScript(tx.vout[vout].scriptPubKey); + + if(!MultiChainTransaction_CheckAssetTransfers(tx,offset,vout,details,reason)) + { + return false; + } + } + } + + if(!mc_CompareAssetQuantities(reason)) // Comparing input/output asset quantities + { + return false; + } + + return true; +} + +bool MultiChainTransaction_ProcessAssetIssuance(const CTransaction& tx, // Tx to check + int offset, // Tx offset in block, -1 if in memppol + bool accept, // Accept to mempools if successful + CMultiChainTxDetails *details, // Tx details object + string& reason) // Error message +{ + int update_mempool; + + + mc_EntityDetails entity; + mc_EntityDetails this_entity; + char asset_name[MC_ENT_MAX_NAME_SIZE+1]; + int multiple; + int err; + int64_t quantity,total; + uint256 txid; + bool new_issue,follow_on,issue_in_output; + unsigned char *ptrOut; + vector issuers; + vector issuer_flags; + uint32_t flags; + uint32_t value_offset; + size_t value_size; + unsigned char short_txid[MC_AST_SHORT_TXID_SIZE]; + CTxDestination addressRet; + + if(tx.IsCoinBase()) + { + return true; + } + + update_mempool=0; + if(accept) + { + update_mempool=1; + } + + total=0; + + asset_name[0]=0; + multiple=1; + new_issue=false; + follow_on=false; + + if(details->details_script_type == 0) // New asset with details script + { + value_offset=mc_FindSpecialParamInDetailsScript(details->details_script,details->details_script_size,MC_ENT_SPRM_NAME,&value_size); + if(value_offset<(uint32_t)details->details_script_size) + { + memcpy(asset_name,details->details_script+value_offset,value_size); + asset_name[value_size]=0x00; + } + value_offset=mc_FindSpecialParamInDetailsScript(details->details_script,details->details_script_size,MC_ENT_SPRM_ASSET_MULTIPLE,&value_size); + if(value_offset<(uint32_t)details->details_script_size) + { + multiple=mc_GetLE(details->details_script+value_offset,value_size); + } + } + + mc_gState->m_TmpAssetsOut->Clear(); + + for (unsigned int vout = 0; vout < tx.vout.size(); vout++) + { + // We already extracted the details, we just have to find entity + if(details->vOutputScriptFlags[vout] & MC_MTX_OUTPUT_DETAIL_FLAG_FOLLOWON_DETAILS) + { + MultiChainTransaction_SetTmpOutputScript(tx.vout[vout].scriptPubKey); + mc_gState->m_TmpScript->SetElement(0); + + if(mc_gState->m_TmpScript->GetEntity(short_txid)) + { + reason="Metadata script rejected - wrong element, should be entityref"; + return false; + } + + } + + if(details->vOutputScriptFlags[vout] & MC_MTX_OUTPUT_DETAIL_FLAG_NOT_OP_RETURN) + { + MultiChainTransaction_SetTmpOutputScript(tx.vout[vout].scriptPubKey); + + mc_gState->m_TmpAssetsTmp->Clear(); + issue_in_output=false; + + for (int e = 0; e < mc_gState->m_TmpScript->GetNumElements(); e++) + { + mc_gState->m_TmpScript->SetElement(e); + err=mc_gState->m_TmpScript->GetAssetGenesis(&quantity); + if(err == 0) // Asset genesis issuance + { + issue_in_output=true; + new_issue=true; + if(quantity+total<0) + { + reason="Asset issue script rejected - overflow"; + return false; + } + + total+=quantity; + + if(details->vOutputDestinations[vout].size() != 1) + { + reason="Asset issue script rejected - wrong destination type"; + return false; + } + + addressRet=details->vOutputDestinations[vout][0]; + + CKeyID *lpKeyID=boost::get (&addressRet); + CScriptID *lpScriptID=boost::get (&addressRet); + if((lpKeyID == NULL) && (lpScriptID == NULL)) + { + reason="Asset issue script rejected - wrong destination type"; + return false; + } + CBitcoinAddress address; + if(lpKeyID != NULL) + { + address=CBitcoinAddress(*lpKeyID); + } + else + { + address=CBitcoinAddress(*lpScriptID); + } + if(update_mempool) + { + if(fDebug)LogPrint("mchn","Found asset issue script in tx %s for %s - (%ld)\n", + tx.GetHash().GetHex().c_str(), + address.ToString().c_str(),quantity); + } + } + else + { + if(err != MC_ERR_WRONG_SCRIPT) + { + reason="Asset issue script rejected - error in script"; + return false; + } + if(mc_gState->m_Features->PerAssetPermissions()) + { + err=mc_gState->m_TmpScript->GetAssetQuantities(mc_gState->m_TmpAssetsTmp,MC_SCR_ASSET_SCRIPT_TYPE_TRANSFER); + if((err != MC_ERR_NOERROR) && (err != MC_ERR_WRONG_SCRIPT)) + { + reason="Script rejected - error in asset transfer script"; + return false; + } + } + + err=mc_gState->m_TmpScript->GetAssetQuantities(mc_gState->m_TmpAssetsOut,MC_SCR_ASSET_SCRIPT_TYPE_FOLLOWON); + if((err != MC_ERR_NOERROR) && (err != MC_ERR_WRONG_SCRIPT)) + { + reason="Asset follow-on script rejected - error in follow-on script"; + return false; + } + if(err == MC_ERR_NOERROR) + { + if(update_mempool) + { + if(fDebug)LogPrint("mchn","Found asset follow-on script in tx %s\n", + tx.GetHash().GetHex().c_str()); + } + } + } + } + if(issue_in_output) + { + if(mc_gState->m_Features->PerAssetPermissions()) + { + if(mc_gState->m_TmpAssetsTmp->GetCount()) + { + reason="Asset issue script rejected - asset transfer in script"; + return false; + } + } + } + } + } + + if(details->details_script_type >= 0) + { + if(details->details_script_type) // Updates are allowed only for assets + { + follow_on=true; + } + else + { + if(details->new_entity_type == MC_ENT_TYPE_ASSET) // If not - we'll deal with it later + { + new_issue=true; + } + } + } + + if(mc_gState->m_TmpAssetsOut->GetCount()) + { + follow_on=true; + } + + if(follow_on) + { + total=0; + if(mc_gState->m_TmpAssetsOut->GetCount() > 1) + { + reason="Asset follow-on script rejected - follow-on for several assets"; + return false; + } + if(new_issue) + { + reason="Asset follow-on script rejected - follow-on and issue in one transaction"; + return false; + } + ptrOut=NULL; + if(mc_gState->m_TmpAssetsOut->GetCount() == 0) + { + if(mc_gState->m_Assets->FindEntityByShortTxID(&entity,short_txid) == 0) + { + reason="Details script rejected - entity not found"; + return false; + } + } + else + { + ptrOut=mc_gState->m_TmpAssetsOut->GetRow(0); + if(mc_gState->m_Assets->FindEntityByFullRef(&entity,ptrOut) == 0) + { + reason="Asset follow-on script rejected - asset not found"; + return false; + } + } + + if(entity.AllowedFollowOns() == 0) + { + reason="Asset follow-on script rejected - follow-ons not allowed for this asset"; + return false; + } + if(details->details_script_type > 0) + { + if(memcmp(entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,short_txid,MC_AST_SHORT_TXID_SIZE)) + { + reason="Asset follow-on script rejected - mismatch in follow-on quantity asset and details script"; + return false; + } + } + if(ptrOut) + { + total=mc_GetABQuantity(ptrOut); + if(total+mc_gState->m_Assets->GetTotalQuantity(&entity) < 0) + { + reason="Asset follow-on script rejected - exceeds maximal value for asset"; + return false; + } + } + } + + if(!new_issue && !follow_on) + { + return true; + } + + if(details->new_entity_output >= 0) + { + if(details->new_entity_type != MC_ENT_TYPE_ASSET) + { + reason="Asset issue script rejected - not allowed in this transaction, conflicts with other entities"; + return false; + } + } + + details->fFullReplayCheckRequired=true; + + issuers.clear(); // Creating issuers list + for (unsigned int i = 0; i < tx.vin.size(); i++) + { + if(details->vInputHashTypes[i] == SIGHASH_ALL) + { + if(details->vInputDestinations[i] != 0) + { + bool can_issue=false; + if(new_issue) + { + if(mc_gState->m_Permissions->CanIssue(NULL,(unsigned char*)&(details->vInputDestinations[i]))) + { + can_issue=true; + } + } + else + { + if(mc_gState->m_Permissions->CanIssue(entity.GetTxID(),(unsigned char*)&(details->vInputDestinations[i]))) + { + can_issue=true; + } + } + if(can_issue) + { + issuers.push_back(details->vInputDestinations[i]); + flags=MC_PFL_NONE; + if(details->vInputScriptTypes[i] == TX_SCRIPTHASH) + { + flags |= MC_PFL_IS_SCRIPTHASH; + } + issuer_flags.push_back(flags); + } + } + } + } + + if(issuers.size() == 0) + { + reason="Inputs don't belong to valid issuer"; + return false; + } + + err=MC_ERR_NOERROR; + + mc_gState->m_TmpScript->Clear(); + mc_gState->m_TmpScript->AddElement(); + unsigned char issuer_buf[24]; + memset(issuer_buf,0,sizeof(issuer_buf)); + flags=MC_PFL_NONE; + uint32_t timestamp=0; + set stored_issuers; + + // First per-entity record in permission database + // We'll need it for scanning asset-related rows + if(new_issue) + { + err=MC_ERR_NOERROR; + + txid=tx.GetHash(); + err=mc_gState->m_Permissions->SetPermission(&txid,issuer_buf,MC_PTP_CONNECT, + (unsigned char*)issuers[0].begin(),0,(uint32_t)(-1),timestamp,flags | MC_PFL_ENTITY_GENESIS ,update_mempool,offset); + } + + uint32_t all_permissions=MC_PTP_ADMIN | MC_PTP_ISSUE; + if(mc_gState->m_Features->PerAssetPermissions()) + { + all_permissions |= MC_PTP_ACTIVATE | MC_PTP_SEND | MC_PTP_RECEIVE; + } + + for (unsigned int i = 0; i < issuers.size(); i++) // Setting per-asset permissions and creating issuers script + { + if(err == MC_ERR_NOERROR) + { + if(stored_issuers.count(issuers[i]) == 0) + { + memcpy(issuer_buf,issuers[i].begin(),sizeof(uint160)); + mc_PutLE(issuer_buf+sizeof(uint160),&issuer_flags[i],4); + if((int)i < mc_gState->m_Assets->MaxStoredIssuers()) // Adding list of issuers to the asset script + { + mc_gState->m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_ISSUER,issuer_buf,sizeof(issuer_buf)); + } + if(new_issue) // Setting first permission record - to scan from + { + err=mc_gState->m_Permissions->SetPermission(&txid,issuer_buf,all_permissions, + (unsigned char*)issuers[0].begin(),0,(uint32_t)(-1),timestamp,flags | MC_PFL_ENTITY_GENESIS ,update_mempool,offset); + } + stored_issuers.insert(issuers[i]); + } + } + } + + memset(issuer_buf,0,sizeof(issuer_buf)); + mc_gState->m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_ISSUER,issuer_buf,1); + if(err) + { + reason="Cannot update permission database for issued asset"; + return false; + } + + const unsigned char *special_script; + size_t special_script_size=0; + special_script=mc_gState->m_TmpScript->GetData(0,&special_script_size); + txid=tx.GetHash(); + if(new_issue) // Updating entity database + { + err=mc_gState->m_Assets->InsertAsset(&txid,offset,total,asset_name,multiple,details->details_script,details->details_script_size,special_script,special_script_size,update_mempool); + } + else + { + err=mc_gState->m_Assets->InsertAssetFollowOn(&txid,offset,total,details->details_script,details->details_script_size,special_script,special_script_size,entity.GetTxID(),update_mempool); + } + + if(err) + { + reason="Asset issue script rejected - could not insert new asset to database"; + if(err == MC_ERR_FOUND) + { + reason="Asset issue script rejected - entity with this name/asset-ref/txid already exists"; + if(mc_gState->m_Assets->FindEntityByTxID(&entity,(unsigned char*)&txid) == 0) + { + if(strlen(asset_name) == 0) + { + mc_gState->m_Assets->FindEntityByName(&entity,asset_name); + } + } + + if(fDebug)LogPrint("mchn","Asset already exists. TxID: %s, AssetRef: %d-%d-%d, Name: %s\n", + tx.GetHash().GetHex().c_str(), + mc_gState->m_Assets->m_Block+1,offset,(int)(*((unsigned char*)&txid+31))+256*(int)(*((unsigned char*)&txid+30)), + entity.GetName()); + } + return false; + } + + + if(update_mempool) + { + if(offset>=0) { - mc_gState->m_TmpScript->SetElement(e); - if(mc_gState->m_TmpScript->GetPermission(&type,&from,&to,×tamp) == 0) + if(mc_gState->m_Assets->FindEntityByTxID(&this_entity,(unsigned char*)&txid)) { - if(type == MC_PTP_GLOBAL_ALL) + if(new_issue) { - CTxDestination addressRet; - if(ExtractDestination(script1, addressRet)) - { - CKeyID *lpKeyID=boost::get (&addressRet); - if(lpKeyID) - { - return *(uint160*)lpKeyID; - } - } + if(fDebug)LogPrint("mchn","New asset. TxID: %s, AssetRef: %d-%d-%d, Name: %s\n", + tx.GetHash().GetHex().c_str(), + mc_gState->m_Assets->m_Block+1,offset,(int)(*((unsigned char*)&txid+0))+256*(int)(*((unsigned char*)&txid+1)), + this_entity.GetName()); + } + else + { + uint256 otxid; + memcpy(&otxid,entity.GetTxID(),32); + if(fDebug)LogPrint("mchn","Follow-on issue. TxID: %s, Original issue txid: %s\n", + tx.GetHash().GetHex().c_str(),otxid.GetHex().c_str()); } } + else + { + reason="Asset issue script rejected - could not insert new asset to database"; + return false; + } } } - return 0; + + return true; } - -bool mc_ExtractInputAssetQuantities(mc_Buffer *assets, const CScript& script1, uint256 hash, string& reason) +bool MultiChainTransaction_ProcessEntityCreation(const CTransaction& tx, // Tx to check + int offset, // Tx offset in block, -1 if in memppol + bool accept, // Accept to mempools if successful + CMultiChainTxDetails *details, // Tx details object + string& reason) // Error message { - int err; - int64_t quantity; - CScript::const_iterator pc1 = script1.begin(); + if(details->new_entity_output < 0) + { + return true; + } + + if(details->new_entity_type == MC_ENT_TYPE_ASSET) // Processed in another place + { + return true; + } + + vector openers; + vector opener_flags; + unsigned char opener_buf[24]; - mc_gState->m_TmpScript->Clear(); - mc_gState->m_TmpScript->SetScript((unsigned char*)(&pc1[0]),(size_t)(script1.end()-pc1),MC_SCR_TYPE_SCRIPTPUBKEY); + int err; + string entity_type_str; + uint32_t flags=MC_PFL_NONE; + uint32_t timestamp=0; + set stored_openers; + int update_mempool; + uint256 txid; + mc_EntityDetails entity; - for (int e = 0; e < mc_gState->m_TmpScript->GetNumElements(); e++) + details->fFullReplayCheckRequired=true; + + update_mempool=0; + if(accept) { - mc_gState->m_TmpScript->SetElement(e); - err=mc_gState->m_TmpScript->GetAssetQuantities(assets,MC_SCR_ASSET_SCRIPT_TYPE_TRANSFER | MC_SCR_ASSET_SCRIPT_TYPE_FOLLOWON); - if((err != MC_ERR_NOERROR) && (err != MC_ERR_WRONG_SCRIPT)) - { - reason="Asset transfer script rejected - error in script"; - return false; - } - err=mc_gState->m_TmpScript->GetAssetGenesis(&quantity); - if(err == 0) + update_mempool=1; + } + + openers.clear(); // List of openers + opener_flags.clear(); + for (unsigned int i = 0; i < tx.vin.size(); i++) + { + if(details->IsRelevantInput(i,details->new_entity_output)) { - mc_EntityDetails entity; - unsigned char buf_amounts[MC_AST_ASSET_FULLREF_BUF_SIZE]; - memset(buf_amounts,0,MC_AST_ASSET_FULLREF_BUF_SIZE); - if(mc_gState->m_Assets->FindEntityByTxID(&entity,(unsigned char*)&hash)) - { - if(mc_gState->m_Features->ShortTxIDInTx() == 0) + if(mc_gState->m_Permissions->CanCreate(NULL,(unsigned char*)&(details->vInputDestinations[i]))) + { + if( (details->new_entity_type != MC_ENT_TYPE_UPGRADE) || (mc_gState->m_Permissions->CanAdmin(NULL,(unsigned char*)&(details->vInputDestinations[i])) != 0) ) { - if(entity.IsUnconfirmedGenesis()) // Not confirmed genesis has -1 in offset field + openers.push_back(details->vInputDestinations[i]); + flags=MC_PFL_NONE; + if(details->vInputScriptTypes[i] == TX_SCRIPTHASH) { - reason="Asset transfer script rejected - using unconfirmed issue"; - return false; + flags |= MC_PFL_IS_SCRIPTHASH; } + opener_flags.push_back(flags); } - memcpy(buf_amounts,entity.GetFullRef(),MC_AST_ASSET_FULLREF_SIZE); - int row=assets->Seek(buf_amounts); - if(row>=0) + } + } + } + + if(openers.size() == 0) + { + reason="Metadata script rejected - Inputs don't belong to valid creator"; + return false; + } + + err=MC_ERR_NOERROR; + + mc_gState->m_TmpScript->Clear(); + mc_gState->m_TmpScript->AddElement(); + txid=tx.GetHash(); // Setting first record in the per-entity permissions list + + if(details->new_entity_type != MC_ENT_TYPE_UPGRADE) + { + memset(opener_buf,0,sizeof(opener_buf)); + err=mc_gState->m_Permissions->SetPermission(&txid,opener_buf,MC_PTP_CONNECT, + (unsigned char*)openers[0].begin(),0,(uint32_t)(-1),timestamp, MC_PFL_ENTITY_GENESIS ,update_mempool,offset); + } + + for (unsigned int i = 0; i < openers.size(); i++) + { + if(err == MC_ERR_NOERROR) + { + if(stored_openers.count(openers[i]) == 0) + { + memcpy(opener_buf,openers[i].begin(),sizeof(uint160)); + mc_PutLE(opener_buf+sizeof(uint160),&opener_flags[i],4); + if((int)i < mc_gState->m_Assets->MaxStoredIssuers()) { - int64_t last=mc_GetABQuantity(assets->GetRow(row)); - quantity+=last; - mc_SetABQuantity(assets->GetRow(row),quantity); + mc_gState->m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_ISSUER,opener_buf,sizeof(opener_buf)); } - else + if(details->new_entity_type != MC_ENT_TYPE_UPGRADE) { - mc_SetABQuantity(buf_amounts,quantity); - assets->Add(buf_amounts); + // Granting default per-entity permissions to openers + err=mc_gState->m_Permissions->SetPermission(&txid,opener_buf,MC_PTP_ADMIN | MC_PTP_ACTIVATE | MC_PTP_WRITE, + (unsigned char*)openers[i].begin(),0,(uint32_t)(-1),timestamp,opener_flags[i] | MC_PFL_ENTITY_GENESIS ,update_mempool,offset); } - } - else - { - reason="Asset transfer script rejected - issue tx not found"; - return false; - } - } - else - { - if(err != MC_ERR_WRONG_SCRIPT) - { - reason="Asset transfer script rejected - error in input issue script"; - return false; + stored_openers.insert(openers[i]); } } + } + + if(err) + { + reason=" Cannot update permission database for new entity"; + return false; } - return true; -} - + memset(opener_buf,0,sizeof(opener_buf)); // Storing opener list in entity metadata + mc_gState->m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_ISSUER,opener_buf,1); -bool mc_CompareAssetQuantities(string& reason) -{ - unsigned char *ptrIn; - unsigned char *ptrOut; - int64_t quantity; - mc_EntityDetails entity; + const unsigned char *special_script; + size_t special_script_size=0; + special_script=mc_gState->m_TmpScript->GetData(0,&special_script_size); + // Updating entity datanase + err=mc_gState->m_Assets->InsertEntity(&txid,offset,details->new_entity_type,details->details_script,details->details_script_size,special_script,special_script_size,update_mempool); + if(err) + { + reason="New entity script rejected - could not insert new entity to database"; + if(err == MC_ERR_ERROR_IN_SCRIPT) + { + reason="New entity script rejected - error in script"; + } + if(err == MC_ERR_FOUND) + { + reason="New entity script rejected - entity with this name already exists"; + } + return false; + } - for(int i=0;im_TmpAssetsIn->GetCount();i++) + if(update_mempool) { - ptrIn=mc_gState->m_TmpAssetsIn->GetRow(i); - int row=mc_gState->m_TmpAssetsOut->Seek(ptrIn); - quantity=mc_GetABQuantity(ptrIn); - if(quantity>0) + if(mc_gState->m_Assets->FindEntityByTxID(&entity,(unsigned char*)&txid)) { - if(row>=0) + entity_type_str="stream"; + if(details->new_entity_type == MC_ENT_TYPE_UPGRADE) { - ptrOut=mc_gState->m_TmpAssetsOut->GetRow(row); - if(memcmp(ptrIn,ptrOut,MC_AST_ASSET_QUANTITY_OFFSET+MC_AST_ASSET_QUANTITY_SIZE)) - { - reason="Asset transfer script rejected - mismatch in input/output quantities"; - return false; - } + entity_type_str="upgrade"; + } + if(offset>=0) + { + LogPrintf("New %s. TxID: %s, StreamRef: %d-%d-%d, Name: %s\n", + entity_type_str.c_str(),tx.GetHash().GetHex().c_str(), + mc_gState->m_Assets->m_Block+1,offset,(int)(*((unsigned char*)&txid+0))+256*(int)(*((unsigned char*)&txid+1)), + entity.GetName()); } else { - reason="Asset transfer script rejected - mismatch in input/output quantities"; - return false; + LogPrintf("New %s. TxID: %s, unconfirmed, Name: %s\n", + entity_type_str.c_str(),tx.GetHash().GetHex().c_str(),entity.GetName()); } } + else + { + reason="New entity script rejected - could not insert new entity to database"; + return false; + } } - for(int i=0;im_TmpAssetsOut->GetCount();i++) - { - ptrOut=mc_gState->m_TmpAssetsOut->GetRow(i); - int row=mc_gState->m_TmpAssetsIn->Seek(ptrOut); - quantity=mc_GetABQuantity(ptrOut); + return true; +} - if(mc_gState->m_Features->ShortTxIDInTx()) +bool AcceptMultiChainTransaction (const CTransaction& tx, // Tx to check + const CCoinsViewCache &inputs, // Tx inputs from UTXO database + int offset, // Tx offset in block, -1 if in memppol + bool accept, // Accept to mempools if successful + string& reason, // Error message + uint32_t *replay) // Replay flag - if tx should be rechecked or only permissions +{ + CMultiChainTxDetails details; + bool fReject=false; + + if(mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) + { + return true; + } + + details.fCheckCachedScript=MultiChainTransaction_CheckCachedScriptFlag(tx); + + mc_gState->m_TmpAssetsIn->Clear(); + + if(!MultiChainTransaction_CheckCoinbaseInputs(tx,&details)) // Inputs + { + if(!MultiChainTransaction_CheckInputs(tx,inputs,&details,reason)) { - if(mc_gState->m_Assets->FindEntityByFullRef(&entity,ptrOut) == 0) - { - reason="Asset transfer script rejected - asset not found"; - return false; - } + return false; } - - if(quantity>0) + } + + mc_gState->m_Permissions->SetCheckPoint(); // if there is an error after this point or it is just check, permission mempool should be restored + + if(!MultiChainTransaction_CheckOutputs(tx,inputs,offset,&details,reason)) // Outputs + { + fReject=true; + goto exitlbl; + } + + // Asset genesis/followon + if(!MultiChainTransaction_ProcessAssetIssuance(tx,offset,accept,&details,reason)) + { + fReject=true; + goto exitlbl; + } + + // Creating of (pseudo)streams/upgrades + if(!MultiChainTransaction_ProcessEntityCreation(tx,offset,accept,&details,reason)) + { + fReject=true; + goto exitlbl; + } + + // Custom filters + fReject=!custom_accept_transacton(tx,inputs,offset,accept,reason,replay); + +exitlbl: + + if(accept) + { + if(details.fSeedNodeInvolved) // Checking if we should disconnect from seed node { - if(row>=0) - { - ptrIn=mc_gState->m_TmpAssetsIn->GetRow(row); - if(memcmp(ptrIn,ptrOut,MC_AST_ASSET_QUANTITY_OFFSET+MC_AST_ASSET_QUANTITY_SIZE)) - { - reason="Asset transfer script rejected - mismatch in input/output quantities"; - return false; - } - } - else + CNode* seed_node; + seed_node=(CNode*)(mc_gState->m_pSeedNode); + + if(!mc_gState->m_Permissions->CanConnect(NULL,seed_node->kAddrRemote.begin())) { - reason="Asset transfer script rejected - mismatch in input/output quantities"; - return false; + LogPrintf("mchn: Seed node lost connect permission \n"); + mc_gState->m_pSeedNode=NULL; } } } - return true; + if(!accept || fReject) // Rolling back permission database if we were just checking or error occurred + { + mc_gState->m_Permissions->RollBackToCheckPoint(); + } + + if(replay) + { + *replay=0; + if(details.fFullReplayCheckRequired) + { + *replay |= MC_PPL_REPLAY; + } + if(details.fAdminMinerGrant) + { + *replay |= MC_PPL_ADMINMINERGRANT; + } + } + + if(fReject) + { + if(fDebug)LogPrint("mchn","mchn: Tx rejected (%s): %s\n",reason.c_str(),EncodeHexTx(tx)); + } + + return !fReject; + } bool AcceptAssetGenesisFromPredefinedIssuers(const CTransaction &tx, @@ -775,7 +2618,8 @@ bool AcceptAssetGenesisFromPredefinedIssuers(const CTransaction &tx, return true; } -bool AcceptMultiChainTransaction(const CTransaction& tx, + +bool AcceptMultiChainTransactionOld(const CTransaction& tx, const CCoinsViewCache &inputs, int offset, bool accept, diff --git a/src/script/standard.cpp b/src/script/standard.cpp index 9f4fc914..2808dc6b 100644 --- a/src/script/standard.cpp +++ b/src/script/standard.cpp @@ -577,13 +577,17 @@ bool CScript::IsUnspendable() const } -bool ExtractDestinations10008(const CScript& scriptPubKey, txnouttype& typeRet, vector& addressRet, int& nRequiredRet) +bool ExtractDestinations10008(const CScript& scriptPubKey, txnouttype& typeRet, vector& addressRet, int& nRequiredRet, bool no_clear, bool *not_cleared) { addressRet.clear(); typeRet = TX_NONSTANDARD; opcodetype opcode; vector vch; + if(not_cleared) + { + *not_cleared=false; + } nRequiredRet=1; CScript::const_iterator pc = scriptPubKey.begin(); @@ -665,7 +669,17 @@ bool ExtractDestinations10008(const CScript& scriptPubKey, txnouttype& typeRet, return true; } - addressRet.clear(); + if(!no_clear) + { + addressRet.clear(); + } + else + { + if(not_cleared) + { + *not_cleared=true; + } + } typeRet = TX_NONSTANDARD; return true; } @@ -691,7 +705,7 @@ bool ExtractDestinations(const CScript& scriptPubKey, txnouttype& typeRet, vecto { if(mc_gState->m_Features->FixedDestinationExtraction() == 0) { - return ExtractDestinations10008(scriptPubKey,typeRet,addressRet,nRequiredRet); + return ExtractDestinations10008(scriptPubKey,typeRet,addressRet,nRequiredRet,false,NULL); } addressRet.clear(); From 0b74405a320cc32c090f0455a7a6cefa7e05f76c Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 8 Feb 2018 08:23:44 +0200 Subject: [PATCH 003/157] Deprecated protocols 10004-10007 --- src/chainparams/buildgenesis.cpp | 80 +- src/chainparams/chainparams.cpp | 75 +- src/chainparams/params.cpp | 265 --- src/chainparams/state.h | 14 - src/core/init-cold.cpp | 40 +- src/core/init.cpp | 42 +- src/core/main.cpp | 180 +- src/entities/asset.cpp | 132 +- src/multichain/multichain-util.cpp | 1 - src/permissions/permission.cpp | 65 +- src/protocol/multichainblock.cpp | 74 +- src/protocol/multichainscript.cpp | 67 +- src/protocol/multichaintx.cpp | 3084 ++-------------------------- src/rpc/rpcassets.cpp | 141 +- src/rpc/rpcmisc.cpp | 1 - src/rpc/rpcpermissions.cpp | 6 +- src/rpc/rpcrawdata.cpp | 263 +-- src/rpc/rpcrawtransaction.cpp | 123 +- src/rpc/rpcstreams.cpp | 138 +- src/rpc/rpcupgrades.cpp | 223 +- src/rpc/rpcutils.cpp | 369 ++-- src/rpc/rpcwalletsend.cpp | 14 - src/script/interpreter.cpp | 87 +- src/script/standard.cpp | 84 +- src/utils/utilparse.cpp | 65 +- src/utils/utilwrapper.cpp | 21 - src/wallet/wallet.cpp | 18 +- src/wallet/walletcoins.cpp | 55 +- src/wallet/wallettxs.cpp | 30 +- 29 files changed, 726 insertions(+), 5031 deletions(-) diff --git a/src/chainparams/buildgenesis.cpp b/src/chainparams/buildgenesis.cpp index 2c8a966d..13a8e145 100644 --- a/src/chainparams/buildgenesis.cpp +++ b/src/chainparams/buildgenesis.cpp @@ -125,14 +125,11 @@ int mc_MultichainParams::Build(const unsigned char* pubkey, int pubkey_size) root_stream_name_size=0; root_stream_name=NULL; - if(mc_gState->m_Features->Streams()) + root_stream_name=(unsigned char *)GetParam("rootstreamname",&root_stream_name_size); + if(IsProtocolMultichain() == 0) { - root_stream_name=(unsigned char *)GetParam("rootstreamname",&root_stream_name_size); - if(IsProtocolMultichain() == 0) - { - root_stream_name_size=0; - } - } + root_stream_name_size=0; + } while(look_for_genesis) { @@ -141,7 +138,7 @@ int mc_MultichainParams::Build(const unsigned char* pubkey, int pubkey_size) txNew.vin.resize(1); - if(root_stream_name_size > ( (mc_gState->m_Features->FixedIn10008() != 0) ? 1 : 0 )) + if(root_stream_name_size > 1) { txNew.vout.resize(2); } @@ -170,14 +167,7 @@ int mc_MultichainParams::Build(const unsigned char* pubkey, int pubkey_size) lpScript=new mc_Script; - if(mc_gState->m_Features->Streams()) - { - lpScript->SetPermission(MC_PTP_GLOBAL_ALL,0,0xffffffff,timestamp); - } - else - { - lpScript->SetPermission(MC_PTP_ALL,0,0xffffffff,timestamp); - } + lpScript->SetPermission(MC_PTP_GLOBAL_ALL,0,0xffffffff,timestamp); elem = lpScript->GetData(0,&elem_size); txNew.vout[0].scriptPubKey << vector(elem, elem + elem_size) << OP_DROP; @@ -185,7 +175,7 @@ int mc_MultichainParams::Build(const unsigned char* pubkey, int pubkey_size) delete lpScript; } - if(root_stream_name_size > ( (mc_gState->m_Features->FixedIn10008() != 0) ? 1 : 0 )) + if(root_stream_name_size > 1) { txNew.vout[1].nValue=0; lpDetails=new mc_Script; @@ -196,12 +186,9 @@ int mc_MultichainParams::Build(const unsigned char* pubkey, int pubkey_size) lpDetails->SetSpecialParamValue(MC_ENT_SPRM_ANYONE_CAN_WRITE,&b,1); } - if(mc_gState->m_Features->FixedIn10007()) + if( (root_stream_name_size > 1) && (root_stream_name[root_stream_name_size - 1] == 0x00) ) { - if( (root_stream_name_size > 1) && (root_stream_name[root_stream_name_size - 1] == 0x00) ) - { - root_stream_name_size--; - } + root_stream_name_size--; } lpDetails->SetSpecialParamValue(MC_ENT_SPRM_NAME,root_stream_name,root_stream_name_size); @@ -212,44 +199,11 @@ int mc_MultichainParams::Build(const unsigned char* pubkey, int pubkey_size) lpDetailsScript=new mc_Script; - if(mc_gState->m_Features->OpDropDetailsScripts()) - { - lpDetailsScript->SetNewEntityType(MC_ENT_TYPE_STREAM,0,script,bytes); + lpDetailsScript->SetNewEntityType(MC_ENT_TYPE_STREAM,0,script,bytes); - elem = lpDetailsScript->GetData(0,&elem_size); - txNew.vout[1].scriptPubKey=CScript(); - txNew.vout[1].scriptPubKey << vector(elem, elem + elem_size) << OP_DROP << OP_RETURN; - } - else - { - lpDetailsScript->SetNewEntityType(MC_ENT_TYPE_STREAM); - - lpDetailsScript->SetGeneralDetails(script,bytes); - txNew.vout[1].scriptPubKey=CScript(); - - for(int e=0;eGetNumElements();e++) - { - elem = lpDetailsScript->GetData(e,&elem_size); - if(e == (lpDetailsScript->GetNumElements() - 1) ) - { - if(elem_size > 0) - { - txNew.vout[1].scriptPubKey << OP_RETURN << vector(elem, elem + elem_size); - } - else - { - txNew.vout[1].scriptPubKey << OP_RETURN; - } - } - else - { - if(elem_size > 0) - { - txNew.vout[1].scriptPubKey << vector(elem, elem + elem_size) << OP_DROP; - } - } - } - } + elem = lpDetailsScript->GetData(0,&elem_size); + txNew.vout[1].scriptPubKey=CScript(); + txNew.vout[1].scriptPubKey << vector(elem, elem + elem_size) << OP_DROP << OP_RETURN; delete lpDetails; delete lpDetailsScript; @@ -310,14 +264,6 @@ int mc_MultichainParams::Build(const unsigned char* pubkey, int pubkey_size) { return err; } - if(mc_gState->m_Features->Streams() == 0) - { - err=SetParam("genesisopreturnscript","[not set]",9); // Some value required to make parameter set valid, but valid value should start from OP_RETURN - if(err) - { - return err; - } - } mc_HexToBin(hash,genesis.GetHash().ToString().c_str(),32); err=SetParam("genesishash",(const char*)hash,32); diff --git a/src/chainparams/chainparams.cpp b/src/chainparams/chainparams.cpp index 8ea27db7..dae7d887 100644 --- a/src/chainparams/chainparams.cpp +++ b/src/chainparams/chainparams.cpp @@ -609,16 +609,12 @@ class CMultiChainParams : public CMainParams { txNew.vin.resize(1); root_stream_name_size=0; - root_stream_name=NULL; - if(mc_gState->m_Features->Streams()) + root_stream_name=(unsigned char *)mc_gState->m_NetworkParams->GetParam("rootstreamname",&root_stream_name_size); + if(mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) { - root_stream_name=(unsigned char *)mc_gState->m_NetworkParams->GetParam("rootstreamname",&root_stream_name_size); - if(mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) - { - root_stream_name_size=0; - } - } - if(root_stream_name_size > ( (mc_gState->m_Features->FixedIn10008() != 0) ? 1 : 0 ) ) + root_stream_name_size=0; + } + if(root_stream_name_size > 1) { txNew.vout.resize(2); } @@ -651,14 +647,7 @@ class CMultiChainParams : public CMainParams { lpScript=new mc_Script; - if(mc_gState->m_Features->Streams()) - { - lpScript->SetPermission(MC_PTP_GLOBAL_ALL,0,0xffffffff,(uint32_t)mc_gState->m_NetworkParams->GetInt64Param("genesistimestamp")); - } - else - { - lpScript->SetPermission(MC_PTP_ALL,0,0xffffffff,(uint32_t)mc_gState->m_NetworkParams->GetInt64Param("genesistimestamp")); - } + lpScript->SetPermission(MC_PTP_GLOBAL_ALL,0,0xffffffff,(uint32_t)mc_gState->m_NetworkParams->GetInt64Param("genesistimestamp")); elem = lpScript->GetData(0,&elem_size); txNew.vout[0].scriptPubKey << vector(elem, elem + elem_size) << OP_DROP; @@ -666,7 +655,7 @@ class CMultiChainParams : public CMainParams { delete lpScript; } - if(root_stream_name_size > ( (mc_gState->m_Features->FixedIn10008() != 0) ? 1 : 0 )) + if(root_stream_name_size > 1) { txNew.vout[1].nValue=0; lpDetails=new mc_Script; @@ -678,13 +667,10 @@ class CMultiChainParams : public CMainParams { } - if(mc_gState->m_Features->FixedIn10007()) + if( (root_stream_name_size > 1) && (root_stream_name[root_stream_name_size - 1] == 0x00) ) { - if( (root_stream_name_size > 1) && (root_stream_name[root_stream_name_size - 1] == 0x00) ) - { - root_stream_name_size--; - } - } + root_stream_name_size--; + } lpDetails->SetSpecialParamValue(MC_ENT_SPRM_NAME,root_stream_name,root_stream_name_size); @@ -695,44 +681,11 @@ class CMultiChainParams : public CMainParams { lpDetailsScript=new mc_Script; - if(mc_gState->m_Features->OpDropDetailsScripts()) - { - lpDetailsScript->SetNewEntityType(MC_ENT_TYPE_STREAM,0,script,bytes); + lpDetailsScript->SetNewEntityType(MC_ENT_TYPE_STREAM,0,script,bytes); - elem = lpDetailsScript->GetData(0,&elem_size); - txNew.vout[1].scriptPubKey=CScript(); - txNew.vout[1].scriptPubKey << vector(elem, elem + elem_size) << OP_DROP << OP_RETURN; - } - else - { - lpDetailsScript->SetNewEntityType(MC_ENT_TYPE_STREAM); - - lpDetailsScript->SetGeneralDetails(script,bytes); - txNew.vout[1].scriptPubKey=CScript(); - - for(int e=0;eGetNumElements();e++) - { - elem = lpDetailsScript->GetData(e,&elem_size); - if(e == (lpDetailsScript->GetNumElements() - 1) ) - { - if(elem_size > 0) - { - txNew.vout[1].scriptPubKey << OP_RETURN << vector(elem, elem + elem_size); - } - else - { - txNew.vout[1].scriptPubKey << OP_RETURN; - } - } - else - { - if(elem_size > 0) - { - txNew.vout[1].scriptPubKey << vector(elem, elem + elem_size) << OP_DROP; - } - } - } - } + elem = lpDetailsScript->GetData(0,&elem_size); + txNew.vout[1].scriptPubKey=CScript(); + txNew.vout[1].scriptPubKey << vector(elem, elem + elem_size) << OP_DROP << OP_RETURN; delete lpDetails; delete lpDetailsScript; diff --git a/src/chainparams/params.cpp b/src/chainparams/params.cpp index 14807a3f..2aa01d48 100644 --- a/src/chainparams/params.cpp +++ b/src/chainparams/params.cpp @@ -1742,216 +1742,11 @@ int mc_Features::MinProtocolVersion() return 10004; } -int mc_Features::ActivatePermission() // This test is eliminated from the code as 10002 is not supported -{ - int ret=0; - int protocol=mc_gState->m_NetworkParams->ProtocolVersion(); - - if(protocol) - { - if(protocol >= 10003) - { - ret=1; - } - } - - return ret; -} - int mc_Features::LastVersionNotSendingProtocolVersionInHandShake() { return 10002; } -int mc_Features::VerifySizeOfOpDropElements() // This test is still in the code to keep protocol!-multichain untouched -{ - - int ret=0; - int protocol=mc_gState->m_NetworkParams->ProtocolVersion(); - - if(mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) - { - return 1; - } - - if(protocol) - { - if(protocol >= 10003) - { - ret=1; - } - } - - return ret; -} - -int mc_Features::PerEntityPermissions() // This test is eliminated from the code as 10002 is not supported -{ - if(mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) - { - return 0; - } - int ret=0; - int protocol=mc_gState->m_NetworkParams->ProtocolVersion(); - - if(protocol) - { - if(protocol >= 10004) - { - ret=1; - } - } - - return ret; -} - -int mc_Features::FollowOnIssues() // This test is eliminated from the code as 10002 is not supported -{ - int ret=0; - if(mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) - { - return 0; - } - int protocol=mc_gState->m_NetworkParams->ProtocolVersion(); - if(protocol) - { - if(protocol >= 10004) - { - ret=1; - } - } - - return ret; -} - -int mc_Features::SpecialParamsInDetailsScript() // This test is eliminated from the code as 10002 is not supported -{ - int ret=0; - int protocol=mc_gState->m_NetworkParams->ProtocolVersion(); - - if(protocol) - { - if(protocol >= 10004) - { - ret=1; - } - } - - return ret; -} - -int mc_Features::FixedGrantsInTheSameTx() // This test is eliminated from the code as 10002 is not supported -{ - int ret=0; - int protocol=mc_gState->m_NetworkParams->ProtocolVersion(); - - if(protocol) - { - if(protocol >= 10004) - { - ret=1; - } - } - - return ret; -} - -int mc_Features::UnconfirmedMinersCannotMine() -{ - int ret=0; - int protocol=mc_gState->m_NetworkParams->ProtocolVersion(); - - if(protocol) - { - if(protocol >= 10004) - { - ret=1; - } - } - - return ret; -} - -int mc_Features::Streams() -{ - int ret=0; - if(mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) - { - return 0; - } - int protocol=mc_gState->m_NetworkParams->ProtocolVersion(); - - if(protocol) - { - if(protocol >= 10006) - { - ret=1; - } - } - - return ret; -} - -int mc_Features::OpDropDetailsScripts() -{ - int ret=0; - if(mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) - { - return 0; - } - int protocol=mc_gState->m_NetworkParams->ProtocolVersion(); - - if(protocol) - { - if(protocol >= 10007) - { - ret=1; - } - } - - return ret; -} - -int mc_Features::ShortTxIDInTx() -{ - int ret=0; - if(mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) - { - return 0; - } - int protocol=mc_gState->m_NetworkParams->ProtocolVersion(); - - if(protocol) - { - if(protocol >= 10007) - { - ret=1; - } - } - - return ret; -} - -int mc_Features::CachedInputScript() -{ - int ret=0; - if(mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) - { - return 0; - } - int protocol=mc_gState->m_NetworkParams->ProtocolVersion(); - - if(protocol) - { - if(protocol >= 10007) - { - ret=1; - } - } - - return ret; -} - int mc_Features::AnyoneCanReceiveEmpty() { int ret=0; @@ -1975,66 +1770,6 @@ int mc_Features::AnyoneCanReceiveEmpty() return ret; } -int mc_Features::FixedIn10007() -{ - int ret=0; - if(mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) - { - return 0; - } - int protocol=mc_gState->m_NetworkParams->ProtocolVersion(); - - if(protocol) - { - if(protocol >= 10007) - { - ret=1; - } - } - - return ret; -} - -int mc_Features::Upgrades() -{ - int ret=0; - if(mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) - { - return 0; - } - int protocol=mc_gState->m_NetworkParams->ProtocolVersion(); - - if(protocol) - { - if(protocol >= 10008) - { - ret=1; - } - } - - return ret; -} - -int mc_Features::FixedIn10008() -{ - int ret=0; - if(mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) - { - return 0; - } - int protocol=mc_gState->m_NetworkParams->ProtocolVersion(); - - if(protocol) - { - if(protocol >= 10008) - { - ret=1; - } - } - - return ret; -} - int mc_Features::FormattedData() { int ret=0; diff --git a/src/chainparams/state.h b/src/chainparams/state.h index 98920cea..f8c3a216 100644 --- a/src/chainparams/state.h +++ b/src/chainparams/state.h @@ -128,22 +128,8 @@ typedef struct mc_UpgradedParameter typedef struct mc_Features { int MinProtocolVersion(); - int ActivatePermission(); int LastVersionNotSendingProtocolVersionInHandShake(); - int VerifySizeOfOpDropElements(); - int PerEntityPermissions(); - int FollowOnIssues(); - int SpecialParamsInDetailsScript(); - int FixedGrantsInTheSameTx(); - int UnconfirmedMinersCannotMine(); - int Streams(); - int OpDropDetailsScripts(); - int ShortTxIDInTx(); - int CachedInputScript(); int AnyoneCanReceiveEmpty(); - int FixedIn10007(); - int Upgrades(); - int FixedIn10008(); int FormattedData(); int FixedDestinationExtraction(); int FixedIn1000920001(); diff --git a/src/core/init-cold.cpp b/src/core/init-cold.cpp index 45ec7255..546e1406 100644 --- a/src/core/init-cold.cpp +++ b/src/core/init-cold.cpp @@ -530,33 +530,31 @@ bool AppInit2_Cold(boost::thread_group& threadGroup,int OutputPipe) vector vSubscribedEntities; if(GetBoolArg("-reindex", false) || GetBoolArg("-rescan", false)) { - if(mc_gState->m_Features->Streams()) + pwalletTxsMain=new mc_WalletTxs; + if(pwalletTxsMain->Initialize(mc_gState->m_NetworkParams->Name(),MC_WMD_TXS | MC_WMD_ADDRESS_TXS) == MC_ERR_NOERROR) { - pwalletTxsMain=new mc_WalletTxs; - if(pwalletTxsMain->Initialize(mc_gState->m_NetworkParams->Name(),MC_WMD_TXS | MC_WMD_ADDRESS_TXS) == MC_ERR_NOERROR) + mc_Buffer *entity_list; + entity_list=pwalletTxsMain->GetEntityList(); + for(int e=0;eGetCount();e++) { - mc_Buffer *entity_list; - entity_list=pwalletTxsMain->GetEntityList(); - for(int e=0;eGetCount();e++) + mc_TxEntityStat *stat; + stat=(mc_TxEntityStat *)entity_list->GetRow(e); + switch(stat->m_Entity.m_EntityType & MC_TET_TYPE_MASK) { - mc_TxEntityStat *stat; - stat=(mc_TxEntityStat *)entity_list->GetRow(e); - switch(stat->m_Entity.m_EntityType & MC_TET_TYPE_MASK) - { - case MC_TET_PUBKEY_ADDRESS: - case MC_TET_SCRIPT_ADDRESS: - case MC_TET_STREAM: - case MC_TET_STREAM_KEY: - case MC_TET_STREAM_PUBLISHER: - vSubscribedEntities.push_back(stat->m_Entity); - break; - } + case MC_TET_PUBKEY_ADDRESS: + case MC_TET_SCRIPT_ADDRESS: + case MC_TET_STREAM: + case MC_TET_STREAM_KEY: + case MC_TET_STREAM_PUBLISHER: + vSubscribedEntities.push_back(stat->m_Entity); + break; } - __US_Sleep(1000); } - pwalletTxsMain->Destroy(); - delete pwalletTxsMain; + __US_Sleep(1000); } + pwalletTxsMain->Destroy(); + delete pwalletTxsMain; + mc_RemoveDir(mc_gState->m_Params->NetworkName(),"wallet"); zap_wallet_txs=true; } diff --git a/src/core/init.cpp b/src/core/init.cpp index fd907707..63ea2f92 100644 --- a/src/core/init.cpp +++ b/src/core/init.cpp @@ -1375,34 +1375,32 @@ bool AppInit2(boost::thread_group& threadGroup,int OutputPipe) vector vSubscribedEntities; if(GetBoolArg("-reindex", false) || GetBoolArg("-rescan", false)) { - if(mc_gState->m_Features->Streams()) + pwalletTxsMain=new mc_WalletTxs; + if(pwalletTxsMain->Initialize(mc_gState->m_NetworkParams->Name(),MC_WMD_TXS | MC_WMD_ADDRESS_TXS) == MC_ERR_NOERROR) { - pwalletTxsMain=new mc_WalletTxs; - if(pwalletTxsMain->Initialize(mc_gState->m_NetworkParams->Name(),MC_WMD_TXS | MC_WMD_ADDRESS_TXS) == MC_ERR_NOERROR) + mc_Buffer *entity_list; + entity_list=pwalletTxsMain->GetEntityList(); + for(int e=0;eGetCount();e++) { - mc_Buffer *entity_list; - entity_list=pwalletTxsMain->GetEntityList(); - for(int e=0;eGetCount();e++) + mc_TxEntityStat *stat; + stat=(mc_TxEntityStat *)entity_list->GetRow(e); + switch(stat->m_Entity.m_EntityType & MC_TET_TYPE_MASK) { - mc_TxEntityStat *stat; - stat=(mc_TxEntityStat *)entity_list->GetRow(e); - switch(stat->m_Entity.m_EntityType & MC_TET_TYPE_MASK) - { - case MC_TET_PUBKEY_ADDRESS: - case MC_TET_SCRIPT_ADDRESS: - case MC_TET_STREAM: - case MC_TET_STREAM_KEY: - case MC_TET_STREAM_PUBLISHER: - case MC_TET_ASSET: - vSubscribedEntities.push_back(stat->m_Entity); - break; - } + case MC_TET_PUBKEY_ADDRESS: + case MC_TET_SCRIPT_ADDRESS: + case MC_TET_STREAM: + case MC_TET_STREAM_KEY: + case MC_TET_STREAM_PUBLISHER: + case MC_TET_ASSET: + vSubscribedEntities.push_back(stat->m_Entity); + break; } - __US_Sleep(1000); } - pwalletTxsMain->Destroy(); - delete pwalletTxsMain; + __US_Sleep(1000); } + pwalletTxsMain->Destroy(); + delete pwalletTxsMain; + mc_RemoveDir(mc_gState->m_Params->NetworkName(),"wallet"); zap_wallet_txs=true; } diff --git a/src/core/main.cpp b/src/core/main.cpp index 9e27ef1f..2ce353e2 100644 --- a/src/core/main.cpp +++ b/src/core/main.cpp @@ -1099,13 +1099,7 @@ bool IsStandardTx(const CTransaction& tx, string& reason,bool check_for_dust) } // only one OP_RETURN txout is permitted -/* MCHN START */ - int max_op_returns=1; - if(mc_gState->m_Features->Streams()) - { - max_op_returns=MCP_MAX_STD_OP_RETURN_COUNT; - } -/* MCHN END */ + int max_op_returns=MCP_MAX_STD_OP_RETURN_COUNT; if ((int)nDataOut > max_op_returns) { reason = "multi-op-return"; @@ -1417,24 +1411,6 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa uint256 hash = tx.GetHash(); if (pool.exists(hash)) return false; - -/* MCHN START */ - if(mc_gState->m_Features->Streams() == 0) - { - if(!AcceptPermissionsAndCheckForDust(tx,false,reason)) - { - return state.DoS(0, - error("AcceptToMemoryPool: : AcceptPermissionsAndCheckForDust failed %s : %s", hash.ToString(),reason), - REJECT_NONSTANDARD, reason); - } - if(!AcceptAssetGenesis(tx,-1,false,reason)) - { - return state.DoS(0, - error("AcceptToMemoryPool: : AcceptAssetGenesis failed %s : %s", hash.ToString(),reason), - REJECT_INVALID, reason); - } - } -/* MCHN END */ // Check for conflicts with in-memory transactions { @@ -1649,35 +1625,11 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa int permissions_from,permissions_to; permissions_from=mc_gState->m_Permissions->m_MempoolPermissions->GetCount(); - if(mc_gState->m_Features->Streams()) + if(!AcceptMultiChainTransaction(tx,view,-1,true,reason, &replay)) { - if(!AcceptMultiChainTransaction(tx,view,-1,true,reason, &replay)) - { - return state.DoS(0, - error("AcceptToMemoryPool: : AcceptMultiChainTransaction failed %s : %s", hash.ToString(),reason), - REJECT_NONSTANDARD, reason); - } - } - else - { - if(!AcceptPermissionsAndCheckForDust(tx,true,reason)) - { - return state.DoS(0, - error("AcceptToMemoryPool: : AcceptPermissionChanges failed when adding to permission db %s - %s", hash.ToString(),reason), - REJECT_INVALID, reason); - } - if(!AcceptAssetGenesis(tx,-1,true,reason)) - { - return state.DoS(0, - error("AcceptToMemoryPool: : AcceptAssetGenesis failed when adding to asset db %s : %s", hash.ToString(),reason), - REJECT_INVALID, reason); - } - if(!AcceptAssetTransfers(tx, view, reason)) - { - return state.DoS(0, - error("AcceptToMemoryPool: : AcceptAssetTransfers failed %s : %s", hash.ToString(),reason), - REJECT_INVALID, reason); - } + return state.DoS(0, + error("AcceptToMemoryPool: : AcceptMultiChainTransaction failed %s : %s", hash.ToString(),reason), + REJECT_NONSTANDARD, reason); } if(fAddToWallet) @@ -2415,56 +2367,44 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin { const CTransaction &tx = block.vtx[i]; string reason; - if(mc_gState->m_Features->Streams()) + if(!AcceptMultiChainTransaction(tx,view,offset,true,reason,NULL)) { - if(!AcceptMultiChainTransaction(tx,view,offset,true,reason,NULL)) - { - return state.DoS(100, error(reason.c_str()), - REJECT_INVALID, "bad-transaction"); - } + return state.DoS(100, error(reason.c_str()), + REJECT_INVALID, "bad-transaction"); + } // unsigned char *root_stream_name; - int root_stream_name_size; - mc_gState->m_NetworkParams->GetParam("rootstreamname",&root_stream_name_size); - if(mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) - { - root_stream_name_size=0; - } - if(root_stream_name_size > 1) + int root_stream_name_size; + mc_gState->m_NetworkParams->GetParam("rootstreamname",&root_stream_name_size); + if(mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) + { + root_stream_name_size=0; + } + if(root_stream_name_size > 1) + { + if(pwalletTxsMain) { - if(pwalletTxsMain) - { - if(mc_gState->m_WalletMode & MC_WMD_TXS) - { - mc_TxEntity entity; - uint256 genesis_hash=block.vtx[0].GetHash(); - entity.Zero(); - - memcpy(entity.m_EntityID,(unsigned char*)&genesis_hash+MC_AST_SHORT_TXID_OFFSET,MC_AST_SHORT_TXID_SIZE); - entity.m_EntityType=MC_TET_STREAM | MC_TET_CHAINPOS; - pwalletTxsMain->AddEntity(&entity,0); - entity.m_EntityType=MC_TET_STREAM | MC_TET_TIMERECEIVED; - pwalletTxsMain->AddEntity(&entity,0); - entity.m_EntityType=MC_TET_STREAM_KEY | MC_TET_CHAINPOS; - pwalletTxsMain->AddEntity(&entity,0); - entity.m_EntityType=MC_TET_STREAM_KEY | MC_TET_TIMERECEIVED; - pwalletTxsMain->AddEntity(&entity,0); - entity.m_EntityType=MC_TET_STREAM_PUBLISHER | MC_TET_CHAINPOS; - pwalletTxsMain->AddEntity(&entity,0); - entity.m_EntityType=MC_TET_STREAM_PUBLISHER | MC_TET_TIMERECEIVED; - pwalletTxsMain->AddEntity(&entity,0); - } + if(mc_gState->m_WalletMode & MC_WMD_TXS) + { + mc_TxEntity entity; + uint256 genesis_hash=block.vtx[0].GetHash(); + entity.Zero(); + + memcpy(entity.m_EntityID,(unsigned char*)&genesis_hash+MC_AST_SHORT_TXID_OFFSET,MC_AST_SHORT_TXID_SIZE); + entity.m_EntityType=MC_TET_STREAM | MC_TET_CHAINPOS; + pwalletTxsMain->AddEntity(&entity,0); + entity.m_EntityType=MC_TET_STREAM | MC_TET_TIMERECEIVED; + pwalletTxsMain->AddEntity(&entity,0); + entity.m_EntityType=MC_TET_STREAM_KEY | MC_TET_CHAINPOS; + pwalletTxsMain->AddEntity(&entity,0); + entity.m_EntityType=MC_TET_STREAM_KEY | MC_TET_TIMERECEIVED; + pwalletTxsMain->AddEntity(&entity,0); + entity.m_EntityType=MC_TET_STREAM_PUBLISHER | MC_TET_CHAINPOS; + pwalletTxsMain->AddEntity(&entity,0); + entity.m_EntityType=MC_TET_STREAM_PUBLISHER | MC_TET_TIMERECEIVED; + pwalletTxsMain->AddEntity(&entity,0); } } } - else - { - if(!AcceptPermissionsAndCheckForDust(tx,true,reason)) - { - return state.DoS(100, error(reason.c_str()), - REJECT_INVALID, "bad-transaction"); - return false; - } - } offset+=tx.GetSerializeSize(SER_NETWORK,tx.nVersion); } if(mc_gState->m_Permissions->Commit(miner_address,&block_hash) != 0) @@ -2586,35 +2526,11 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin string reason; if(!fJustCheck) { - if(mc_gState->m_Features->Streams()) - { - if(!AcceptMultiChainTransaction(tx,view,offset,true,reason,NULL)) - { - return state.DoS(0, - error("ConnectBlock: : AcceptMultiChainTransaction failed %s : %s", tx.GetHash().ToString(),reason), - REJECT_NONSTANDARD, reason); - } - } - else + if(!AcceptMultiChainTransaction(tx,view,offset,true,reason,NULL)) { - if(!AcceptPermissionsAndCheckForDust(tx,true,reason)) - { - return state.DoS(0, - error("ConnectBlock: AcceptPermissionChanges failed when adding to permission db %s - %s", tx.GetHash().ToString(),reason), - REJECT_INVALID, reason); - } - if(!AcceptAssetGenesis(tx,offset,true,reason)) - { - return state.DoS(0, - error("ConnectBlock: AcceptAssetGenesis failed when adding to asset db %s : %s", tx.GetHash().ToString(),reason), - REJECT_INVALID, reason); - } - if(!AcceptAssetTransfers(tx, view, reason)) - { - return state.DoS(0, - error("ConnectBlock: AcceptAssetTransfers failed %s : %s", tx.GetHash().ToString(),reason), - REJECT_INVALID, reason); - } + return state.DoS(0, + error("ConnectBlock: : AcceptMultiChainTransaction failed %s : %s", tx.GetHash().ToString(),reason), + REJECT_NONSTANDARD, reason); } } /* MCHN END */ @@ -2641,19 +2557,9 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin string reason; if(!fJustCheck) { - if(mc_gState->m_Features->Streams()) - { - if(!AcceptMultiChainTransaction(tx,view,coinbase_offset,true,reason,NULL)) - { - return false; - } - } - else + if(!AcceptMultiChainTransaction(tx,view,coinbase_offset,true,reason,NULL)) { - if(!AcceptPermissionsAndCheckForDust(tx,true,reason)) - { - return false; - } + return false; } } } diff --git a/src/entities/asset.cpp b/src/entities/asset.cpp index 89b26090..3ac3c255 100644 --- a/src/entities/asset.cpp +++ b/src/entities/asset.cpp @@ -551,12 +551,9 @@ void mc_EntityDetails::Set(mc_EntityLedgerRow* row) m_Permissions |= MC_PTP_ADMIN | MC_PTP_ACTIVATE | MC_PTP_WRITE; break; default: - if(mc_gState->m_Features->FixedIn10007()) + if(m_LedgerRow.m_EntityType <= MC_ENT_TYPE_STREAM_MAX) { - if(m_LedgerRow.m_EntityType <= MC_ENT_TYPE_STREAM_MAX) - { - m_Permissions = MC_PTP_WRITE | MC_PTP_ACTIVATE; - } + m_Permissions = MC_PTP_WRITE | MC_PTP_ACTIVATE; } break; } @@ -570,17 +567,14 @@ void mc_EntityDetails::Set(mc_EntityLedgerRow* row) dname_buf[0]=0xff; value_offset=mc_FindNamedParamInDetailsScript(m_LedgerRow.m_Script,m_LedgerRow.m_ScriptSize,(char*)dname_buf,&value_size); } - if(mc_gState->m_Features->Streams()) - { - if(value_offset < m_LedgerRow.m_ScriptSize) - { - if(value_size == 2) + if(value_offset < m_LedgerRow.m_ScriptSize) + { + if(value_size == 2) + { + if((char)m_LedgerRow.m_Script[value_offset] == '*') { - if((char)m_LedgerRow.m_Script[value_offset] == '*') - { - value_offset=m_LedgerRow.m_ScriptSize; - value_size=0; - } + value_offset=m_LedgerRow.m_ScriptSize; + value_size=0; } } } @@ -603,16 +597,8 @@ void mc_EntityDetails::Set(mc_EntityLedgerRow* row) } mc_ZeroABRaw(m_FullRef); - if(mc_gState->m_Features->ShortTxIDInTx()) - { - memcpy(m_FullRef+MC_AST_SHORT_TXID_OFFSET,m_LedgerRow.m_Key+MC_AST_SHORT_TXID_OFFSET,MC_AST_SHORT_TXID_SIZE); - mc_SetABRefType(m_FullRef,MC_AST_ASSET_REF_TYPE_SHORT_TXID); - } - else - { - memcpy(m_FullRef,m_Ref,MC_AST_ASSET_REF_SIZE); - mc_SetABRefType(m_FullRef,MC_AST_ASSET_REF_TYPE_REF); - } + memcpy(m_FullRef+MC_AST_SHORT_TXID_OFFSET,m_LedgerRow.m_Key+MC_AST_SHORT_TXID_OFFSET,MC_AST_SHORT_TXID_SIZE); + mc_SetABRefType(m_FullRef,MC_AST_ASSET_REF_TYPE_SHORT_TXID); } int mc_AssetDB::InsertEntity(const void* txid, int offset, int entity_type, const void *script,size_t script_size, const void* special_script, size_t special_script_size,int update_mempool) @@ -669,37 +655,34 @@ int mc_AssetDB::InsertEntity(const void* txid, int offset, int entity_type, cons } upgrade_start_block=0; - if(mc_gState->m_Features->Upgrades()) + if(entity_type == MC_ENT_TYPE_UPGRADE) { - if(entity_type == MC_ENT_TYPE_UPGRADE) + if(script) { - if(script) + if(mc_gState->m_Features->ParameterUpgrades() == 0) { - if(mc_gState->m_Features->ParameterUpgrades() == 0) + value_offset=mc_FindSpecialParamInDetailsScript((unsigned char*)script,script_size,MC_ENT_SPRM_UPGRADE_PROTOCOL_VERSION,&value_size); + if(value_offset == script_size) { - value_offset=mc_FindSpecialParamInDetailsScript((unsigned char*)script,script_size,MC_ENT_SPRM_UPGRADE_PROTOCOL_VERSION,&value_size); - if(value_offset == script_size) - { - return MC_ERR_ERROR_IN_SCRIPT; - } - if( (value_size <=0) || (value_size > 4) ) - { - return MC_ERR_ERROR_IN_SCRIPT; - } - if((int)mc_GetLE((unsigned char*)script+value_offset,value_size) < 0) - { - return MC_ERR_ERROR_IN_SCRIPT; - } + return MC_ERR_ERROR_IN_SCRIPT; } - value_offset=mc_FindSpecialParamInDetailsScript((unsigned char*)script,script_size,MC_ENT_SPRM_UPGRADE_START_BLOCK,&value_size); - if(value_offset != script_size) + if( (value_size <=0) || (value_size > 4) ) { - if( (value_size <=0) || (value_size > 4) ) - { - return MC_ERR_ERROR_IN_SCRIPT; - } - upgrade_start_block=(uint32_t)mc_GetLE((unsigned char*)script+value_offset,value_size); + return MC_ERR_ERROR_IN_SCRIPT; + } + if((int)mc_GetLE((unsigned char*)script+value_offset,value_size) < 0) + { + return MC_ERR_ERROR_IN_SCRIPT; + } + } + value_offset=mc_FindSpecialParamInDetailsScript((unsigned char*)script,script_size,MC_ENT_SPRM_UPGRADE_START_BLOCK,&value_size); + if(value_offset != script_size) + { + if( (value_size <=0) || (value_size > 4) ) + { + return MC_ERR_ERROR_IN_SCRIPT; } + upgrade_start_block=(uint32_t)mc_GetLE((unsigned char*)script+value_offset,value_size); } } } @@ -793,12 +776,9 @@ int mc_AssetDB::InsertEntity(const void* txid, int offset, int entity_type, cons } } - if(mc_gState->m_Features->Upgrades()) + if(entity_type == MC_ENT_TYPE_UPGRADE) { - if(entity_type == MC_ENT_TYPE_UPGRADE) - { - return mc_gState->m_Permissions->SetApproval((unsigned char*)txid+MC_AST_SHORT_TXID_OFFSET,1,NULL,upgrade_start_block,mc_TimeNowAsUInt(),MC_PFL_ENTITY_GENESIS,update_mempool,offset); - } + return mc_gState->m_Permissions->SetApproval((unsigned char*)txid+MC_AST_SHORT_TXID_OFFSET,1,NULL,upgrade_start_block,mc_TimeNowAsUInt(),MC_PFL_ENTITY_GENESIS,update_mempool,offset); } return MC_ERR_NOERROR; @@ -841,14 +821,11 @@ int mc_AssetDB::InsertAsset(const void* txid, int offset, uint64_t quantity, con } add_param=true; - if(mc_gState->m_Features->OpDropDetailsScripts()) + if(script) { - if(script) + if(mc_FindSpecialParamInDetailsScript((unsigned char*)script,script_size,MC_ENT_SPRM_ASSET_MULTIPLE,&value_size) != script_size) { - if(mc_FindSpecialParamInDetailsScript((unsigned char*)script,script_size,MC_ENT_SPRM_ASSET_MULTIPLE,&value_size) != script_size) - { - add_param=false; - } + add_param=false; } } @@ -993,14 +970,6 @@ int mc_AssetDB::InsertAssetFollowOn(const void* txid, int offset, uint64_t quant return MC_ERR_NOT_FOUND; } - if(mc_gState->m_Features->ShortTxIDInTx() == 0) - { - if(aldRow.m_PrevPos < 0) // Unconfirmed genesis for protocol < 10007 - { - return MC_ERR_NOT_FOUND; - } - } - value_offset=mc_FindSpecialParamInDetailsScript(aldRow.m_Script,aldRow.m_ScriptSize,MC_ENT_SPRM_FOLLOW_ONS,&value_size); if(value_offset == aldRow.m_ScriptSize) { @@ -1656,17 +1625,14 @@ const char* mc_EntityDetails::GetName() dname_buf[0]=0xff; value_offset=mc_FindNamedParamInDetailsScript(m_LedgerRow.m_Script,m_LedgerRow.m_ScriptSize,(char*)dname_buf,&value_size); } - if(mc_gState->m_Features->Streams()) - { - if(value_offset < m_LedgerRow.m_ScriptSize) - { - if(value_size == 2) + if(value_offset < m_LedgerRow.m_ScriptSize) + { + if(value_size == 2) + { + if((char)m_LedgerRow.m_Script[value_offset] == '*') { - if((char)m_LedgerRow.m_Script[value_offset] == '*') - { - value_offset=m_LedgerRow.m_ScriptSize; - value_size=0; - } + value_offset=m_LedgerRow.m_ScriptSize; + value_size=0; } } } @@ -1701,11 +1667,7 @@ const unsigned char* mc_EntityDetails::GetFullRef() const unsigned char* mc_EntityDetails::GetShortRef() { - if(mc_gState->m_Features->ShortTxIDInTx()) - { - return GetTxID()+MC_AST_SHORT_TXID_OFFSET; - } - return m_Ref; + return GetTxID()+MC_AST_SHORT_TXID_OFFSET; } const unsigned char* mc_EntityDetails::GetScript() @@ -1942,10 +1904,6 @@ void mc_AssetDB::Dump() uint32_t mc_AssetDB::MaxEntityType() { - if(mc_gState->m_Features->Upgrades() == 0) - { - return MC_ENT_TYPE_STREAM_MAX; - } return MC_ENT_TYPE_MAX; } diff --git a/src/multichain/multichain-util.cpp b/src/multichain/multichain-util.cpp index 97f6a549..06879e68 100644 --- a/src/multichain/multichain-util.cpp +++ b/src/multichain/multichain-util.cpp @@ -43,7 +43,6 @@ int main(int argc, char* argv[]) if(mc_gState->m_Params->m_NumArguments>2) { v=atoi(mc_gState->m_Params->m_Arguments[2]); -// if( (v>=mc_gState->m_Features->MinProtocolVersion()) && (v<=version) ) if(mc_gState->IsSupported(v)) { version=v; diff --git a/src/permissions/permission.cpp b/src/permissions/permission.cpp index 3c36752a..0c8f27f5 100644 --- a/src/permissions/permission.cpp +++ b/src/permissions/permission.cpp @@ -725,10 +725,6 @@ uint32_t mc_Permissions::GetPossiblePermissionTypes(const void* entity_details) } full_type = MC_PTP_GLOBAL_ALL; - if(mc_gState->m_Features->Streams() == 0) - { - full_type-=MC_PTP_CREATE; - } return full_type; } @@ -748,18 +744,11 @@ uint32_t mc_Permissions::GetPossiblePermissionTypes(uint32_t entity_type) break; case MC_ENT_TYPE_NONE: full_type = MC_PTP_GLOBAL_ALL; - if(mc_gState->m_Features->Streams() == 0) - { - full_type-=MC_PTP_CREATE; - } break; default: - if(mc_gState->m_Features->FixedIn10007()) + if(entity_type <= MC_ENT_TYPE_STREAM_MAX) { - if(entity_type <= MC_ENT_TYPE_STREAM_MAX) - { - full_type = MC_PTP_WRITE | MC_PTP_ACTIVATE | MC_PTP_ADMIN; - } + full_type = MC_PTP_WRITE | MC_PTP_ACTIVATE | MC_PTP_ADMIN; } break; } @@ -809,11 +798,8 @@ uint32_t mc_Permissions::GetPermissionType(const char *str,uint32_t full_type) if(mc_MemcmpCheckSize(start,"mine", ptr-start) == 0)perm_type = MC_PTP_MINE; if(mc_MemcmpCheckSize(start,"admin", ptr-start) == 0)perm_type = MC_PTP_ADMIN; if(mc_MemcmpCheckSize(start,"activate", ptr-start) == 0)perm_type = MC_PTP_ACTIVATE; - if(mc_gState->m_Features->Streams()) - { - if(mc_MemcmpCheckSize(start,"create", ptr-start) == 0)perm_type = MC_PTP_CREATE; - if(mc_MemcmpCheckSize(start,"write", ptr-start) == 0)perm_type = MC_PTP_WRITE; - } + if(mc_MemcmpCheckSize(start,"create", ptr-start) == 0)perm_type = MC_PTP_CREATE; + if(mc_MemcmpCheckSize(start,"write", ptr-start) == 0)perm_type = MC_PTP_WRITE; if(perm_type == 0) { @@ -1102,18 +1088,12 @@ int mc_Permissions::CanSend(const void* lpEntity,const void* lpAddress) if(result == 0) { - if(mc_gState->m_Features->Streams()) - { - result |= GetPermission(lpEntity,lpAddress,MC_PTP_ISSUE); - } + result |= GetPermission(lpEntity,lpAddress,MC_PTP_ISSUE); } if(result == 0) { - if(mc_gState->m_Features->Streams()) - { - result |= GetPermission(lpEntity,lpAddress,MC_PTP_CREATE); - } + result |= GetPermission(lpEntity,lpAddress,MC_PTP_CREATE); } if(result == 0) @@ -1640,19 +1620,9 @@ int mc_Permissions::IsBarredByDiversity(uint32_t block,uint32_t last,int miner_c int mc_Permissions::CanAdmin(const void* lpEntity,const void* lpAddress) { - if(mc_gState->m_Features->Streams()) + if(m_Block == -1) { - if(m_Block == -1) - { - return MC_PTP_ADMIN; - } - } - else - { - if(m_AdminCount == 0) - { - return MC_PTP_ADMIN; - } + return MC_PTP_ADMIN; } if(mc_IsNullEntity(lpEntity)) @@ -3053,19 +3023,13 @@ int mc_Permissions::SetPermissionInternal(const void* lpEntity,const void* lpAdd types[num_types]=MC_PTP_CONNECT;num_types++; types[num_types]=MC_PTP_SEND;num_types++; types[num_types]=MC_PTP_RECEIVE;num_types++; - if(mc_gState->m_Features->Streams()) - { - types[num_types]=MC_PTP_WRITE;num_types++; - types[num_types]=MC_PTP_CREATE;num_types++; - } + types[num_types]=MC_PTP_WRITE;num_types++; + types[num_types]=MC_PTP_CREATE;num_types++; types[num_types]=MC_PTP_ISSUE;num_types++; types[num_types]=MC_PTP_MINE;num_types++; types[num_types]=MC_PTP_ACTIVATE;num_types++; types[num_types]=MC_PTP_ADMIN;num_types++; - if(mc_gState->m_Features->Upgrades()) - { - types[num_types]=MC_PTP_UPGRADE;num_types++; - } + types[num_types]=MC_PTP_UPGRADE;num_types++; err=MC_ERR_NOERROR; @@ -3836,12 +3800,7 @@ int mc_Permissions::StoreBlockInfoInternal(const void* lpMiner,const void* lpHas mc_PermissionLedgerRow pldRow; mc_BlockMinerDBRow pdbBlockMinerRow; mc_AdminMinerGrantDBRow pdbAdminMinerGrantRow; - - if(mc_gState->m_Features->CachedInputScript() == 0) - { - return MC_ERR_NOERROR; - } - + if(mc_gState->m_NetworkParams->GetInt64Param("supportminerprecheck") == 0) { return MC_ERR_NOERROR; diff --git a/src/protocol/multichainblock.cpp b/src/protocol/multichainblock.cpp index 90f11268..a72754b6 100644 --- a/src/protocol/multichainblock.cpp +++ b/src/protocol/multichainblock.cpp @@ -360,64 +360,31 @@ bool ReplayMemPool(CTxMemPool& pool, int from,bool accept) } else { - - if(mc_gState->m_Features->Streams()) + int permissions_from,permissions_to; + permissions_from=mc_gState->m_Permissions->m_MempoolPermissions->GetCount(); + if(entry.FullReplayRequired()) { - int permissions_from,permissions_to; - permissions_from=mc_gState->m_Permissions->m_MempoolPermissions->GetCount(); - if(entry.FullReplayRequired()) - { - LOCK(pool.cs); - CCoinsView dummy; - CCoinsViewCache view(&dummy); - CCoinsViewMemPool viewMemPool(pcoinsTip, pool); - view.SetBackend(viewMemPool); - if(!AcceptMultiChainTransaction(tx,view,-1,accept,reason,NULL)) - { - removed_type="rejected"; - } - } - else + LOCK(pool.cs); + CCoinsView dummy; + CCoinsViewCache view(&dummy); + CCoinsViewMemPool viewMemPool(pcoinsTip, pool); + view.SetBackend(viewMemPool); + if(!AcceptMultiChainTransaction(tx,view,-1,accept,reason,NULL)) { - if(mc_gState->m_Permissions->MempoolPermissionsCheck(entry.ReplayPermissionFrom(),entry.ReplayPermissionTo()) == 0) - { - removed_type="rejected"; - } - } - if(removed_type.size() == 0) - { - permissions_to=mc_gState->m_Permissions->m_MempoolPermissions->GetCount(); - pool.mapTx[hash].SetReplayNodeParams(entry.FullReplayRequired(),permissions_from,permissions_to); + removed_type="rejected"; } } else - { - if(removed_type.size() == 0) - { - if(!AcceptPermissionsAndCheckForDust(tx,accept,reason)) - { - removed_type="permissions"; - } - } - if(removed_type.size() == 0) - { - if(!AcceptAssetGenesis(tx,-1,true,reason)) - { - removed_type="issue"; - } - } - if(removed_type.size() == 0) - { - LOCK(pool.cs); - CCoinsView dummy; - CCoinsViewCache view(&dummy); - CCoinsViewMemPool viewMemPool(pcoinsTip, pool); - view.SetBackend(viewMemPool); - if(!AcceptAssetTransfers(tx, view, reason)) - { - removed_type="transfer"; - } - } + { + if(mc_gState->m_Permissions->MempoolPermissionsCheck(entry.ReplayPermissionFrom(),entry.ReplayPermissionTo()) == 0) + { + removed_type="rejected"; + } + } + if(removed_type.size() == 0) + { + permissions_to=mc_gState->m_Permissions->m_MempoolPermissions->GetCount(); + pool.mapTx[hash].SetReplayNodeParams(entry.FullReplayRequired(),permissions_from,permissions_to); } } @@ -611,7 +578,6 @@ bool ReadTxFromDisk(CBlockIndex* pindex,int32_t offset,CTransaction& tx) bool VerifyBlockMiner(CBlock *block_in,CBlockIndex* pindexNew) { if( (mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) || - (mc_gState->m_Features->CachedInputScript() == 0) || (mc_gState->m_NetworkParams->GetInt64Param("supportminerprecheck") == 0) || (MCP_ANYONE_CAN_MINE) ) { diff --git a/src/protocol/multichainscript.cpp b/src/protocol/multichainscript.cpp index c0f5b87f..07789ec8 100644 --- a/src/protocol/multichainscript.cpp +++ b/src/protocol/multichainscript.cpp @@ -934,30 +934,23 @@ int mc_Script::SetScript(const unsigned char* src,const size_t bytes,int type) { if(lastSize == 0) { - if(mc_gState->m_Features->FixedIn10007()) + if(opcode > MC_DCT_SCRIPT_OP_16) // Not push data { - if(opcode > MC_DCT_SCRIPT_OP_16) // Not push data - { - lastSize=-1; - } - else + lastSize=-1; + } + else + { + if(opcode > MC_DCT_SCRIPT_OP_PUSHDATA4) { - if(opcode > MC_DCT_SCRIPT_OP_PUSHDATA4) + if(mc_gState->m_Features->FormattedData()) { - if(mc_gState->m_Features->FormattedData()) + if( (m_ScriptType & MC_DCT_SCRIPT_TYPE_OP_RETURN ) == 0) { - if( (m_ScriptType & MC_DCT_SCRIPT_TYPE_OP_RETURN ) == 0) - { - m_ScriptType |= MC_DCT_SCRIPT_TYPE_DIRTY_OP_RETURN; - } + m_ScriptType |= MC_DCT_SCRIPT_TYPE_DIRTY_OP_RETURN; } } } } - else - { - lastSize=-1; - } } } @@ -1777,20 +1770,10 @@ int mc_Script::GetNewEntityType(uint32_t *type) return MC_ERR_INVALID_PARAMETER_VALUE; } - if(mc_gState->m_Features->OpDropDetailsScripts()) - { - if(m_lpCoord[m_CurrentElement*2+1] < MC_DCT_SCRIPT_IDENTIFIER_LEN+1+1) - { - return MC_ERR_WRONG_SCRIPT; - } - } - else + if(m_lpCoord[m_CurrentElement*2+1] < MC_DCT_SCRIPT_IDENTIFIER_LEN+1+1) { - if(m_lpCoord[m_CurrentElement*2+1] != MC_DCT_SCRIPT_IDENTIFIER_LEN+1+1) - { - return MC_ERR_WRONG_SCRIPT; - } - } + return MC_ERR_WRONG_SCRIPT; + } ptr=m_lpData+m_lpCoord[m_CurrentElement*2+0]; @@ -1881,13 +1864,8 @@ int mc_Script::GetFullRef(unsigned char *ref,uint32_t *script_type) } new_ref=1; - shift=0; - ref_type=MC_AST_ASSET_REF_TYPE_REF; - if(mc_gState->m_Features->ShortTxIDInTx()) - { - shift=MC_AST_SHORT_TXID_OFFSET; - ref_type=MC_AST_ASSET_REF_TYPE_SHORT_TXID; - } + shift=MC_AST_SHORT_TXID_OFFSET; + ref_type=MC_AST_ASSET_REF_TYPE_SHORT_TXID; memset(ref,0,MC_AST_ASSET_FULLREF_SIZE); for(i=0;im_Features->ShortTxIDInTx()) - { - shift=MC_AST_SHORT_TXID_OFFSET; - ref_type=MC_AST_ASSET_REF_TYPE_SHORT_TXID; - } + shift=MC_AST_SHORT_TXID_OFFSET; + ref_type=MC_AST_ASSET_REF_TYPE_SHORT_TXID; items=(m_lpCoord[m_CurrentElement*2+1] - (MC_DCT_SCRIPT_IDENTIFIER_LEN+1)) / (mc_gState->m_NetworkParams->m_AssetRefSize + MC_AST_ASSET_QUANTITY_SIZE); @@ -2064,11 +2037,7 @@ int mc_Script::SetAssetQuantities(mc_Buffer *amounts,uint32_t script_type) return err; } - shift=0; - if(mc_gState->m_Features->ShortTxIDInTx()) - { - shift=MC_AST_SHORT_TXID_OFFSET; - } + shift=MC_AST_SHORT_TXID_OFFSET; for(i=0;iGetCount();i++) @@ -2294,7 +2263,6 @@ int mc_Script::SetRawData(const unsigned char *data,const int size) int mc_Script::GetDataFormat(uint32_t *format) { unsigned char *ptr; - unsigned char *ptrEnd; unsigned char f; if(format) @@ -2313,7 +2281,6 @@ int mc_Script::GetDataFormat(uint32_t *format) } ptr=m_lpData+m_lpCoord[m_CurrentElement*2+0]; - ptrEnd=ptr+m_lpCoord[m_CurrentElement*2+1]; if(memcmp(ptr,MC_DCT_SCRIPT_MULTICHAIN_IDENTIFIER,MC_DCT_SCRIPT_IDENTIFIER_LEN) != 0) { diff --git a/src/protocol/multichaintx.cpp b/src/protocol/multichaintx.cpp index b2363258..5fe4e608 100644 --- a/src/protocol/multichaintx.cpp +++ b/src/protocol/multichaintx.cpp @@ -825,6 +825,8 @@ bool MultiChainTransaction_CheckEntityItem(const CTransaction& tx, unsigned char short_txid[MC_AST_SHORT_TXID_SIZE]; mc_EntityDetails entity; + mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(NULL); // Format scripts are eliminated for protocol checks + mc_gState->m_TmpScript->SetElement(0); // Should be spke if(mc_gState->m_TmpScript->GetEntity(short_txid)) // Entity element @@ -2030,3012 +2032,276 @@ bool AcceptMultiChainTransaction (const CTransaction& tx, } -bool AcceptAssetGenesisFromPredefinedIssuers(const CTransaction &tx, - int offset,bool accept, - vector vInputDestinations, - vector vInputHashTypes, - vector vInputScriptTypes, - bool fReturnFalseIfFound, - string& reason, - bool *fFullReplayCheckRequired) -{ - int update_mempool=0; - mc_EntityDetails entity; - mc_EntityDetails this_entity; - unsigned char details_script[MC_ENT_MAX_SCRIPT_SIZE]; - char asset_name[MC_ENT_MAX_NAME_SIZE+1]; - int multiple; - int details_script_size; +bool AcceptAdminMinerPermissions(const CTransaction& tx, + int offset, + bool verify_signatures, + string& reason, + uint32_t *result) +{ + vector vInputScriptTypes; + vector vInputDestinations; + vector vInputHashTypes; + vector vInputCanGrantAdminMine; + vector vInputHadAdminPermissionBeforeThisTx; + vector vInputPrevOutputScripts; + bool fIsEntity; + bool fReject; + bool fAdminFound; int err; - int64_t quantity,total; - uint256 txid; - bool new_issue,follow_on,issue_in_output; - int details_script_type; - unsigned char *ptrOut; - vector issuers; - vector issuer_flags; - uint32_t flags; - uint32_t value_offset; - size_t value_size; - unsigned char short_txid[MC_AST_SHORT_TXID_SIZE]; - - - if(mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) + + if(result) { - return true; - } + *result=0; + } + if(tx.IsCoinBase()) { return true; } - if(accept) - { - update_mempool=1; + for (unsigned int i = 0; i < tx.vin.size(); i++) + { + vInputCanGrantAdminMine.push_back(false); + vInputPrevOutputScripts.push_back(CScript()); + vInputDestinations.push_back(0); } - total=0; - - asset_name[0]=0; - multiple=1; - new_issue=false; - follow_on=false; - details_script_type=-1; - mc_gState->m_TmpAssetsOut->Clear(); - - details_script_size=0; + fReject=false; + fAdminFound=false; for (unsigned int j = 0; j < tx.vout.size(); j++) { - mc_gState->m_TmpScript->Clear(); - + int cs_offset,cs_new_offset,cs_size,cs_vin; + unsigned char *cs_script; + const CScript& script1 = tx.vout[j].scriptPubKey; CScript::const_iterator pc1 = script1.begin(); - uint32_t new_entity_type; - int entity_update; - - CTxDestination addressRet; - + + + mc_gState->m_TmpScript->Clear(); mc_gState->m_TmpScript->SetScript((unsigned char*)(&pc1[0]),(size_t)(script1.end()-pc1),MC_SCR_TYPE_SCRIPTPUBKEY); - - if(mc_gState->m_TmpScript->IsOpReturnScript()) - { - if(mc_gState->m_Features->OpDropDetailsScripts()) + + if(mc_gState->m_TmpScript->IsOpReturnScript()) + { + if( mc_gState->m_TmpScript->GetNumElements() == 2 ) { - if(mc_gState->m_TmpScript->GetNumElements() == 2 ) // One OP_DROP + OP_RETRUN - new entity + mc_gState->m_TmpScript->SetElement(0); + + cs_offset=0; + while( (err=mc_gState->m_TmpScript->GetCachedScript(cs_offset,&cs_new_offset,&cs_vin,&cs_script,&cs_size)) != MC_ERR_WRONG_SCRIPT ) { - mc_gState->m_TmpScript->SetElement(0); - err=mc_gState->m_TmpScript->GetNewEntityType(&new_entity_type,&entity_update,details_script,&details_script_size); - if(err == 0) // New entity element + if(err != MC_ERR_NOERROR) { - if(new_entity_type == MC_ENT_TYPE_ASSET) - { - if(details_script_type >= 0) - { - reason="Metadata script rejected - too many new entities"; - return false; - } - if(entity_update == 0) - { - details_script_type=entity_update; - *asset_name=0x00; - multiple=1; - value_offset=mc_FindSpecialParamInDetailsScript(details_script,details_script_size,MC_ENT_SPRM_NAME,&value_size); - if(value_offset<(uint32_t)details_script_size) - { - memcpy(asset_name,details_script+value_offset,value_size); - asset_name[value_size]=0x00; - } - value_offset=mc_FindSpecialParamInDetailsScript(details_script,details_script_size,MC_ENT_SPRM_ASSET_MULTIPLE,&value_size); - if(value_offset<(uint32_t)details_script_size) - { - multiple=mc_GetLE(details_script+value_offset,value_size); - } - } - } - } - else - { - if(err != MC_ERR_WRONG_SCRIPT) - { - reason="Asset details script rejected - error in script"; - return false; - } + reason="Metadata script rejected - error in cached script"; + fReject=true; + goto exitlbl; } - - } - if(mc_gState->m_TmpScript->GetNumElements()) // 2 OP_DROPs + OP_RETURN - entity update - { - mc_gState->m_TmpScript->SetElement(1); - err=mc_gState->m_TmpScript->GetNewEntityType(&new_entity_type,&entity_update,details_script,&details_script_size); - if(err == 0) // New entity element - { - if(entity_update == 0) - { - reason="Metadata script rejected - wrong element, should be entity update"; - return false; - } - if(details_script_type >= 0) - { - reason="Metadata script rejected - too many new new entities/entity updates"; - return false; - } - if(new_entity_type == MC_ENT_TYPE_ASSET) - { - details_script_type=entity_update; - mc_gState->m_TmpScript->SetElement(0); - // Should be spke - if(mc_gState->m_TmpScript->GetEntity(short_txid)) // Entity element - { - reason="Metadata script rejected - wrong element, should be entityref"; - return false; - } - if(mc_gState->m_Assets->FindEntityByShortTxID(&entity,short_txid) == 0) - { - reason="Metadata script rejected - entity not found"; - return false; - } - } - else - { - reason="Metadata script rejected - wrong entity type, should be asset"; - return false; - } - } - } - } - else - { - if(details_script_size == 0) - { - int e=mc_gState->m_TmpScript->GetNumElements()-1; - if(e == 0) + if(cs_offset) { - mc_gState->m_TmpScript->SetElement(e); - err=mc_gState->m_TmpScript->GetAssetDetails(asset_name,&multiple,details_script,&details_script_size); - if(err) + if( cs_vin >= (int)tx.vin.size() ) { - if(err != MC_ERR_WRONG_SCRIPT) - { - reason="Asset details script rejected - error in script"; - return false; - } - details_script_size=0; + reason="Metadata script rejected - invalid input in cached script"; + fReject=true; + goto exitlbl; } + vInputPrevOutputScripts[cs_vin]=CScript(cs_script,cs_script+cs_size); + vInputCanGrantAdminMine[cs_vin]=true; } + cs_offset=cs_new_offset; } } } - else + } + + for (unsigned int i = 0; i < tx.vin.size(); i++) + { + if(vInputCanGrantAdminMine[i]) { - mc_gState->m_TmpAssetsTmp->Clear(); - issue_in_output=false; - - for (int e = 0; e < mc_gState->m_TmpScript->GetNumElements(); e++) + vInputCanGrantAdminMine[i]=false; + const CScript& script2 = tx.vin[i].scriptSig; + CScript::const_iterator pc2 = script2.begin(); + if (!script2.IsPushOnly()) { - mc_gState->m_TmpScript->SetElement(e); - err=mc_gState->m_TmpScript->GetAssetGenesis(&quantity); - if(err == 0) + reason="sigScript should be push-only"; + fReject=true; + goto exitlbl; + } + + const CScript& script1 = vInputPrevOutputScripts[i]; + CScript::const_iterator pc1 = script1.begin(); + + txnouttype typeRet; + int nRequiredRet; + vector addressRets; + int op_addr_offset,op_addr_size,is_redeem_script,sighash_type,check_last; + + sighash_type=SIGHASH_NONE; + if(ExtractDestinations(script1,typeRet,addressRets,nRequiredRet)) + { + if ( (typeRet != TX_NULL_DATA) && (typeRet != TX_MULTISIG) ) { - issue_in_output=true; - new_issue=true; - if(quantity+total<0) - { - reason="Asset issue script rejected - overflow"; - return false; - } - - total+=quantity; - - if(!ExtractDestination(script1, addressRet)) - { - reason="Asset issue script rejected - wrong destination type"; - return false; - } - - CKeyID *lpKeyID=boost::get (&addressRet); - CScriptID *lpScriptID=boost::get (&addressRet); - if((lpKeyID == NULL) && (lpScriptID == NULL)) - { - reason="Asset issue script rejected - wrong destination type"; - return false; - } - CBitcoinAddress address; - if(lpKeyID != NULL) + CKeyID *lpKeyID=boost::get (&addressRets[0]); + CScriptID *lpScriptID=boost::get (&addressRets[0]); + if( (lpKeyID == NULL) && (lpScriptID == NULL) ) { - address=CBitcoinAddress(*lpKeyID); + fReject=true; + goto exitlbl; } - else + if(lpKeyID) { - address=CBitcoinAddress(*lpScriptID); + vInputDestinations[i]=*(uint160*)lpKeyID; } - if(update_mempool) + if(lpScriptID) { - if(fDebug)LogPrint("mchn","Found asset issue script in tx %s for %s - (%ld)\n", - tx.GetHash().GetHex().c_str(), - address.ToString().c_str(),quantity); + vInputDestinations[i]=*(uint160*)lpScriptID; } - } - else - { - if(err != MC_ERR_WRONG_SCRIPT) + + check_last=0; + if( typeRet == TX_PUBKEY ) { - reason="Asset issue script rejected - error in script"; - return false; + check_last=1; } - if(mc_gState->m_Features->PerAssetPermissions()) - { - err=mc_gState->m_TmpScript->GetAssetQuantities(mc_gState->m_TmpAssetsTmp,MC_SCR_ASSET_SCRIPT_TYPE_TRANSFER); - if((err != MC_ERR_NOERROR) && (err != MC_ERR_WRONG_SCRIPT)) - { - reason="Script rejected - error in asset transfer script"; - return false; - } - } - err=mc_gState->m_TmpScript->GetAssetQuantities(mc_gState->m_TmpAssetsOut,MC_SCR_ASSET_SCRIPT_TYPE_FOLLOWON); - if((err != MC_ERR_NOERROR) && (err != MC_ERR_WRONG_SCRIPT)) - { - reason="Asset follow-on script rejected - error in follow-on script"; - return false; - } - if(err == MC_ERR_NOERROR) + // Find sighash_type + mc_ExtractAddressFromInputScript((unsigned char*)(&pc2[0]),(int)(script2.end()-pc2),&op_addr_offset,&op_addr_size,&is_redeem_script,&sighash_type,check_last); + if(sighash_type == SIGHASH_ALL) { - if(update_mempool) + if(mc_gState->m_Permissions->CanAdmin(NULL,(unsigned char*)&vInputDestinations[i])) { - if(fDebug)LogPrint("mchn","Found asset follow-on script in tx %s\n", - tx.GetHash().GetHex().c_str()); - } + vInputCanGrantAdminMine[i]=true; + if(verify_signatures) + { + vInputCanGrantAdminMine[i]=false; + if(VerifyScript(script2, script1, STANDARD_SCRIPT_VERIFY_FLAGS, CachingTransactionSignatureChecker(&tx, i, false))) + { + vInputCanGrantAdminMine[i]=true; + } + else + { + reason="Signature verification error"; + fReject=true; + goto exitlbl; + } + } + } } - } - } - if(issue_in_output) - { - if(mc_gState->m_Features->PerAssetPermissions()) - { - if(mc_gState->m_TmpAssetsTmp->GetCount()) - { - reason="Asset issue script rejected - asset transfer in script"; - return false; - } } - } - } + } + } } - - if(details_script_type >= 0) - { - if(details_script_type) - { - follow_on=true; - } - else - { - new_issue=true; - } - } - - if(mc_gState->m_TmpAssetsOut->GetCount()) - { - follow_on=true; - } - - if(follow_on) - { - total=0; - if(mc_gState->m_TmpAssetsOut->GetCount() > 1) - { - reason="Asset follow-on script rejected - follow-on for several assets"; - return false; - } - if(new_issue) - { - reason="Asset follow-on script rejected - follow-on and issue in one transaction"; - return false; - } - ptrOut=NULL; - if(mc_gState->m_TmpAssetsOut->GetCount() == 0) - { - if(mc_gState->m_Assets->FindEntityByShortTxID(&entity,short_txid) == 0) - { - reason="Details script rejected - entity not found"; - return false; - } - } - else - { - ptrOut=mc_gState->m_TmpAssetsOut->GetRow(0); - if(mc_gState->m_Assets->FindEntityByFullRef(&entity,ptrOut) == 0) - { - reason="Asset follow-on script rejected - asset not found"; - return false; - } - } - if(mc_gState->m_Features->ShortTxIDInTx() == 0) - { - if(entity.IsUnconfirmedGenesis()) - { - reason="Asset follow-on script rejected - unconfirmed asset"; - return false; - } - } - if(entity.AllowedFollowOns() == 0) - { - reason="Asset follow-on script rejected - follow-ons not allowed for this asset"; - return false; - } - if(details_script_type >= 0) - { - if(memcmp(entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,short_txid,MC_AST_SHORT_TXID_SIZE)) - { - reason="Asset follow-on script rejected - mismatch in follow-on quantity asset and details script"; - return false; - } - } - if(ptrOut) - { - total=mc_GetABQuantity(ptrOut); - if(total+mc_gState->m_Assets->GetTotalQuantity(&entity) < 0) - { - reason="Asset follow-on script rejected - exceeds maximal value for asset"; - return false; - } - } - } - - if(!new_issue && !follow_on) - { - return true; - } - - if(fReturnFalseIfFound) - { - { - reason="Asset issue script rejected - not allowed in this transaction, conflicts with other entities"; - return false; - } - } - - *fFullReplayCheckRequired=true; - - issuers.clear(); - if(vInputDestinations.size()) - { - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - if(vInputHashTypes[i] == SIGHASH_ALL) - { - if(vInputDestinations[i] != 0) - { - bool can_issue=false; - if(new_issue) - { - if(mc_gState->m_Permissions->CanIssue(NULL,(unsigned char*)&vInputDestinations[i])) - { - can_issue=true; - } - } - else - { - if(mc_gState->m_Permissions->CanIssue(entity.GetTxID(),(unsigned char*)&vInputDestinations[i])) - { - can_issue=true; - } - } - if(can_issue) - { - issuers.push_back(vInputDestinations[i]); - flags=MC_PFL_NONE; - if(vInputScriptTypes[i] == TX_SCRIPTHASH) - { - flags |= MC_PFL_IS_SCRIPTHASH; - } - issuer_flags.push_back(flags); - } - } - } - } - } - else - { - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - const CScript& script2 = tx.vin[i].scriptSig; - CScript::const_iterator pc2 = script2.begin(); - - mc_gState->m_TmpScript->Clear(); - mc_gState->m_TmpScript->SetScript((unsigned char*)(&pc2[0]),(size_t)(script2.end()-pc2),MC_SCR_TYPE_SCRIPTSIG); - - if(mc_gState->m_TmpScript->GetNumElements() == 2) - { - size_t elem_size; - const unsigned char *elem; - - elem = mc_gState->m_TmpScript->GetData(0,&elem_size); - - if(elem_size > 1) // If this is multisig with one signature it should be OP_0 - { - unsigned char hash_type=elem[elem_size-1] & 0x1f; - - if(hash_type == SIGHASH_ALL) - { - elem = mc_gState->m_TmpScript->GetData(1,&elem_size); - const unsigned char *pubkey_hash=(unsigned char *)Hash160(elem,elem+elem_size).begin(); - - if(new_issue) - { - if(mc_gState->m_Permissions->CanIssue(NULL,pubkey_hash)) - { - issuers.push_back(Hash160(elem,elem+elem_size)); - } - } - else - { - if(mc_gState->m_Permissions->CanIssue(entity.GetTxID(),pubkey_hash)) - { - issuers.push_back(Hash160(elem,elem+elem_size)); - } - } - } - } - } - } - } - - if(issuers.size() == 0) - { - reason="Inputs don't belong to valid issuer"; - return false; - } - - err=MC_ERR_NOERROR; - - mc_gState->m_TmpScript->Clear(); - mc_gState->m_TmpScript->AddElement(); - unsigned char issuer_buf[24]; - memset(issuer_buf,0,sizeof(issuer_buf)); - flags=MC_PFL_NONE; - uint32_t timestamp=0; - set stored_issuers; - - if(new_issue) - { - mc_gState->m_Permissions->SetCheckPoint(); - err=MC_ERR_NOERROR; - - txid=tx.GetHash(); - err=mc_gState->m_Permissions->SetPermission(&txid,issuer_buf,MC_PTP_CONNECT, - (unsigned char*)issuers[0].begin(),0,(uint32_t)(-1),timestamp,flags | MC_PFL_ENTITY_GENESIS ,update_mempool,offset); - } - - uint32_t all_permissions=MC_PTP_ADMIN | MC_PTP_ISSUE; - if(mc_gState->m_Features->PerAssetPermissions()) - { - all_permissions |= MC_PTP_ACTIVATE | MC_PTP_SEND | MC_PTP_RECEIVE; - } - - for (unsigned int i = 0; i < issuers.size(); i++) - { - if(err == MC_ERR_NOERROR) - { - if(stored_issuers.count(issuers[i]) == 0) - { - memcpy(issuer_buf,issuers[i].begin(),sizeof(uint160)); - mc_PutLE(issuer_buf+sizeof(uint160),&issuer_flags[i],4); - if((int)i < mc_gState->m_Assets->MaxStoredIssuers()) - { - mc_gState->m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_ISSUER,issuer_buf,sizeof(issuer_buf)); - } - if(new_issue) - { - err=mc_gState->m_Permissions->SetPermission(&txid,issuer_buf,all_permissions, - (unsigned char*)issuers[0].begin(),0,(uint32_t)(-1),timestamp,flags | MC_PFL_ENTITY_GENESIS ,update_mempool,offset); - } - stored_issuers.insert(issuers[i]); - } - } - } - - memset(issuer_buf,0,sizeof(issuer_buf)); - mc_gState->m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_ISSUER,issuer_buf,1); - if(new_issue) - { - if((err != MC_ERR_NOERROR) || !accept) - { - mc_gState->m_Permissions->RollBackToCheckPoint(); - } - } - if(err) - { - reason="Cannot update permission database for issued asset"; - return false; - } - - const unsigned char *special_script; - size_t special_script_size=0; - special_script=mc_gState->m_TmpScript->GetData(0,&special_script_size); - txid=tx.GetHash(); - if(new_issue) - { - err=mc_gState->m_Assets->InsertAsset(&txid,offset,total,asset_name,multiple,details_script,details_script_size,special_script,special_script_size,update_mempool); - } - else - { - err=mc_gState->m_Assets->InsertAssetFollowOn(&txid,offset,total,details_script,details_script_size,special_script,special_script_size,entity.GetTxID(),update_mempool); - } - - if(err) - { - reason="Asset issue script rejected - could not insert new asset to database"; - if(err == MC_ERR_FOUND) - { - reason="Asset issue script rejected - entity with this name/asset-ref/txid already exists"; - if(mc_gState->m_Assets->FindEntityByTxID(&entity,(unsigned char*)&txid) == 0) - { - if(strlen(asset_name) == 0) - { - mc_gState->m_Assets->FindEntityByName(&entity,asset_name); - } - } - - if(fDebug)LogPrint("mchn","Asset already exists. TxID: %s, AssetRef: %d-%d-%d, Name: %s\n", - tx.GetHash().GetHex().c_str(), - mc_gState->m_Assets->m_Block+1,offset,(int)(*((unsigned char*)&txid+31))+256*(int)(*((unsigned char*)&txid+30)), - entity.GetName()); - } - return false; - } - - - if(update_mempool) - { - if(offset>=0) - { - if(mc_gState->m_Assets->FindEntityByTxID(&this_entity,(unsigned char*)&txid)) - { - if(new_issue) - { - if(fDebug)LogPrint("mchn","New asset. TxID: %s, AssetRef: %d-%d-%d, Name: %s\n", - tx.GetHash().GetHex().c_str(), - mc_gState->m_Assets->m_Block+1,offset,(int)(*((unsigned char*)&txid+0))+256*(int)(*((unsigned char*)&txid+1)), - this_entity.GetName()); - } - else - { - uint256 otxid; - memcpy(&otxid,entity.GetTxID(),32); - if(fDebug)LogPrint("mchn","Follow-on issue. TxID: %s, Original issue txid: %s\n", - tx.GetHash().GetHex().c_str(),otxid.GetHex().c_str()); - } - } - else - { - reason="Asset issue script rejected - could not insert new asset to database"; - return false; - } - } - } - - return true; -} - - -bool AcceptMultiChainTransactionOld(const CTransaction& tx, - const CCoinsViewCache &inputs, - int offset, - bool accept, - string& reason, - uint32_t *replay) -{ - bool fScriptHashAllFound; - bool fSeedNodeInvolved; - bool fReject; - bool fShouldHaveDestination; - bool fRejectIfOpDropOpReturn; - bool fCheckCachedScript; - bool fFullReplayCheckRequired; - bool fAdminMinerGrant; - int nNewEntityOutput; - unsigned char details_script[MC_ENT_MAX_SCRIPT_SIZE]; - int details_script_size; - uint32_t new_entity_type; - int err; - - vector vInputScriptTypes; - vector vInputDestinations; - vector vInputHashTypes; - vector vInputCanGrantAdminMine; - vector vInputHadAdminPermissionBeforeThisTx; - set vAllowedAdmins; - set vAllowedActivators; - - if(mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) - { - return true; - } - - details_script_size=0; - new_entity_type=MC_ENT_TYPE_NONE; - - fCheckCachedScript=false; - if(mc_gState->m_Features->CachedInputScript()) - { - if(mc_gState->m_NetworkParams->GetInt64Param("supportminerprecheck")) - { - fCheckCachedScript=true; - } - } - - fFullReplayCheckRequired=false; - fAdminMinerGrant=false; - fScriptHashAllFound=false; - fRejectIfOpDropOpReturn=false; - mc_gState->m_TmpAssetsIn->Clear(); - for (unsigned int i = 0; i < tx.vin.size(); i++) // Processing input scripts - { - if(tx.IsCoinBase()) - { - if(i == 0) - { - if(mc_gState->m_Permissions->m_Block == -1) - { - vInputScriptTypes.push_back(TX_PUBKEYHASH); - vInputDestinations.push_back(mc_GenesisAdmin(tx)); // Genesis admin is considered to be admin/opener of everything in genesis block - fCheckCachedScript=false; - } - else - { - vInputScriptTypes.push_back(TX_NONSTANDARD); - vInputDestinations.push_back(0); - } - fScriptHashAllFound=true; // Allows to transfer OP_RETURN metadata in this tx. - vInputHashTypes.push_back(SIGHASH_ALL); - } - else - { - vInputScriptTypes.push_back(TX_NONSTANDARD); - vInputDestinations.push_back(0); - vInputHashTypes.push_back(SIGHASH_NONE); - } - } - else - { - const CScript& script2 = tx.vin[i].scriptSig; - CScript::const_iterator pc2 = script2.begin(); - if(mc_gState->m_Features->FixedIn10007()) - { - if (!script2.IsPushOnly()) - { - reason="sigScript should be push-only"; - return false; - } - } - - const COutPoint &prevout = tx.vin[i].prevout; - const CCoins *coins = inputs.AccessCoins(prevout.hash); - assert(coins); - - const CScript& script1 = coins->vout[prevout.n].scriptPubKey; - CScript::const_iterator pc1 = script1.begin(); - - txnouttype typeRet; - int nRequiredRet; - vector addressRets; - int op_addr_offset,op_addr_size,is_redeem_script,sighash_type,check_last; - - sighash_type=SIGHASH_NONE; - if(ExtractDestinations(script1,typeRet,addressRets,nRequiredRet)) // Is Standard transaction - { - if (typeRet != TX_NULL_DATA) // Single-address destinations - { - CKeyID *lpKeyID=boost::get (&addressRets[0]); - CScriptID *lpScriptID=boost::get (&addressRets[0]); - if( (lpKeyID == NULL) && (lpScriptID == NULL) ) - { - reason="Internal error: cannot extract address from input scriptPubKey"; - return false; - } - else - { - if(typeRet != TX_MULTISIG) - { - if(lpKeyID) - { - vInputDestinations.push_back(*(uint160*)lpKeyID); - } - if(lpScriptID) - { - vInputDestinations.push_back(*(uint160*)lpScriptID); - } - } - else - { - vInputDestinations.push_back(0); - } - } - - check_last=0; - if( (typeRet == TX_PUBKEY) || (typeRet == TX_MULTISIG) ) - { - check_last=1; - } - - // Find sighash_type - mc_ExtractAddressFromInputScript((unsigned char*)(&pc2[0]),(int)(script2.end()-pc2),&op_addr_offset,&op_addr_size,&is_redeem_script,&sighash_type,check_last); - if(sighash_type == SIGHASH_ALL) - { - fScriptHashAllFound=true; - } - if(mc_gState->m_Features->FixedIn10008()) - { - if(sighash_type == SIGHASH_SINGLE) - { - if(i >= tx.vout.size()) - { - reason="SIGHASH_SINGLE input without matching output"; - return false; - } - } - } - if(check_last) - { - fRejectIfOpDropOpReturn=true; // pay-to-pubkey and bare multisig script cannot be considered "publisher" for the stream, because we cannot - // extract it from the input script. Though we can still accept this transaction it is rejected - // for consistency with principle "each input which signs the stream item must contain a permitted writer" - } - } - else - { - fRejectIfOpDropOpReturn=true; // Null data scripts cannot be used in txs with OP_DROP+OP_RETURN - // We should not be there at all as null data scripts cannot be signed generally speaking - vInputDestinations.push_back(0); - } - vInputScriptTypes.push_back(typeRet); - } - else - { - fRejectIfOpDropOpReturn=true; // Non-standard inputs cannot be used in txs with OP_DROP+OP_RETURN - // Otherwise we cannot be sure where are the signatures in input script - vInputScriptTypes.push_back(TX_NONSTANDARD); - vInputDestinations.push_back(0); - } - - vInputHashTypes.push_back(sighash_type); - - if(mc_gState->m_Features->PerAssetPermissions()) - { - mc_gState->m_TmpAssetsTmp->Clear(); - if(!mc_ExtractInputAssetQuantities(mc_gState->m_TmpAssetsTmp,script1,prevout.hash,reason)) // Filling input asset quantity list - { - return false; - } - if(!mc_VerifyAssetPermissions(mc_gState->m_TmpAssetsTmp,addressRets,1,MC_PTP_SEND,reason)) - { - return false; - } - } - - if(!mc_ExtractInputAssetQuantities(mc_gState->m_TmpAssetsIn,script1,prevout.hash,reason)) - { - return false; - } - } - - vInputCanGrantAdminMine.push_back(!fCheckCachedScript); - } - - fReject=false; - nNewEntityOutput=-1; - fSeedNodeInvolved=false; - fShouldHaveDestination=false; - fShouldHaveDestination |= (MCP_ALLOW_ARBITRARY_OUTPUTS == 0); - fShouldHaveDestination |= (MCP_ANYONE_CAN_RECEIVE == 0); - fShouldHaveDestination |= (MCP_ALLOW_MULTISIG_OUTPUTS == 0); - fShouldHaveDestination |= (MCP_ALLOW_P2SH_OUTPUTS == 0); - - mc_gState->m_Permissions->SetCheckPoint(); - mc_gState->m_TmpAssetsOut->Clear(); - - // Processing output scripts - // pass 0 - all permissions, except mine, admin and activate + script type checks - // pass 1 - mine permissions (to retrieve cached scripts in pass=0) - // pass 2 - admin and activate permissions (to retrive list of admins in pass=0) - // pass 3 - everything except permissions - balance, assets, streams - for(int pass=0;pass<4;pass++) - { - for (unsigned int j = 0; j < tx.vout.size(); j++) - { - bool fNoDestinationInOutput; - bool fIsPurePermission; - bool fAdminRequired; - bool fCheckAdminList; - bool fAllValidPublishers; - bool fScriptParsed; - unsigned char short_txid[MC_AST_SHORT_TXID_SIZE]; - mc_EntityDetails entity; - int receive_required; - uint32_t type,from,to,timestamp,flags,approval; - unsigned char item_key[MC_ENT_MAX_ITEM_KEY_SIZE]; - int item_key_size,entity_update; - int cs_offset,cs_new_offset,cs_size,cs_vin; - unsigned char *cs_script; - - const CScript& script1 = tx.vout[j].scriptPubKey; - CScript::const_iterator pc1 = script1.begin(); - - - mc_gState->m_TmpScript->Clear(); - mc_gState->m_TmpScript->SetScript((unsigned char*)(&pc1[0]),(size_t)(script1.end()-pc1),MC_SCR_TYPE_SCRIPTPUBKEY); - - if(mc_gState->m_TmpScript->IsOpReturnScript()) // OP_RETURN - { - mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(NULL); - - if( (pass == 0) && !fScriptHashAllFound) - { - if(mc_gState->m_Features->FixedIn10007()) - { - if( (vInputHashTypes.size() <= j) || (vInputHashTypes[j] != SIGHASH_SINGLE) ) - { - reason="Output with metadata should be properly signed"; - fReject=true; - goto exitlbl; - } - } - else - { - reason="Tx with metadata should have at least one SIGHASH_ALL output"; - fReject=true; - goto exitlbl; - } - } - - if( (pass == 0) && (mc_gState->m_TmpScript->GetNumElements() > 1) ) // We have at least one OP_DROP element - { - if(fRejectIfOpDropOpReturn) // We cannot extract sighash_type properly from the input script - // as we don't know where are the signatures - { - reason="Non-standard, P2PK or bare multisig inputs cannot be used in this tx"; - fReject=true; - goto exitlbl; - } - - if(mc_gState->m_TmpScript->IsDirtyOpReturnScript()) - { - reason="Non-standard, Only OP_DROP elements are allowed in metadata outputs with OP_DROP"; - fReject=true; - goto exitlbl; - } - } - - if( (pass == 0) && (mc_gState->m_TmpScript->GetNumElements() == 2) ) // One OP_DROP + OP_RETURN - new entity or cached script - { - mc_gState->m_TmpScript->SetElement(0); - fScriptParsed=false; - - if(mc_gState->m_Features->CachedInputScript()) - { - cs_offset=0; - while( (err=mc_gState->m_TmpScript->GetCachedScript(cs_offset,&cs_new_offset,&cs_vin,&cs_script,&cs_size)) != MC_ERR_WRONG_SCRIPT ) - { - fScriptParsed=true; - if(err != MC_ERR_NOERROR) - { - reason="Metadata script rejected - error in cached script"; - fReject=true; - goto exitlbl; - } - if(cs_offset) - { - if( cs_vin >= (int)tx.vin.size() ) - { - reason="Metadata script rejected - invalid input in cached script"; - fReject=true; - goto exitlbl; - } - const COutPoint &prevout = tx.vin[cs_vin].prevout; - const CCoins *coins = inputs.AccessCoins(prevout.hash); - - const CScript& script3 = coins->vout[prevout.n].scriptPubKey; - CScript::const_iterator pc3 = script3.begin(); - - if(cs_size != (int)script3.size()) - { - reason="Metadata script rejected - cached script mismatch"; - fReject=true; - goto exitlbl; - } - if(memcmp(cs_script,(unsigned char*)&pc3[0],cs_size)) - { - reason="Metadata script rejected - cached script mismatch"; - fReject=true; - goto exitlbl; - } - if(fCheckCachedScript) - { - if(vInputHashTypes[cs_vin] == SIGHASH_ALL) - { - vInputCanGrantAdminMine[cs_vin]=true; - } - } - } - cs_offset=cs_new_offset; - } - } - - if(mc_gState->m_Features->OpDropDetailsScripts()) - { - err=mc_gState->m_TmpScript->GetNewEntityType(&new_entity_type,&entity_update,details_script,&details_script_size); - if(err == 0) // New entity element - { - fScriptParsed=true; - if(nNewEntityOutput >= 0) - { - reason="Metadata script rejected - too many new entities"; - fReject=true; - goto exitlbl; - } - if(entity_update) - { - reason="Metadata script rejected - entity update script should be preceded by entityref"; - fReject=true; - goto exitlbl; - } - if(new_entity_type <= mc_gState->m_Assets->MaxEntityType()) - { - if(new_entity_type != MC_ENT_TYPE_ASSET) - { - nNewEntityOutput=j; - } - } - else - { - reason="Metadata script rejected - unsupported new entity type"; - fReject=true; - goto exitlbl; - } - } - else - { - if(err != MC_ERR_WRONG_SCRIPT) - { - reason="Entity details script rejected - error in script"; - fReject=true; - goto exitlbl; - } - } - } - else - { - if(mc_gState->m_TmpScript->GetNewEntityType(&new_entity_type) == 0) // New entity element - { - fScriptParsed=true; - if(nNewEntityOutput >= 0) - { - reason="Metadata script rejected - too many new entities"; - fReject=true; - goto exitlbl; - } - if(new_entity_type != MC_ENT_TYPE_STREAM) - { - reason="Metadata script rejected - unsupported new entity type"; - fReject=true; - goto exitlbl; - } - nNewEntityOutput=j; - // Should be SPKc - mc_gState->m_TmpScript->SetElement(1); - err=mc_gState->m_TmpScript->GetGeneralDetails(details_script,&details_script_size); - if(err) - { - if(err != MC_ERR_WRONG_SCRIPT) - { - reason="Entity details script rejected - error in script"; - fReject=true; - goto exitlbl; - } - details_script_size=0; - } - } - } - if(!fScriptParsed) - { - reason="Metadata script rejected - Unrecognized script, should be new entity"; - if(mc_gState->m_Features->CachedInputScript()) - { - reason+=" or input script cache"; - } - fReject=true; - goto exitlbl; - } - } - - if( (pass == 0) && - (mc_gState->m_TmpScript->GetNumElements() > 3) && - (mc_gState->m_Features->MultipleStreamKeys() == 0) ) // More than 2 OP_DROPs - { - reason="Metadata script rejected - too many elements"; - fReject=true; - goto exitlbl; - } - - if( (pass == 0) && (mc_gState->m_TmpScript->GetNumElements() == 3) ) // 2 OP_DROPs + OP_RETURN - possible upgrade approval - { - if(mc_gState->m_Features->Upgrades()) - { - mc_gState->m_TmpScript->SetElement(1); - - if(mc_gState->m_TmpScript->GetApproval(&approval,×tamp) == 0) - { - if(vInputHadAdminPermissionBeforeThisTx.size() == 0) - { - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - vInputHadAdminPermissionBeforeThisTx.push_back(mc_gState->m_Permissions->CanAdmin(NULL,(unsigned char*)&vInputDestinations[i]) != 0); - } - } - } - } - } - - if( (pass == 3) && (mc_gState->m_TmpScript->GetNumElements() >= 3) ) // 2 OP_DROPs + OP_RETURN - item key or entity update or upgrade approval - { - mc_gState->m_TmpScript->SetElement(0); - // Should be spke - if(mc_gState->m_TmpScript->GetEntity(short_txid)) // Entity element - { - reason="Metadata script rejected - wrong element, should be entityref"; - fReject=true; - goto exitlbl; - } - if(mc_gState->m_Assets->FindEntityByShortTxID(&entity,short_txid) == 0) - { - reason="Metadata script rejected - entity not found"; - fReject=true; - goto exitlbl; - } - - if(mc_gState->m_Features->OpDropDetailsScripts()) - { - if(entity.GetEntityType() == MC_ENT_TYPE_ASSET) - { - if(mc_gState->m_TmpScript->GetNumElements() > 3) - { - reason="Metadata script rejected - too many elements in asset update script"; - fReject=true; - goto exitlbl; - } - mc_gState->m_TmpScript->SetElement(1); - err=mc_gState->m_TmpScript->GetNewEntityType(&new_entity_type,&entity_update,details_script,&details_script_size); - if(err == 0) // New entity element - { - if(entity_update == 0) - { - reason="Metadata script rejected - wrong element, should be entity update"; - fReject=true; - goto exitlbl; - } - if(new_entity_type != MC_ENT_TYPE_ASSET) - { - reason="Metadata script rejected - entity type mismatch in update script"; - fReject=true; - goto exitlbl; - } - } - else - { - reason="Metadata script rejected - wrong element, should be entity update"; - fReject=true; - goto exitlbl; - } - } - else // (pseudo)stream or upgrade - { - if((mc_gState->m_Features->Upgrades() == 0) || (entity.GetEntityType() != MC_ENT_TYPE_UPGRADE)) // (pseudo)stream - { - for (int e = 1; e < mc_gState->m_TmpScript->GetNumElements()-1; e++) - { - mc_gState->m_TmpScript->SetElement(e); - // Should be spkk - if(mc_gState->m_TmpScript->GetItemKey(item_key,&item_key_size)) // Item key - { - reason="Metadata script rejected - wrong element, should be item key"; - fReject=true; - goto exitlbl; - } - } - } - else - { - if(mc_gState->m_TmpScript->GetNumElements() > 3) - { - reason="Metadata script rejected - too many elements in upgrade approval script"; - fReject=true; - goto exitlbl; - } - mc_gState->m_TmpScript->SetElement(1); // Upgrade approval - - if(mc_gState->m_TmpScript->GetApproval(&approval,×tamp)) - { - reason="Metadata script rejected - wrong element, should be upgrade approval"; - fReject=true; - goto exitlbl; - } - uint256 upgrade_hash=*(uint256*)(entity.GetTxID()); - if(approval) - { - LogPrintf("Found approval script in tx %s for %s\n", - tx.GetHash().GetHex().c_str(), - upgrade_hash.ToString().c_str()); - } - else - { - LogPrintf("Found disapproval script in tx %s for %s\n", - tx.GetHash().GetHex().c_str(), - upgrade_hash.ToString().c_str()); - } - - bool fAdminFound=false; - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - if(vInputHadAdminPermissionBeforeThisTx[i]) - { - if( (vInputHashTypes[i] == SIGHASH_ALL) || ( (vInputHashTypes[i] == SIGHASH_SINGLE) && (i == j) ) ) - { - if(mc_gState->m_Permissions->SetApproval(entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,approval, - (unsigned char*)&vInputDestinations[i],entity.UpgradeStartBlock(),timestamp,MC_PFL_NONE,1,offset) == 0) - { - fAdminFound=true; - } - } - } - } - if(!fAdminFound) - { - reason="Inputs don't belong to valid admin for approval script"; - fReject=true; - goto exitlbl; - } - } - } - } - else - { - if(entity.GetEntityType() != MC_ENT_TYPE_STREAM) - { - reason="Metadata script rejected - not stream entity"; - fReject=true; - goto exitlbl; - } - mc_gState->m_TmpScript->SetElement(1); - // Should be spkk - if(mc_gState->m_TmpScript->GetItemKey(item_key,&item_key_size)) // Item key - { - reason="Metadata script rejected - wrong element, should be item key"; - fReject=true; - goto exitlbl; - } - } - - if( (entity.GetEntityType() != MC_ENT_TYPE_ASSET) && (entity.GetEntityType() != MC_ENT_TYPE_UPGRADE) ) - { - fAllValidPublishers=true; - if(entity.AnyoneCanWrite() == 0) - { - if(mc_gState->m_Features->FixedIn10007()) - { - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - if( (vInputHashTypes[i] == SIGHASH_ALL) || ( (vInputHashTypes[i] == SIGHASH_SINGLE) && (i == j) ) ) - { - if(fAllValidPublishers) - { - if(mc_gState->m_Permissions->CanWrite(entity.GetTxID(),(unsigned char*)&vInputDestinations[i]) == 0) - { - fAllValidPublishers=false; - } - } - } - } - } - else - { - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - if(fAllValidPublishers) - { - fAllValidPublishers=false; - if( (vInputHashTypes[i] == SIGHASH_ALL) || ( (vInputHashTypes[i] == SIGHASH_SINGLE) && (i == j) ) ) - { - // Only in these two cases we can extract address from the input script - // When subscribing to the stream we don't have output scripts - if( (vInputScriptTypes[i] == TX_PUBKEYHASH) || (vInputScriptTypes[i] == TX_SCRIPTHASH) ) - { - if(mc_gState->m_Permissions->CanWrite(entity.GetTxID(),(unsigned char*)&vInputDestinations[i])) - { - fAllValidPublishers=true; - } - } - } - } - } - } - } - if(!fAllValidPublishers) - { - reason="Metadata script rejected - Inputs don't belong to valid publisher"; - fReject=true; - goto exitlbl; - } - } - } - } - else // Not OP_RETURN - { - txnouttype typeRet; - int nRequiredRet; - vector addressRets; - - fNoDestinationInOutput=false; - - if(!ExtractDestinations(script1,typeRet,addressRets,nRequiredRet)) - { - fNoDestinationInOutput=true; - } - - if( (pass == 0) && fShouldHaveDestination ) // Some setting in the protocol require address can be extracted - { - if(fNoDestinationInOutput && - ( (MCP_ANYONE_CAN_RECEIVE == 0) || (MCP_ALLOW_ARBITRARY_OUTPUTS == 0) ) ) - { - reason="Script rejected - destination required "; - fReject=true; - goto exitlbl; - } - - if((MCP_ALLOW_ARBITRARY_OUTPUTS == 0) || (mc_gState->m_Features->FixedDestinationExtraction() == 0) ) - { - if((typeRet == TX_MULTISIG) && (MCP_ALLOW_MULTISIG_OUTPUTS == 0)) - { - reason="Script rejected - multisig is not allowed"; - fReject=true; - goto exitlbl; - } - - if((typeRet == TX_SCRIPTHASH) && (MCP_ALLOW_P2SH_OUTPUTS == 0)) - { - reason="Script rejected - P2SH is not allowed"; - fReject=true; - goto exitlbl; - } - } - } - - receive_required=0; // Number of required receive permissions - if( (pass == 3) && fShouldHaveDestination ) - { - receive_required=addressRets.size(); - if(typeRet == TX_MULTISIG) - { - receive_required-=(pc1[0]-0x50); - receive_required+=1; - if(receive_required>(int)addressRets.size()) - { - receive_required=addressRets.size(); - } - } - } - - fIsPurePermission=false; - if(mc_gState->m_TmpScript->GetNumElements()) - { - fIsPurePermission=true; - } - - entity.Zero(); // Permission processing - for (int e = 0; e < mc_gState->m_TmpScript->GetNumElements(); e++) - { - mc_gState->m_TmpScript->SetElement(e); - if(mc_gState->m_TmpScript->GetEntity(short_txid) == 0) // Entity element - { - if(entity.GetEntityType()) - { - reason="Script rejected - duplicate entity script"; - fReject=true; - goto exitlbl; - } - if(mc_gState->m_Assets->FindEntityByShortTxID(&entity,short_txid) == 0) - { - reason="Script rejected - entity not found"; - fReject=true; - goto exitlbl; - } - } - else // Not entity element - { - if(mc_gState->m_TmpScript->GetPermission(&type,&from,&to,×tamp) == 0)// Permission script - { - if(fNoDestinationInOutput) - { - reason="Script rejected - wrong destination type in output with permission script"; - fReject=true; - goto exitlbl; - } - fCheckAdminList=false; - if(mc_gState->m_Features->CachedInputScript()) - { - fCheckAdminList=true; - } - else - { - if(type & ( MC_PTP_ACTIVATE | MC_PTP_ADMIN )) - { - fCheckAdminList=true; - } - } - fAdminRequired=false; - switch(pass) - { - case 0: // Not admin or activate - fAdminRequired=fCheckAdminList; - if(mc_gState->m_Features->CachedInputScript()) - { - type &= ( MC_PTP_CONNECT | MC_PTP_SEND | MC_PTP_RECEIVE | MC_PTP_WRITE); - } - else - { - type &= ~( MC_PTP_ACTIVATE | MC_PTP_ADMIN); - } - break; - case 1: // Admin or activate - if(mc_gState->m_Features->CachedInputScript()) - { - type &= ( MC_PTP_CREATE | MC_PTP_ISSUE | MC_PTP_ACTIVATE ); - } - else - { - type = 0; - } - break; - case 2: // Admin or activate - if(mc_gState->m_Features->CachedInputScript()) - { - type &= ( MC_PTP_MINE | MC_PTP_ADMIN ); - if(type) - { - fAdminMinerGrant=true; - } - } - else - { - type &= ( MC_PTP_ACTIVATE | MC_PTP_ADMIN ); - } - break; - case 3: // This pass is not about permissions - type=0; - break; - } - if(fAdminRequired) // this is done for pass=2, calculating list of valid admins before this tx has effect - { - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - if( (vInputHashTypes[i] == SIGHASH_ALL) || ( (vInputHashTypes[i] == SIGHASH_SINGLE) && (i == j) ) ) - { - if(mc_gState->m_Permissions->CanAdmin(entity.GetTxID(),(unsigned char*)&vInputDestinations[i])) - { - vAllowedAdmins.insert(strprintf("%d-%d-%d",i,j,e)); - } - if(mc_gState->m_Permissions->CanActivate(entity.GetTxID(),(unsigned char*)&vInputDestinations[i])) - { - vAllowedActivators.insert(strprintf("%d-%d-%d",i,j,e)); - } - } - } - } - - if(type) - { - CKeyID *lpKeyID=boost::get (&addressRets[0]); - CScriptID *lpScriptID=boost::get (&addressRets[0]); - // Permissions cannot be granted to nonstandard outputs or bare multisigs - if(((lpKeyID == NULL) && (lpScriptID == NULL)) || (addressRets.size() > 1)) - { - reason="Permission script rejected - wrong destination type"; - fReject=true; - goto exitlbl; - } - CBitcoinAddress address; - unsigned char* ptr=NULL; - flags=MC_PFL_NONE; - if(lpKeyID != NULL) - { - address=CBitcoinAddress(*lpKeyID); - ptr=(unsigned char*)(lpKeyID); - if(type & MC_PTP_CONNECT) - { - if(mc_gState->m_pSeedNode) - { - CNode* seed_node; - seed_node=(CNode*)(mc_gState->m_pSeedNode); - - // If connect permission of seed node was involved, we may want to disconnect from it - if(memcmp(ptr,seed_node->kAddrRemote.begin(),20) == 0) - { - fSeedNodeInvolved=true; - } - } - } - } - else - { - flags=MC_PFL_IS_SCRIPTHASH; - address=CBitcoinAddress(*lpScriptID); - ptr=(unsigned char*)(lpScriptID); - } - - if(fDebug)LogPrint("mchn","Found permission script in tx %s for %s - (%08x: %d - %d)\n", - tx.GetHash().GetHex().c_str(), - address.ToString().c_str(), - type, from, to); - - bool fAdminFound=false; - bool fAdminFoundWithoutCachedScript=false; - bool fActivateIsEnough=mc_gState->m_Permissions->IsActivateEnough(type); - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - if( ( fCheckAdminList && !fActivateIsEnough && (vAllowedAdmins.count(strprintf("%d-%d-%d",i,j,e)) > 0)) || - ( fCheckAdminList && fActivateIsEnough && (vAllowedActivators.count(strprintf("%d-%d-%d",i,j,e)) > 0)) || - (!fCheckAdminList && ( (vInputHashTypes[i] == SIGHASH_ALL) || ( (vInputHashTypes[i] == SIGHASH_SINGLE) && (i == j) ) ))) - { - if(vInputDestinations[i] != 0) - { - if( ( (type & (MC_PTP_ADMIN | MC_PTP_MINE)) == 0) || vInputCanGrantAdminMine[i] || (entity.GetEntityType() > 0) ) - { - fFullReplayCheckRequired=true; - if(mc_gState->m_Permissions->SetPermission(entity.GetTxID(),ptr,type,(unsigned char*)&vInputDestinations[i],from,to,timestamp,flags,1,offset) == 0) - { - fAdminFound=true; - } - } - else - { - fAdminFoundWithoutCachedScript=true; - } - } - } - } - if(!fAdminFound) - { - reason="Inputs don't belong to valid admin"; - if(fAdminFoundWithoutCachedScript) - { - reason="Inputs require scriptPubKey cache to support miner precheck"; - } - fReject=true; - goto exitlbl; - } - } - entity.Zero(); // Entity element in the non-op-return output should be followed by permission element - // So only permission can nullify it - } - else // Not permission script - { - if(entity.GetEntityType()) - { - reason="Script rejected - entity script should be followed by permission"; - fReject=true; - goto exitlbl; - } - fIsPurePermission=false; - } - } - } // End of permission script processing - - if(entity.GetEntityType()) - { - reason="Script rejected - incomplete entity script"; - fReject=true; - goto exitlbl; - } - - if(tx.vout[j].nValue > 0) - { - fIsPurePermission=false; - } - - if(pass == 3) - { - if(mc_gState->m_Features->PerAssetPermissions()) - { - mc_gState->m_TmpAssetsTmp->Clear(); - if(!mc_ExtractOutputAssetQuantities(mc_gState->m_TmpAssetsTmp,reason,true)) - { - fReject=true; - goto exitlbl; - } - if(!mc_VerifyAssetPermissions(mc_gState->m_TmpAssetsTmp,addressRets,receive_required,MC_PTP_RECEIVE,reason)) - { - return false; - } - } - if(!mc_ExtractOutputAssetQuantities(mc_gState->m_TmpAssetsOut,reason,false)) // Filling output asset quantity list - { - fReject=true; - goto exitlbl; - } - } - - if( (pass == 3) && !fIsPurePermission ) // Check for dust and receive permission - { - if((offset < 0) && Params().RequireStandard()) // If not in block - part of IsStandard check - { - if (tx.vout[j].IsDust(::minRelayTxFee)) - { - if(!tx.IsCoinBase()) - { - reason="Transaction amount too small"; - fReject=true; - goto exitlbl; - } - } - } - for(int a=0;a<(int)addressRets.size();a++) - { - CKeyID *lpKeyID=boost::get (&addressRets[a]); - CScriptID *lpScriptID=boost::get (&addressRets[a]); - if((lpKeyID == NULL) && (lpScriptID == NULL)) - { - reason="Script rejected - wrong destination type"; - fReject=true; - goto exitlbl; - } - unsigned char* ptr=NULL; - if(lpKeyID != NULL) - { - ptr=(unsigned char*)(lpKeyID); - } - else - { - ptr=(unsigned char*)(lpScriptID); - } - - bool fCanReceive=mc_gState->m_Permissions->CanReceive(NULL,ptr); - - if(tx.IsCoinBase()) // Miner can send funds to himself in coinbase even without receive permission - { - fCanReceive |= mc_gState->m_Permissions->CanMine(NULL,ptr); - } - if(fCanReceive) - { - receive_required--; - } - } - if(receive_required>0) - { - if( (tx.vout[j].nValue > 0) || - (mc_gState->m_TmpScript->GetNumElements() > 0) || - (mc_gState->m_Features->AnyoneCanReceiveEmpty() == 0) ) - { - reason="One of the outputs doesn't have receive permission"; - fReject=true; - goto exitlbl; - } - } - } - } // End of non-OP_RETURN output - } // End of output processing - } // End of pass loop - - if(!mc_CompareAssetQuantities(reason)) // Comparing input/output asset quantities - { - fReject=true; - goto exitlbl; - } - - // Checking asset geneses and follow-ons - // If new entity is found - asset genesis is forbidden, otherwise they will have the same reference - // If we fail here only permission db was updated. it will be rolled back - if(!AcceptAssetGenesisFromPredefinedIssuers(tx,offset,accept,vInputDestinations,vInputHashTypes,vInputScriptTypes,(nNewEntityOutput >= 0),reason,&fFullReplayCheckRequired)) - { - fReject=true; - goto exitlbl; - } - - if(nNewEntityOutput >= 0) // (pseudo)streams and upgrade - { - vector openers; - vector opener_flags; - unsigned char opener_buf[24]; - - string entity_type_str; - uint32_t flags=MC_PFL_NONE; - uint32_t timestamp=0; - set stored_openers; - int update_mempool; - uint256 txid; - mc_EntityDetails entity; - - fFullReplayCheckRequired=true; - - update_mempool=0; - if(accept) - { - update_mempool=1; - } - - openers.clear(); // List of openers - opener_flags.clear(); - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - if( (vInputHashTypes[i] == SIGHASH_ALL) || ( (vInputHashTypes[i] == SIGHASH_SINGLE) && ((int)i == nNewEntityOutput) ) ) - { - if(mc_gState->m_Permissions->CanCreate(NULL,(unsigned char*)&vInputDestinations[i])) - { - if( (new_entity_type != MC_ENT_TYPE_UPGRADE) || (mc_gState->m_Permissions->CanAdmin(NULL,(unsigned char*)&vInputDestinations[i]) != 0) ) - { - openers.push_back(vInputDestinations[i]); - flags=MC_PFL_NONE; - if(vInputScriptTypes[i] == TX_SCRIPTHASH) - { - flags |= MC_PFL_IS_SCRIPTHASH; - } - opener_flags.push_back(flags); - } - } - } - } - - if(openers.size() == 0) - { - reason="Metadata script rejected - Inputs don't belong to valid creator"; - fReject=true; - goto exitlbl; - } - - err=MC_ERR_NOERROR; - - mc_gState->m_TmpScript->Clear(); - mc_gState->m_TmpScript->AddElement(); - txid=tx.GetHash(); // Setting first record in the per-entity permissions list - - if(new_entity_type != MC_ENT_TYPE_UPGRADE) - { - memset(opener_buf,0,sizeof(opener_buf)); - err=mc_gState->m_Permissions->SetPermission(&txid,opener_buf,MC_PTP_CONNECT, - (unsigned char*)openers[0].begin(),0,(uint32_t)(-1),timestamp, MC_PFL_ENTITY_GENESIS ,update_mempool,offset); - } - for (unsigned int i = 0; i < openers.size(); i++) - { - if(err == MC_ERR_NOERROR) - { - if(stored_openers.count(openers[i]) == 0) - { - memcpy(opener_buf,openers[i].begin(),sizeof(uint160)); - mc_PutLE(opener_buf+sizeof(uint160),&opener_flags[i],4); - if((int)i < mc_gState->m_Assets->MaxStoredIssuers()) - { - mc_gState->m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_ISSUER,opener_buf,sizeof(opener_buf)); - } - if(new_entity_type != MC_ENT_TYPE_UPGRADE) - { - // Granting default permissions to openers - err=mc_gState->m_Permissions->SetPermission(&txid,opener_buf,MC_PTP_ADMIN | MC_PTP_ACTIVATE | MC_PTP_WRITE, - (unsigned char*)openers[i].begin(),0,(uint32_t)(-1),timestamp,opener_flags[i] | MC_PFL_ENTITY_GENESIS ,update_mempool,offset); - } - stored_openers.insert(openers[i]); - } - } - } - if(err) - { - reason=" Cannot update permission database for new entity"; - fReject=true; - goto exitlbl; - } - - memset(opener_buf,0,sizeof(opener_buf)); // Storing opener list in entity metadata - mc_gState->m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_ISSUER,opener_buf,1); - - const unsigned char *special_script; - size_t special_script_size=0; - special_script=mc_gState->m_TmpScript->GetData(0,&special_script_size); - err=mc_gState->m_Assets->InsertEntity(&txid,offset,new_entity_type,details_script,details_script_size,special_script,special_script_size,update_mempool); - if(err) - { - reason="New entity script rejected - could not insert new entity to database"; - if(err == MC_ERR_ERROR_IN_SCRIPT) - { - reason="New entity script rejected - error in script"; - } - if(err == MC_ERR_FOUND) - { - reason="New entity script rejected - entity with this name already exists"; - } - fReject=true; - goto exitlbl; - } - - if(update_mempool) - { - if(mc_gState->m_Assets->FindEntityByTxID(&entity,(unsigned char*)&txid)) - { - entity_type_str="stream"; - if(new_entity_type == MC_ENT_TYPE_UPGRADE) - { - entity_type_str="upgrade"; - } - if(offset>=0) - { - LogPrintf("New %s. TxID: %s, StreamRef: %d-%d-%d, Name: %s\n", - entity_type_str.c_str(),tx.GetHash().GetHex().c_str(), - mc_gState->m_Assets->m_Block+1,offset,(int)(*((unsigned char*)&txid+0))+256*(int)(*((unsigned char*)&txid+1)), - entity.GetName()); - } - else - { - LogPrintf("New %s. TxID: %s, unconfirmed, Name: %s\n", - entity_type_str.c_str(),tx.GetHash().GetHex().c_str(),entity.GetName()); - } - } - else - { - reason="New entity script rejected - could not insert new entity to database"; - fReject=true; - goto exitlbl; - } - } - } - - fReject=!custom_accept_transacton(tx,inputs,offset,accept,reason,replay); - -exitlbl: - - if(accept) - { - if(fSeedNodeInvolved) // Checking if we should disconnect from seed node - { - CNode* seed_node; - seed_node=(CNode*)(mc_gState->m_pSeedNode); - - if(!mc_gState->m_Permissions->CanConnect(NULL,seed_node->kAddrRemote.begin())) - { - LogPrintf("mchn: Seed node lost connect permission \n"); - mc_gState->m_pSeedNode=NULL; - } - } - } - - if(!accept || fReject) // Rolling back permission database if we were just checking or error occurred - { - mc_gState->m_Permissions->RollBackToCheckPoint(); - } - - if(replay) - { - *replay=0; - if(fFullReplayCheckRequired) - { - *replay |= MC_PPL_REPLAY; - } - if(fAdminMinerGrant) - { - *replay |= MC_PPL_ADMINMINERGRANT; - } - } - - if(fReject) - { - if(fDebug)LogPrint("mchn","mchn: Tx rejected: %s\n",EncodeHexTx(tx)); - } - - return !fReject; -} - -bool AcceptAdminMinerPermissions(const CTransaction& tx, - int offset, - bool verify_signatures, - string& reason, - uint32_t *result) -{ - vector vInputScriptTypes; - vector vInputDestinations; - vector vInputHashTypes; - vector vInputCanGrantAdminMine; - vector vInputHadAdminPermissionBeforeThisTx; - vector vInputPrevOutputScripts; - bool fIsEntity; - bool fReject; - bool fAdminFound; - int err; - - if(result) - { - *result=0; - } - - if(tx.IsCoinBase()) - { - return true; - } - - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - vInputCanGrantAdminMine.push_back(false); - vInputPrevOutputScripts.push_back(CScript()); - vInputDestinations.push_back(0); - } - - fReject=false; - fAdminFound=false; - for (unsigned int j = 0; j < tx.vout.size(); j++) - { - int cs_offset,cs_new_offset,cs_size,cs_vin; - unsigned char *cs_script; - - const CScript& script1 = tx.vout[j].scriptPubKey; - CScript::const_iterator pc1 = script1.begin(); - - - mc_gState->m_TmpScript->Clear(); - mc_gState->m_TmpScript->SetScript((unsigned char*)(&pc1[0]),(size_t)(script1.end()-pc1),MC_SCR_TYPE_SCRIPTPUBKEY); - - if(mc_gState->m_TmpScript->IsOpReturnScript()) - { - if( mc_gState->m_TmpScript->GetNumElements() == 2 ) - { - mc_gState->m_TmpScript->SetElement(0); - - if(mc_gState->m_Features->CachedInputScript()) - { - cs_offset=0; - while( (err=mc_gState->m_TmpScript->GetCachedScript(cs_offset,&cs_new_offset,&cs_vin,&cs_script,&cs_size)) != MC_ERR_WRONG_SCRIPT ) - { - if(err != MC_ERR_NOERROR) - { - reason="Metadata script rejected - error in cached script"; - fReject=true; - goto exitlbl; - } - if(cs_offset) - { - if( cs_vin >= (int)tx.vin.size() ) - { - reason="Metadata script rejected - invalid input in cached script"; - fReject=true; - goto exitlbl; - } - vInputPrevOutputScripts[cs_vin]=CScript(cs_script,cs_script+cs_size); - vInputCanGrantAdminMine[cs_vin]=true; -// fAdminFound=true; - } - cs_offset=cs_new_offset; - } - } - } - } - } -/* - if(!fAdminFound) - { - goto exitlbl; - } - - fAdminFound=false; -*/ - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - if(vInputCanGrantAdminMine[i]) - { - vInputCanGrantAdminMine[i]=false; - const CScript& script2 = tx.vin[i].scriptSig; - CScript::const_iterator pc2 = script2.begin(); - if(mc_gState->m_Features->FixedIn10007()) - { - if (!script2.IsPushOnly()) - { - reason="sigScript should be push-only"; - fReject=true; - goto exitlbl; - } - } - - const CScript& script1 = vInputPrevOutputScripts[i]; - CScript::const_iterator pc1 = script1.begin(); - - txnouttype typeRet; - int nRequiredRet; - vector addressRets; - int op_addr_offset,op_addr_size,is_redeem_script,sighash_type,check_last; - - sighash_type=SIGHASH_NONE; - if(ExtractDestinations(script1,typeRet,addressRets,nRequiredRet)) - { - if ( (typeRet != TX_NULL_DATA) && (typeRet != TX_MULTISIG) ) - { - CKeyID *lpKeyID=boost::get (&addressRets[0]); - CScriptID *lpScriptID=boost::get (&addressRets[0]); - if( (lpKeyID == NULL) && (lpScriptID == NULL) ) - { - fReject=true; - goto exitlbl; - } - if(lpKeyID) - { - vInputDestinations[i]=*(uint160*)lpKeyID; - } - if(lpScriptID) - { - vInputDestinations[i]=*(uint160*)lpScriptID; - } - - check_last=0; - if( typeRet == TX_PUBKEY ) - { - check_last=1; - } - - // Find sighash_type - mc_ExtractAddressFromInputScript((unsigned char*)(&pc2[0]),(int)(script2.end()-pc2),&op_addr_offset,&op_addr_size,&is_redeem_script,&sighash_type,check_last); - if(sighash_type == SIGHASH_ALL) - { - if(mc_gState->m_Permissions->CanAdmin(NULL,(unsigned char*)&vInputDestinations[i])) - { - vInputCanGrantAdminMine[i]=true; - if(verify_signatures) - { - vInputCanGrantAdminMine[i]=false; - if(VerifyScript(script2, script1, STANDARD_SCRIPT_VERIFY_FLAGS, CachingTransactionSignatureChecker(&tx, i, false))) - { - vInputCanGrantAdminMine[i]=true; - } - else - { - reason="Signature verification error"; - fReject=true; - goto exitlbl; - } - } -/* - if(vInputCanGrantAdminMine[i]) - { - fAdminFound=true; - } - */ - } - } - } - } - } - } -/* - if(!fAdminFound) - { - goto exitlbl; - } -*/ - for (unsigned int j = 0; j < tx.vout.size(); j++) - { - unsigned char short_txid[MC_AST_SHORT_TXID_SIZE]; - uint32_t type,from,to,timestamp,flags; - - const CScript& script1 = tx.vout[j].scriptPubKey; - CScript::const_iterator pc1 = script1.begin(); - - - mc_gState->m_TmpScript->Clear(); - mc_gState->m_TmpScript->SetScript((unsigned char*)(&pc1[0]),(size_t)(script1.end()-pc1),MC_SCR_TYPE_SCRIPTPUBKEY); - - CTxDestination addressRet; - - if(ExtractDestination(script1,addressRet)) - { - fIsEntity=false; - - for (int e = 0; e < mc_gState->m_TmpScript->GetNumElements(); e++) - { - mc_gState->m_TmpScript->SetElement(e); - if(mc_gState->m_TmpScript->GetEntity(short_txid) == 0) - { - if(fIsEntity) - { - reason="Script rejected - duplicate entity script"; - fReject=true; - goto exitlbl; - } - if(mc_gState->m_Features->FixedIn1000920001()) - { - fIsEntity=true; - } - } - else - { - if(mc_gState->m_TmpScript->GetPermission(&type,&from,&to,×tamp) == 0) - { - type &= ( MC_PTP_MINE | MC_PTP_ADMIN ); - if(fIsEntity) - { - type=0; - } - if(type) - { - CKeyID *lpKeyID=boost::get (&addressRet); - CScriptID *lpScriptID=boost::get (&addressRet); - if((lpKeyID == NULL) && (lpScriptID == NULL)) - { - reason="Permission script rejected - wrong destination type"; - fReject=true; - goto exitlbl; - } - unsigned char* ptr=NULL; - flags=MC_PFL_NONE; - if(lpKeyID != NULL) - { - ptr=(unsigned char*)(lpKeyID); - } - else - { - flags=MC_PFL_IS_SCRIPTHASH; - ptr=(unsigned char*)(lpScriptID); - } - fAdminFound=false; - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - if(vInputCanGrantAdminMine[i]) - { - if(mc_gState->m_Permissions->SetPermission(NULL,ptr,type,(unsigned char*)&vInputDestinations[i],from,to,timestamp,flags,1,offset) == 0) - { - fAdminFound=true; - } - } - } - if(!fAdminFound) - { - reason="Inputs don't belong to valid admin"; - fReject=true; - goto exitlbl; - } - else - { - if(result) - { - *result |= MC_PPL_ADMINMINERGRANT; - } - } - } - fIsEntity=false; - } - else - { - if(fIsEntity) - { - reason="Script rejected - entity script should be followed by permission"; - fReject=true; - goto exitlbl; - } - } - } - } - } - } - -exitlbl: - - if(fReject) - { - if(fDebug)LogPrint("mchn","mchn: AcceptAdminMinerPermissions: Tx rejected: %s\n",EncodeHexTx(tx)); - } - - return !fReject; -} - - -/** - * Used only for protocols < 10006 - */ - -bool AcceptAssetTransfers(const CTransaction& tx, const CCoinsViewCache &inputs, string& reason) -{ - unsigned char *ptrIn; - unsigned char *ptrOut; - int err; - int64_t quantity; - - if(mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) - { - return true; - } - if(tx.IsCoinBase()) - { - return true; - } - - mc_gState->m_TmpAssetsIn->Clear(); - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - const COutPoint &prevout = tx.vin[i].prevout; - const CCoins *coins = inputs.AccessCoins(prevout.hash); - assert(coins); - - mc_gState->m_TmpScript->Clear(); - - const CScript& script1 = coins->vout[prevout.n].scriptPubKey; - CScript::const_iterator pc1 = script1.begin(); - - mc_gState->m_TmpScript->SetScript((unsigned char*)(&pc1[0]),(size_t)(script1.end()-pc1),MC_SCR_TYPE_SCRIPTPUBKEY); - - for (int e = 0; e < mc_gState->m_TmpScript->GetNumElements(); e++) - { - mc_gState->m_TmpScript->SetElement(e); - err=mc_gState->m_TmpScript->GetAssetQuantities(mc_gState->m_TmpAssetsIn,MC_SCR_ASSET_SCRIPT_TYPE_TRANSFER | MC_SCR_ASSET_SCRIPT_TYPE_FOLLOWON); - if((err != MC_ERR_NOERROR) && (err != MC_ERR_WRONG_SCRIPT)) - { - reason="Asset transfer script rejected - error in script"; - return false; - } - err=mc_gState->m_TmpScript->GetAssetGenesis(&quantity); - if(err == 0) - { - uint256 hash=prevout.hash; - mc_EntityDetails entity; - unsigned char buf_amounts[MC_AST_ASSET_FULLREF_BUF_SIZE]; - memset(buf_amounts,0,MC_AST_ASSET_FULLREF_BUF_SIZE); - if(mc_gState->m_Assets->FindEntityByTxID(&entity,(unsigned char*)&hash)) - { - if(entity.IsUnconfirmedGenesis()) // Not confirmed genesis has -1 in offset field - { - reason="Asset transfer script rejected - using unconfirmed issue"; - return false; - } - memcpy(buf_amounts,entity.GetFullRef(),MC_AST_ASSET_FULLREF_SIZE); - int row=mc_gState->m_TmpAssetsIn->Seek(buf_amounts); - if(row>=0) - { - int64_t last=mc_GetABQuantity(mc_gState->m_TmpAssetsIn->GetRow(row)); - quantity+=last; - mc_SetABQuantity(mc_gState->m_TmpAssetsIn->GetRow(row),quantity); - } - else - { - mc_SetABQuantity(buf_amounts,quantity); - mc_gState->m_TmpAssetsIn->Add(buf_amounts); - } - } - else - { - reason="Asset transfer script rejected - issue tx not found"; - return false; - } - } - else - { - if(err != MC_ERR_WRONG_SCRIPT) - { - reason="Asset transfer script rejected - error in input issue script"; - return false; - } - } - } - } - - mc_gState->m_TmpAssetsOut->Clear(); - for (unsigned int j = 0; j < tx.vout.size(); j++) - { - mc_gState->m_TmpScript->Clear(); - - const CScript& script1 = tx.vout[j].scriptPubKey; - - CScript::const_iterator pc1 = script1.begin(); - - mc_gState->m_TmpScript->SetScript((unsigned char*)(&pc1[0]),(size_t)(script1.end()-pc1),MC_SCR_TYPE_SCRIPTPUBKEY); - for (int e = 0; e < mc_gState->m_TmpScript->GetNumElements(); e++) - { - mc_gState->m_TmpScript->SetElement(e); - err=mc_gState->m_TmpScript->GetAssetQuantities(mc_gState->m_TmpAssetsOut,MC_SCR_ASSET_SCRIPT_TYPE_TRANSFER); - if((err != MC_ERR_NOERROR) && (err != MC_ERR_WRONG_SCRIPT)) - { - reason="Asset transfer script rejected - error in output transfer script"; - return false; - } - if(err == MC_ERR_NOERROR) - { - } - } - } - - if(fDebug)LogPrint("mchnminor","Found asset transfer script in tx %s, %d assets\n", - tx.GetHash().GetHex().c_str(),mc_gState->m_TmpAssetsOut->GetCount()); - for(int i=0;im_TmpAssetsIn->GetCount();i++) - { - ptrIn=mc_gState->m_TmpAssetsIn->GetRow(i); - int row=mc_gState->m_TmpAssetsOut->Seek(ptrIn); - quantity=mc_GetABQuantity(ptrIn); - if(quantity>0) - { - if(row>=0) - { - ptrOut=mc_gState->m_TmpAssetsOut->GetRow(row); - if(memcmp(ptrIn,ptrOut,MC_AST_ASSET_QUANTITY_OFFSET+MC_AST_ASSET_QUANTITY_SIZE)) - { - reason="Asset transfer script rejected - mismatch in input/output quantities"; - return false; - } - } - else - { - reason="Asset transfer script rejected - mismatch in input/output quantities"; - return false; - } - } - } - - for(int i=0;im_TmpAssetsOut->GetCount();i++) - { - ptrOut=mc_gState->m_TmpAssetsOut->GetRow(i); - int row=mc_gState->m_TmpAssetsIn->Seek(ptrOut); - quantity=mc_GetABQuantity(ptrOut); - if(quantity>0) - { - if(row>=0) - { - ptrIn=mc_gState->m_TmpAssetsIn->GetRow(row); - if(memcmp(ptrIn,ptrOut,MC_AST_ASSET_QUANTITY_OFFSET+MC_AST_ASSET_QUANTITY_SIZE)) - { - reason="Asset transfer script rejected - mismatch in input/output quantities"; - return false; - } - } - else - { - reason="Asset transfer script rejected - mismatch in input/output quantities"; - return false; - } - } - } - - return true; -} - -/** - * Used only for protocols < 10006 - */ - - -bool AcceptAssetGenesis(const CTransaction &tx,int offset,bool accept,string& reason) -{ - int update_mempool=0; - mc_EntityDetails entity; - mc_EntityDetails this_entity; - unsigned char details_script[MC_ENT_MAX_SCRIPT_SIZE]; - char asset_name[MC_ENT_MAX_NAME_SIZE+1]; - int multiple; - int details_script_size; - int err; - int64_t quantity,total; - uint256 txid; - bool new_issue; - unsigned char *ptrOut; - vector issuers; - - - if(mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) - { - return true; - } - if(tx.IsCoinBase()) - { - return true; - } - - if(accept) - { - update_mempool=1; - } - - total=0; - - asset_name[0]=0; - multiple=1; - new_issue=false; - mc_gState->m_TmpAssetsOut->Clear(); - - details_script_size=0; - for (unsigned int j = 0; j < tx.vout.size(); j++) - { - mc_gState->m_TmpScript->Clear(); - - const CScript& script1 = tx.vout[j].scriptPubKey; - CScript::const_iterator pc1 = script1.begin(); - - CTxDestination addressRet; - - mc_gState->m_TmpScript->SetScript((unsigned char*)(&pc1[0]),(size_t)(script1.end()-pc1),MC_SCR_TYPE_SCRIPTPUBKEY); - - if(mc_gState->m_TmpScript->IsOpReturnScript()) - { - int e=mc_gState->m_TmpScript->GetNumElements()-1; - if(e != 0) - { - reason="Asset details script rejected - error in script, too many script elements"; - return false; - } - { - mc_gState->m_TmpScript->SetElement(e); - err=mc_gState->m_TmpScript->GetAssetDetails(asset_name,&multiple,details_script,&details_script_size); - if(err) - { - if(err != MC_ERR_WRONG_SCRIPT) - { - reason="Asset details script rejected - error in script"; - return false; - } - details_script_size=0; - } - - } - } - else - { - for (int e = 0; e < mc_gState->m_TmpScript->GetNumElements(); e++) - { - mc_gState->m_TmpScript->SetElement(e); - err=mc_gState->m_TmpScript->GetAssetGenesis(&quantity); - if(err == 0) - { - new_issue=true; - if(quantity+total<0) - { - reason="Asset issue script rejected - negative total amount"; - return false; - } - - total+=quantity; - - if(!ExtractDestination(script1, addressRet)) - { - reason="Asset issue script rejected - wrong destination type"; - return false; - } - - CKeyID *lpKeyID=boost::get (&addressRet); - CScriptID *lpScriptID=boost::get (&addressRet); - if((lpKeyID == NULL) && (lpScriptID == NULL)) - { - reason="Asset issue script rejected - wrong destination type"; - return false; - } - CBitcoinAddress address; - if(lpKeyID != NULL) - { - address=CBitcoinAddress(*lpKeyID); - } - else - { - address=CBitcoinAddress(*lpScriptID); - } - if(update_mempool) - { - if(fDebug)LogPrint("mchn","Found asset issue script in tx %s for %s - (%ld)\n", - tx.GetHash().GetHex().c_str(), - address.ToString().c_str(),quantity); - } - } - else - { - if(err != MC_ERR_WRONG_SCRIPT) - { - reason="Asset issue script rejected - error in script"; - return false; - } - err=mc_gState->m_TmpScript->GetAssetQuantities(mc_gState->m_TmpAssetsOut,MC_SCR_ASSET_SCRIPT_TYPE_FOLLOWON); - if((err != MC_ERR_NOERROR) && (err != MC_ERR_WRONG_SCRIPT)) - { - reason="Asset follow-on script rejected - error in follow-on script"; - return false; - } - if(err == MC_ERR_NOERROR) - { - if(update_mempool) - { - if(fDebug)LogPrint("mchn","Found asset follow-on script in tx %s\n", - tx.GetHash().GetHex().c_str()); - } - } - } - } - - } - } - - if(mc_gState->m_TmpAssetsOut->GetCount()) - { - if(mc_gState->m_TmpAssetsOut->GetCount() > 1) - { - reason="Asset follow-on script rejected - follow-on for several assets"; - return false; - } - if(new_issue) - { - reason="Asset follow-on script rejected - follow-on and issue in one transaction"; - return false; - } - ptrOut=mc_gState->m_TmpAssetsOut->GetRow(0); - if(mc_gState->m_Assets->FindEntityByRef(&entity,ptrOut) == 0) - { - reason="Asset follow-on script rejected - asset not found"; - return false; - } - if(entity.AllowedFollowOns() == 0) - { - reason="Asset follow-on script rejected - follow-ons not allowed for this asset"; - return false; - } - total=mc_GetABQuantity(ptrOut); - } - - if(!new_issue && (mc_gState->m_TmpAssetsOut->GetCount() == 0)) - { - return true; - } - - issuers.clear(); - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - const CScript& script2 = tx.vin[i].scriptSig; - CScript::const_iterator pc2 = script2.begin(); - - mc_gState->m_TmpScript->Clear(); - mc_gState->m_TmpScript->SetScript((unsigned char*)(&pc2[0]),(size_t)(script2.end()-pc2),MC_SCR_TYPE_SCRIPTSIG); - - if(mc_gState->m_TmpScript->GetNumElements() == 2) - { - size_t elem_size; - const unsigned char *elem; - - elem = mc_gState->m_TmpScript->GetData(0,&elem_size); - - if(elem_size > 1) // If this is multisig with one signature it should be OP_0 - { - unsigned char hash_type=elem[elem_size-1] & 0x1f; - - if(hash_type == SIGHASH_ALL) - { - elem = mc_gState->m_TmpScript->GetData(1,&elem_size); - const unsigned char *pubkey_hash=(unsigned char *)Hash160(elem,elem+elem_size).begin(); - - if(new_issue) - { - if(mc_gState->m_Permissions->CanIssue(NULL,pubkey_hash)) - { - issuers.push_back(Hash160(elem,elem+elem_size)); - } - } - else - { - if(mc_gState->m_Permissions->CanIssue(entity.GetTxID(),pubkey_hash)) - { - issuers.push_back(Hash160(elem,elem+elem_size)); - } - } - } - } - } - } - - if(issuers.size() == 0) - { - reason="Inputs don't belong to valid issuer"; - return false; - } - - err=MC_ERR_NOERROR; - - mc_gState->m_TmpScript->Clear(); - mc_gState->m_TmpScript->AddElement(); - unsigned char issuer_buf[20]; - memset(issuer_buf,0,sizeof(issuer_buf)); - uint32_t flags=MC_PFL_NONE; - uint32_t timestamp=0; - set stored_issuers; - - if(new_issue) - { - mc_gState->m_Permissions->SetCheckPoint(); - err=MC_ERR_NOERROR; - - txid=tx.GetHash(); - err=mc_gState->m_Permissions->SetPermission(&txid,issuer_buf,MC_PTP_CONNECT, - (unsigned char*)issuers[0].begin(),0,(uint32_t)(-1),timestamp,flags | MC_PFL_ENTITY_GENESIS ,update_mempool,offset); - } - - for (unsigned int i = 0; i < issuers.size(); i++) - { - if(err == MC_ERR_NOERROR) - { - if(stored_issuers.count(issuers[i]) == 0) - { - memcpy(issuer_buf,issuers[i].begin(),sizeof(uint160)); - if((int)i < mc_gState->m_Assets->MaxStoredIssuers()) - { - mc_gState->m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_ISSUER,issuer_buf,sizeof(issuer_buf)); - } - if(new_issue) - { - err=mc_gState->m_Permissions->SetPermission(&txid,issuer_buf,MC_PTP_ADMIN | MC_PTP_ISSUE, - (unsigned char*)issuers[0].begin(),0,(uint32_t)(-1),timestamp,flags | MC_PFL_ENTITY_GENESIS ,update_mempool,offset); - } - stored_issuers.insert(issuers[i]); - } - } - } - - memset(issuer_buf,0,sizeof(issuer_buf)); - mc_gState->m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_ISSUER,issuer_buf,1); - if(new_issue) - { - if((err != MC_ERR_NOERROR) || !accept) - { - mc_gState->m_Permissions->RollBackToCheckPoint(); - } - } - if(err) - { - reason="Cannot update permission database for issued asset"; - return false; - } - const unsigned char *special_script; - size_t special_script_size=0; - special_script=mc_gState->m_TmpScript->GetData(0,&special_script_size); - txid=tx.GetHash(); - if(new_issue) - { - err=mc_gState->m_Assets->InsertAsset(&txid,offset,total,asset_name,multiple,details_script,details_script_size,special_script,special_script_size,update_mempool); - } - else - { - err=mc_gState->m_Assets->InsertAssetFollowOn(&txid,offset,total,details_script,details_script_size,special_script,special_script_size,entity.GetTxID(),update_mempool); - } - - if(err) + for (unsigned int j = 0; j < tx.vout.size(); j++) { - reason="Asset issue script rejected - could not insert new asset to database"; - if(err == MC_ERR_FOUND) - { - reason="Asset issue script rejected - asset with such name/asset-ref/txid already exists"; - if(mc_gState->m_Assets->FindEntityByTxID(&entity,(unsigned char*)&txid) == 0) - { - if(strlen(asset_name) == 0) - { - mc_gState->m_Assets->FindEntityByName(&entity,asset_name); - } - } + unsigned char short_txid[MC_AST_SHORT_TXID_SIZE]; + uint32_t type,from,to,timestamp,flags; - if(fDebug)LogPrint("mchn","Asset already exists. TxID: %s, AssetRef: %d-%d-%d, Name: %s\n", - tx.GetHash().GetHex().c_str(), - mc_gState->m_Assets->m_Block+1,offset,(int)(*((unsigned char*)&txid+31))+256*(int)(*((unsigned char*)&txid+30)), - entity.GetName()); - } - return false; - } - - - if(update_mempool) - { - if(offset>=0) - { - if(mc_gState->m_Assets->FindEntityByTxID(&this_entity,(unsigned char*)&txid)) - { - if(new_issue) - { - if(fDebug)LogPrint("mchn","New asset. TxID: %s, AssetRef: %d-%d-%d, Name: %s\n", - tx.GetHash().GetHex().c_str(), - mc_gState->m_Assets->m_Block+1,offset,(int)(*((unsigned char*)&txid+0))+256*(int)(*((unsigned char*)&txid+1)), - this_entity.GetName()); - } - else - { - uint256 otxid; - memcpy(&otxid,entity.GetTxID(),32); - if(fDebug)LogPrint("mchn","Follow-on issue. TxID: %s, Original issue txid: %s\n", - tx.GetHash().GetHex().c_str(),otxid.GetHex().c_str()); - } - } - else - { - reason="Asset issue script rejected - could not insert new asset to database"; - return false; - } - } - } - - return true; -} - -/** - * Used only for protocols < 10006 - */ + const CScript& script1 = tx.vout[j].scriptPubKey; + CScript::const_iterator pc1 = script1.begin(); -bool AcceptPermissionsAndCheckForDust(const CTransaction &tx,bool accept,string& reason) -{ - uint32_t type,from,to,timestamp,flags; - int receive_required; - bool is_op_return; - bool is_pure_permission; - bool seed_node_involved; - bool reject; - bool sighashall_found; - int pass; - mc_Script *lpInputScript=NULL; - - if(mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) - { - return true; - } - - sighashall_found=false; - - std::vector admin_outputs; - admin_outputs.resize(tx.vin.size()); - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - const CScript& script2 = tx.vin[i].scriptSig; - CScript::const_iterator pc2 = script2.begin(); mc_gState->m_TmpScript->Clear(); - mc_gState->m_TmpScript->SetScript((unsigned char*)(&pc2[0]),(size_t)(script2.end()-pc2),MC_SCR_TYPE_SCRIPTSIG); - - admin_outputs[i]=tx.vout.size(); - if(tx.IsCoinBase()) - { - admin_outputs[i]=-1; - } - else - { - bool is_pay_to_pubkeyhash=false; - if(mc_gState->m_TmpScript->GetNumElements() == 2) - { - size_t elem_size; - const unsigned char *elem; - unsigned char hash_type; - - elem = mc_gState->m_TmpScript->GetData(0,&elem_size); - - if(elem_size > 1) - { - is_pay_to_pubkeyhash=true; - hash_type=elem[elem_size-1] & 0x1f; - - if(hash_type == SIGHASH_ALL) - { - admin_outputs[i]=-1; - sighashall_found=true; - } - else - { - if(hash_type == SIGHASH_SINGLE) - { - admin_outputs[i]=i; - } - else - { - reason="Permission script rejected - invalid signature hash type"; - return false; - } - } - } - } - if(!is_pay_to_pubkeyhash) - { - int first_sig=1; - int this_sig; - if(mc_gState->m_TmpScript->GetNumElements() == 1) // pay to pubkey - { - first_sig=0; - } - this_sig=first_sig; - while(this_sigm_TmpScript->GetNumElements()) - { - size_t elem_size; - const unsigned char *elem; - unsigned char hash_type; - - elem = mc_gState->m_TmpScript->GetData(this_sig,&elem_size); - if(elem_size > 1) - { - hash_type=elem[elem_size-1] & 0x1f; - - if(hash_type == SIGHASH_ALL) - { - sighashall_found=true; - } - this_sig++; - } - else // First element of redeemScript - { - this_sig=mc_gState->m_TmpScript->GetNumElements(); - } - } - } - } - } - - reject=false; - seed_node_involved=false; - mc_gState->m_Permissions->SetCheckPoint(); - - lpInputScript=new mc_Script; - - for(pass=0;pass<3;pass++) - { - for (unsigned int j = 0; j < tx.vout.size(); j++) - { - mc_gState->m_TmpScript->Clear(); - - const CScript& script1 = tx.vout[j].scriptPubKey; - CScript::const_iterator pc1 = script1.begin(); - - CTxDestination addressRet; + mc_gState->m_TmpScript->SetScript((unsigned char*)(&pc1[0]),(size_t)(script1.end()-pc1),MC_SCR_TYPE_SCRIPTPUBKEY); + + CTxDestination addressRet; - mc_gState->m_TmpScript->SetScript((unsigned char*)(&pc1[0]),(size_t)(script1.end()-pc1),MC_SCR_TYPE_SCRIPTPUBKEY); + if(ExtractDestination(script1,addressRet)) + { + fIsEntity=false; - is_op_return=false; - if(mc_gState->m_TmpScript->IsOpReturnScript()) + for (int e = 0; e < mc_gState->m_TmpScript->GetNumElements(); e++) { - is_op_return=true; - if(!sighashall_found) + mc_gState->m_TmpScript->SetElement(e); + if(mc_gState->m_TmpScript->GetEntity(short_txid) == 0) { - if(!tx.IsCoinBase()) + if(fIsEntity) { - reason="Tx with metadata should have at least one SIGHASH_ALL output"; - reject=true; - goto exitlbl; + reason="Script rejected - duplicate entity script"; + fReject=true; + goto exitlbl; } - } - } - else - { - txnouttype typeRet; - int nRequiredRet; - vector addressRets; - - if(!ExtractDestinations(script1,typeRet,addressRets,nRequiredRet)) - { - reason="Script rejected - wrong destination type"; - reject=true; - goto exitlbl; - } - - if((typeRet == TX_MULTISIG) && (MCP_ALLOW_MULTISIG_OUTPUTS == 0)) - { - reason="Script rejected - multisig is not allowed"; - reject=true; - goto exitlbl; - } - - if((typeRet == TX_SCRIPTHASH) && (MCP_ALLOW_P2SH_OUTPUTS == 0)) - { - reason="Script rejected - P2SH is not allowed"; - reject=true; - goto exitlbl; - } - - receive_required=addressRets.size(); - if(typeRet == TX_MULTISIG) - { - receive_required-=(pc1[0]-0x50); - receive_required+=1; - if(receive_required>(int)addressRets.size()) + if(mc_gState->m_Features->FixedIn1000920001()) { - receive_required=addressRets.size(); + fIsEntity=true; } } - - is_pure_permission=false; - if(mc_gState->m_TmpScript->GetNumElements()) - { - is_pure_permission=true; - } - for (int e = 0; e < mc_gState->m_TmpScript->GetNumElements(); e++) - { - mc_gState->m_TmpScript->SetElement(e); + else + { if(mc_gState->m_TmpScript->GetPermission(&type,&from,&to,×tamp) == 0) - { - switch(pass) + { + type &= ( MC_PTP_MINE | MC_PTP_ADMIN ); + if(fIsEntity) { - case 0: - type &= ~( MC_PTP_ACTIVATE | MC_PTP_ADMIN ); - break; - case 1: - type &= ( MC_PTP_ACTIVATE | MC_PTP_ADMIN ); - break; - case 2: - type=0; - break; + type=0; } if(type) { - CKeyID *lpKeyID=boost::get (&addressRets[0]); - CScriptID *lpScriptID=boost::get (&addressRets[0]); - if(((lpKeyID == NULL) && (lpScriptID == NULL)) || (addressRets.size() > 1)) + CKeyID *lpKeyID=boost::get (&addressRet); + CScriptID *lpScriptID=boost::get (&addressRet); + if((lpKeyID == NULL) && (lpScriptID == NULL)) { reason="Permission script rejected - wrong destination type"; - reject=true; + fReject=true; goto exitlbl; } - if(lpScriptID) - { - if(type != MC_PTP_RECEIVE) - { - reason="Permission script rejected - only receive permission can be set for P2SH"; - reject=true; - goto exitlbl; - } - } - - CBitcoinAddress address; unsigned char* ptr=NULL; flags=MC_PFL_NONE; if(lpKeyID != NULL) { - address=CBitcoinAddress(*lpKeyID); ptr=(unsigned char*)(lpKeyID); - if(type & MC_PTP_CONNECT) - { - if(mc_gState->m_pSeedNode) - { - CNode* seed_node; - seed_node=(CNode*)(mc_gState->m_pSeedNode); - - if(memcmp(ptr,seed_node->kAddrRemote.begin(),20) == 0) - { - seed_node_involved=true; - } - } - } } else { flags=MC_PFL_IS_SCRIPTHASH; - address=CBitcoinAddress(*lpScriptID); ptr=(unsigned char*)(lpScriptID); } - if(fDebug)LogPrint("mchn","Found permission script in tx %s for %s - (%08x: %d - %d)\n", - tx.GetHash().GetHex().c_str(), - address.ToString().c_str(), - type, from, to); - - bool admin_found=false; + fAdminFound=false; for (unsigned int i = 0; i < tx.vin.size(); i++) { - if((admin_outputs[i] < 0) || (admin_outputs[i] == (int)j)) + if(vInputCanGrantAdminMine[i]) { - const CScript& script2 = tx.vin[i].scriptSig; - CScript::const_iterator pc2 = script2.begin(); - - lpInputScript->Clear(); - lpInputScript->SetScript((unsigned char*)(&pc2[0]),(size_t)(script2.end()-pc2),MC_SCR_TYPE_SCRIPTSIG); - - if(lpInputScript->GetNumElements() > 1) + if(mc_gState->m_Permissions->SetPermission(NULL,ptr,type,(unsigned char*)&vInputDestinations[i],from,to,timestamp,flags,1,offset) == 0) { - size_t elem_size; - const unsigned char *elem; - - elem = lpInputScript->GetData(0,&elem_size); - if(elem_size > 1) // it is not multisig with one signature - { - elem = lpInputScript->GetData(1,&elem_size); - const unsigned char *pubkey_hash=(unsigned char *)Hash160(elem,elem+elem_size).begin(); - if(mc_gState->m_Permissions->SetPermission(NULL,ptr,type,pubkey_hash,from,to,timestamp,flags,1,-1) == 0) - { - admin_found=true; - } - } - } + fAdminFound=true; + } } - } - if(!admin_found) + } + if(!fAdminFound) { reason="Inputs don't belong to valid admin"; - reject=true; + fReject=true; goto exitlbl; + } + else + { + if(result) + { + *result |= MC_PPL_ADMINMINERGRANT; + } } } - } - else - { - is_pure_permission=false; + fIsEntity=false; } - } - - if(tx.vout[j].nValue > 0) - { - is_pure_permission=false; - } - - - if( (pass == 2) && !is_op_return && !is_pure_permission ) - { - if (tx.vout[j].IsDust(::minRelayTxFee)) + else { - if(!tx.IsCoinBase()) - { - reason="Transaction amount too small"; - reject=true; - goto exitlbl; - } - } - - for(int a=0;a<(int)addressRets.size();a++) - { - CKeyID *lpKeyID=boost::get (&addressRets[a]); - CScriptID *lpScriptID=boost::get (&addressRets[a]); - if((lpKeyID == NULL) && (lpScriptID == NULL)) - { - reason="Script rejected - wrong destination type"; - reject=true; - goto exitlbl; - } - unsigned char* ptr=NULL; - if(lpKeyID != NULL) - { - ptr=(unsigned char*)(lpKeyID); - } - else - { - ptr=(unsigned char*)(lpScriptID); - } - - bool can_receive=mc_gState->m_Permissions->CanReceive(NULL,ptr); - - if(tx.IsCoinBase()) + if(fIsEntity) { - can_receive |= mc_gState->m_Permissions->CanMine(NULL,ptr); + reason="Script rejected - entity script should be followed by permission"; + fReject=true; + goto exitlbl; } - - if(can_receive) - { - receive_required--; - } - } - - if(receive_required>0) - { - reason="One of the outputs doesn't have receive permission"; - reject=true; - goto exitlbl; } - } - } + } + } } } - -exitlbl: - - if(accept) - { - if(seed_node_involved) - { - CNode* seed_node; - seed_node=(CNode*)(mc_gState->m_pSeedNode); - - if(!mc_gState->m_Permissions->CanConnect(NULL,seed_node->kAddrRemote.begin())) - { - LogPrintf("mchn: Seed node lost connect permission \n"); - mc_gState->m_pSeedNode=NULL; - } - } - } - - if(!accept || reject) - { - mc_gState->m_Permissions->RollBackToCheckPoint(); - } - - if(lpInputScript) + +exitlbl: + + if(fReject) { - delete lpInputScript; + if(fDebug)LogPrint("mchn","mchn: AcceptAdminMinerPermissions: Tx rejected: %s\n",EncodeHexTx(tx)); } - return !reject; + return !fReject; } diff --git a/src/rpc/rpcassets.cpp b/src/rpc/rpcassets.cpp index 52721c19..9ddd5c04 100644 --- a/src/rpc/rpcassets.cpp +++ b/src/rpc/rpcassets.cpp @@ -9,11 +9,6 @@ void mergeGenesisWithAssets(mc_Buffer *genesis_amounts, mc_Buffer *asset_amounts) { - if(mc_gState->m_Features->ShortTxIDInTx() == 0) - { - return; - } - unsigned char buf[MC_AST_ASSET_FULLREF_BUF_SIZE]; int64_t quantity; memset(buf,0,MC_AST_ASSET_FULLREF_BUF_SIZE); @@ -189,12 +184,9 @@ Value issuefromcmd(const Array& params, bool fHelp) } } - if(mc_gState->m_Features->Streams()) + if(asset_name == "*") { - if(asset_name == "*") - { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid asset name: *"); - } + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid asset name: *"); } unsigned char buf_a[MC_AST_ASSET_REF_SIZE]; @@ -222,14 +214,11 @@ Value issuefromcmd(const Array& params, bool fHelp) lpDetails->Clear(); lpDetails->AddElement(); - if(mc_gState->m_Features->OpDropDetailsScripts()) + if(asset_name.size()) { - if(asset_name.size()) - { - lpDetails->SetSpecialParamValue(MC_ENT_SPRM_NAME,(const unsigned char*)(asset_name.c_str()),asset_name.size());//+1); - } - lpDetails->SetSpecialParamValue(MC_ENT_SPRM_ASSET_MULTIPLE,(unsigned char*)&multiple,4); - } + lpDetails->SetSpecialParamValue(MC_ENT_SPRM_NAME,(const unsigned char*)(asset_name.c_str()),asset_name.size());//+1); + } + lpDetails->SetSpecialParamValue(MC_ENT_SPRM_ASSET_MULTIPLE,(unsigned char*)&multiple,4); if(is_open) { @@ -285,32 +274,16 @@ Value issuefromcmd(const Array& params, bool fHelp) script=lpDetails->GetData(0,&bytes); lpDetailsScript->Clear(); - if(mc_gState->m_Features->OpDropDetailsScripts()) + err=lpDetailsScript->SetNewEntityType(MC_ENT_TYPE_ASSET,0,script,bytes); + if(err) { - err=lpDetailsScript->SetNewEntityType(MC_ENT_TYPE_ASSET,0,script,bytes); - if(err) - { - strError= "Invalid custom fields or asset name, too long"; - goto exitlbl; - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid custom fields or asset name, too long"); - } - - elem = lpDetailsScript->GetData(0,&elem_size); - scriptOpReturn << vector(elem, elem + elem_size) << OP_DROP << OP_RETURN; + strError= "Invalid custom fields or asset name, too long"; + goto exitlbl; + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid custom fields or asset name, too long"); } - else - { - err=lpDetailsScript->SetAssetDetails(asset_name.c_str(),multiple,script,bytes); - if(err) - { - strError= "Invalid custom fields or asset name, too long"; - goto exitlbl; - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid custom fields or asset name, too long"); - } - elem = lpDetailsScript->GetData(0,&elem_size); - scriptOpReturn << OP_RETURN << vector(elem, elem + elem_size); - } + elem = lpDetailsScript->GetData(0,&elem_size); + scriptOpReturn << vector(elem, elem + elem_size) << OP_DROP << OP_RETURN; @@ -428,13 +401,6 @@ Value issuemorefromcmd(const Array& params, bool fHelp) { ParseEntityIdentifier(params[2],&entity, MC_ENT_TYPE_ASSET); memcpy(buf,entity.GetFullRef(),MC_AST_ASSET_FULLREF_SIZE); - if(mc_gState->m_Features->ShortTxIDInTx() == 0) - { - if(entity.IsUnconfirmedGenesis()) - { - throw JSONRPCError(RPC_UNCONFIRMED_ENTITY, string("Unconfirmed asset: ")+params[2].get_str()); - } - } multiple=entity.GetAssetMultiple(); } else @@ -519,29 +485,18 @@ Value issuemorefromcmd(const Array& params, bool fHelp) script=lpDetails->GetData(0,&bytes); if(bytes > 0) { -// mc_DumpSize("script",script,bytes,bytes); - if(mc_gState->m_Features->OpDropDetailsScripts()) - { - lpDetailsScript->SetEntity(entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET); - err=lpDetailsScript->SetNewEntityType(MC_ENT_TYPE_ASSET,1,script,bytes); - if(err) - { - strError= "Invalid custom fields, too long"; - goto exitlbl; -// throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid custom fields, too long"); - } - - elem = lpDetailsScript->GetData(0,&elem_size); - scriptOpReturn << vector(elem, elem + elem_size) << OP_DROP; - elem = lpDetailsScript->GetData(1,&elem_size); - scriptOpReturn << vector(elem, elem + elem_size) << OP_DROP << OP_RETURN; - } - else + lpDetailsScript->SetEntity(entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET); + err=lpDetailsScript->SetNewEntityType(MC_ENT_TYPE_ASSET,1,script,bytes); + if(err) { - lpDetailsScript->SetGeneralDetails(script,bytes); - elem = lpDetailsScript->GetData(0,&elem_size); - scriptOpReturn << OP_RETURN << vector(elem, elem + elem_size); + strError= "Invalid custom fields, too long"; + goto exitlbl; } + + elem = lpDetailsScript->GetData(0,&elem_size); + scriptOpReturn << vector(elem, elem + elem_size) << OP_DROP; + elem = lpDetailsScript->GetData(1,&elem_size); + scriptOpReturn << vector(elem, elem + elem_size) << OP_DROP << OP_RETURN; } @@ -1217,26 +1172,19 @@ Value getaddressbalances(const Array& params, bool fHelp) quantity=mc_GetABQuantity(asset_amounts->GetRow(a)); if(mc_gState->m_Assets->FindEntityByTxID(&entity,(unsigned char*)&hash)) { - if((entity.IsUnconfirmedGenesis() != 0) && (mc_gState->m_Features->ShortTxIDInTx() == 0) ) + ptr=(unsigned char *)entity.GetFullRef(); + memcpy(buf,ptr,MC_AST_ASSET_FULLREF_SIZE); + int row=asset_amounts->Seek(buf); + if(row >= 0) { - is_genesis=true; + int64_t last=mc_GetABQuantity(asset_amounts->GetRow(row)); + quantity+=last; + mc_SetABQuantity(asset_amounts->GetRow(row),quantity); } else { - ptr=(unsigned char *)entity.GetFullRef(); - memcpy(buf,ptr,MC_AST_ASSET_FULLREF_SIZE); - int row=asset_amounts->Seek(buf); - if(row >= 0) - { - int64_t last=mc_GetABQuantity(asset_amounts->GetRow(row)); - quantity+=last; - mc_SetABQuantity(asset_amounts->GetRow(row),quantity); - } - else - { - mc_SetABQuantity(buf,quantity); - asset_amounts->Add(buf); - } + mc_SetABQuantity(buf,quantity); + asset_amounts->Add(buf); } } @@ -1440,26 +1388,19 @@ Value getassetbalances(const Array& params, bool fHelp) quantity=mc_GetABQuantity(asset_amounts->GetRow(a)); if(mc_gState->m_Assets->FindEntityByTxID(&entity,(unsigned char*)&hash)) { - if((entity.IsUnconfirmedGenesis() != 0) && (mc_gState->m_Features->ShortTxIDInTx() == 0) ) + ptr=(unsigned char *)entity.GetFullRef(); + memcpy(buf,ptr,MC_AST_ASSET_FULLREF_SIZE); + int row=asset_amounts->Seek(buf); + if(row >= 0) { - is_genesis=true; + int64_t last=mc_GetABQuantity(asset_amounts->GetRow(row)); + quantity+=last; + mc_SetABQuantity(asset_amounts->GetRow(row),quantity); } else { - ptr=(unsigned char *)entity.GetFullRef(); - memcpy(buf,ptr,MC_AST_ASSET_FULLREF_SIZE); - int row=asset_amounts->Seek(buf); - if(row >= 0) - { - int64_t last=mc_GetABQuantity(asset_amounts->GetRow(row)); - quantity+=last; - mc_SetABQuantity(asset_amounts->GetRow(row),quantity); - } - else - { - mc_SetABQuantity(buf,quantity); - asset_amounts->Add(buf); - } + mc_SetABQuantity(buf,quantity); + asset_amounts->Add(buf); } } diff --git a/src/rpc/rpcmisc.cpp b/src/rpc/rpcmisc.cpp index b7f4ca98..a7f258ed 100644 --- a/src/rpc/rpcmisc.cpp +++ b/src/rpc/rpcmisc.cpp @@ -1058,7 +1058,6 @@ Value createmultisig(const Array& params, bool fHelp) /* MCHN START */ if(mc_gState->m_NetworkParams->IsProtocolMultichain()) { -// if(MCP_ALLOW_ARBITRARY_OUTPUTS == 0) if((MCP_ALLOW_ARBITRARY_OUTPUTS == 0) || (mc_gState->m_Features->FixedDestinationExtraction() == 0) ) { if(MCP_ALLOW_P2SH_OUTPUTS == 0) diff --git a/src/rpc/rpcpermissions.cpp b/src/rpc/rpcpermissions.cpp index 8e6e688e..8b9dcc25 100644 --- a/src/rpc/rpcpermissions.cpp +++ b/src/rpc/rpcpermissions.cpp @@ -10,11 +10,7 @@ string AllowedPermissions() { - string ret="connect,send,receive,issue,mine,admin,activate"; - if(mc_gState->m_Features->Streams()) - { - ret += ",create"; - } + string ret="connect,send,receive,issue,mine,admin,activate,create"; return ret; } diff --git a/src/rpc/rpcrawdata.cpp b/src/rpc/rpcrawdata.cpp index 6562ce2f..36be5970 100644 --- a/src/rpc/rpcrawdata.cpp +++ b/src/rpc/rpcrawdata.cpp @@ -29,12 +29,6 @@ uint32_t ParseRawDataParamType(Value *param,mc_EntityDetails *given_entity,mc_En this_param_type=MC_DATA_API_PARAM_TYPE_NONE; if(d.name_ == "inputcache") { - if( mc_gState->m_Features->CachedInputScript() == 0 ) - { - *errorCode=RPC_NOT_SUPPORTED; - *strError=string("Cached input scripts are not supported by this protocol version"); - goto exitlbl; - } this_param_type=MC_DATA_API_PARAM_TYPE_CIS; } if(d.name_ == "create") @@ -43,12 +37,6 @@ uint32_t ParseRawDataParamType(Value *param,mc_EntityDetails *given_entity,mc_En { if(d.value_.get_str() == "stream") { - if( mc_gState->m_Features->Streams() == 0 ) - { - *errorCode=RPC_NOT_SUPPORTED; - *strError=string("Streams are not supported by this protocol version"); - goto exitlbl; - } this_param_type=MC_DATA_API_PARAM_TYPE_CREATE_STREAM; } if(d.value_.get_str() == "asset") @@ -57,12 +45,6 @@ uint32_t ParseRawDataParamType(Value *param,mc_EntityDetails *given_entity,mc_En } if(d.value_.get_str() == "upgrade") { - if( mc_gState->m_Features->Upgrades() == 0 ) - { - *errorCode=RPC_NOT_SUPPORTED; - *strError=string("Upgrades are not supported by this protocol version"); - goto exitlbl; - } this_param_type=MC_DATA_API_PARAM_TYPE_CREATE_UPGRADE; } } @@ -93,22 +75,10 @@ uint32_t ParseRawDataParamType(Value *param,mc_EntityDetails *given_entity,mc_En } if(entity->GetEntityType() == MC_ENT_TYPE_STREAM) { - if( mc_gState->m_Features->Streams() == 0 ) - { - *errorCode=RPC_NOT_SUPPORTED; - *strError=string("Upgrades are not supported by this protocol version"); - goto exitlbl; - } this_param_type=MC_DATA_API_PARAM_TYPE_PUBLISH; } if(entity->GetEntityType() == MC_ENT_TYPE_UPGRADE) { - if( mc_gState->m_Features->Upgrades() == 0 ) - { - *errorCode=RPC_NOT_SUPPORTED; - *strError=string("Upgrades are not supported by this protocol version"); - goto exitlbl; - } this_param_type=MC_DATA_API_PARAM_TYPE_APPROVAL; } if(this_param_type == MC_DATA_API_PARAM_TYPE_NONE) @@ -117,25 +87,6 @@ uint32_t ParseRawDataParamType(Value *param,mc_EntityDetails *given_entity,mc_En goto exitlbl; } } -/* - if(d.name_ == "data") - { - if(!missing_data) - { - *strError=string("data field can appear only once in the object"); - goto exitlbl; - } - missing_data=false; - if(d.value_.type() != str_type) - { - if(d.value_.type() != obj_type) - { - *strError=string("data should be string or object"); - goto exitlbl; - } - } - } - */ if( (d.name_ == "text") || (d.name_ == "json") ) { if( mc_gState->m_Features->FormattedData() == 0 ) @@ -151,54 +102,11 @@ uint32_t ParseRawDataParamType(Value *param,mc_EntityDetails *given_entity,mc_En } missing_data=false; } -/* - if(d.name_ == "format") - { - if( mc_gState->m_Features->FormattedData() == 0 ) - { - *errorCode=RPC_NOT_SUPPORTED; - *strError=string("Formatted data is not supported by this protocol version"); - goto exitlbl; - } - if(*data_format != MC_SCR_DATA_FORMAT_UNKNOWN) - { - *strError=string("format field can appear only once in the object"); - goto exitlbl; - } - if(d.value_.type() != null_type && !d.value_.get_str().empty()) - { - if(d.value_.get_str() == "hex") - { - *data_format=MC_SCR_DATA_FORMAT_RAW; - } - if(d.value_.get_str() == "text") - { - *data_format=MC_SCR_DATA_FORMAT_UTF8; - } - if(d.value_.get_str() == "json") - { - *data_format=MC_SCR_DATA_FORMAT_UBJSON; - } - } - if(*data_format == MC_SCR_DATA_FORMAT_UNKNOWN) - { - *strError=string("Invalid format"); - } - } - */ if(this_param_type != MC_DATA_API_PARAM_TYPE_NONE) { if(param_type != MC_DATA_API_PARAM_TYPE_NONE) { - *strError=string("Only one of the following keywords can appear in the object: create, update"); - if(mc_gState->m_Features->Streams()) - { - *strError += string(", for"); - } - if(mc_gState->m_Features->Streams()) - { - *strError += string(", json, text"); - } + *strError=string("Only one of the following keywords can appear in the object: create, update, for, json, text"); goto exitlbl; } } @@ -375,50 +283,6 @@ vector ParseRawFormattedData(const Value *value,uint32_t *data_fo } } -/* - switch(data_format) - { - case MC_SCR_DATA_FORMAT_RAW: - case MC_SCR_DATA_FORMAT_UNKNOWN: - if(value->type() != null_type && (value->type()==str_type)) - { - bool fIsHex; - vValue=ParseHex(value->get_str().c_str(),fIsHex); - if(!fIsHex) - { - *strError=string("value should be hexadecimal string"); - } - } - else - { - *strError=string("Invalid value in data field for this format"); - } - break; - case MC_SCR_DATA_FORMAT_UTF8: - if(value->type() != null_type && (value->type()==str_type)) - { - vValue=vector (value->get_str().begin(),value->get_str().end()); - } - else - { - *strError=string("Invalid value in data field for this format"); - } - break; - case MC_SCR_DATA_FORMAT_UBJSON: - size_t bytes; - int err; - const unsigned char *script; - lpDetailsScript->Clear(); - lpDetailsScript->AddElement(); - if((err = ubjson_write(*value,lpDetailsScript,MAX_FORMATTED_DATA_DEPTH)) != MC_ERR_NOERROR) - { - *strError=string("Couldn't transfer JSON object to internal UBJSON format"); - } - script = lpDetailsScript->GetData(0,&bytes); - vValue=vector (script,script+bytes); - break; - } -*/ return vValue; } @@ -552,10 +416,7 @@ CScript RawDataScriptIssue(Value *param,mc_Script *lpDetails,mc_Script *lpDetail entity_name=d.value_.get_str(); if(entity_name.size()) { - if(mc_gState->m_Features->OpDropDetailsScripts()) - { - lpDetails->SetSpecialParamValue(MC_ENT_SPRM_NAME,(const unsigned char*)(entity_name.c_str()),entity_name.size()); - } + lpDetails->SetSpecialParamValue(MC_ENT_SPRM_NAME,(const unsigned char*)(entity_name.c_str()),entity_name.size()); } } else @@ -580,10 +441,7 @@ CScript RawDataScriptIssue(Value *param,mc_Script *lpDetails,mc_Script *lpDetail } else { - if(mc_gState->m_Features->OpDropDetailsScripts()) - { - lpDetails->SetSpecialParamValue(MC_ENT_SPRM_ASSET_MULTIPLE,(unsigned char*)&multiple,4); - } + lpDetails->SetSpecialParamValue(MC_ENT_SPRM_ASSET_MULTIPLE,(unsigned char*)&multiple,4); } } else @@ -663,37 +521,17 @@ CScript RawDataScriptIssue(Value *param,mc_Script *lpDetails,mc_Script *lpDetail if(strError->size() == 0) { lpDetailsScript->Clear(); - if(mc_gState->m_Features->OpDropDetailsScripts()) + script=lpDetails->GetData(0,&bytes); + err=lpDetailsScript->SetNewEntityType(MC_ENT_TYPE_ASSET,0,script,bytes); + if(err) { - script=lpDetails->GetData(0,&bytes); - err=lpDetailsScript->SetNewEntityType(MC_ENT_TYPE_ASSET,0,script,bytes); - if(err) - { - *strError=string("Invalid custom fields, too long"); - } - else - { - script = lpDetailsScript->GetData(0,&bytes); - scriptOpReturn << vector(script, script + bytes) << OP_DROP << OP_RETURN; - } + *strError=string("Invalid custom fields, too long"); } else { - script=lpDetails->GetData(0,&bytes); - err=lpDetailsScript->SetAssetDetails(entity_name.c_str(),multiple,script,bytes); script = lpDetailsScript->GetData(0,&bytes); - if(err) - { - *strError=string("Invalid custom fields, too long"); - } - else - { - if(bytes > 0) - { - scriptOpReturn << OP_RETURN << vector(script, script + bytes); - } - } - } + scriptOpReturn << vector(script, script + bytes) << OP_DROP << OP_RETURN; + } } return scriptOpReturn; @@ -731,44 +569,22 @@ CScript RawDataScriptFollowOn(Value *param,mc_EntityDetails *entity,mc_Script *l } } - if(mc_gState->m_Features->OpDropDetailsScripts()) - { - int err; - lpDetailsScript->Clear(); - lpDetailsScript->SetEntity(entity->GetTxID()+MC_AST_SHORT_TXID_OFFSET); - script = lpDetailsScript->GetData(0,&bytes); - scriptOpReturn << vector(script, script + bytes) << OP_DROP; + lpDetailsScript->Clear(); + lpDetailsScript->SetEntity(entity->GetTxID()+MC_AST_SHORT_TXID_OFFSET); + script = lpDetailsScript->GetData(0,&bytes); + scriptOpReturn << vector(script, script + bytes) << OP_DROP; - lpDetailsScript->Clear(); - script=lpDetails->GetData(0,&bytes); - err=lpDetailsScript->SetNewEntityType(MC_ENT_TYPE_ASSET,1,script,bytes); - if(err) - { - *strError=string("Invalid custom fields, too long"); - } - else - { - script = lpDetailsScript->GetData(0,&bytes); - scriptOpReturn << vector(script, script + bytes) << OP_DROP << OP_RETURN; - } + lpDetailsScript->Clear(); + script=lpDetails->GetData(0,&bytes); + err=lpDetailsScript->SetNewEntityType(MC_ENT_TYPE_ASSET,1,script,bytes); + if(err) + { + *strError=string("Invalid custom fields, too long"); } else { - lpDetailsScript->Clear(); - script=lpDetails->GetData(0,&bytes); - err=lpDetailsScript->SetGeneralDetails(script,bytes); - if(err) - { - *strError=string("Invalid custom fields, too long"); - } - else - { - script = lpDetailsScript->GetData(0,&bytes); - if(bytes > 0) - { - scriptOpReturn << OP_RETURN << vector(script, script + bytes); - } - } + script = lpDetailsScript->GetData(0,&bytes); + scriptOpReturn << vector(script, script + bytes) << OP_DROP << OP_RETURN; } return scriptOpReturn; @@ -853,42 +669,17 @@ CScript RawDataScriptCreateStream(Value *param,mc_Script *lpDetails,mc_Script *l if(strError->size() == 0) { lpDetailsScript->Clear(); - if(mc_gState->m_Features->OpDropDetailsScripts()) + script=lpDetails->GetData(0,&bytes); + err=lpDetailsScript->SetNewEntityType(MC_ENT_TYPE_STREAM,0,script,bytes); + if(err) { - script=lpDetails->GetData(0,&bytes); - err=lpDetailsScript->SetNewEntityType(MC_ENT_TYPE_STREAM,0,script,bytes); - if(err) - { - *strError=string("Invalid custom fields, too long"); - } - else - { - script = lpDetailsScript->GetData(0,&bytes); - scriptOpReturn << vector(script, script + bytes) << OP_DROP << OP_RETURN; - } + *strError=string("Invalid custom fields, too long"); } else { - lpDetailsScript->SetNewEntityType(MC_ENT_TYPE_STREAM); script = lpDetailsScript->GetData(0,&bytes); - scriptOpReturn << vector(script, script + bytes) << OP_DROP; - - lpDetailsScript->Clear(); - script=lpDetails->GetData(0,&bytes); - err=lpDetailsScript->SetGeneralDetails(script,bytes); - if(err) - { - *strError=string("Invalid custom fields, too long"); - } - else - { - script = lpDetailsScript->GetData(0,&bytes); - if(bytes > 0) - { - scriptOpReturn << OP_RETURN << vector(script, script + bytes); - } - } - } + scriptOpReturn << vector(script, script + bytes) << OP_DROP << OP_RETURN; + } } return scriptOpReturn; diff --git a/src/rpc/rpcrawtransaction.cpp b/src/rpc/rpcrawtransaction.cpp index 2d5fe3b2..0735a6e8 100644 --- a/src/rpc/rpcrawtransaction.cpp +++ b/src/rpc/rpcrawtransaction.cpp @@ -160,61 +160,45 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, Object& entry) // int e=mc_gState->m_TmpScript->GetNumElements()-1; lpScript->ExtractAndDeleteDataFormat(&format); - int e=lpScript->GetNumElements()-1; + lpScript->SetElement(0); + err=lpScript->GetNewEntityType(&new_entity_type,&asset_update,details_script,&details_script_size); + if((err == 0) && (new_entity_type == MC_ENT_TYPE_ASSET)) { - if(mc_gState->m_Features->OpDropDetailsScripts()) + if(asset_update == 0) { - - lpScript->SetElement(0); + detals_script_found=true; + is_issuefirst=true; + *asset_name=0x00; + multiple=1; + value_offset=mc_FindSpecialParamInDetailsScript(details_script,details_script_size,MC_ENT_SPRM_NAME,&value_size); + if(value_offset<(uint32_t)details_script_size) + { + memcpy(asset_name,details_script+value_offset,value_size); + asset_name[value_size]=0x00; + } + value_offset=mc_FindSpecialParamInDetailsScript(details_script,details_script_size,MC_ENT_SPRM_ASSET_MULTIPLE,&value_size); + if(value_offset<(uint32_t)details_script_size) + { + multiple=mc_GetLE(details_script+value_offset,value_size); + } + } + } + else + { + err=lpScript->GetEntity(short_txid); + if(err == 0) + { + lpScript->SetElement(1); err=lpScript->GetNewEntityType(&new_entity_type,&asset_update,details_script,&details_script_size); if((err == 0) && (new_entity_type == MC_ENT_TYPE_ASSET)) { - if(asset_update == 0) + if(asset_update) { detals_script_found=true; - is_issuefirst=true; - *asset_name=0x00; - multiple=1; - value_offset=mc_FindSpecialParamInDetailsScript(details_script,details_script_size,MC_ENT_SPRM_NAME,&value_size); - if(value_offset<(uint32_t)details_script_size) - { - memcpy(asset_name,details_script+value_offset,value_size); - asset_name[value_size]=0x00; - } - value_offset=mc_FindSpecialParamInDetailsScript(details_script,details_script_size,MC_ENT_SPRM_ASSET_MULTIPLE,&value_size); - if(value_offset<(uint32_t)details_script_size) - { - multiple=mc_GetLE(details_script+value_offset,value_size); - } + is_issuemore=true; } - } - else - { - err=lpScript->GetEntity(short_txid); - if(err == 0) - { - lpScript->SetElement(1); - err=lpScript->GetNewEntityType(&new_entity_type,&asset_update,details_script,&details_script_size); - if((err == 0) && (new_entity_type == MC_ENT_TYPE_ASSET)) - { - if(asset_update) - { - detals_script_found=true; - is_issuemore=true; - } - } - } - } - } - else - { - lpScript->SetElement(e); - err=lpScript->GetAssetDetails(asset_name,&multiple,details_script,&details_script_size); - if(err == 0) - { - detals_script_found=true; - } - } + } + } } size_t elem_size; @@ -844,19 +828,13 @@ Value appendrawchange(const Array& params, bool fHelp) } } - if(mc_gState->m_Features->VerifySizeOfOpDropElements()) - { - int assets_per_opdrop=(MCP_STD_OP_DROP_SIZE-4)/(mc_gState->m_NetworkParams->m_AssetRefSize+MC_AST_ASSET_QUANTITY_SIZE); - - if(mc_gState->m_Features->VerifySizeOfOpDropElements()) - { - assets_per_opdrop=(MAX_SCRIPT_ELEMENT_SIZE-4)/(mc_gState->m_NetworkParams->m_AssetRefSize+MC_AST_ASSET_QUANTITY_SIZE); - } + int assets_per_opdrop; - if(amounts->GetCount() > assets_per_opdrop) - { - throw JSONRPCError(RPC_NOT_ALLOWED, strprintf("Too many assets, maximal number for this chain - %d",assets_per_opdrop)); - } + assets_per_opdrop=(MAX_SCRIPT_ELEMENT_SIZE-4)/(mc_gState->m_NetworkParams->m_AssetRefSize+MC_AST_ASSET_QUANTITY_SIZE); + + if(amounts->GetCount() > assets_per_opdrop) + { + throw JSONRPCError(RPC_NOT_ALLOWED, strprintf("Too many assets, maximal number for this chain - %d",assets_per_opdrop)); } CScript scriptChange=GetScriptForDestination(address.Get()); @@ -1013,16 +991,13 @@ void AddCacheInputScriptIfNeeded(CMutableTransaction& rawTx,Array inputs, bool f { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, cache must be boolean"); } - if(mc_gState->m_Features->CachedInputScript()) + if(cache_this.get_bool()) { - if(cache_this.get_bool()) + if(scriptPubKeyString.is_null()) { - if(scriptPubKeyString.is_null()) - { - fMissingScript=true; - } - cache_value=1; - } + fMissingScript=true; + } + cache_value=1; } } cache_array.push_back(cache_value); @@ -1266,12 +1241,9 @@ Value appendrawtransaction(const Array& params, bool fHelp) if( required & (MC_PTP_ADMIN | MC_PTP_MINE) ) { - if(mc_gState->m_Features->CachedInputScript()) + if(mc_gState->m_NetworkParams->GetInt64Param("supportminerprecheck")) { - if(mc_gState->m_NetworkParams->GetInt64Param("supportminerprecheck")) - { - fCachedInputScriptRequired=true; - } + fCachedInputScriptRequired=true; } } @@ -1433,12 +1405,9 @@ Value createrawtransaction(const Array& params, bool fHelp) if( required & (MC_PTP_ADMIN | MC_PTP_MINE) ) { - if(mc_gState->m_Features->CachedInputScript()) + if(mc_gState->m_NetworkParams->GetInt64Param("supportminerprecheck")) { - if(mc_gState->m_NetworkParams->GetInt64Param("supportminerprecheck")) - { - fCachedInputScriptRequired=true; - } + fCachedInputScriptRequired=true; } } diff --git a/src/rpc/rpcstreams.cpp b/src/rpc/rpcstreams.cpp index d93db3ae..f001de36 100644 --- a/src/rpc/rpcstreams.cpp +++ b/src/rpc/rpcstreams.cpp @@ -168,15 +168,7 @@ Value liststreams(const Array& params, bool fHelp) int root_stream_name_size; mc_gState->m_NetworkParams->GetParam("rootstreamname",&root_stream_name_size); - if( (root_stream_name_size <= 1) && (inputStrings.size() == 0) && (mc_gState->m_Features->FixedIn10008() == 0) ) // Patch, to be removed in 10008 - { - mc_AdjustStartAndCount(&count,&start,streams->GetCount()-1); - start++; - } - else - { - mc_AdjustStartAndCount(&count,&start,streams->GetCount()); - } + mc_AdjustStartAndCount(&count,&start,streams->GetCount()); Array partial_results; @@ -238,10 +230,6 @@ Value liststreams(const Array& params, bool fHelp) { return_partial=true; } - if( (root_stream_name_size <= 1) && (inputStrings.size() == 0) && (mc_gState->m_Features->FixedIn10008() == 0) ) // Patch, to be removed in 10008 - { - return_partial=true; - } mc_gState->m_Assets->FreeEntityList(streams); if(return_partial) { @@ -260,11 +248,6 @@ Value createstreamfromcmd(const Array& params, bool fHelp) if (fHelp || params.size() < 4) throw runtime_error("Help message not found\n"); - if(mc_gState->m_Features->Streams() == 0) - { - throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported for this protocol version"); - } - if (strcmp(params[1].get_str().c_str(),"stream")) throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid entity type, should be stream"); @@ -291,12 +274,9 @@ Value createstreamfromcmd(const Array& params, bool fHelp) if(params[3].type() != bool_type) throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid open flag, should be boolean"); - if(mc_gState->m_Features->Streams()) + if(stream_name == "*") { - if(stream_name == "*") - { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid stream name: *"); - } + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid stream name: *"); } unsigned char buf_a[MC_AST_ASSET_REF_SIZE]; @@ -381,54 +361,16 @@ Value createstreamfromcmd(const Array& params, bool fHelp) size_t elem_size; const unsigned char *elem; - if(mc_gState->m_Features->OpDropDetailsScripts()) + err=lpDetailsScript->SetNewEntityType(MC_ENT_TYPE_STREAM,0,script,bytes); + if(err) { - err=lpDetailsScript->SetNewEntityType(MC_ENT_TYPE_STREAM,0,script,bytes); - if(err) - { - strError= "Invalid custom fields or stream name, too long"; - goto exitlbl; + strError= "Invalid custom fields or stream name, too long"; + goto exitlbl; // throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid custom fields or stream name, too long"); - } - - elem = lpDetailsScript->GetData(0,&elem_size); - scriptOpReturn << vector(elem, elem + elem_size) << OP_DROP << OP_RETURN; } - else - { - lpDetailsScript->SetNewEntityType(MC_ENT_TYPE_STREAM); - err=lpDetailsScript->SetGeneralDetails(script,bytes); - if(err) - { - strError= "Invalid custom fields or stream name, too long"; - goto exitlbl; -// throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid custom fields or stream name, too long"); - } - - for(int e=0;eGetNumElements();e++) - { - elem = lpDetailsScript->GetData(e,&elem_size); - if(e == (lpDetailsScript->GetNumElements() - 1) ) - { - if(elem_size > 0) - { - scriptOpReturn << OP_RETURN << vector(elem, elem + elem_size); - } - else - { - scriptOpReturn << OP_RETURN; - } - } - else - { - if(elem_size > 0) - { - scriptOpReturn << vector(elem, elem + elem_size) << OP_DROP; - } - } - } - } + elem = lpDetailsScript->GetData(0,&elem_size); + scriptOpReturn << vector(elem, elem + elem_size) << OP_DROP << OP_RETURN; if(params[0].get_str() != "*") @@ -501,10 +443,6 @@ Value createfromcmd(const Array& params, bool fHelp) if (fHelp || params.size() < 4) throw runtime_error("Help message not found\n"); - if(mc_gState->m_Features->Streams() == 0) - { - throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported for this protocol version"); - } if (strcmp(params[1].get_str().c_str(),"stream") == 0) { return createstreamfromcmd(params,fHelp); @@ -522,11 +460,6 @@ Value createcmd(const Array& params, bool fHelp) { if (fHelp || params.size() < 3) throw runtime_error("Help message not found\n"); - - if(mc_gState->m_Features->Streams() == 0) - { - throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported for this protocol version"); - } Array ext_params; ext_params.push_back("*"); @@ -542,12 +475,7 @@ Value publish(const Array& params, bool fHelp) { if (fHelp || params.size() < 3 || params.size() > 4) throw runtime_error("Help message not found\n"); - - if(mc_gState->m_Features->Streams() == 0) - { - throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported for this protocol version"); - } - + Array ext_params; ext_params.push_back("*"); BOOST_FOREACH(const Value& value, params) @@ -563,11 +491,6 @@ Value publishfrom(const Array& params, bool fHelp) if (fHelp || params.size() != 4) throw runtime_error("Help message not found\n"); - if(mc_gState->m_Features->Streams() == 0) - { - throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported for this protocol version"); - } - mc_Script *lpScript=mc_gState->m_TmpBuffers->m_RpcScript3; lpScript->Clear(); @@ -722,10 +645,6 @@ Value subscribe(const Array& params, bool fHelp) if (fHelp || params.size() < 1 || params.size() > 2) throw runtime_error("Help message not found\n"); - if(mc_gState->m_Features->Streams() == 0) - { - throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported for this protocol version"); - } if((mc_gState->m_WalletMode & MC_WMD_TXS) == 0) { throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported with this wallet version. To get this functionality, run \"multichaind -walletdbversion=2 -rescan\" "); @@ -811,10 +730,6 @@ Value unsubscribe(const Array& params, bool fHelp) if (fHelp || params.size() < 1 || params.size() > 2) throw runtime_error("Help message not found\n"); - if(mc_gState->m_Features->Streams() == 0) - { - throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported for this protocol version"); - } if((mc_gState->m_WalletMode & MC_WMD_TXS) == 0) { throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported with this wallet version. To get this functionality, run \"multichaind -walletdbversion=2 -rescan\" "); @@ -898,10 +813,6 @@ Value liststreamtxitems(const Array& params, bool fHelp) if (fHelp || params.size() < 2 || params.size() > 3) throw runtime_error("Help message not found\n"); - if(mc_gState->m_Features->Streams() == 0) - { - throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported for this protocol version"); - } if((mc_gState->m_WalletMode & MC_WMD_TXS) == 0) { throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported with this wallet version. For full streams functionality, run \"multichaind -walletdbversion=2 -rescan\" "); @@ -1002,10 +913,6 @@ Value liststreamitems(const Array& params, bool fHelp) if (fHelp || params.size() < 1 || params.size() > 5) throw runtime_error("Help message not found\n"); - if(mc_gState->m_Features->Streams() == 0) - { - throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported for this protocol version"); - } if((mc_gState->m_WalletMode & MC_WMD_TXS) == 0) { throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported with this wallet version. For full streams functionality, run \"multichaind -walletdbversion=2 -rescan\" "); @@ -1113,10 +1020,6 @@ Value liststreamblockitems(const Array& params, bool fHelp) if (fHelp || params.size() < 2 || params.size() > 5) throw runtime_error("Help message not found\n"); - if(mc_gState->m_Features->Streams() == 0) - { - throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported for this protocol version"); - } if((mc_gState->m_WalletMode & MC_WMD_TXS) == 0) { throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported with this wallet version. For full streams functionality, run \"multichaind -walletdbversion=2 -rescan\" "); @@ -1258,10 +1161,6 @@ void getSubKeyEntityFromPublisher(string str,mc_TxEntityStat entStat,mc_TxEntity Value getstreamsummary(const Array& params, bool fPublisher) { - if(mc_gState->m_Features->Streams() == 0) - { - throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported for this protocol version"); - } if((mc_gState->m_WalletMode & MC_WMD_TXS) == 0) { throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported with this wallet version. For full streams functionality, run \"multichaind -walletdbversion=2 -rescan\" "); @@ -1527,10 +1426,6 @@ Value liststreamkeyitems(const Array& params, bool fHelp) if (fHelp || params.size() < 2 || params.size() > 6) throw runtime_error("Help message not found\n"); - if(mc_gState->m_Features->Streams() == 0) - { - throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported for this protocol version"); - } if((mc_gState->m_WalletMode & MC_WMD_TXS) == 0) { throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported with this wallet version. For full streams functionality, run \"multichaind -walletdbversion=2 -rescan\" "); @@ -1633,10 +1528,6 @@ Value liststreampublisheritems(const Array& params, bool fHelp) if (fHelp || params.size() < 2 || params.size() > 6) throw runtime_error("Help message not found\n"); - if(mc_gState->m_Features->Streams() == 0) - { - throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported for this protocol version"); - } if((mc_gState->m_WalletMode & MC_WMD_TXS) == 0) { throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported with this wallet version. For full streams functionality, run \"multichaind -walletdbversion=2 -rescan\" "); @@ -1968,10 +1859,6 @@ Value liststreamkeys(const Array& params, bool fHelp) if (fHelp || params.size() < 1 || params.size() > 6) throw runtime_error("Help message not found\n"); - if(mc_gState->m_Features->Streams() == 0) - { - throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported for this protocol version"); - } if((mc_gState->m_WalletMode & MC_WMD_TXS) == 0) { throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported with this wallet version. For full streams functionality, run \"multichaind -walletdbversion=2 -rescan\" "); @@ -1989,11 +1876,6 @@ Value liststreampublishers(const Array& params, bool fHelp) throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported with this wallet version. For full streams functionality, run \"multichaind -walletdbversion=2 -rescan\" "); } - if(mc_gState->m_Features->Streams() == 0) - { - throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported for this protocol version"); - } - return liststreamkeys_or_publishers(params,true); } diff --git a/src/rpc/rpcupgrades.cpp b/src/rpc/rpcupgrades.cpp index 3fb04023..69a317e2 100644 --- a/src/rpc/rpcupgrades.cpp +++ b/src/rpc/rpcupgrades.cpp @@ -14,11 +14,6 @@ Value createupgradefromcmd(const Array& params, bool fHelp) if (fHelp || params.size() < 5) throw runtime_error("Help message not found\n"); - if(mc_gState->m_Features->Upgrades() == 0) - { - throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported for this protocol version"); - } - if (strcmp(params[1].get_str().c_str(),"upgrade")) throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid entity type, should be stream"); @@ -44,12 +39,9 @@ Value createupgradefromcmd(const Array& params, bool fHelp) upgrade_name=params[2].get_str(); } - if(mc_gState->m_Features->Streams()) + if(upgrade_name == "*") { - if(upgrade_name == "*") - { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid upgrade name: *"); - } + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid upgrade name: *"); } unsigned char buf_a[MC_AST_ASSET_REF_SIZE]; @@ -447,7 +439,7 @@ Value listupgrades(const json_spirit::Array& params, bool fHelp) mc_PermissionDetails *plsRow; mc_PermissionDetails *plsDet; mc_EntityDetails upgrade_entity; - int flags,consensus,remaining; + int consensus; Value null_value; string param_name; @@ -524,7 +516,6 @@ Value listupgrades(const json_spirit::Array& params, bool fHelp) if(upgrades->GetCount()) { plsRow=(mc_PermissionDetails *)(upgrades->GetRow(upgrades->GetCount()-1)); - flags=plsRow->m_Flags; consensus=plsRow->m_RequiredAdmins; if(plsRow->m_Type != MC_PTP_UPGRADE) { @@ -549,7 +540,6 @@ Value listupgrades(const json_spirit::Array& params, bool fHelp) for(int j=0;jGetCount();j++) { plsDet=(mc_PermissionDetails *)(details->GetRow(j)); - remaining=plsDet->m_RequiredAdmins; if(plsDet->m_BlockFrom < plsDet->m_BlockTo) { uint160 addr; @@ -584,211 +574,4 @@ Value listupgrades(const json_spirit::Array& params, bool fHelp) return results; } -Value listupgrades_old(const json_spirit::Array& params, bool fHelp) -{ - if (fHelp || params.size() > 1) - throw runtime_error("Help message not found\n"); - - Array results; - mc_Buffer *upgrades; - int latest_version; - - upgrades=NULL; - - - vector inputStrings; - if (params.size() > 0 && params[0].type() != null_type && ((params[0].type() != str_type) || (params[0].get_str() !="*" ) ) ) - { - if(params[0].type() == str_type) - { - inputStrings.push_back(params[0].get_str()); - if(params[0].get_str() == "") - { - return results; - } - } - else - { - inputStrings=ParseStringList(params[0]); - if(inputStrings.size() == 0) - { - return results; - } - } - } - - - if(inputStrings.size()) - { - { - LOCK(cs_main); - for(int is=0;is<(int)inputStrings.size();is++) - { - string param=inputStrings[is]; - - mc_EntityDetails upgrade_entity; - ParseEntityIdentifier(param,&upgrade_entity, MC_ENT_TYPE_UPGRADE); - - upgrades=mc_gState->m_Permissions->GetUpgradeList(upgrade_entity.GetTxID() + MC_AST_SHORT_TXID_OFFSET,upgrades); - } - } - } - else - { - { - LOCK(cs_main); - upgrades=mc_gState->m_Permissions->GetUpgradeList(NULL,upgrades); - } - } - - if(upgrades == NULL) - throw JSONRPCError(RPC_INTERNAL_ERROR, "Cannot open entity database"); - - set stored_upgrades; - map map_sorted; - uint160 hash; - - for(int i=0;iGetCount();i++) - { - mc_PermissionDetails *plsRow; - plsRow=(mc_PermissionDetails *)(upgrades->GetRow(i)); - if(plsRow->m_Type == MC_PTP_UPGRADE) - { - memcpy(&hash,plsRow->m_Address,sizeof(uint160)); - stored_upgrades.insert(hash); - map_sorted.insert(std::make_pair(plsRow->m_LastRow,i)); - } - } - for(int i=0;iGetCount();i++) - { - mc_PermissionDetails *plsRow; - plsRow=(mc_PermissionDetails *)(upgrades->GetRow(i)); - if(plsRow->m_Type != MC_PTP_UPGRADE) - { - memcpy(&hash,plsRow->m_Address,sizeof(uint160)); - if(stored_upgrades.count(hash) == 0) - { -// plsRow->m_Type = MC_PTP_UPGRADE; - plsRow->m_BlockTo = 0; - map_sorted.insert(std::make_pair(plsRow->m_LastRow,i)); - } - } - } - - latest_version=(int)mc_gState->m_NetworkParams->GetInt64Param("protocolversion"); - BOOST_FOREACH(PAIRTYPE(const uint64_t, int)& item, map_sorted) - { - int i=item.second; - Object entry; - mc_PermissionDetails *plsRow; - mc_PermissionDetails *plsDet; - mc_EntityDetails upgrade_entity; - bool take_it,approved; - int flags,consensus,remaining; - plsRow=(mc_PermissionDetails *)(upgrades->GetRow(i)); - - upgrade_entity.Zero(); - mc_gState->m_Assets->FindEntityByShortTxID(&upgrade_entity,plsRow->m_Address); - - entry=UpgradeEntry(upgrade_entity.GetTxID()); - approved=true; - if(plsRow->m_BlockFrom >= plsRow->m_BlockTo) - { - approved=false; - entry.push_back(Pair("approved", false)); - Value null_value; - entry.push_back(Pair("appliedblock",null_value)); - } - else - { - entry.push_back(Pair("approved", true)); - int current_height=chainActive.Height(); - int applied_height=upgrade_entity.UpgradeStartBlock(); - if((int)plsRow->m_BlockReceived > applied_height) - { - applied_height=plsRow->m_BlockReceived; - } - if(current_height >=applied_height) - { - if(upgrade_entity.UpgradeProtocolVersion()) - { - if((latest_version < mc_gState->MinProtocolForbiddenDowngradeVersion()) || (upgrade_entity.UpgradeProtocolVersion() >= latest_version)) - { - latest_version=upgrade_entity.UpgradeProtocolVersion(); - entry.push_back(Pair("appliedblock",(int64_t)applied_height)); - } - else - { - Value null_value; - entry.push_back(Pair("appliedblock",null_value)); - } - } - else - { - entry.push_back(Pair("appliedblock",(int64_t)applied_height)); - } - } - else - { - Value null_value; - entry.push_back(Pair("appliedblock",null_value)); - } - } - - take_it=true; - flags=plsRow->m_Flags; - consensus=plsRow->m_RequiredAdmins; - - if(take_it) - { - Array admins; - Array pending; - mc_Buffer *details; - - if(plsRow->m_Type == MC_PTP_UPGRADE) - { - details=mc_gState->m_Permissions->GetPermissionDetails(plsRow); - } - else - { - details=NULL; - } - - if(details) - { - for(int j=0;jGetCount();j++) - { - plsDet=(mc_PermissionDetails *)(details->GetRow(j)); - remaining=plsDet->m_RequiredAdmins; - if(plsDet->m_BlockFrom < plsDet->m_BlockTo) - { - uint160 addr; - memcpy(&addr,plsDet->m_LastAdmin,sizeof(uint160)); - CKeyID lpKeyID=CKeyID(addr); - admins.push_back(CBitcoinAddress(lpKeyID).ToString()); - } - } - consensus=plsRow->m_RequiredAdmins; - } - if(admins.size() == 0) - { - if(plsRow->m_BlockFrom < plsRow->m_BlockTo) - { - uint160 addr; - memcpy(&addr,plsRow->m_LastAdmin,sizeof(uint160)); - CKeyID lpKeyID=CKeyID(addr); - admins.push_back(CBitcoinAddress(lpKeyID).ToString()); - } - } - - entry.push_back(Pair("admins", admins)); - entry.push_back(Pair("required", (int64_t)(consensus-admins.size()))); - results.push_back(entry); - } - } - - mc_gState->m_Permissions->FreePermissionList(upgrades); - - return results; -} diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index e2b3ae9f..d36d7f85 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -275,22 +275,15 @@ int ParseAssetKey(const char* asset_key,unsigned char *txid,unsigned char *asset int ParseAssetKeyToFullAssetRef(const char* asset_key,unsigned char *full_asset_ref,int *multiple,int *type,int entity_type) { int ret; - if(mc_gState->m_Features->ShortTxIDInTx()) + unsigned char txid[MC_ENT_KEY_SIZE]; + ret=ParseAssetKey(asset_key,txid,NULL,NULL,multiple,type,entity_type); + if(ret == MC_ASSET_KEY_UNCONFIRMED_GENESIS) { - unsigned char txid[MC_ENT_KEY_SIZE]; - ret=ParseAssetKey(asset_key,txid,NULL,NULL,multiple,type,entity_type); - if(ret == MC_ASSET_KEY_UNCONFIRMED_GENESIS) - { - ret=0; - } - memcpy(full_asset_ref+MC_AST_SHORT_TXID_OFFSET,txid+MC_AST_SHORT_TXID_OFFSET,MC_AST_SHORT_TXID_SIZE); - - mc_SetABRefType(full_asset_ref,MC_AST_ASSET_REF_TYPE_SHORT_TXID); - } - else - { - ret=ParseAssetKey(asset_key,NULL,full_asset_ref,NULL,multiple,type,entity_type); + ret=0; } + memcpy(full_asset_ref+MC_AST_SHORT_TXID_OFFSET,txid+MC_AST_SHORT_TXID_OFFSET,MC_AST_SHORT_TXID_SIZE); + + mc_SetABRefType(full_asset_ref,MC_AST_ASSET_REF_TYPE_SHORT_TXID); return ret; } @@ -527,11 +520,8 @@ Array PermissionEntries(const CTxOut& txout,mc_Script *lpScript,bool fLong) if(full_type & MC_PTP_CONNECT)entry.push_back(Pair("connect", (type & MC_PTP_CONNECT) ? true : false)); if(full_type & MC_PTP_SEND)entry.push_back(Pair("send", (type & MC_PTP_SEND) ? true : false)); if(full_type & MC_PTP_RECEIVE)entry.push_back(Pair("receive", (type & MC_PTP_RECEIVE) ? true : false)); - if(mc_gState->m_Features->Streams()) - { - if(full_type & MC_PTP_WRITE)entry.push_back(Pair("write", (type & MC_PTP_WRITE) ? true : false)); - if(full_type & MC_PTP_CREATE)entry.push_back(Pair("create", (type & MC_PTP_CREATE) ? true : false)); - } + if(full_type & MC_PTP_WRITE)entry.push_back(Pair("write", (type & MC_PTP_WRITE) ? true : false)); + if(full_type & MC_PTP_CREATE)entry.push_back(Pair("create", (type & MC_PTP_CREATE) ? true : false)); if(full_type & MC_PTP_ISSUE)entry.push_back(Pair("issue", (type & MC_PTP_ISSUE) ? true : false)); if(full_type & MC_PTP_MINE)entry.push_back(Pair("mine", (type & MC_PTP_MINE) ? true : false)); if(full_type & MC_PTP_ADMIN)entry.push_back(Pair("admin", (type & MC_PTP_ADMIN) ? true : false)); @@ -1416,13 +1406,7 @@ string ParseRawOutputObject(Value param,CAmount& nAmount,mc_Script *lpScript, in memset(buf,0,MC_AST_ASSET_FULLREF_BUF_SIZE); - if(mc_gState->m_Features->VerifySizeOfOpDropElements()) - { - if(mc_gState->m_Features->VerifySizeOfOpDropElements()) - { - assets_per_opdrop=(MAX_SCRIPT_ELEMENT_SIZE-4)/(mc_gState->m_NetworkParams->m_AssetRefSize+MC_AST_ASSET_QUANTITY_SIZE); - } - } + assets_per_opdrop=(MAX_SCRIPT_ELEMENT_SIZE-4)/(mc_gState->m_NetworkParams->m_AssetRefSize+MC_AST_ASSET_QUANTITY_SIZE); BOOST_FOREACH(const Pair& a, param.get_obj()) { @@ -2188,92 +2172,85 @@ CScript ParseRawMetadataNotRefactored(Value param,uint32_t allowed_objects,mc_En if(d.name_ == "inputcache") { new_type=-3; - if( ((allowed_objects & 0x0200) == 0) || (mc_gState->m_Features->CachedInputScript() == 0) ) + if(d.value_.type() != array_type) { - strError=string("Keyword not allowed in this API"); + strError=string("Array should be specified for inputcache"); } else { - if(d.value_.type() != array_type) - { - strError=string("Array should be specified for inputcache"); - } - else + int cs_offset,cs_vin,cs_size; + string cs_script=""; + Array csa=d.value_.get_array(); + lpDetails->Clear(); + lpDetails->SetCachedScript(0,&cs_offset,-1,NULL,-1); + for(int csi=0;csi<(int)csa.size();csi++) { - int cs_offset,cs_vin,cs_size; - string cs_script=""; - Array csa=d.value_.get_array(); - lpDetails->Clear(); - lpDetails->SetCachedScript(0,&cs_offset,-1,NULL,-1); - for(int csi=0;csi<(int)csa.size();csi++) + if(strError.size() == 0) { - if(strError.size() == 0) + if(csa[csi].type() != obj_type) { - if(csa[csi].type() != obj_type) + strError=string("Elements of inputcache should be objects"); + } + cs_vin=-1; + cs_size=-1; + BOOST_FOREACH(const Pair& csf, csa[csi].get_obj()) + { + bool cs_parsed=false; + if(csf.name_ == "vin") { - strError=string("Elements of inputcache should be objects"); - } - cs_vin=-1; - cs_size=-1; - BOOST_FOREACH(const Pair& csf, csa[csi].get_obj()) - { - bool cs_parsed=false; - if(csf.name_ == "vin") - { - cs_parsed=true; - if(csf.value_.type() != int_type) - { - strError=string("vin should be integer"); - } - else - { - cs_vin=csf.value_.get_int(); - } - } - if(csf.name_ == "scriptPubKey") + cs_parsed=true; + if(csf.value_.type() != int_type) { - cs_parsed=true; - if(csf.value_.type() != str_type) - { - strError=string("scriptPubKey should be string"); - } - else - { - cs_script=csf.value_.get_str(); - cs_size=cs_script.size()/2; - } + strError=string("vin should be integer"); } - if(!cs_parsed) + else { - strError=string("Invalid field: ") + csf.name_; - } + cs_vin=csf.value_.get_int(); + } } - if(strError.size() == 0) + if(csf.name_ == "scriptPubKey") { - if(cs_vin<0) + cs_parsed=true; + if(csf.value_.type() != str_type) { - strError=string("Missing vin field"); + strError=string("scriptPubKey should be string"); } + else + { + cs_script=csf.value_.get_str(); + cs_size=cs_script.size()/2; + } } - if(strError.size() == 0) + if(!cs_parsed) { - if(cs_size<0) - { - strError=string("Missing scriptPubKey field"); - } - } - if(strError.size() == 0) + strError=string("Invalid field: ") + csf.name_; + } + } + if(strError.size() == 0) + { + if(cs_vin<0) { - bool fIsHex; - vector dataData(ParseHex(cs_script.c_str(),fIsHex)); - if(!fIsHex) - { - strError=string("scriptPubKey should be hexadecimal string"); - } - else - { - lpDetails->SetCachedScript(cs_offset,&cs_offset,cs_vin,&dataData[0],cs_size); - } + strError=string("Missing vin field"); + } + } + if(strError.size() == 0) + { + if(cs_size<0) + { + strError=string("Missing scriptPubKey field"); + } + } + if(strError.size() == 0) + { + bool fIsHex; + vector dataData(ParseHex(cs_script.c_str(),fIsHex)); + if(!fIsHex) + { + strError=string("scriptPubKey should be hexadecimal string"); + } + else + { + lpDetails->SetCachedScript(cs_offset,&cs_offset,cs_vin,&dataData[0],cs_size); } } } @@ -2723,17 +2700,11 @@ CScript ParseRawMetadataNotRefactored(Value param,uint32_t allowed_objects,mc_En } if(multiple_is_set) { - if(mc_gState->m_Features->OpDropDetailsScripts()) - { - lpDetails->SetSpecialParamValue(MC_ENT_SPRM_ASSET_MULTIPLE,(unsigned char*)&multiple,4); - } + lpDetails->SetSpecialParamValue(MC_ENT_SPRM_ASSET_MULTIPLE,(unsigned char*)&multiple,4); } if(entity_name.size()) { - if(mc_gState->m_Features->OpDropDetailsScripts()) - { - lpDetails->SetSpecialParamValue(MC_ENT_SPRM_NAME,(const unsigned char*)(entity_name.c_str()),entity_name.size());//+1); - } + lpDetails->SetSpecialParamValue(MC_ENT_SPRM_NAME,(const unsigned char*)(entity_name.c_str()),entity_name.size());//+1); } if(vKey.size()) { @@ -2812,69 +2783,32 @@ CScript ParseRawMetadataNotRefactored(Value param,uint32_t allowed_objects,mc_En { if(new_type == MC_ENT_TYPE_UPGRADE) { - if(mc_gState->m_Features->Upgrades()) + if((allowed_objects & 0x0040) == 0) { - if((allowed_objects & 0x0040) == 0) - { - strError=string("Creating new upgrades not allowed in this API"); - } - else - { - if(lpDetails->m_Size) - { - strError=string("Invalid fields in details object"); - } - if(entity_name.size()) - { - lpDetails->SetSpecialParamValue(MC_ENT_SPRM_NAME,(const unsigned char*)(entity_name.c_str()),entity_name.size());//+1); - } - if(protocol_version > 0) - { - lpDetails->SetSpecialParamValue(MC_ENT_SPRM_UPGRADE_PROTOCOL_VERSION,(unsigned char*)&protocol_version,4); - } - else - { - strError=string("Missing protocol-version"); - } - if(startblock > 0) - { - lpDetails->SetSpecialParamValue(MC_ENT_SPRM_UPGRADE_START_BLOCK,(unsigned char*)&startblock,4); - } - if(multiple_is_set) - { - strError=string("Invalid field: multiple"); - } - if(open_is_set) - { - strError=string("Invalid field: open"); - } - if(vKey.size()) - { - strError=string("Invalid field: key"); - } - if(vValue.size()) - { - strError=string("Invalid field: value"); - } - } + strError=string("Creating new upgrades not allowed in this API"); } else - { - strError=string("Upgrades are not supported by this protocol version"); - } - } - } - - if(strError.size() == 0) - { - if(new_type == -5) - { - if(mc_gState->m_Features->Upgrades()) { if(lpDetails->m_Size) { - strError=string("Invalid field: details"); + strError=string("Invalid fields in details object"); + } + if(entity_name.size()) + { + lpDetails->SetSpecialParamValue(MC_ENT_SPRM_NAME,(const unsigned char*)(entity_name.c_str()),entity_name.size());//+1); + } + if(protocol_version > 0) + { + lpDetails->SetSpecialParamValue(MC_ENT_SPRM_UPGRADE_PROTOCOL_VERSION,(unsigned char*)&protocol_version,4); } + else + { + strError=string("Missing protocol-version"); + } + if(startblock > 0) + { + lpDetails->SetSpecialParamValue(MC_ENT_SPRM_UPGRADE_START_BLOCK,(unsigned char*)&startblock,4); + } if(multiple_is_set) { strError=string("Invalid field: multiple"); @@ -2892,9 +2826,32 @@ CScript ParseRawMetadataNotRefactored(Value param,uint32_t allowed_objects,mc_En strError=string("Invalid field: value"); } } - else + } + } + + if(strError.size() == 0) + { + if(new_type == -5) + { + if(lpDetails->m_Size) + { + strError=string("Invalid field: details"); + } + if(multiple_is_set) + { + strError=string("Invalid field: multiple"); + } + if(open_is_set) { - strError=string("Upgrades are not supported by this protocol version"); + strError=string("Invalid field: open"); + } + if(vKey.size()) + { + strError=string("Invalid field: key"); + } + if(vValue.size()) + { + strError=string("Invalid field: value"); } } } @@ -2907,64 +2864,32 @@ CScript ParseRawMetadataNotRefactored(Value param,uint32_t allowed_objects,mc_En if(new_type == MC_ENT_TYPE_ASSET) { - if(mc_gState->m_Features->OpDropDetailsScripts()) + script=lpDetails->GetData(0,&bytes); + err=lpDetailsScript->SetNewEntityType(MC_ENT_TYPE_ASSET,0,script,bytes); + if(err) { - script=lpDetails->GetData(0,&bytes); - err=lpDetailsScript->SetNewEntityType(MC_ENT_TYPE_ASSET,0,script,bytes); - if(err) - { - strError=string("Invalid custom fields, too long"); - } - else - { - script = lpDetailsScript->GetData(0,&bytes); - scriptOpReturn << vector(script, script + bytes) << OP_DROP << OP_RETURN; - } + strError=string("Invalid custom fields, too long"); } else { - script=lpDetails->GetData(0,&bytes); - lpDetailsScript->SetAssetDetails(entity_name.c_str(),multiple,script,bytes); script = lpDetailsScript->GetData(0,&bytes); - if(bytes > 0) - { - scriptOpReturn << OP_RETURN << vector(script, script + bytes); - } + scriptOpReturn << vector(script, script + bytes) << OP_DROP << OP_RETURN; } } if(new_type == MC_ENT_TYPE_STREAM) { - if(mc_gState->m_Features->OpDropDetailsScripts()) + int err; + script=lpDetails->GetData(0,&bytes); + err=lpDetailsScript->SetNewEntityType(MC_ENT_TYPE_STREAM,0,script,bytes); + if(err) { - int err; - script=lpDetails->GetData(0,&bytes); - err=lpDetailsScript->SetNewEntityType(MC_ENT_TYPE_STREAM,0,script,bytes); - if(err) - { - strError=string("Invalid custom fields, too long"); - } - else - { - script = lpDetailsScript->GetData(0,&bytes); - scriptOpReturn << vector(script, script + bytes) << OP_DROP << OP_RETURN; - } + strError=string("Invalid custom fields, too long"); } else { - lpDetailsScript->Clear(); - lpDetailsScript->SetNewEntityType(MC_ENT_TYPE_STREAM); script = lpDetailsScript->GetData(0,&bytes); - scriptOpReturn << vector(script, script + bytes) << OP_DROP; - - lpDetailsScript->Clear(); - script=lpDetails->GetData(0,&bytes); - lpDetailsScript->SetGeneralDetails(script,bytes); - script = lpDetailsScript->GetData(0,&bytes); - if(bytes > 0) - { - scriptOpReturn << OP_RETURN << vector(script, script + bytes); - } + scriptOpReturn << vector(script, script + bytes) << OP_DROP << OP_RETURN; } } @@ -2986,37 +2911,23 @@ CScript ParseRawMetadataNotRefactored(Value param,uint32_t allowed_objects,mc_En if(new_type == -2) { - if(mc_gState->m_Features->OpDropDetailsScripts()) - { - int err; - lpDetailsScript->Clear(); - lpDetailsScript->SetEntity(entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET); - script = lpDetailsScript->GetData(0,&bytes); - scriptOpReturn << vector(script, script + bytes) << OP_DROP; + int err; + lpDetailsScript->Clear(); + lpDetailsScript->SetEntity(entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET); + script = lpDetailsScript->GetData(0,&bytes); + scriptOpReturn << vector(script, script + bytes) << OP_DROP; - lpDetailsScript->Clear(); - script=lpDetails->GetData(0,&bytes); - err=lpDetailsScript->SetNewEntityType(MC_ENT_TYPE_ASSET,1,script,bytes); - if(err) - { - strError=string("Invalid custom fields, too long"); - } - else - { - script = lpDetailsScript->GetData(0,&bytes); - scriptOpReturn << vector(script, script + bytes) << OP_DROP << OP_RETURN; - } + lpDetailsScript->Clear(); + script=lpDetails->GetData(0,&bytes); + err=lpDetailsScript->SetNewEntityType(MC_ENT_TYPE_ASSET,1,script,bytes); + if(err) + { + strError=string("Invalid custom fields, too long"); } else { - lpDetailsScript->Clear(); - script=lpDetails->GetData(0,&bytes); - lpDetailsScript->SetGeneralDetails(script,bytes); script = lpDetailsScript->GetData(0,&bytes); - if(bytes > 0) - { - scriptOpReturn << OP_RETURN << vector(script, script + bytes); - } + scriptOpReturn << vector(script, script + bytes) << OP_DROP << OP_RETURN; } } diff --git a/src/rpc/rpcwalletsend.cpp b/src/rpc/rpcwalletsend.cpp index c2f3e925..550a732b 100644 --- a/src/rpc/rpcwalletsend.cpp +++ b/src/rpc/rpcwalletsend.cpp @@ -863,13 +863,6 @@ Value sendassetfrom(const Array& params, bool fHelp) mc_EntityDetails entity; ParseEntityIdentifier(params[2],&entity, MC_ENT_TYPE_ASSET); memcpy(buf,entity.GetFullRef(),MC_AST_ASSET_FULLREF_SIZE); - if(mc_gState->m_Features->ShortTxIDInTx() == 0) - { - if(entity.IsUnconfirmedGenesis()) - { - throw JSONRPCError(RPC_UNCONFIRMED_ENTITY, "Unconfirmed asset: "+params[2].get_str()); - } - } multiple=entity.GetAssetMultiple(); } else @@ -994,13 +987,6 @@ Value sendassettoaddress(const Array& params, bool fHelp) mc_EntityDetails entity; ParseEntityIdentifier(params[1],&entity, MC_ENT_TYPE_ASSET); memcpy(buf,entity.GetFullRef(),MC_AST_ASSET_FULLREF_SIZE); - if(mc_gState->m_Features->ShortTxIDInTx() == 0) - { - if(entity.IsUnconfirmedGenesis()) - { - throw JSONRPCError(RPC_UNCONFIRMED_ENTITY, "Unconfirmed asset: "+params[1].get_str()); - } - } multiple=entity.GetAssetMultiple(); } else diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index 5091557e..8f08b69b 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -257,10 +257,6 @@ bool EvalScript(vector >& stack, const CScript& script, un int nOpCount = 0; bool fRequireMinimal = (flags & SCRIPT_VERIFY_MINIMALDATA) != 0; -/* MCHN START */ - bool fLongElement=false; -/* MCHN END */ - try { while (pc < pend) @@ -272,40 +268,7 @@ bool EvalScript(vector >& stack, const CScript& script, un // if (!script.GetOp(pc, opcode, vchPushValue)) return set_error(serror, SCRIPT_ERR_BAD_OPCODE); -/* MCHN START */ - if(fLongElement) - { - if(mc_gState->m_Features->FixedIn10007() == 0) - { - if(opcode != OP_DROP) - { - return set_error(serror, SCRIPT_ERR_PUSH_SIZE); - } - } - } - if(mc_gState->m_Features->Streams()) // It is now part of the IsStandard check - { - fLongElement=false; - } - else - { - if (vchPushValue.size() > MAX_SCRIPT_ELEMENT_SIZE) - { - if(mc_gState->m_Features->VerifySizeOfOpDropElements() == 0) - { - fLongElement=true; - } - else - { - return set_error(serror, SCRIPT_ERR_PUSH_SIZE); - } - } - else - { - fLongElement=false; - } - } -/* MCHN END */ + // Note how OP_RESERVED does not count towards the opcode limit. if (opcode > OP_16 && ++nOpCount > 201) return set_error(serror, SCRIPT_ERR_OP_COUNT); @@ -1186,14 +1149,11 @@ bool VerifyScript(const CScript& scriptSig, const CScript& scriptPubKey, unsigne // return set_error(serror, SCRIPT_ERR_EVAL_FALSE); /* MCHN START */ - if(mc_gState->m_Features->Streams()) + if(!scriptSig.HasSmallIntegerInTheBeginning()) { - if(!scriptSig.HasSmallIntegerInTheBeginning()) + if(stack.size() != 1) { - if(stack.size() != 1) - { - return set_error(serror, SCRIPT_ERR_EVAL_FALSE); - } + return set_error(serror, SCRIPT_ERR_EVAL_FALSE); } } /* MCHN END */ @@ -1208,31 +1168,28 @@ bool VerifyScript(const CScript& scriptSig, const CScript& scriptPubKey, unsigne unsigned int p2shflags=flags; CTxDestination addressRet; - if(mc_gState->m_Features->Streams()) + if(!scriptSig.HasSmallIntegerInTheBeginning()) { - if(!scriptSig.HasSmallIntegerInTheBeginning()) - { - return set_error(serror, SCRIPT_ERR_EVAL_FALSE); - } - if(!ExtractDestination(scriptPubKey, addressRet)) - { - return set_error(serror, SCRIPT_ERR_VERIFY); - } + return set_error(serror, SCRIPT_ERR_EVAL_FALSE); + } + if(!ExtractDestination(scriptPubKey, addressRet)) + { + return set_error(serror, SCRIPT_ERR_VERIFY); + } - if( (p2shflags & SCRIPT_VERIFY_SKIP_SEND_PERMISSION_CHECK) == 0) + if( (p2shflags & SCRIPT_VERIFY_SKIP_SEND_PERMISSION_CHECK) == 0) + { + CScriptID *lpScriptID=boost::get (&addressRet); + if(lpScriptID) { - CScriptID *lpScriptID=boost::get (&addressRet); - if(lpScriptID) - { - if(mc_gState->m_Permissions->CanSend(NULL,(unsigned char*)(lpScriptID))) - { - p2shflags |= SCRIPT_VERIFY_SKIP_SEND_PERMISSION_CHECK; - } - } - else + if(mc_gState->m_Permissions->CanSend(NULL,(unsigned char*)(lpScriptID))) { - return set_error(serror, SCRIPT_ERR_VERIFY); - } + p2shflags |= SCRIPT_VERIFY_SKIP_SEND_PERMISSION_CHECK; + } + } + else + { + return set_error(serror, SCRIPT_ERR_VERIFY); } } diff --git a/src/script/standard.cpp b/src/script/standard.cpp index 2808dc6b..ce040d90 100644 --- a/src/script/standard.cpp +++ b/src/script/standard.cpp @@ -64,14 +64,10 @@ bool Solver(const CScript& scriptPubKey, txnouttype& typeRet, vectorm_NetworkParams->IsProtocolMultichain()) { - if(mc_gState->m_Features->Streams()) - { - mTemplates.insert(make_pair(TX_NULL_DATA, CScript() << OP_DROPDATA << OP_DROP << OP_RETURN)); - mTemplates.insert(make_pair(TX_NULL_DATA, CScript() << OP_DROPDATA << OP_DROP << OP_RETURN << OP_SMALLDATA)); - mTemplates.insert(make_pair(TX_NULL_DATA, CScript() << OP_DROPDATA << OP_DROP << OP_DROPDATA << OP_DROP << OP_RETURN)); - mTemplates.insert(make_pair(TX_NULL_DATA, CScript() << OP_DROPDATA << OP_DROP << OP_DROPDATA << OP_DROP << OP_RETURN << OP_SMALLDATA)); - } - + mTemplates.insert(make_pair(TX_NULL_DATA, CScript() << OP_DROPDATA << OP_DROP << OP_RETURN)); + mTemplates.insert(make_pair(TX_NULL_DATA, CScript() << OP_DROPDATA << OP_DROP << OP_RETURN << OP_SMALLDATA)); + mTemplates.insert(make_pair(TX_NULL_DATA, CScript() << OP_DROPDATA << OP_DROP << OP_DROPDATA << OP_DROP << OP_RETURN)); + mTemplates.insert(make_pair(TX_NULL_DATA, CScript() << OP_DROPDATA << OP_DROP << OP_DROPDATA << OP_DROP << OP_RETURN << OP_SMALLDATA)); } mTemplates.insert(make_pair(TX_SCRIPTHASH, CScript() << OP_HASH160 << OP_PUBKEYHASH << OP_EQUAL)); @@ -190,11 +186,8 @@ bool Solver(const CScript& scriptPubKey, txnouttype& typeRet, vectorm_Features->VerifySizeOfOpDropElements()) - { - if (vch1.size() > MAX_SCRIPT_ELEMENT_SIZE) - break; - } + if (vch1.size() > MAX_SCRIPT_ELEMENT_SIZE) + break; } /* MCHN END */ else if (opcode1 != opcode2 || vch1 != vch2) @@ -388,23 +381,13 @@ bool IsStandardNullData(const CScript& scriptPubKey,bool standard_check) int max_op_drop_count=2; bool check_sizes=false; - if(mc_gState->m_Features->VerifySizeOfOpDropElements()) + if( !fixed || standard_check ) { - if( !fixed || standard_check ) - { - check_sizes=true; - } + check_sizes=true; } -/* - unsigned int sizes[3]; - sizes[0]=0; - sizes[1]=0; - sizes[2]=0; -*/ if(mc_gState->m_Features->FormattedData()) - { - + { max_op_drop_count=MAX_OP_RETURN_OP_DROP_COUNT; } @@ -980,54 +963,27 @@ bool IsStandard(const CScript& scriptPubKey, txnouttype& whichType) max_op_drops=MCP_STD_OP_DROP_COUNT; - if(mc_gState->m_Features->VerifySizeOfOpDropElements()) + for(int d=0;d MAX_SCRIPT_ELEMENT_SIZE) ) { - if ( !scriptPubKey.GetOp(pc, opcode, vch) || (vch.size() > MAX_SCRIPT_ELEMENT_SIZE) ) - { - whichType = TX_NONSTANDARD; - return false; - } - if ( !scriptPubKey.GetOp(pc, opcode) || (opcode != OP_DROP) ) - { - whichType = TX_NONSTANDARD; - return false; - } + whichType = TX_NONSTANDARD; + return false; } - else + if ( !scriptPubKey.GetOp(pc, opcode) || (opcode != OP_DROP) ) { - return true; + whichType = TX_NONSTANDARD; + return false; } } - } - else - { - for(int d=0;dm_Features->ShortTxIDInTx() == 0) && (entity.IsUnconfirmedGenesis() != 0) ) + memcpy(buf,entity.GetFullRef(),MC_AST_ASSET_FULLREF_SIZE); + row=amounts->Seek(buf); + last=0; + if(row >= 0) { - if(required) // Unconfirmed genesis in protocol < 10007, cannot be spent - { - memset(buf,0,MC_AST_ASSET_FULLREF_SIZE); - mc_SetABRefType(buf,MC_AST_ASSET_REF_TYPE_GENESIS); - mc_SetABQuantity(buf,total); - amounts->Add(buf); - *required |= MC_PTP_ISSUE; - } + last=mc_GetABQuantity(amounts->GetRow(row)); + total+=last; + mc_SetABQuantity(amounts->GetRow(row),total); } - else + else { - memcpy(buf,entity.GetFullRef(),MC_AST_ASSET_FULLREF_SIZE); - row=amounts->Seek(buf); - last=0; - if(row >= 0) - { - last=mc_GetABQuantity(amounts->GetRow(row)); - total+=last; - mc_SetABQuantity(amounts->GetRow(row),total); - } - else - { - mc_SetABQuantity(buf,total); - amounts->Add(buf); - } - - if(required) + mc_SetABQuantity(buf,total); + amounts->Add(buf); + } + + if(required) + { + if(expected_required == 0) { - if(expected_required == 0) - { - *required |= MC_PTP_ISSUE; - } - } + *required |= MC_PTP_ISSUE; + } } } else // Asset not found, no error but the caller should check required field @@ -451,14 +437,6 @@ bool ParseMultichainTxOutToBuffer(uint256 hash, } } } - else - { - if(mc_gState->m_Features->OpDropDetailsScripts() == 0)// May be Follow-on details from v10007 - { - strFailReason="Invalid publish script, not stream"; - return false; - } - } } else { @@ -512,13 +490,10 @@ bool ParseMultichainTxOutToBuffer(uint256 hash, *required |= admin_type; if( type & (MC_PTP_ADMIN | MC_PTP_MINE) ) { - if(mc_gState->m_Features->CachedInputScript()) + if(mc_gState->m_NetworkParams->GetInt64Param("supportminerprecheck")) { - if(mc_gState->m_NetworkParams->GetInt64Param("supportminerprecheck")) - { - *required |= MC_PTP_CACHED_SCRIPT_REQUIRED; - } - } + *required |= MC_PTP_CACHED_SCRIPT_REQUIRED; + } } if(hash == 0) diff --git a/src/utils/utilwrapper.cpp b/src/utils/utilwrapper.cpp index 53223018..9232fedd 100644 --- a/src/utils/utilwrapper.cpp +++ b/src/utils/utilwrapper.cpp @@ -738,10 +738,6 @@ int mc_BuildDescription(int build, char *desc) int mc_MultichainParams::SetProtocolGlobals() { - if(mc_gState->m_Features->ShortTxIDInTx() == 0) - { - m_AssetRefSize=MC_AST_ASSET_REF_SIZE; - } MCP_ALLOW_ARBITRARY_OUTPUTS=1; if(mc_gState->m_Features->FixedDestinationExtraction() != 0) { @@ -798,12 +794,6 @@ int mc_MultichainParams::SetGlobals() CENT=0; MAX_MONEY=0; } -/* - if(mc_gState->m_Features->ShortTxIDInTx() == 0) - { - m_AssetRefSize=MC_AST_ASSET_REF_SIZE; - } -*/ MCP_MAX_STD_OP_RETURN_COUNT=mc_gState->m_NetworkParams->GetInt64Param("maxstdopreturnscount"); MCP_INITIAL_BLOCK_REWARD=mc_gState->m_NetworkParams->GetInt64Param("initialblockreward"); MCP_FIRST_BLOCK_REWARD=mc_gState->m_NetworkParams->GetInt64Param("firstblockreward"); @@ -815,17 +805,6 @@ int mc_MultichainParams::SetGlobals() MCP_ANYONE_CAN_RECEIVE=mc_gState->m_NetworkParams->GetInt64Param("anyonecanreceive"); MCP_ANYONE_CAN_ACTIVATE=mc_gState->m_NetworkParams->GetInt64Param("anyonecanactivate"); MCP_MINIMUM_PER_OUTPUT=mc_gState->m_NetworkParams->GetInt64Param("minimumperoutput"); -/* - MCP_ALLOW_ARBITRARY_OUTPUTS=1; - if(mc_gState->m_Features->FixedDestinationExtraction() != 0) - { - int aao=mc_gState->m_NetworkParams->GetInt64Param("allowarbitraryoutputs"); - if(aao>=0) - { - MCP_ALLOW_ARBITRARY_OUTPUTS=aao; - } - } - */ MCP_ALLOW_MULTISIG_OUTPUTS=mc_gState->m_NetworkParams->GetInt64Param("allowmultisigoutputs"); MCP_ALLOW_P2SH_OUTPUTS=mc_gState->m_NetworkParams->GetInt64Param("allowp2shoutputs"); MCP_WITH_NATIVE_CURRENCY=0; diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index cce28d7b..f1af8731 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1417,22 +1417,10 @@ mc_TxImport *StartImport(CWallet *lpWallet,bool fOnlyUnsynced, int block, int *e for(unsigned int i=0;im_Features->ShortTxIDInTx() == 0) ) + if(mc_gState->m_Assets->FindEntityByShortTxID(&stream_entity,vStreamsToImport[i].m_EntityID)) { - if(mc_gState->m_Assets->FindEntityByRef(&stream_entity,vStreamsToImport[i].m_EntityID)) - { - take_it=true; - } - } - else - { - if(mc_gState->m_Assets->FindEntityByShortTxID(&stream_entity,vStreamsToImport[i].m_EntityID)) - { - take_it=true; - } - } -// if(mc_gState->m_Assets->FindEntityByShortTxID(&stream_entity,vStreamsToImport[i].m_EntityID)) + take_it=true; + } if(take_it) { entity.Zero(); diff --git a/src/wallet/walletcoins.cpp b/src/wallet/walletcoins.cpp index 9636ae6e..0a3d8219 100644 --- a/src/wallet/walletcoins.cpp +++ b/src/wallet/walletcoins.cpp @@ -559,46 +559,21 @@ void DebugPrintAssetTxOut(uint256 hash,int index,unsigned char* assetrefbin,int6 { string txid=hash.GetHex(); - - if(mc_gState->m_Features->ShortTxIDInTx()) + if(debug_print) { - if(debug_print) + printf("TxOut: %s-%d ",txid.c_str(),index); + if(mc_GetABRefType(assetrefbin) == MC_AST_ASSET_REF_TYPE_SPECIAL) { - printf("TxOut: %s-%d ",txid.c_str(),index); - if(mc_GetABRefType(assetrefbin) == MC_AST_ASSET_REF_TYPE_SPECIAL) - { - printf("Special: %08x%08x",(uint32_t)mc_GetLE(assetrefbin,4),(uint32_t)mc_GetLE(assetrefbin+4,4)); - } - else - { - for(int i=MC_AST_SHORT_TXID_OFFSET+MC_AST_SHORT_TXID_SIZE-1;i>=MC_AST_SHORT_TXID_OFFSET;i--) - { - printf("%02x",assetrefbin[i]); - } - } - printf(" %ld\n",quantity); + printf("Special: %08x%08x",(uint32_t)mc_GetLE(assetrefbin,4),(uint32_t)mc_GetLE(assetrefbin+4,4)); } - } - else - { - string assetref=""; - if(mc_GetABRefType(assetrefbin) == MC_AST_ASSET_REF_TYPE_SHORT_TXID) + else { - for(int i=0;i<8;i++) + for(int i=MC_AST_SHORT_TXID_OFFSET+MC_AST_SHORT_TXID_SIZE-1;i>=MC_AST_SHORT_TXID_OFFSET;i--) { - assetref += strprintf("%02x",assetrefbin[MC_AST_SHORT_TXID_OFFSET+MC_AST_SHORT_TXID_SIZE-i-1]); + printf("%02x",assetrefbin[i]); } } - else - { - assetref += itostr((int)mc_GetLE(assetrefbin,4)); - assetref += "-"; - assetref += itostr((int)mc_GetLE(assetrefbin+4,4)); - assetref += "-"; - assetref += itostr((int)mc_GetLE(assetrefbin+8,2)); - } - - if(debug_print)printf("TxOut: %s-%d %s %ld\n",txid.c_str(),index,assetref.c_str(),quantity); + printf(" %ld\n",quantity); } } @@ -1641,12 +1616,9 @@ CAmount BuildAssetTransaction(CWallet *lpWallet, txNew.vout.push_back(txout); } - int assets_per_opdrop=(MAX_SCRIPT_ELEMENT_SIZE-4)/(mc_gState->m_NetworkParams->m_AssetRefSize+MC_AST_ASSET_QUANTITY_SIZE); + int assets_per_opdrop; - if(mc_gState->m_Features->VerifySizeOfOpDropElements()) - { - assets_per_opdrop=(MAX_SCRIPT_ELEMENT_SIZE-4)/(mc_gState->m_NetworkParams->m_AssetRefSize+MC_AST_ASSET_QUANTITY_SIZE); - } + assets_per_opdrop=(MAX_SCRIPT_ELEMENT_SIZE-4)/(mc_gState->m_NetworkParams->m_AssetRefSize+MC_AST_ASSET_QUANTITY_SIZE); size_t elem_size; const unsigned char *elem; @@ -2703,12 +2675,9 @@ bool CWallet::InitializeUnspentList() lpAssetGroups=new CAssetGroupTree; - int assets_per_opdrop=(MAX_SCRIPT_ELEMENT_SIZE-4)/(mc_gState->m_NetworkParams->m_AssetRefSize+MC_AST_ASSET_QUANTITY_SIZE); + int assets_per_opdrop; - if(mc_gState->m_Features->VerifySizeOfOpDropElements()) - { - assets_per_opdrop=(MAX_SCRIPT_ELEMENT_SIZE-4)/(mc_gState->m_NetworkParams->m_AssetRefSize+MC_AST_ASSET_QUANTITY_SIZE); - } + assets_per_opdrop=(MAX_SCRIPT_ELEMENT_SIZE-4)/(mc_gState->m_NetworkParams->m_AssetRefSize+MC_AST_ASSET_QUANTITY_SIZE); int max_assets_per_group=assets_per_opdrop*MCP_STD_OP_DROP_COUNT; diff --git a/src/wallet/wallettxs.cpp b/src/wallet/wallettxs.cpp index fd600e84..39dae314 100644 --- a/src/wallet/wallettxs.cpp +++ b/src/wallet/wallettxs.cpp @@ -2440,25 +2440,8 @@ int mc_WalletTxs::AddTx(mc_TxImport *import,const CWalletTx& tx,int block,CDiskT if(fNewAsset) { entity.Zero(); - if(mc_gState->m_Features->ShortTxIDInTx()) - { - entity.m_EntityType=MC_TET_ASSET | MC_TET_CHAINPOS; - memcpy(entity.m_EntityID,(unsigned char*)&hash+MC_AST_SHORT_TXID_OFFSET,MC_AST_SHORT_TXID_SIZE); - } - else - { - if(block >= 0) - { - entity.m_EntityType=MC_TET_ASSET | MC_TET_CHAINPOS; - int tx_offset=sizeof(CBlockHeader)+block_pos->nTxOffset; - mc_PutLE(entity.m_EntityID,&block,4); - mc_PutLE(entity.m_EntityID+4,&tx_offset,4); - for(i=0;iFindEntity(&entity) >= 0) @@ -2493,14 +2476,7 @@ int mc_WalletTxs::AddTx(mc_TxImport *import,const CWalletTx& tx,int block,CDiskT { ptrOut=mc_gState->m_TmpAssetsOut->GetRow(i); entity.Zero(); - if(mc_gState->m_Features->ShortTxIDInTx()) - { - memcpy(entity.m_EntityID,ptrOut+MC_AST_SHORT_TXID_OFFSET,MC_AST_SHORT_TXID_SIZE); - } - else - { - memcpy(entity.m_EntityID,ptrOut,MC_AST_ASSET_REF_SIZE); - } + memcpy(entity.m_EntityID,ptrOut+MC_AST_SHORT_TXID_OFFSET,MC_AST_SHORT_TXID_SIZE); entity.m_EntityType=MC_TET_ASSET | MC_TET_CHAINPOS; fRelevantEntity=false; if(imp->FindEntity(&entity) >= 0) From a968425a51ec9222b7f0d292fbca55c699db14d7 Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 14 Feb 2018 12:31:33 +0200 Subject: [PATCH 004/157] Fixed -shortoutput message --- src/core/init-cold.cpp | 7 +++++-- src/core/init.cpp | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/core/init-cold.cpp b/src/core/init-cold.cpp index 546e1406..96438fb9 100644 --- a/src/core/init-cold.cpp +++ b/src/core/init-cold.cpp @@ -1015,8 +1015,11 @@ bool AppInit2_Cold(boost::thread_group& threadGroup,int OutputPipe) if(mapMultiArgs.count("-rpcallowip") == 0) { - sprintf(bufOutput,"Listening for API requests on port %d (local only - see rpcallowip setting)\n\n",(int)GetArg("-rpcport", BaseParams().RPCPort())); - bytes_written=write(OutputPipe,bufOutput,strlen(bufOutput)); + if(!GetBoolArg("-shortoutput", false)) + { + sprintf(bufOutput,"Listening for API requests on port %d (local only - see rpcallowip setting)\n\n",(int)GetArg("-rpcport", BaseParams().RPCPort())); + bytes_written=write(OutputPipe,bufOutput,strlen(bufOutput)); + } } diff --git a/src/core/init.cpp b/src/core/init.cpp index 63ea2f92..4e73de8c 100644 --- a/src/core/init.cpp +++ b/src/core/init.cpp @@ -2141,8 +2141,11 @@ bool AppInit2(boost::thread_group& threadGroup,int OutputPipe) if(mapMultiArgs.count("-rpcallowip") == 0) { - sprintf(bufOutput,"Listening for API requests on port %d (local only - see rpcallowip setting)\n\n",(int)GetArg("-rpcport", BaseParams().RPCPort())); - bytes_written=write(OutputPipe,bufOutput,strlen(bufOutput)); + if(!GetBoolArg("-shortoutput", false)) + { + sprintf(bufOutput,"Listening for API requests on port %d (local only - see rpcallowip setting)\n\n",(int)GetArg("-rpcport", BaseParams().RPCPort())); + bytes_written=write(OutputPipe,bufOutput,strlen(bufOutput)); + } } // int version=mc_gState->m_NetworkParams->GetInt64Param("protocolversion"); From c8386c15a824ed658386bae7aad43c1d9ce79656 Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 14 Feb 2018 15:02:49 +0200 Subject: [PATCH 005/157] Off-chain items, protocol --- src/chainparams/params.cpp | 21 ++++ src/chainparams/state.h | 1 + src/protocol/multichainscript.cpp | 202 +++++++++++++++++++++++++++++- src/protocol/multichainscript.h | 3 + src/protocol/multichaintx.cpp | 49 +++++++- src/version/version.cpp | 4 +- 6 files changed, 274 insertions(+), 6 deletions(-) diff --git a/src/chainparams/params.cpp b/src/chainparams/params.cpp index 2aa01d48..bfd7b701 100644 --- a/src/chainparams/params.cpp +++ b/src/chainparams/params.cpp @@ -1911,3 +1911,24 @@ int mc_Features::ParameterUpgrades() return ret; } +int mc_Features::OffChainData() +{ + int ret=0; + if(mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) + { + return 0; + } + int protocol=mc_gState->m_NetworkParams->ProtocolVersion(); + + if(protocol) + { + if(protocol >= 20003) + { + ret=1; + } + } + + return ret; +} + + diff --git a/src/chainparams/state.h b/src/chainparams/state.h index f8c3a216..ffd4fd15 100644 --- a/src/chainparams/state.h +++ b/src/chainparams/state.h @@ -137,6 +137,7 @@ typedef struct mc_Features int FixedIsUnspendable(); int PerAssetPermissions(); int ParameterUpgrades(); + int OffChainData(); } mc_Features; typedef struct mc_BlockHeaderInfo diff --git a/src/protocol/multichainscript.cpp b/src/protocol/multichainscript.cpp index 07789ec8..0295f0eb 100644 --- a/src/protocol/multichainscript.cpp +++ b/src/protocol/multichainscript.cpp @@ -6,6 +6,8 @@ #define MC_DCT_SCRIPT_ALLOC_BUFFER_CHUNK 4096 #define MC_DCT_SCRIPT_ALLOC_INDEX_CHUNK 16 +#define MC_DCT_SCRIPT_CHUNK_HASH_SIZE 32 + #define MC_DCT_SCRIPT_OP_PUSHDATA1 0x4c #define MC_DCT_SCRIPT_OP_PUSHDATA2 0x4d #define MC_DCT_SCRIPT_OP_PUSHDATA4 0x4e @@ -33,13 +35,15 @@ #define MC_DCT_SCRIPT_MULTICHAIN_DATA_FORMAT_PREFIX 'f' #define MC_DCT_SCRIPT_MULTICHAIN_RAW_DATA_PREFIX 'd' +#define MC_DCT_SCRIPT_EXTENDED_TYPE_DATA_FORMAT 1 +#define MC_DCT_SCRIPT_EXTENDED_TYPE_CHUNK_DEF 2 + #define MC_DCT_SCRIPT_TYPE_REGULAR 0x00 #define MC_DCT_SCRIPT_TYPE_OP_RETURN 0x01 #define MC_DCT_SCRIPT_TYPE_OP_DROP 0x02 #define MC_DCT_SCRIPT_TYPE_DIRTY_OP_RETURN 0x04 - int mc_Script::Zero() { m_Size=0; @@ -2341,6 +2345,197 @@ int mc_Script::SetDataFormat(const uint32_t format) return MC_ERR_NOERROR; } +int mc_Script::GetChunkDef(uint32_t *format,unsigned char** hashes,uint64_t *size,uint32_t *chunk_size_log) +{ + unsigned char *ptr; + unsigned char *ptrEnd; + unsigned char l,f,extended_type; + uint64_t s,c,t; + int format_found=0; + int chunkdef_found=0; + + if(format) + { + *format=MC_SCR_DATA_FORMAT_UNKNOWN; + } + + if(m_CurrentElement<0) + { + return MC_ERR_INVALID_PARAMETER_VALUE; + } + + if(m_lpCoord[m_CurrentElement*2+1] < MC_DCT_SCRIPT_IDENTIFIER_LEN+1+1+1) + { + return MC_ERR_WRONG_SCRIPT; + } + + ptr=m_lpData+m_lpCoord[m_CurrentElement*2+0]; + ptrEnd=ptr+m_lpCoord[m_CurrentElement*2+1]; + + if(memcmp(ptr,MC_DCT_SCRIPT_MULTICHAIN_IDENTIFIER,MC_DCT_SCRIPT_IDENTIFIER_LEN) != 0) + { + return MC_ERR_WRONG_SCRIPT; + } + + + if(ptr[MC_DCT_SCRIPT_IDENTIFIER_LEN] != MC_DCT_SCRIPT_MULTICHAIN_DATA_FORMAT_PREFIX) + { + return MC_ERR_WRONG_SCRIPT; + } + + ptr+=MC_DCT_SCRIPT_IDENTIFIER_LEN+1; + + f=(unsigned char)(*ptr); + + if( (f & MC_SCR_DATA_FORMAT_EXTENDED_MASK) == 0) + { + return MC_ERR_WRONG_SCRIPT; + } + + ptr++; + while(ptr < ptrEnd) + { + extended_type=(unsigned char)(*ptr); + ptr++; + switch(extended_type) + { + case MC_DCT_SCRIPT_EXTENDED_TYPE_DATA_FORMAT: + if(format_found) + { + return MC_ERR_ERROR_IN_SCRIPT; + } + format_found=1; + if(ptr63) + { + return MC_ERR_ERROR_IN_SCRIPT; + } + c=1 << l; + s=mc_GetLE(ptr,8); + ptr+=8; + t=0; + if(s) + { + t=(s-1)/c + 1; + } + if( (t >> 24) != 0 ) + { + return MC_ERR_ERROR_IN_SCRIPT; + } + if(ptr+t*MC_DCT_SCRIPT_CHUNK_HASH_SIZE <= ptrEnd) + { + if(size) + { + *size=s; + } + if(chunk_size_log) + { + *chunk_size_log=l; + } + if(hashes) + { + *hashes=ptr; + } + ptr+=t*MC_DCT_SCRIPT_CHUNK_HASH_SIZE; + } + else + { + return MC_ERR_ERROR_IN_SCRIPT; + } + } + else + { + return MC_ERR_ERROR_IN_SCRIPT; + } + } + else + { + return MC_ERR_ERROR_IN_SCRIPT; + } + break; + } + } + + return MC_ERR_NOERROR; +} + +int mc_Script::SetChunkDef(const uint32_t format,unsigned char* hashes,uint64_t size,uint32_t chunk_size_log) +{ + int err; + unsigned char buf[MC_DCT_SCRIPT_IDENTIFIER_LEN+14]; + uint64_t t,c; + + if(chunk_size_log>63) + { + return MC_ERR_INVALID_PARAMETER_VALUE; + } + c=1 << chunk_size_log; + t=0; + if(size) + { + t=(size-1)/c + 1; + } + if( (t >> 24) != 0 ) + { + return MC_ERR_INVALID_PARAMETER_VALUE; + } + + err=AddElement(); + if(err) + { + return err; + } + + memcpy(buf,MC_DCT_SCRIPT_MULTICHAIN_IDENTIFIER,MC_DCT_SCRIPT_IDENTIFIER_LEN); + buf[MC_DCT_SCRIPT_IDENTIFIER_LEN]=MC_DCT_SCRIPT_MULTICHAIN_DATA_FORMAT_PREFIX; + buf[MC_DCT_SCRIPT_IDENTIFIER_LEN+1]=MC_SCR_DATA_FORMAT_EXTENDED_MASK; + buf[MC_DCT_SCRIPT_IDENTIFIER_LEN+2]=(unsigned char)MC_DCT_SCRIPT_EXTENDED_TYPE_DATA_FORMAT; + buf[MC_DCT_SCRIPT_IDENTIFIER_LEN+3]=(unsigned char)format; + buf[MC_DCT_SCRIPT_IDENTIFIER_LEN+4]=(unsigned char)MC_DCT_SCRIPT_EXTENDED_TYPE_CHUNK_DEF; + buf[MC_DCT_SCRIPT_IDENTIFIER_LEN+5]=(unsigned char)chunk_size_log; + mc_PutLE(buf+MC_DCT_SCRIPT_IDENTIFIER_LEN+6,&size,8); + + err=SetData(buf,MC_DCT_SCRIPT_IDENTIFIER_LEN+14); + if(err) + { + return err; + } + + err=SetData(hashes,t*MC_DCT_SCRIPT_CHUNK_HASH_SIZE); + if(err) + { + return err; + } + + + return MC_ERR_NOERROR; +} + + int mc_Script::ExtractAndDeleteDataFormat(uint32_t *format) { int elem,err; @@ -2365,7 +2560,10 @@ int mc_Script::ExtractAndDeleteDataFormat(uint32_t *format) SetElement(elem); if( (err=GetDataFormat(format)) != MC_ERR_WRONG_SCRIPT ) { - DeleteElement(elem); + if( (mc_gState->m_Features->OffChainData() == 0) || (GetChunkDef(NULL,NULL,NULL,NULL) == MC_ERR_WRONG_SCRIPT) ) + { + DeleteElement(elem); + } return err; } diff --git a/src/protocol/multichainscript.h b/src/protocol/multichainscript.h index 1d651e0b..dc1f1542 100644 --- a/src/protocol/multichainscript.h +++ b/src/protocol/multichainscript.h @@ -105,6 +105,9 @@ typedef struct mc_Script int GetDataFormat(uint32_t *format); int SetDataFormat(const uint32_t format); + int GetChunkDef(uint32_t *format,unsigned char** hashes,uint64_t *size,uint32_t *chunk_size_log); + int SetChunkDef(const uint32_t format,unsigned char* hashes,uint64_t size,uint32_t chunk_size_log); + int ExtractAndDeleteDataFormat(uint32_t *format); int DeleteDuplicatesInRange(int from,int to); diff --git a/src/protocol/multichaintx.cpp b/src/protocol/multichaintx.cpp index 5fe4e608..54997fd7 100644 --- a/src/protocol/multichaintx.cpp +++ b/src/protocol/multichaintx.cpp @@ -559,6 +559,43 @@ void MultiChainTransaction_FillAdminPermissionsBeforeTx(const CTransaction& tx, } } +bool MultiChainTransaction_VerifyAndDeleteDataFormatElements(string& reason) +{ + int elem,err; + + mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(NULL); // Format scripts are eliminated for protocol checks + + if(mc_gState->m_Features->OffChainData() == 0) + { + return true; + } + + if(mc_gState->m_TmpScript->m_NumElements < 2) + { + return true; + } + + elem=mc_gState->m_TmpScript->m_NumElements-2; + + while( (elem >= 0 ) && ((err=mc_gState->m_TmpScript->GetChunkDef(NULL,NULL,NULL,NULL)) == MC_ERR_NOERROR) ) + { + mc_gState->m_TmpScript->DeleteElement(elem); + elem--; + if(elem < 0) + { + err=MC_ERR_WRONG_SCRIPT; + } + } + + if(err != MC_ERR_WRONG_SCRIPT) + { + reason="Error in data format script"; + return false; + } + + return true; +} + bool MultiChainTransaction_CheckOpReturnScript(const CTransaction& tx, const CCoinsViewCache &inputs, int vout, @@ -570,7 +607,11 @@ bool MultiChainTransaction_CheckOpReturnScript(const CTransaction& tx, vector addressRets; CTxDestination single_address; - mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(NULL); // Format scripts are eliminated for protocol checks +// mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(NULL); // Format scripts are eliminated for protocol checks + if(!MultiChainTransaction_VerifyAndDeleteDataFormatElements(reason)) + { + return false; + } if(!details->fScriptHashAllFound) { @@ -825,7 +866,11 @@ bool MultiChainTransaction_CheckEntityItem(const CTransaction& tx, unsigned char short_txid[MC_AST_SHORT_TXID_SIZE]; mc_EntityDetails entity; - mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(NULL); // Format scripts are eliminated for protocol checks +// mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(NULL); // Format scripts are eliminated for protocol checks + if(!MultiChainTransaction_VerifyAndDeleteDataFormatElements(reason)) + { + return false; + } mc_gState->m_TmpScript->SetElement(0); // Should be spke diff --git a/src/version/version.cpp b/src/version/version.cpp index ac342acf..e8749e24 100644 --- a/src/version/version.cpp +++ b/src/version/version.cpp @@ -13,8 +13,8 @@ int mc_State::VersionInfo(int version) return custom_version; } - int this_build=20000102; - int this_protocol=20002; + int this_build=20000103; + int this_protocol=20003; if(version < 0) { From 3434314a769c915e34234d1c1688b222e979bf88 Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 15 Feb 2018 11:42:13 +0200 Subject: [PATCH 006/157] Parameter description tweak in params.dat --- src/chainparams/paramlist.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/chainparams/paramlist.h b/src/chainparams/paramlist.h index 52035c70..4521d7c9 100644 --- a/src/chainparams/paramlist.h +++ b/src/chainparams/paramlist.h @@ -127,7 +127,7 @@ static const mc_OneMultichainParam MultichainParamArray[] = { "adminconsensuscreate" , "admin-consensus-create" , MC_PRM_UINT32 | MC_PRM_USER | MC_PRM_CLONE | MC_PRM_DECIMAL , -1, 0, 0, 1000000, 0.0, 10006, 0, "-mc-adminconsensuscreate", "adminconsensusissue","", - "* to change create permissions."}, + "* to change create permissions."}, { "adminconsensusissue" , "admin-consensus-issue" , MC_PRM_UINT32 | MC_PRM_USER | MC_PRM_CLONE | MC_PRM_DECIMAL , -1, 0, 0, 1000000, 0.0, 10003, 0, "-mc-adminconsensusissue", "lockadminminerounds","", From 719ef6a4461bc271eb03067cbe6db0ed54b098ef Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 15 Feb 2018 17:06:59 +0200 Subject: [PATCH 007/157] Chunk DB, entities --- src/Makefile.am | 2 + src/chainparams/params.cpp | 20 ++ src/chainparams/state.h | 1 + src/utils/declare.h | 1 + src/utils/utilwrapper.cpp | 5 + src/wallet/chunkdb.cpp | 390 +++++++++++++++++++++++++++++++++++++ src/wallet/chunkdb.h | 170 ++++++++++++++++ src/wallet/wallettxdb.cpp | 2 +- src/wallet/wallettxs.cpp | 33 +++- src/wallet/wallettxs.h | 2 + 10 files changed, 622 insertions(+), 4 deletions(-) create mode 100644 src/wallet/chunkdb.cpp create mode 100644 src/wallet/chunkdb.h diff --git a/src/Makefile.am b/src/Makefile.am index 95ab04dc..4c5da512 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -259,6 +259,7 @@ multichain_libbitcoin_multichain_a_SOURCES = \ protocol/multichainscript.cpp \ utils/dbwrapper.cpp \ wallet/wallettxdb.cpp \ + wallet/chunkdb.cpp \ permissions/permission.cpp \ entities/asset.cpp @@ -522,6 +523,7 @@ libbitcoinconsensus_la_SOURCES = \ protocol/multichainscript.cpp \ utils/dbwrapper.cpp \ wallet/wallettxdb.cpp \ + wallet/chunkdb.cpp \ permissions/permission.cpp \ entities/asset.cpp \ structs/hash.cpp \ diff --git a/src/chainparams/params.cpp b/src/chainparams/params.cpp index bfd7b701..4adb1388 100644 --- a/src/chainparams/params.cpp +++ b/src/chainparams/params.cpp @@ -1931,4 +1931,24 @@ int mc_Features::OffChainData() return ret; } +int mc_Features::Chunks() +{ + int ret=0; + if(mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) + { + return 0; + } + int protocol=mc_gState->m_NetworkParams->ProtocolVersion(); + + if(protocol) + { + if(protocol >= 20003) + { + ret=1; + } + } + + return ret; +} + diff --git a/src/chainparams/state.h b/src/chainparams/state.h index ffd4fd15..299477d4 100644 --- a/src/chainparams/state.h +++ b/src/chainparams/state.h @@ -138,6 +138,7 @@ typedef struct mc_Features int PerAssetPermissions(); int ParameterUpgrades(); int OffChainData(); + int Chunks(); } mc_Features; typedef struct mc_BlockHeaderInfo diff --git a/src/utils/declare.h b/src/utils/declare.h index ac6e15a0..5dafb493 100644 --- a/src/utils/declare.h +++ b/src/utils/declare.h @@ -235,6 +235,7 @@ void mc_GetCompoundHash160(void *result,const void *hash1,const void *hash2); int mc_SetIPv4ServerAddress(const char* host); int mc_FindIPv4ServerAddress(uint32_t *all_ips,int max_ips); int mc_GenerateConfFiles(const char *network_name); +void mc_CreateDir(const char *dir_name); void mc_RemoveDataDir(const char *network_name); void mc_RemoveDir(const char *network_name,const char *dir_name); int mc_GetDataDirArg(char *buf); diff --git a/src/utils/utilwrapper.cpp b/src/utils/utilwrapper.cpp index 9232fedd..6baafdb1 100644 --- a/src/utils/utilwrapper.cpp +++ b/src/utils/utilwrapper.cpp @@ -387,6 +387,11 @@ const boost::filesystem::path mc_GetDataDir(const char *network_name,int create) return path; } +void mc_CreateDir(const char *dir_name) +{ + boost::filesystem::create_directories(boost::filesystem::path(dir_name)); +} + void mc_RemoveDataDir(const char *network_name) { boost::filesystem::path path; diff --git a/src/wallet/chunkdb.cpp b/src/wallet/chunkdb.cpp new file mode 100644 index 00000000..44307ff8 --- /dev/null +++ b/src/wallet/chunkdb.cpp @@ -0,0 +1,390 @@ +// Copyright (c) 2014-2017 Coin Sciences Ltd +// MultiChain code distributed under the GPLv3 license, see COPYING file. + +#include "multichain/multichain.h" +#include "wallet/chunkdb.h" + +#define MC_CDB_MAX_FILE_SIZE 0x8000000 // Maximal data file size, 128MB + +void sprintf_hex(char *hex,const unsigned char *bin,int size); +/* +{ + int i; + for(i=0;iClose(); + delete m_DB; + m_DB=NULL; + } + + if(m_Subscriptions) + { + delete m_Subscriptions; + } + + Zero(); + return MC_ERR_NOERROR; +} + +int mc_ChunkDB::AddSubscription(mc_SubscriptionDBRow *subscription) +{ + char enthex[33]; + char dir_name[64]; + switch(subscription->m_Entity.m_EntityType) + { + case MC_TET_STREAM: + sprintf_hex(enthex,subscription->m_Entity.m_EntityID,MC_AST_SHORT_TXID_SIZE); + sprintf(dir_name,"chunks/data/stream-%s",enthex); + break; + default: + return MC_ERR_NOT_SUPPORTED; + } + mc_GetFullFileName(m_Name,dir_name,"",MC_FOM_RELATIVE_TO_DATADIR | MC_FOM_CREATE_DIR,subscription->m_DirName); + mc_CreateDir(subscription->m_DirName); + + return m_Subscriptions->Add(subscription,(char*)subscription+m_ValueOffset); +} + +mc_SubscriptionDBRow *mc_ChunkDB::FindSubscription(mc_TxEntity* entity) +{ + int row; + mc_SubscriptionDBRow subscription; + + subscription.Zero(); + + subscription.m_RecordType=MC_CDB_TYPE_SUBSCRIPTION; + memcpy(&subscription.m_Entity,entity,sizeof(mc_TxEntity)); + + row=m_Subscriptions->Seek(&subscription); + if(row >= 0) + { + return (mc_SubscriptionDBRow *)m_Subscriptions->GetRow(row); + } + + return NULL; +} + +int mc_ChunkDB::AddEntity(mc_TxEntity* entity, uint32_t flags) +{ + int err; + mc_SubscriptionDBRow subscription; + char msg[256]; + char enthex[65]; + + err=MC_ERR_NOERROR; + + subscription.Zero(); + + subscription.m_RecordType=MC_CDB_TYPE_SUBSCRIPTION; + memcpy(&subscription.m_Entity,entity,sizeof(mc_TxEntity)); + + subscription.m_Entity.m_EntityType &= MC_TET_TYPE_MASK; + + if(FindSubscription(&subscription.m_Entity)) + { + return MC_ERR_NOERROR; + } + + subscription.m_Flags=flags; + subscription.m_SubscriptionID=m_DBStat.m_LastSubscription+1; + + err=AddSubscription(&subscription); + + if(err == MC_ERR_NOERROR) + { + m_DBStat.m_LastSubscription+=1; + + err=m_DB->Write((char*)&subscription+m_KeyOffset,m_KeySize,(char*)&subscription+m_ValueOffset,m_ValueSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + + if(err == MC_ERR_NOERROR) + { + err=m_DB->Write((char*)&m_DBStat+m_KeyOffset,m_KeySize,(char*)&m_DBStat+m_ValueOffset,m_ValueSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + } + + if(err == MC_ERR_NOERROR) + { + err=m_DB->Commit(MC_OPT_DB_DATABASE_TRANSACTIONAL); + } + + if(err) + { + m_DBStat.m_LastSubscription-=1; + m_Subscriptions->SetCount(m_Subscriptions->GetCount()-1); + } + } + + sprintf_hex(enthex,entity->m_EntityID,MC_TDB_ENTITY_ID_SIZE); + + if(err) + { + sprintf(msg,"Could not add entity (%08X, %s), error: %d",entity->m_EntityType,enthex,err); + LogString(msg); + } + else + { + sprintf(msg,"Entity (%08X, %s) added successfully",entity->m_EntityType,enthex); + LogString(msg); + } + + return err; +} + +int mc_ChunkDB::Initialize(const char *name,uint32_t mode) +{ + int err,value_len,i; + char msg[256]; + + mc_SubscriptionDBRow subscription; + + unsigned char *ptr; + + err=MC_ERR_NOERROR; + + strcpy(m_Name,name); + + m_DB=new mc_Database; + + mc_GetFullFileName(name,"chunks","",MC_FOM_RELATIVE_TO_DATADIR | MC_FOM_CREATE_DIR,m_DirName); + mc_CreateDir(m_DirName); + mc_GetFullFileName(name,"chunks/chunks",".db",MC_FOM_RELATIVE_TO_DATADIR | MC_FOM_CREATE_DIR,m_DBName); + mc_GetFullFileName(name,"chunks/chunks",".log",MC_FOM_RELATIVE_TO_DATADIR,m_LogFileName); + + m_DB->SetOption("KeySize",0,m_KeySize); + m_DB->SetOption("ValueSize",0,m_ValueSize); + + + err=m_DB->Open(m_DBName,MC_OPT_DB_DATABASE_CREATE_IF_MISSING | MC_OPT_DB_DATABASE_TRANSACTIONAL | MC_OPT_DB_DATABASE_LEVELDB); + + if(err) + { + LogString("Initialize: Cannot open database"); + return err; + } + + m_Subscriptions=new mc_Buffer; + + m_Subscriptions->Initialize(m_KeySize,sizeof(mc_SubscriptionDBRow),MC_BUF_MODE_MAP); + + m_DBStat.Zero(); + + ptr=(unsigned char*)m_DB->Read((char*)&m_DBStat+m_KeyOffset,m_KeySize,&value_len,MC_OPT_DB_DATABASE_SEEK_ON_READ,&err); + if(err) + { + LogString("Initialize: Cannot read from database"); + return err; + } + + if(ptr) + { + memcpy((char*)&m_DBStat+m_ValueOffset,ptr,m_ValueSize); + + subscription.Zero(); + + ptr=(unsigned char*)m_DB->MoveNext(&err); +// ptr=(unsigned char*)m_DB->Read((char*)&subscription+m_KeyOffset,m_KeySize,&value_len,MC_OPT_DB_DATABASE_SEEK_ON_READ,&err); + if(err) + { + return err; + } + + if(ptr) + { + memcpy((char*)&subscription,ptr,m_TotalSize); + } + + while(ptr) + { + if( (subscription.m_RecordType != MC_CDB_TYPE_SUBSCRIPTION) || + (subscription.m_Zero != 0) || (subscription.m_Zero1 != 0) ) + { + ptr=NULL; + } + if(ptr) + { + AddSubscription(&subscription); + if(subscription.m_SubscriptionID > m_DBStat.m_LastSubscription) + { + m_DBStat.m_LastSubscription=subscription.m_SubscriptionID; + } + ptr=(unsigned char*)m_DB->MoveNext(&err); + if(err) + { + LogString("Error on MoveNext"); + return MC_ERR_CORRUPTED; + } + if(ptr) + { + memcpy((char*)&subscription,ptr,m_TotalSize); + } + } + } + } + else + { + err=m_DB->Write((char*)&m_DBStat+m_KeyOffset,m_KeySize,(char*)&m_DBStat+m_ValueOffset,m_ValueSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + if(err) + { + return err; + } + + + err=m_DB->Commit(MC_OPT_DB_DATABASE_TRANSACTIONAL); + if(err) + { + return err; + } + } + + m_DBStat.m_InitMode |= mode; + + Dump("Initialize"); + + sprintf(msg, "Initialized. Chunks: %d",m_DBStat.m_Count); + LogString(msg); + + return err; +} + +void mc_ChunkDB::Dump(const char *message) +{ + if((m_DBStat.m_InitMode & MC_WMD_DEBUG) == 0) + { + return; + } + mc_ChunkDBRow dbrow; + + unsigned char *ptr; + int dbvalue_len,err,i; + char ShortName[65]; + char FileName[MC_DCT_DB_MAX_PATH]; + FILE *fHan; + + sprintf(ShortName,"chunks/chunks"); + mc_GetFullFileName(m_Name,ShortName,".dmp",MC_FOM_RELATIVE_TO_DATADIR | MC_FOM_CREATE_DIR,FileName); + + fHan=fopen(FileName,"a"); + if(fHan == NULL) + { + return; + } + + mc_LogString(fHan,message); + + fprintf(fHan,"Entities\n"); + dbrow.Zero(); + ptr=(unsigned char*)m_DB->Read((char*)&dbrow+m_KeyOffset,m_KeySize,&dbvalue_len,MC_OPT_DB_DATABASE_SEEK_ON_READ,&err); + if(err) + { + return; + } + + if(ptr) + { + memcpy((char*)&dbrow+m_ValueOffset,ptr,m_ValueSize); + while(ptr) + { + mc_MemoryDumpCharSizeToFile(fHan,(char*)&dbrow+m_KeyOffset,0,m_TotalSize,m_TotalSize); + ptr=(unsigned char*)m_DB->MoveNext(&err); + if(ptr) + { + memcpy((char*)&dbrow+m_KeyOffset,ptr,m_TotalSize); + } + } + } + +/* + for(i=0;iGetCount()) + { + fprintf(fHan,"RawMemPool %d\n",m_Imports[i].m_ImportID); + mc_MemoryDumpCharSizeToFile(fHan,m_RawMemPools[i]->GetRow(0),0,m_RawMemPools[i]->GetCount()*m_Database->m_TotalSize,m_Database->m_TotalSize); + } + } + if(m_MemPools[i]) + { + if(m_MemPools[i]->GetCount()) + { + fprintf(fHan,"MemPool %d\n",m_Imports[i].m_ImportID); + mc_MemoryDumpCharSizeToFile(fHan,m_MemPools[i]->GetRow(0),0,m_MemPools[i]->GetCount()*m_Database->m_TotalSize,m_Database->m_TotalSize); + } + } + } + if(m_RawUpdatePool) + { + if(m_RawUpdatePool->GetCount()) + { + fprintf(fHan,"RawUpdatePool\n"); + mc_MemoryDumpCharSizeToFile(fHan,m_RawUpdatePool->GetRow(0),0,m_RawUpdatePool->GetCount()*m_Database->m_TotalSize,m_Database->m_TotalSize); + } + } +*/ + fprintf(fHan,"\n<<<<<< \tChain height: %6d\t%s\n\n",mc_gState->m_Permissions->m_Block,message); + fclose(fHan); +} diff --git a/src/wallet/chunkdb.h b/src/wallet/chunkdb.h new file mode 100644 index 00000000..4037a60c --- /dev/null +++ b/src/wallet/chunkdb.h @@ -0,0 +1,170 @@ +// Copyright (c) 2014-2017 Coin Sciences Ltd +// MultiChain code distributed under the GPLv3 license, see COPYING file. + +#ifndef MULTICHAIN_CHUNKDB_H +#define MULTICHAIN_CHUNKDB_H + +#include "utils/declare.h" +#include "wallet/wallettxdb.h" + +#define MC_CDB_CHUNK_HASH_SIZE 32 +#define MC_CDB_ROW_SIZE 80 +#define MC_CDB_HEADER_SIZE 40 + +#define MC_TDB_TXID_SIZE 32 +#define MC_TDB_ENTITY_KEY_SIZE 32 +#define MC_TDB_ENTITY_ID_SIZE 20 +#define MC_TDB_ENTITY_TYPE_SIZE 4 +#define MC_TDB_GENERATION_SIZE 4 +#define MC_TDB_POS_SIZE 4 +#define MC_TDB_ROW_SIZE 80 + +#define MC_CDB_TYPE_DB_STAT 0 +#define MC_CDB_TYPE_SUBSCRIPTION 1 +#define MC_CDB_TYPE_FILE 2 + + +/** File DB Row*/ + +typedef struct mc_SubscriptionFileDBRow +{ + uint32_t m_SubscriptionID; // Should be Zero + uint32_t m_RecordType; // Should be MC_CDB_TYPE_FILE + uint32_t m_FileID; // File ID + mc_TxEntity m_Entity; // Parent Entity + uint32_t m_Size; // File size + uint32_t m_StorageFlags; // Internal flags + uint32_t m_Count; // Total count + uint32_t m_FirstTimestamp; // Timestamp of the first record + uint32_t m_FirstOffset; // First data offset + uint32_t m_LastTimeStamp; // Timestamp of the last record + uint32_t m_LastOffset; // Last data file size + void Zero(); +} mc_SubscriptionFileDBRow; + + +/** Entity DB Row*/ + +typedef struct mc_SubscriptionDBRow +{ + uint32_t m_Zero; // Should be Zero + uint32_t m_RecordType; // Should be MC_CDB_TYPE_SUBSCRIPTION + uint32_t m_Zero1; // Should be Zero + mc_TxEntity m_Entity; // Parent Entity + uint32_t m_Flags; // Flags passed from higher level + uint32_t m_SubscriptionID; // Subscription ID + uint32_t m_Count; // Total chunk count + uint32_t m_FirstFileID; // First data file ID/Reserved + uint32_t m_FirstFileOffset; // First data file offset/Reserved + uint32_t m_LastFileID; // Last data file ID + uint32_t m_LastFileSize; // Last data file size + + char m_DirName[MC_DCT_DB_MAX_PATH]; // Full file name + + void Zero(); +} mc_SubscriptionDBRow; + +typedef struct mc_ChunkDBStat +{ + uint32_t m_Zero; // Should be Zero + uint32_t m_RecordType; // Should be MC_CDB_TYPE_DB_STAT + uint32_t m_Zero1; // Should be Zero + mc_TxEntity m_ZeroEntity; // Zero Entity + uint32_t m_InitMode; + uint32_t m_ChunkDBVersion; + uint32_t m_Count; // Total tx count + uint32_t m_FullSize; // Total tx size + uint32_t m_LastSubscription; + uint32_t m_Reserved1; + uint32_t m_Reserved2; + void Zero(); +} mc_ChunkDBStat; + + +/** Chunk DB Row **/ + +typedef struct mc_ChunkDBRow +{ + uint32_t m_SubscriptionID; // Subscription ID + unsigned char m_Hash[MC_CDB_CHUNK_HASH_SIZE]; // Chunk hash + uint32_t m_Size; // Chunk Size + uint32_t m_Flags; // Flags passed from higher level + uint32_t m_StorageFlags; // Internal flags + uint32_t m_InternalFileID; // Data file ID + uint32_t m_InternalFileOffset; // Offset in the data file + uint32_t m_PrevSubscriptionID; // Prev Subscription ID for this hash + uint32_t m_NextSubscriptionID; // Next Subscription ID for this hash + + void Zero(); +} mc_ChunkDBRow; + + +/** Chunk DB **/ + +typedef struct mc_ChunkDB +{ + mc_Database *m_DB; // Database object + uint32_t m_KeyOffset; + uint32_t m_KeySize; + uint32_t m_ValueOffset; + uint32_t m_ValueSize; + uint32_t m_TotalSize; + + mc_ChunkDBStat m_DBStat; + + char m_Name[MC_PRM_NETWORK_NAME_MAX_SIZE+1]; // Chain name + char m_DirName[MC_DCT_DB_MAX_PATH]; // Chunk directory name + char m_DBName[MC_DCT_DB_MAX_PATH]; // Full database name + char m_LogFileName[MC_DCT_DB_MAX_PATH]; // Full log file name + + mc_Buffer *m_Subscriptions; // List of import entities (mc_TxEntityStat) + + mc_ChunkDB() + { + Zero(); + } + + ~mc_ChunkDB() + { + Destroy(); + } + + int Initialize( // Initialization + const char *name, // Chain name + uint32_t mode); // Unused + + int AddSubscription(mc_SubscriptionDBRow *subscription); + int AddEntity(mc_TxEntity *entity,uint32_t flags); // Adds entity + + mc_SubscriptionDBRow *FindSubscription(mc_TxEntity *entity); // Finds subscription + + int AddChunk( // Adds chunk to mempool + const unsigned char *hash, // Chunk hash (before chopping) + const mc_TxEntity *entity, // Parent entity + const unsigned char *chunk, // Chunk data + const unsigned char *metadata, // Chunk metadata + const uint32_t chunk_size, // Chunk size + const uint32_t metadata_size, // Chunk metadata size + const uint32_t flags); // Flags + + int GetChunk( + const unsigned char *hash, // Chunk hash (before chopping) + const unsigned char *entity, // Parent entity + unsigned char **chunk, // Chunk data + unsigned char **metadata, // Metadata + uint32_t *size, // Chunk Size + uint32_t *metadata_size, // Chunk Size + const uint32_t flags); // Flags + + int Commit(); // Commit mempool to disk + + void Zero(); + int Destroy(); + void Dump(const char *message); + + void LogString(const char *message); + +} mc_ChunkDB; + +#endif /* MULTICHAIN_CHUNKDB_H */ + diff --git a/src/wallet/wallettxdb.cpp b/src/wallet/wallettxdb.cpp index 9b6d9f20..b10f5496 100644 --- a/src/wallet/wallettxdb.cpp +++ b/src/wallet/wallettxdb.cpp @@ -14,7 +14,7 @@ void sprintf_hex(char *hex,const unsigned char *bin,int size) sprintf(hex+(i*2),"%02x",bin[size-1-i]); } - hex[64]=0; + hex[size*2]=0; } void mc_TxEntityRowExtension::Zero() diff --git a/src/wallet/wallettxs.cpp b/src/wallet/wallettxs.cpp index 39dae314..855b05c9 100644 --- a/src/wallet/wallettxs.cpp +++ b/src/wallet/wallettxs.cpp @@ -207,6 +207,7 @@ void mc_WalletTxs::Zero() { int i; m_Database=NULL; + m_ChunkDB=NULL; m_lpWallet=NULL; for(i=0;iInitialize(name,mode); + + if(err) + { + return err; + } m_Database=new mc_TxDB; - m_Mode=mode; err=m_Database->Initialize(name,mode); - + if(err == MC_ERR_NOERROR) { m_Mode=m_Database->m_DBStat.m_InitMode; @@ -290,6 +298,11 @@ int mc_WalletTxs::Destroy() { delete m_Database; } + + if(m_ChunkDB) + { + delete m_ChunkDB; + } Zero(); return MC_ERR_NOERROR; @@ -324,8 +337,22 @@ int mc_WalletTxs::AddEntity(mc_TxEntity *entity,uint32_t flags) return MC_ERR_INTERNAL_ERROR; } m_Database->Lock(1,0); - err=m_Database->AddEntity(entity,flags); + err=m_Database->AddEntity(entity,flags); m_Database->UnLock(); + + if(err == MC_ERR_NOERROR) + { + if(mc_gState->m_Features->Chunks()) + { + switch(entity->m_EntityType & MC_TET_TYPE_MASK) + { + case MC_TET_STREAM: + err=m_ChunkDB->AddEntity(entity,flags); + break; + } + } + } + return err; } diff --git a/src/wallet/wallettxs.h b/src/wallet/wallettxs.h index adba12f8..54f26f0f 100644 --- a/src/wallet/wallettxs.h +++ b/src/wallet/wallettxs.h @@ -11,6 +11,7 @@ #include "wallet/wallet.h" #include "multichain/multichain.h" #include "wallet/wallettxdb.h" +#include "wallet/chunkdb.h" #define MC_TDB_MAX_OP_RETURN_SIZE 256 @@ -46,6 +47,7 @@ typedef struct mc_WalletCachedAddTx typedef struct mc_WalletTxs { mc_TxDB *m_Database; + mc_ChunkDB *m_ChunkDB; CWallet *m_lpWallet; uint32_t m_Mode; std::map m_UTXOs[MC_TDB_MAX_IMPORTS]; From ad7c05d563846f92cd0fbb8f33c4ccd8b70bb893 Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 18 Feb 2018 10:17:34 +0200 Subject: [PATCH 008/157] Fixed, wallet semaphore initialization --- src/wallet/wallettxdb.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/wallet/wallettxdb.cpp b/src/wallet/wallettxdb.cpp index b10f5496..77c720f4 100644 --- a/src/wallet/wallettxdb.cpp +++ b/src/wallet/wallettxdb.cpp @@ -457,6 +457,13 @@ int mc_TxDB::Initialize(const char *name,uint32_t mode) m_RawUpdatePool=new mc_Buffer; err=m_RawUpdatePool->Initialize(MC_TDB_TXID_SIZE,m_Database->m_TotalSize,MC_BUF_MODE_MAP); + m_Semaphore=__US_SemCreate(); + if(m_Semaphore == NULL) + { + LogString("Initialize: Cannot initialize semaphore"); + return MC_ERR_INTERNAL_ERROR; + } + Dump("Initialize"); sprintf(msg, "Initialized. Chain height: %d, Txs: %d",m_DBStat.m_Block,m_DBStat.m_Count); LogString(msg); From df58d6229b6eec34b863a6b49d9138d237d971a3 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 19 Feb 2018 14:40:07 +0200 Subject: [PATCH 009/157] Fixed anyone-can-create --- src/chainparams/globals.h | 1 + src/chainparams/params.cpp | 29 +++++++++++++++++++++++++++++ src/chainparams/params.h | 1 + src/chainparams/state.h | 1 + src/permissions/permission.cpp | 14 ++++++++++++-- src/utils/utilwrapper.cpp | 1 + 6 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/chainparams/globals.h b/src/chainparams/globals.h index f3feff15..b333a9c1 100644 --- a/src/chainparams/globals.h +++ b/src/chainparams/globals.h @@ -34,6 +34,7 @@ int MCP_ANYONE_CAN_MINE=0; int MCP_ANYONE_CAN_CONNECT=0; int MCP_ANYONE_CAN_SEND=0; int MCP_ANYONE_CAN_RECEIVE=0; +int MCP_ANYONE_CAN_CREATE=0; int MCP_ANYONE_CAN_ACTIVATE=0; int64_t MCP_MINIMUM_PER_OUTPUT=0; int MCP_ALLOW_ARBITRARY_OUTPUTS=1; diff --git a/src/chainparams/params.cpp b/src/chainparams/params.cpp index 4adb1388..ffcf6b69 100644 --- a/src/chainparams/params.cpp +++ b/src/chainparams/params.cpp @@ -1952,3 +1952,32 @@ int mc_Features::Chunks() } +int mc_Features::FixedIn1001020003() +{ + int ret=0; + if(mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) + { + return 1; + } + int protocol=mc_gState->m_NetworkParams->ProtocolVersion(); + + if(protocol) + { + if(protocol < 20000) + { + if(protocol >= 10010) + { + ret=1; + } + } + else + { + if(protocol >= 20003) + { + ret=1; + } + } + } + + return ret; +} diff --git a/src/chainparams/params.h b/src/chainparams/params.h index 08ee0d62..f5c8c612 100644 --- a/src/chainparams/params.h +++ b/src/chainparams/params.h @@ -71,6 +71,7 @@ extern int MCP_ANYONE_CAN_MINE; extern int MCP_ANYONE_CAN_CONNECT; extern int MCP_ANYONE_CAN_SEND; extern int MCP_ANYONE_CAN_RECEIVE; +extern int MCP_ANYONE_CAN_CREATE; extern int MCP_ANYONE_CAN_ACTIVATE; extern int64_t MCP_MINIMUM_PER_OUTPUT; extern int MCP_ALLOW_ARBITRARY_OUTPUTS; diff --git a/src/chainparams/state.h b/src/chainparams/state.h index 299477d4..7fba20ee 100644 --- a/src/chainparams/state.h +++ b/src/chainparams/state.h @@ -139,6 +139,7 @@ typedef struct mc_Features int ParameterUpgrades(); int OffChainData(); int Chunks(); + int FixedIn1001020003(); } mc_Features; typedef struct mc_BlockHeaderInfo diff --git a/src/permissions/permission.cpp b/src/permissions/permission.cpp index 0c8f27f5..85f88bc9 100644 --- a/src/permissions/permission.cpp +++ b/src/permissions/permission.cpp @@ -1228,9 +1228,19 @@ int mc_Permissions::CanCreate(const void* lpEntity,const void* lpAddress) // if(lpEntity == NULL) if(mc_IsNullEntity(lpEntity)) { - if(MCP_ANYONE_CAN_RECEIVE) + if(mc_gState->m_Features->FixedIn1001020003()) + { + if(MCP_ANYONE_CAN_CREATE) + { + return MC_PTP_CREATE; + } + } + else { - return MC_PTP_CREATE; + if(MCP_ANYONE_CAN_RECEIVE) + { + return MC_PTP_CREATE; + } } } diff --git a/src/utils/utilwrapper.cpp b/src/utils/utilwrapper.cpp index 6baafdb1..ef8df17c 100644 --- a/src/utils/utilwrapper.cpp +++ b/src/utils/utilwrapper.cpp @@ -808,6 +808,7 @@ int mc_MultichainParams::SetGlobals() MCP_ANYONE_CAN_CONNECT=mc_gState->m_NetworkParams->GetInt64Param("anyonecanconnect"); MCP_ANYONE_CAN_SEND=mc_gState->m_NetworkParams->GetInt64Param("anyonecansend"); MCP_ANYONE_CAN_RECEIVE=mc_gState->m_NetworkParams->GetInt64Param("anyonecanreceive"); + MCP_ANYONE_CAN_CREATE=mc_gState->m_NetworkParams->GetInt64Param("anyonecancreate"); MCP_ANYONE_CAN_ACTIVATE=mc_gState->m_NetworkParams->GetInt64Param("anyonecanactivate"); MCP_MINIMUM_PER_OUTPUT=mc_gState->m_NetworkParams->GetInt64Param("minimumperoutput"); MCP_ALLOW_MULTISIG_OUTPUTS=mc_gState->m_NetworkParams->GetInt64Param("allowmultisigoutputs"); From 624119aa845d23e0fd953e2eb65b7db3ac90ab8e Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 19 Feb 2018 18:53:08 +0200 Subject: [PATCH 010/157] Off-chain data, storage, not debugged --- src/entities/asset.h | 7 + src/wallet/chunkdb.cpp | 922 +++++++++++++++++++++++++++++++++++++++- src/wallet/chunkdb.h | 128 ++++-- src/wallet/wallettxdb.h | 1 + 4 files changed, 1014 insertions(+), 44 deletions(-) diff --git a/src/entities/asset.h b/src/entities/asset.h index 60fa458c..a0b2c1e1 100644 --- a/src/entities/asset.h +++ b/src/entities/asset.h @@ -62,6 +62,13 @@ #define MC_ENT_SPRM_UPGRADE_START_BLOCK 0x43 #define MC_ENT_SPRM_UPGRADE_CHAIN_PARAMS 0x44 +#define MC_ENT_SPRM_TIMESTAMP 0x81 +#define MC_ENT_SPRM_SOURCE_TXID 0x82 +#define MC_ENT_SPRM_SOURCE_VOUT 0x83 +#define MC_ENT_SPRM_CHUNK_DETAILS 0x84 +#define MC_ENT_SPRM_CHUNK_DATA 0x85 + + #define MC_ENT_FLAG_OFFSET_IS_SET 0x00000001 #define MC_ENT_FLAG_NAME_IS_SET 0x00000010 diff --git a/src/wallet/chunkdb.cpp b/src/wallet/chunkdb.cpp index 44307ff8..85becae5 100644 --- a/src/wallet/chunkdb.cpp +++ b/src/wallet/chunkdb.cpp @@ -4,7 +4,8 @@ #include "multichain/multichain.h" #include "wallet/chunkdb.h" -#define MC_CDB_MAX_FILE_SIZE 0x8000000 // Maximal data file size, 128MB +#define MC_CDB_TMP_FLAG_SHOULD_COMMIT 0x00000001 + void sprintf_hex(char *hex,const unsigned char *bin,int size); /* @@ -64,12 +65,18 @@ void mc_ChunkDB::Zero() m_LogFileName[0]=0; m_DB=NULL; m_KeyOffset=0; - m_KeySize=36; - m_ValueOffset=36; - m_ValueSize=28; + m_KeySize=40; + m_ValueOffset=40; + m_ValueSize=40; m_TotalSize=m_KeySize+m_ValueSize; m_Subscriptions=NULL; + m_MemPool=NULL; + m_ChunkData=NULL; + m_TmpScript=NULL; + + m_Semaphore=NULL; + m_LockedBy=0; } int mc_ChunkDB::Destroy() @@ -86,33 +93,115 @@ int mc_ChunkDB::Destroy() delete m_Subscriptions; } + if(m_MemPool) + { + delete m_MemPool; + } + + if(m_ChunkData) + { + delete m_ChunkData; + } + + if(m_TmpScript) + { + delete m_TmpScript; + } + + if(m_Semaphore) + { + __US_SemDestroy(m_Semaphore); + } + + Zero(); return MC_ERR_NOERROR; } +int mc_ChunkDB::Lock(int write_mode,int allow_secondary) +{ + uint64_t this_thread; + this_thread=__US_ThreadID(); + + if(this_thread == m_LockedBy) + { + if(allow_secondary == 0) + { + LogString("Secondary lock!!!"); + } + return allow_secondary; + } + + __US_SemWait(m_Semaphore); + m_LockedBy=this_thread; + + return 0; +} + +void mc_ChunkDB::UnLock() +{ + m_LockedBy=0; + __US_SemPost(m_Semaphore); +} + +int mc_ChunkDB::Lock() +{ + return Lock(1,0); +} + int mc_ChunkDB::AddSubscription(mc_SubscriptionDBRow *subscription) { char enthex[33]; char dir_name[64]; + int err; switch(subscription->m_Entity.m_EntityType) { case MC_TET_STREAM: sprintf_hex(enthex,subscription->m_Entity.m_EntityID,MC_AST_SHORT_TXID_SIZE); sprintf(dir_name,"chunks/data/stream-%s",enthex); break; + case MC_TET_AUTHOR: + sprintf(dir_name,"chunks/data/author"); + break; + case MC_TET_NONE: + break; default: return MC_ERR_NOT_SUPPORTED; } - mc_GetFullFileName(m_Name,dir_name,"",MC_FOM_RELATIVE_TO_DATADIR | MC_FOM_CREATE_DIR,subscription->m_DirName); - mc_CreateDir(subscription->m_DirName); - return m_Subscriptions->Add(subscription,(char*)subscription+m_ValueOffset); + if(subscription->m_Entity.m_EntityType != MC_TET_NONE) + { + mc_GetFullFileName(m_Name,dir_name,"",MC_FOM_RELATIVE_TO_DATADIR | MC_FOM_CREATE_DIR,subscription->m_DirName); + mc_CreateDir(subscription->m_DirName); + } + else + { + subscription->m_DirName[0]=0x00; + } + + if(subscription->m_SubscriptionID >= m_Subscriptions->GetCount()) + { + err=m_Subscriptions->SetCount(subscription->m_SubscriptionID+1); + if(err) + { + return err; + } + } + + return m_Subscriptions->PutRow(subscription->m_SubscriptionID,subscription,(char*)subscription+m_ValueOffset); + +// return m_Subscriptions->Add(subscription,(char*)subscription+m_ValueOffset); } -mc_SubscriptionDBRow *mc_ChunkDB::FindSubscription(mc_TxEntity* entity) +mc_SubscriptionDBRow *mc_ChunkDB::FindSubscription(const mc_TxEntity* entity) { int row; mc_SubscriptionDBRow subscription; + + if(entity == NULL) + { + return (mc_SubscriptionDBRow *)m_Subscriptions->GetRow(0); + } subscription.Zero(); @@ -128,7 +217,7 @@ mc_SubscriptionDBRow *mc_ChunkDB::FindSubscription(mc_TxEntity* entity) return NULL; } -int mc_ChunkDB::AddEntity(mc_TxEntity* entity, uint32_t flags) +int mc_ChunkDB::AddEntityInternal(mc_TxEntity* entity, uint32_t flags) { int err; mc_SubscriptionDBRow subscription; @@ -193,9 +282,37 @@ int mc_ChunkDB::AddEntity(mc_TxEntity* entity, uint32_t flags) return err; } +int mc_ChunkDB::AddEntity(mc_TxEntity* entity, uint32_t flags) +{ + int err; + + Lock(); + err=AddEntityInternal(entity,flags); + UnLock(); + + return err; +} + +int mc_ChunkDB::FindSubscription(const mc_TxEntity *entity,mc_SubscriptionDBRow *subscription) +{ + mc_SubscriptionDBRow *found; + + Lock(); + found=FindSubscription(entity); + if(found) + { + memcpy(subscription,found,sizeof(mc_SubscriptionDBRow)); + } + UnLock(); + + return found ? MC_ERR_NOERROR : MC_ERR_NOT_FOUND; +} + + + int mc_ChunkDB::Initialize(const char *name,uint32_t mode) { - int err,value_len,i; + int err,value_len; char msg[256]; mc_SubscriptionDBRow subscription; @@ -229,6 +346,9 @@ int mc_ChunkDB::Initialize(const char *name,uint32_t mode) m_Subscriptions->Initialize(m_KeySize,sizeof(mc_SubscriptionDBRow),MC_BUF_MODE_MAP); + + subscription.Zero(); + m_DBStat.Zero(); ptr=(unsigned char*)m_DB->Read((char*)&m_DBStat+m_KeyOffset,m_KeySize,&value_len,MC_OPT_DB_DATABASE_SEEK_ON_READ,&err); @@ -242,7 +362,9 @@ int mc_ChunkDB::Initialize(const char *name,uint32_t mode) { memcpy((char*)&m_DBStat+m_ValueOffset,ptr,m_ValueSize); + m_Subscriptions->SetCount(m_DBStat.m_LastSubscription+1); subscription.Zero(); + AddSubscription(&subscription); ptr=(unsigned char*)m_DB->MoveNext(&err); // ptr=(unsigned char*)m_DB->Read((char*)&subscription+m_KeyOffset,m_KeySize,&value_len,MC_OPT_DB_DATABASE_SEEK_ON_READ,&err); @@ -259,7 +381,7 @@ int mc_ChunkDB::Initialize(const char *name,uint32_t mode) while(ptr) { if( (subscription.m_RecordType != MC_CDB_TYPE_SUBSCRIPTION) || - (subscription.m_Zero != 0) || (subscription.m_Zero1 != 0) ) + (subscription.m_Zero != 0) || (subscription.m_Zero1 != 0) || (subscription.m_Zero2 != 0)) { ptr=NULL; } @@ -285,6 +407,10 @@ int mc_ChunkDB::Initialize(const char *name,uint32_t mode) } else { + m_Subscriptions->SetCount(m_DBStat.m_LastSubscription+1); + subscription.Zero(); + AddSubscription(&subscription); + err=m_DB->Write((char*)&m_DBStat+m_KeyOffset,m_KeySize,(char*)&m_DBStat+m_ValueOffset,m_ValueSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); if(err) { @@ -301,6 +427,22 @@ int mc_ChunkDB::Initialize(const char *name,uint32_t mode) m_DBStat.m_InitMode |= mode; + m_MemPool=new mc_Buffer; // Key - entity with m_Pos set to 0 + txid + err=m_MemPool->Initialize(m_KeySize,m_TotalSize,MC_BUF_MODE_MAP); + + m_ChunkData=new mc_Script(); + m_ChunkData->Clear(); + + m_TmpScript=new mc_Script; + m_TmpScript->Clear(); + + m_Semaphore=__US_SemCreate(); + if(m_Semaphore == NULL) + { + LogString("Initialize: Cannot initialize semaphore"); + return MC_ERR_INTERNAL_ERROR; + } + Dump("Initialize"); sprintf(msg, "Initialized. Chunks: %d",m_DBStat.m_Count); @@ -318,7 +460,7 @@ void mc_ChunkDB::Dump(const char *message) mc_ChunkDBRow dbrow; unsigned char *ptr; - int dbvalue_len,err,i; + int dbvalue_len,err; char ShortName[65]; char FileName[MC_DCT_DB_MAX_PATH]; FILE *fHan; @@ -388,3 +530,759 @@ void mc_ChunkDB::Dump(const char *message) fprintf(fHan,"\n<<<<<< \tChain height: %6d\t%s\n\n",mc_gState->m_Permissions->m_Block,message); fclose(fHan); } + +int mc_ScriptMatchesTxIDAndVOut(unsigned char *ptr,size_t bytes,const unsigned char *txid,const int vout) +{ + size_t value_size; + uint32_t offset; + + offset=mc_FindSpecialParamInDetailsScript(ptr,bytes,MC_ENT_SPRM_SOURCE_TXID,&value_size); + if(offset != bytes) + { + if(memcmp(ptr+offset,txid,MC_TDB_TXID_SIZE) == 0) + { + if(vout >= 0) + { + offset=mc_FindSpecialParamInDetailsScript(ptr,bytes,MC_ENT_SPRM_SOURCE_VOUT,&value_size); + if(offset != bytes) + { + if(mc_GetLE(ptr+offset,value_size) == vout) + { + return MC_ERR_NOERROR; + } + } + } + else + { + return MC_ERR_NOERROR; + } + } + } + + return MC_ERR_NOERROR; +} + +int mc_ChunkDB::GetChunkDefInternal( + mc_ChunkDBRow *chunk_def, + const unsigned char *hash, + const void *entity, + const unsigned char *txid, + const int vout, + int *mempool_row) +{ + mc_SubscriptionDBRow *subscription; + int err,value_len,mprow; + size_t bytes; + uint32_t on_disk_items=0; + uint32_t total_items=0; + unsigned char *ptr; + + err=MC_ERR_NOERROR; + + chunk_def->Zero(); + + subscription=FindSubscription((mc_TxEntity*)entity); + + if(subscription == NULL) + { + return MC_ERR_NOT_FOUND; + } + + memcpy(chunk_def->m_Hash,hash,MC_CDB_CHUNK_HASH_SIZE); + chunk_def->m_SubscriptionID=subscription->m_SubscriptionID; + chunk_def->m_Pos=0; + + mprow=m_MemPool->Seek((unsigned char*)chunk_def); + if(mprow >= 0) + { + if(mempool_row) + { + *mempool_row=mprow; + } + memcpy(chunk_def,(mc_ChunkDBRow *)m_MemPool->GetRow(mprow),sizeof(mc_ChunkDBRow)); + if(txid) + { + on_disk_items=chunk_def->m_TmpOnDiskItems; + total_items=chunk_def->m_ItemCount; + while(chunk_def->m_Pos < total_items) + { +// ptr=(unsigned char *)m_ChunkData->GetData(chunk_def->m_InternalFileOffset,&bytes); + ptr=GetChunkInternal(chunk_def,-1,-1,&bytes); + if(mc_ScriptMatchesTxIDAndVOut(ptr,bytes,txid,vout) == MC_ERR_NOERROR) + { + return MC_ERR_NOERROR; + } + chunk_def->m_Pos+=1; + if(chunk_def->m_Pos < total_items) + { + mprow=m_MemPool->Seek((unsigned char*)chunk_def); + if(mprow >= 0) + { + memcpy(chunk_def,(mc_ChunkDBRow *)m_MemPool->GetRow(mprow),sizeof(mc_ChunkDBRow)); + } + else + { + chunk_def->m_Pos=total_items; + } + } + } + } + else + { + if(entity == NULL) + { + if(mempool_row) + { + while(chunk_def->m_NextSubscriptionID) + { + chunk_def->m_SubscriptionID=chunk_def->m_NextSubscriptionID; + mprow=m_MemPool->Seek((unsigned char*)chunk_def); + if(mprow >= 0) + { + if(mempool_row) + { + *mempool_row=mprow; + } + memcpy(chunk_def,(mc_ChunkDBRow *)m_MemPool->GetRow(mprow),sizeof(mc_ChunkDBRow)); + } + else + { + return MC_ERR_INTERNAL_ERROR; + } + } + } + } + return MC_ERR_NOERROR; + } + if(on_disk_items == 0) + { + err=MC_ERR_NOT_FOUND; + } + } + else + { + ptr=(unsigned char*)m_DB->Read((char*)chunk_def+m_KeyOffset,m_KeySize,&value_len,0,&err); + if(err) + { + return err; + } + + if(ptr) + { + memcpy((char*)chunk_def+m_ValueOffset,ptr,m_ValueSize); + if(txid) + { + ptr=GetChunkInternal(chunk_def,-1,-1,&bytes); + total_items=chunk_def->m_ItemCount; + while(ptr) + { + if(mc_ScriptMatchesTxIDAndVOut(ptr,bytes,txid,vout) == MC_ERR_NOERROR) + { + return MC_ERR_NOERROR; + } + chunk_def->m_Pos+=1; + + if(chunk_def->m_Pos < total_items) + { + ptr=(unsigned char*)m_DB->Read((char*)chunk_def+m_KeyOffset,m_KeySize,&value_len,0,&err); + if(err) + { + return err; + } + + if(ptr) + { + memcpy((char*)chunk_def+m_ValueOffset,ptr,m_ValueSize); + ptr=GetChunkInternal(chunk_def,-1,-1,&bytes); + } + else + { + chunk_def->m_Pos=total_items; + } + } + } + err=MC_ERR_NOT_FOUND; + } + else + { + if(entity == NULL) + { + if(mempool_row) + { + while(chunk_def->m_NextSubscriptionID) + { + chunk_def->m_SubscriptionID=chunk_def->m_NextSubscriptionID; + ptr=(unsigned char*)m_DB->Read((char*)chunk_def+m_KeyOffset,m_KeySize,&value_len,0,&err); + if(err) + { + return err; + } + + if(ptr) + { + memcpy((char*)chunk_def+m_ValueOffset,ptr,m_ValueSize); + } + else + { + return MC_ERR_CORRUPTED; + } + } + } + } + return MC_ERR_NOERROR; + } + } + else + { + err=MC_ERR_NOT_FOUND; + } + } + + return MC_ERR_NOT_FOUND; +} + +int mc_ChunkDB::GetChunkDef( + mc_ChunkDBRow *chunk_def, + const unsigned char *hash, + const void *entity, + const unsigned char *txid, + const int vout) +{ + int err; + + Lock(); + err=GetChunkDefInternal(chunk_def,hash,entity,txid,vout,NULL); + UnLock(); + + return err; +} + +int mc_ChunkDB::AddChunkInternal( + const unsigned char *hash, + const mc_TxEntity *entity, + const unsigned char *txid, + const int vout, + const unsigned char *chunk, + const unsigned char *details, + const uint32_t chunk_size, + const uint32_t details_size, + const uint32_t flags) +{ + int err; + int add_null_row,add_entity_row; + int total_items,on_disk_items; + int mempool_entity_row; + int mempool_last_null_row; + uint32_t timestamp; + size_t bytes; + const unsigned char *ptr; + + mc_ChunkDBRow chunk_def; + mc_ChunkDBRow entity_chunk_def; + mc_ChunkDBRow null_chunk_def; + mc_SubscriptionDBRow *subscription; + + chunk_def.Zero(); + + err=GetChunkDefInternal(&chunk_def,hash,entity,txid,vout,NULL); + if(err == MC_ERR_NOERROR) + { + return MC_ERR_FOUND; + } + if(err != MC_ERR_NOT_FOUND) + { + return err; + } + + if( (m_ChunkData->m_Size + chunk_size + details_size + MC_CDB_MAX_CHUNK_EXTRA_SIZE > MC_CDB_MAX_CHUNK_DATA_POOL_SIZE) || + (m_MemPool->GetCount() + 2 > MC_CDB_MAX_MEMPOOL_SIZE ) ) + { + CommitInternal(-1); + } + + subscription=FindSubscription(entity); + if(subscription == NULL) + { + LogString("Internal error: trying to add chunk to unsubscribed entity"); + return MC_ERR_INTERNAL_ERROR; + } + if(subscription->m_SubscriptionID == 0) + { + LogString("Internal error: trying to add chunk to null entity"); + return MC_ERR_INTERNAL_ERROR; + } + + add_null_row=0; + add_entity_row=0; + total_items=0; + on_disk_items=0; + mempool_entity_row=-1; + + if(txid) + { + err=GetChunkDefInternal(&entity_chunk_def,hash,entity,NULL,-1,&mempool_entity_row); + if(err == MC_ERR_NOERROR) + { + total_items=entity_chunk_def.m_ItemCount; + on_disk_items=total_items; + if(entity_chunk_def.m_InternalFileID < 0) + { + on_disk_items=entity_chunk_def.m_TmpOnDiskItems; + } + } + else + { + if(err != MC_ERR_NOT_FOUND) + { + return err; + } + entity_chunk_def.m_SubscriptionID=0; + add_entity_row=1; + } + } + else + { + add_entity_row=1; + } + + if(add_entity_row) + { + err=GetChunkDefInternal(&null_chunk_def,hash,NULL,NULL,-1,&mempool_last_null_row); + if(err == MC_ERR_NOT_FOUND) + { + add_null_row=1; + } + if(err != MC_ERR_NOERROR) + { + return err; + } + } + + chunk_def.Zero(); + memcpy(chunk_def.m_Hash,hash,MC_CDB_CHUNK_HASH_SIZE); + chunk_def.m_InternalFileID=-1; + chunk_def.m_InternalFileOffset=m_ChunkData->m_NumElements; + + m_TmpScript->Clear(); + m_TmpScript->AddElement(); + + timestamp=mc_TimeNowAsUInt(); + m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_TIMESTAMP,(unsigned char*)×tamp,sizeof(timestamp)); + if(txid) + { + m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_SOURCE_TXID,hash,MC_CDB_CHUNK_HASH_SIZE); + if(vout >= 0) + { + m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_SOURCE_TXID,(unsigned char*)&vout,sizeof(vout)); + } + } + if(details_size) + { + m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_CHUNK_DETAILS,details,details_size); + } + if(chunk_size) + { + m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_CHUNK_DATA,chunk,chunk_size); + } + + chunk_def.m_Size=chunk_size; + chunk_def.m_HeaderSize=m_TmpScript->m_Size-chunk_size; + chunk_def.m_Flags=flags; + + ptr=m_TmpScript->GetData(0,&bytes); + + m_ChunkData->AddElement(); + err=m_ChunkData->SetData(ptr,bytes); + if(err) + { + return err; + } + + if(add_null_row) + { + chunk_def.m_SubscriptionID=0; + chunk_def.m_Pos=0; + chunk_def.m_ItemCount=1; + chunk_def.m_TmpOnDiskItems=0; + chunk_def.m_PrevSubscriptionID=0; + chunk_def.m_NextSubscriptionID=subscription->m_SubscriptionID; + m_MemPool->Add(&chunk_def,(char*)&chunk_def+m_ValueOffset); + } + + chunk_def.m_SubscriptionID=subscription->m_SubscriptionID; + chunk_def.m_Pos=total_items-on_disk_items; + chunk_def.m_ItemCount=total_items+1; + chunk_def.m_TmpOnDiskItems=on_disk_items; + if(add_null_row) + { + chunk_def.m_PrevSubscriptionID=0; + chunk_def.m_NextSubscriptionID=0; + } + else + { + if(add_entity_row) + { + chunk_def.m_PrevSubscriptionID=entity_chunk_def.m_SubscriptionID; + chunk_def.m_NextSubscriptionID=0; + } + else + { + chunk_def.m_PrevSubscriptionID=entity_chunk_def.m_PrevSubscriptionID; + chunk_def.m_NextSubscriptionID=entity_chunk_def.m_NextSubscriptionID; + } + } + + m_MemPool->Add(&chunk_def,(char*)&chunk_def+m_ValueOffset); + + + if(mempool_entity_row >= 0) + { + ((mc_ChunkDBRow *)m_MemPool->GetRow(mempool_entity_row))->m_ItemCount += 1; + } + + if(mempool_last_null_row >= 0) + { + if(add_entity_row) + { + ((mc_ChunkDBRow *)m_MemPool->GetRow(mempool_entity_row))->m_NextSubscriptionID = subscription->m_SubscriptionID; + } + } + + return MC_ERR_NOERROR; +} + +int mc_ChunkDB::AddChunk( + const unsigned char *hash, + const mc_TxEntity *entity, + const unsigned char *txid, + const int vout, + const unsigned char *chunk, + const unsigned char *details, + const uint32_t chunk_size, + const uint32_t details_size, + const uint32_t flags) +{ + int err; + + Lock(); + err=AddChunkInternal(hash,entity,txid,vout,chunk,details,chunk_size,details_size,flags); + UnLock(); + + return err; +} + +void mc_ChunkDB::SetFileName(char *FileName, + mc_SubscriptionDBRow *subscription, + uint32_t fileid) +{ + sprintf(FileName,"%s/chunks%06u.dat",subscription->m_DirName,fileid); +} + + + + +unsigned char *mc_ChunkDB::GetChunkInternal(mc_ChunkDBRow *chunk_def, + int32_t offset, + int32_t len, + size_t *bytes) +{ + unsigned char *ptr; + size_t bytes_to_read; + mc_SubscriptionDBRow *subscription; + char FileName[MC_DCT_DB_MAX_PATH]; + int FileHan; + uint32_t read_from; + + ptr=NULL; + FileHan=0; + if(chunk_def->m_InternalFileID < 0) + { + ptr=(unsigned char *)m_ChunkData->GetData(chunk_def->m_InternalFileOffset,&bytes_to_read); + read_from=0; + if(offset >= 0) + { + read_from+=chunk_def->m_HeaderSize+offset; + if(offset >= (int)chunk_def->m_Size) + { + return NULL; + } + bytes_to_read=chunk_def->m_Size-offset; + } + if(bytes) + { + *bytes=bytes_to_read; + } + return ptr+read_from; + } + else + { + subscription=(mc_SubscriptionDBRow *)m_Subscriptions->GetRow(chunk_def->m_SubscriptionID); + SetFileName(FileName,subscription,chunk_def->m_InternalFileID); + + FileHan=open(FileName,_O_BINARY | O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); + + if(FileHan<=0) + { + return NULL; + } + + read_from=chunk_def->m_InternalFileOffset; + bytes_to_read=chunk_def->m_HeaderSize; + if(offset >= 0) + { + read_from+=chunk_def->m_HeaderSize+offset; + bytes_to_read=chunk_def->m_Size; + if(len>0) + { + bytes_to_read=len; + } + } + + if(lseek64(FileHan,read_from,SEEK_SET) != (int)read_from) + { + goto exitlbl; + } + + m_TmpScript->Clear(); + if(m_TmpScript->Resize(bytes_to_read,1)) + { + goto exitlbl; + } + + if(read(FileHan,m_TmpScript->m_lpData,bytes_to_read) != (int)bytes_to_read) + { + goto exitlbl; + } + + ptr=m_TmpScript->m_lpData; + if(bytes) + { + *bytes=bytes_to_read; + } + } + +exitlbl: + + if(FileHan) + { + close(FileHan); + } + + return ptr; +} + +unsigned char *mc_ChunkDB::GetChunk(mc_ChunkDBRow *chunk_def, + int32_t offset, + int32_t len, + size_t *bytes) +{ + unsigned char *ptr; + + Lock(); + ptr=GetChunkInternal(chunk_def,offset,len,bytes); + UnLock(); + + return ptr; +} + +int mc_ChunkDB::AddToFile(const void* chunk, + uint32_t size, + mc_SubscriptionDBRow *subscription, + uint32_t fileid, + uint32_t offset) +{ + char FileName[MC_DCT_DB_MAX_PATH]; + int FileHan,err; + + SetFileName(FileName,subscription,fileid); + err=MC_ERR_NOERROR; + + FileHan=open(FileName,_O_BINARY | O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); + + if(FileHan<=0) + { + return MC_ERR_INTERNAL_ERROR; + } + + if(lseek64(FileHan,offset,SEEK_SET) != offset) + { + err=MC_ERR_INTERNAL_ERROR; + goto exitlbl; + } + + if(write(FileHan,chunk,size) != size) + { + err=MC_ERR_INTERNAL_ERROR; + return MC_ERR_INTERNAL_ERROR; + } + +exitlbl: + + close(FileHan); + return err; +} + +int mc_ChunkDB::FlushDataFile(mc_SubscriptionDBRow *subscription,uint32_t fileid) +{ + char FileName[MC_DCT_DB_MAX_PATH]; + int FileHan; + SetFileName(FileName,subscription,fileid); + + FileHan=open(FileName,_O_BINARY | O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); + + if(FileHan<=0) + { + return MC_ERR_INTERNAL_ERROR; + } + __US_FlushFile(FileHan); + close(FileHan); + return MC_ERR_NOERROR; +} + + +int mc_ChunkDB::CommitInternal(int block) +{ + int r,s; + int err; + int last_file_id, last_file_offset; + uint32_t size; + char msg[256]; + + mc_SubscriptionDBRow *subscription; + mc_ChunkDBRow *chunk_def; + + Dump("Before Commit"); + if(m_MemPool->GetCount() == 0) + { + goto exitlbl; + } + + last_file_id=-1; + last_file_offset=0; + + for(r=0;rGetCount();r++) + { + chunk_def=(mc_ChunkDBRow *)m_MemPool->GetRow(r); + + s=chunk_def->m_SubscriptionID; + if(s == 0) + { + s=chunk_def->m_NextSubscriptionID; + } + + subscription=(mc_SubscriptionDBRow *)m_Subscriptions->GetRow(s); + subscription->m_TmpFlags |= MC_CDB_TMP_FLAG_SHOULD_COMMIT; + + if(last_file_id < 0) + { + size=chunk_def->m_Size+chunk_def->m_HeaderSize; + if(subscription->m_LastFileSize+size > MC_CDB_MAX_FILE_SIZE) // New file is needed + { + FlushDataFile(subscription,subscription->m_LastFileID); + subscription->m_LastFileID+=1; + subscription->m_LastFileSize=0; + } + + err=AddToFile(GetChunkInternal(chunk_def,-1,-1,NULL),size, + subscription,subscription->m_LastFileID,subscription->m_LastFileSize); + if(err) + { + sprintf(msg,"Couldn't store key in file, error: %d",err); + LogString(msg); + return err; + } + + chunk_def->m_InternalFileID=subscription->m_LastFileID; + chunk_def->m_InternalFileOffset=subscription->m_LastFileSize; + chunk_def->m_Pos+=chunk_def->m_TmpOnDiskItems; + chunk_def->m_TmpOnDiskItems=0; + + if(chunk_def->m_SubscriptionID == 0) + { + last_file_id=subscription->m_LastFileID; + last_file_offset=subscription->m_LastFileSize; + } + + subscription->m_LastFileSize+=size; + subscription->m_Count+=1; + subscription->m_FullSize+=chunk_def->m_Size; + + m_DBStat.m_Count+=1; + m_DBStat.m_FullSize+=chunk_def->m_Size; + } + else + { + chunk_def->m_InternalFileID=last_file_id; + chunk_def->m_InternalFileOffset=last_file_offset; + chunk_def->m_Pos+=chunk_def->m_TmpOnDiskItems; + chunk_def->m_TmpOnDiskItems=0; + last_file_id=-1; + } + + err=m_DB->Write((char*)chunk_def+m_KeyOffset,m_KeySize,(char*)chunk_def+m_ValueOffset,m_ValueSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + if(err) + { + goto exitlbl; + } + } + + for(s=0;sGetCount();s++) + { + subscription=(mc_SubscriptionDBRow *)m_Subscriptions->GetRow(s); + if(subscription->m_TmpFlags & MC_CDB_TMP_FLAG_SHOULD_COMMIT) + { + subscription->m_TmpFlags=0; + + FlushDataFile(subscription,subscription->m_LastFileID); + + err=m_DB->Write((char*)subscription+m_KeyOffset,m_KeySize,(char*)subscription+m_ValueOffset,m_ValueSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + if(err) + { + goto exitlbl; + } + + } + } + + err=m_DB->Write((char*)&m_DBStat+m_KeyOffset,m_KeySize,(char*)&m_DBStat+m_ValueOffset,m_ValueSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + if(err) + { + goto exitlbl; + } + + err=m_DB->Commit(MC_OPT_DB_DATABASE_TRANSACTIONAL); + if(err) + { + goto exitlbl; + } + + +exitlbl: + if(err) + { + sprintf(msg,"Could not commit new Block %d, Chunks: %d, Error: %d",block,m_MemPool->GetCount(),err); + LogString(msg); + } + else + { + sprintf(msg,"NewBlock %d, Chunks: %d,",block,m_MemPool->GetCount()); + LogString(msg); + m_MemPool->Clear(); + m_ChunkData->Clear(); + } + Dump("After Commit"); + + return MC_ERR_NOERROR; +} + +int mc_ChunkDB::Commit(int block) +{ + int err; + + Lock(); + err=CommitInternal(block); + UnLock(); + + return err; +} + diff --git a/src/wallet/chunkdb.h b/src/wallet/chunkdb.h index 4037a60c..ec8dd165 100644 --- a/src/wallet/chunkdb.h +++ b/src/wallet/chunkdb.h @@ -11,17 +11,15 @@ #define MC_CDB_ROW_SIZE 80 #define MC_CDB_HEADER_SIZE 40 -#define MC_TDB_TXID_SIZE 32 -#define MC_TDB_ENTITY_KEY_SIZE 32 -#define MC_TDB_ENTITY_ID_SIZE 20 -#define MC_TDB_ENTITY_TYPE_SIZE 4 -#define MC_TDB_GENERATION_SIZE 4 -#define MC_TDB_POS_SIZE 4 -#define MC_TDB_ROW_SIZE 80 -#define MC_CDB_TYPE_DB_STAT 0 -#define MC_CDB_TYPE_SUBSCRIPTION 1 -#define MC_CDB_TYPE_FILE 2 +#define MC_CDB_TYPE_DB_STAT 0x00000000 +#define MC_CDB_TYPE_SUBSCRIPTION 0x01000000 +#define MC_CDB_TYPE_FILE 0x02000000 + +#define MC_CDB_MAX_FILE_SIZE 0x40000000 // Maximal data file size, 1GB +#define MC_CDB_MAX_CHUNK_DATA_POOL_SIZE 0x8000000 // Maximal size of chunk pool before commit, 128MB +#define MC_CDB_MAX_CHUNK_EXTRA_SIZE 1024 +#define MC_CDB_MAX_MEMPOOL_SIZE 1024 /** File DB Row*/ @@ -50,14 +48,17 @@ typedef struct mc_SubscriptionDBRow uint32_t m_Zero; // Should be Zero uint32_t m_RecordType; // Should be MC_CDB_TYPE_SUBSCRIPTION uint32_t m_Zero1; // Should be Zero + uint32_t m_Zero2; // Should be Zero mc_TxEntity m_Entity; // Parent Entity uint32_t m_Flags; // Flags passed from higher level - uint32_t m_SubscriptionID; // Subscription ID + int32_t m_SubscriptionID; // Subscription ID uint32_t m_Count; // Total chunk count - uint32_t m_FirstFileID; // First data file ID/Reserved + int32_t m_FirstFileID; // First data file ID/Reserved uint32_t m_FirstFileOffset; // First data file offset/Reserved - uint32_t m_LastFileID; // Last data file ID + int32_t m_LastFileID; // Last data file ID uint32_t m_LastFileSize; // Last data file size + uint32_t m_TmpFlags; + uint64_t m_FullSize; // Total tx size char m_DirName[MC_DCT_DB_MAX_PATH]; // Full file name @@ -69,14 +70,16 @@ typedef struct mc_ChunkDBStat uint32_t m_Zero; // Should be Zero uint32_t m_RecordType; // Should be MC_CDB_TYPE_DB_STAT uint32_t m_Zero1; // Should be Zero + uint32_t m_Zero2; // Should be Zero mc_TxEntity m_ZeroEntity; // Zero Entity uint32_t m_InitMode; uint32_t m_ChunkDBVersion; - uint32_t m_Count; // Total tx count - uint32_t m_FullSize; // Total tx size - uint32_t m_LastSubscription; + int32_t m_LastSubscription; uint32_t m_Reserved1; + uint32_t m_Count; // Total tx count + uint64_t m_FullSize; // Total tx size uint32_t m_Reserved2; + uint32_t m_Reserved3; void Zero(); } mc_ChunkDBStat; @@ -85,15 +88,19 @@ typedef struct mc_ChunkDBStat typedef struct mc_ChunkDBRow { - uint32_t m_SubscriptionID; // Subscription ID unsigned char m_Hash[MC_CDB_CHUNK_HASH_SIZE]; // Chunk hash + int32_t m_SubscriptionID; // Subscription ID + uint32_t m_Pos; // Position of this record for subscription/hash uint32_t m_Size; // Chunk Size uint32_t m_Flags; // Flags passed from higher level + uint32_t m_HeaderSize; // Header size uint32_t m_StorageFlags; // Internal flags - uint32_t m_InternalFileID; // Data file ID + int32_t m_ItemCount; // Number of times this chunk appears in subscription (if m_Pos=0) + int32_t m_TmpOnDiskItems; + int32_t m_InternalFileID; // Data file ID uint32_t m_InternalFileOffset; // Offset in the data file - uint32_t m_PrevSubscriptionID; // Prev Subscription ID for this hash - uint32_t m_NextSubscriptionID; // Next Subscription ID for this hash + int32_t m_PrevSubscriptionID; // Prev Subscription ID for this hash + int32_t m_NextSubscriptionID; // Next Subscription ID for this hash void Zero(); } mc_ChunkDBRow; @@ -118,6 +125,12 @@ typedef struct mc_ChunkDB char m_LogFileName[MC_DCT_DB_MAX_PATH]; // Full log file name mc_Buffer *m_Subscriptions; // List of import entities (mc_TxEntityStat) + mc_Buffer *m_MemPool; + mc_Script *m_ChunkData; + mc_Script *m_TmpScript; + + void *m_Semaphore; // mc_TxDB object semaphore + uint64_t m_LockedBy; // ID of the thread locking it mc_ChunkDB() { @@ -135,28 +148,75 @@ typedef struct mc_ChunkDB int AddSubscription(mc_SubscriptionDBRow *subscription); int AddEntity(mc_TxEntity *entity,uint32_t flags); // Adds entity + int AddEntityInternal(mc_TxEntity *entity,uint32_t flags); - mc_SubscriptionDBRow *FindSubscription(mc_TxEntity *entity); // Finds subscription + mc_SubscriptionDBRow *FindSubscription(const mc_TxEntity *entity); // Finds subscription + int FindSubscription(const mc_TxEntity *entity,mc_SubscriptionDBRow *subscription); // Finds subscription int AddChunk( // Adds chunk to mempool const unsigned char *hash, // Chunk hash (before chopping) const mc_TxEntity *entity, // Parent entity + const unsigned char *txid, + const int vout, const unsigned char *chunk, // Chunk data - const unsigned char *metadata, // Chunk metadata + const unsigned char *details, // Chunk metadata const uint32_t chunk_size, // Chunk size - const uint32_t metadata_size, // Chunk metadata size + const uint32_t details_size, // Chunk metadata size const uint32_t flags); // Flags - - int GetChunk( + + int AddChunkInternal( // Adds chunk to mempool const unsigned char *hash, // Chunk hash (before chopping) - const unsigned char *entity, // Parent entity - unsigned char **chunk, // Chunk data - unsigned char **metadata, // Metadata - uint32_t *size, // Chunk Size - uint32_t *metadata_size, // Chunk Size + const mc_TxEntity *entity, // Parent entity + const unsigned char *txid, + const int vout, + const unsigned char *chunk, // Chunk data + const unsigned char *details, // Chunk metadata + const uint32_t chunk_size, // Chunk size + const uint32_t details_size, // Chunk metadata size const uint32_t flags); // Flags - - int Commit(); // Commit mempool to disk + + int GetChunkDefInternal( + mc_ChunkDBRow *chunk_def, + const unsigned char *hash, // Chunk hash (before chopping) + const void *entity, + const unsigned char *txid, + const int vout, + int *mempool_entity_row); + + + int GetChunkDef( + mc_ChunkDBRow *chunk_def, + const unsigned char *hash, // Chunk hash (before chopping) + const void *entity, + const unsigned char *txid, + const int vout); + + unsigned char *GetChunkInternal(mc_ChunkDBRow *chunk_def, + int32_t offset, + int32_t len, + size_t *bytes); + + unsigned char *GetChunk(mc_ChunkDBRow *chunk_def, + int32_t offset, + int32_t len, + size_t *bytes); + + void SetFileName(char *FileName, + mc_SubscriptionDBRow *subscription, + uint32_t fileid); + + int FlushDataFile(mc_SubscriptionDBRow *subscription, + uint32_t fileid); + + + int AddToFile(const void *chunk, + uint32_t size, + mc_SubscriptionDBRow *subscription, + uint32_t fileid, + uint32_t offset); + + int Commit(int block); // Commit mempool to disk + int CommitInternal(int block); void Zero(); int Destroy(); @@ -164,6 +224,10 @@ typedef struct mc_ChunkDB void LogString(const char *message); + int Lock(); + int Lock(int write_mode, int allow_secondary); + void UnLock(); + } mc_ChunkDB; #endif /* MULTICHAIN_CHUNKDB_H */ diff --git a/src/wallet/wallettxdb.h b/src/wallet/wallettxdb.h index 04fe09c6..f101c642 100644 --- a/src/wallet/wallettxdb.h +++ b/src/wallet/wallettxdb.h @@ -29,6 +29,7 @@ #define MC_TET_STREAM_KEY 0x00000006 #define MC_TET_STREAM_PUBLISHER 0x00000007 #define MC_TET_ASSET 0x00000008 +#define MC_TET_AUTHOR 0x00000009 #define MC_TET_SUBKEY_STREAM_KEY 0x00000046 #define MC_TET_SUBKEY_STREAM_PUBLISHER 0x00000047 #define MC_TET_SUBKEY 0x00000040 From 2e2f2ac9da7915bb800134aea0d85a34cb1d0cfe Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 20 Feb 2018 09:32:22 +0200 Subject: [PATCH 011/157] Skipping writing best block in wallet.dat for walletdbversion=2 --- src/wallet/wallet.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index f1af8731..cd25d3b5 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -355,8 +355,11 @@ bool CWallet::ChangeWalletPassphrase(const SecureString& strOldWalletPassphrase, void CWallet::SetBestChain(const CBlockLocator& loc) { - CWalletDB walletdb(strWalletFile); - walletdb.WriteBestBlock(loc); + if( (mc_gState->m_WalletMode & MC_WMD_TXS) == 0 ) + { + CWalletDB walletdb(strWalletFile); + walletdb.WriteBestBlock(loc); + } } bool CWallet::SetMinVersion(enum WalletFeature nVersion, CWalletDB* pwalletdbIn, bool fExplicit) From 2b996d2d0bd322d05798951dbe0d6edcd5822230 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 20 Feb 2018 16:51:35 +0200 Subject: [PATCH 012/157] Off-chain items, protocol --- src/chainparams/globals.h | 2 + src/chainparams/paramlist.h | 10 +- src/chainparams/params.h | 2 + src/protocol/multichainscript.cpp | 214 +++++++++++++----------------- src/protocol/multichainscript.h | 5 +- src/utils/utilwrapper.cpp | 2 + src/wallet/chunkdb.cpp | 58 ++++---- src/wallet/chunkdb.h | 9 +- src/wallet/wallettxs.cpp | 7 + 9 files changed, 151 insertions(+), 158 deletions(-) diff --git a/src/chainparams/globals.h b/src/chainparams/globals.h index b333a9c1..fd1baabe 100644 --- a/src/chainparams/globals.h +++ b/src/chainparams/globals.h @@ -24,6 +24,8 @@ int MAX_OP_RETURN_SHOWN=16384; int MAX_FORMATTED_DATA_DEPTH=100; unsigned int MAX_OP_RETURN_OP_DROP_COUNT=100000000; uint32_t JSON_NO_DOUBLE_FORMATTING=0; +int MAX_CHUNK_SIZE = 16777216; +int MAX_CHUNK_COUNT = 128; int MCP_MAX_STD_OP_RETURN_COUNT=0; int64_t MCP_INITIAL_BLOCK_REWARD=0; diff --git a/src/chainparams/paramlist.h b/src/chainparams/paramlist.h index 4521d7c9..e7c0a8ba 100644 --- a/src/chainparams/paramlist.h +++ b/src/chainparams/paramlist.h @@ -32,8 +32,16 @@ static const mc_OneMultichainParam MultichainParamArray[] = "Target time between blocks (transaction confirmation delay), seconds."}, { "maximumblocksize" , "maximum-block-size" , MC_PRM_UINT32 | MC_PRM_USER | MC_PRM_CLONE , -1, 8388608, 5000,1000000000, 0.0, 10001, 0, "-mc-maximumblocksize", - "timingupgrademingap","", + "maximumchunksize","", "Maximum block size in bytes."}, + { "maximumchunksize" , "maximum-chunk-size" , + MC_PRM_UINT32 | MC_PRM_USER | MC_PRM_CLONE , -1, 16777216, 256, 67108864, 0.0, 20003, 0, "-mc-maximumchunksize", + "maximumchunkcount","", + "Maximum chunk size in bytes."}, + { "maximumchunkcount" , "maximum-chunk-count" , + MC_PRM_UINT32 | MC_PRM_USER | MC_PRM_CLONE , -1, 128, 16, 1024, 0.0, 20003, 0, "-mc-maximumchunkcount", + "timingupgrademingap","", + "Maximum number of chunks in one off-chain item."}, { "timingupgrademingap" , "timing-upgrade-min-gap" , MC_PRM_UINT32 | MC_PRM_USER | MC_PRM_CLONE | MC_PRM_HIDDEN , -1, 100, 1, 31536000, 0.0, 20002, 0, "-mc-timingupgrademingap", "defaultnetworkport","", diff --git a/src/chainparams/params.h b/src/chainparams/params.h index f5c8c612..e89a3109 100644 --- a/src/chainparams/params.h +++ b/src/chainparams/params.h @@ -66,6 +66,8 @@ extern int MCP_MAX_STD_OP_RETURN_COUNT; extern int64_t MCP_INITIAL_BLOCK_REWARD; extern int64_t MCP_FIRST_BLOCK_REWARD; extern int MCP_TARGET_BLOCK_TIME; +extern int MAX_CHUNK_SIZE; +extern int MAX_CHUNK_COUNT; extern int MCP_ANYONE_CAN_ADMIN; extern int MCP_ANYONE_CAN_MINE; extern int MCP_ANYONE_CAN_CONNECT; diff --git a/src/protocol/multichainscript.cpp b/src/protocol/multichainscript.cpp index 0295f0eb..e3fae82f 100644 --- a/src/protocol/multichainscript.cpp +++ b/src/protocol/multichainscript.cpp @@ -35,8 +35,7 @@ #define MC_DCT_SCRIPT_MULTICHAIN_DATA_FORMAT_PREFIX 'f' #define MC_DCT_SCRIPT_MULTICHAIN_RAW_DATA_PREFIX 'd' -#define MC_DCT_SCRIPT_EXTENDED_TYPE_DATA_FORMAT 1 -#define MC_DCT_SCRIPT_EXTENDED_TYPE_CHUNK_DEF 2 +#define MC_DCT_SCRIPT_EXTENDED_TYPE_CHUNK_DEF 0xF0 #define MC_DCT_SCRIPT_TYPE_REGULAR 0x00 #define MC_DCT_SCRIPT_TYPE_OP_RETURN 0x01 @@ -2345,14 +2344,12 @@ int mc_Script::SetDataFormat(const uint32_t format) return MC_ERR_NOERROR; } -int mc_Script::GetChunkDef(uint32_t *format,unsigned char** hashes,uint64_t *size,uint32_t *chunk_size_log) +int mc_Script::GetChunkDef(uint32_t *format,unsigned char** hashes,int *chunk_count,uint64_t *total_size) { unsigned char *ptr; unsigned char *ptrEnd; - unsigned char l,f,extended_type; - uint64_t s,c,t; - int format_found=0; - int chunkdef_found=0; + unsigned char f; + int c,count,shift,size; if(format) { @@ -2364,7 +2361,7 @@ int mc_Script::GetChunkDef(uint32_t *format,unsigned char** hashes,uint64_t *siz return MC_ERR_INVALID_PARAMETER_VALUE; } - if(m_lpCoord[m_CurrentElement*2+1] < MC_DCT_SCRIPT_IDENTIFIER_LEN+1+1+1) + if(m_lpCoord[m_CurrentElement*2+1] < MC_DCT_SCRIPT_IDENTIFIER_LEN+1+1) { return MC_ERR_WRONG_SCRIPT; } @@ -2387,123 +2384,91 @@ int mc_Script::GetChunkDef(uint32_t *format,unsigned char** hashes,uint64_t *siz f=(unsigned char)(*ptr); - if( (f & MC_SCR_DATA_FORMAT_EXTENDED_MASK) == 0) + if(f != MC_DCT_SCRIPT_EXTENDED_TYPE_CHUNK_DEF) { return MC_ERR_WRONG_SCRIPT; } ptr++; - while(ptr < ptrEnd) + + if(ptr+2 > ptrEnd) { - extended_type=(unsigned char)(*ptr); - ptr++; - switch(extended_type) - { - case MC_DCT_SCRIPT_EXTENDED_TYPE_DATA_FORMAT: - if(format_found) - { - return MC_ERR_ERROR_IN_SCRIPT; - } - format_found=1; - if(ptr63) - { - return MC_ERR_ERROR_IN_SCRIPT; - } - c=1 << l; - s=mc_GetLE(ptr,8); - ptr+=8; - t=0; - if(s) - { - t=(s-1)/c + 1; - } - if( (t >> 24) != 0 ) - { - return MC_ERR_ERROR_IN_SCRIPT; - } - if(ptr+t*MC_DCT_SCRIPT_CHUNK_HASH_SIZE <= ptrEnd) - { - if(size) - { - *size=s; - } - if(chunk_size_log) - { - *chunk_size_log=l; - } - if(hashes) - { - *hashes=ptr; - } - ptr+=t*MC_DCT_SCRIPT_CHUNK_HASH_SIZE; - } - else - { - return MC_ERR_ERROR_IN_SCRIPT; - } - } - else - { - return MC_ERR_ERROR_IN_SCRIPT; - } - } - else - { - return MC_ERR_ERROR_IN_SCRIPT; - } - break; - } - } + return MC_ERR_ERROR_IN_SCRIPT; + } - return MC_ERR_NOERROR; -} + if(format) + { + *format=(uint32_t)(*ptr); + } + + ptr++; + + count=(int)mc_GetVarInt(ptr,ptrEnd-ptr,-1,&shift); + + if(count<0) + { + return MC_ERR_ERROR_IN_SCRIPT; + } -int mc_Script::SetChunkDef(const uint32_t format,unsigned char* hashes,uint64_t size,uint32_t chunk_size_log) -{ - int err; - unsigned char buf[MC_DCT_SCRIPT_IDENTIFIER_LEN+14]; - uint64_t t,c; + ptr+=shift; + + if(count > MAX_CHUNK_COUNT) + { + return MC_ERR_ERROR_IN_SCRIPT; + } - if(chunk_size_log>63) + if(chunk_count) { - return MC_ERR_INVALID_PARAMETER_VALUE; + *chunk_count=count; } - c=1 << chunk_size_log; - t=0; - if(size) + + if(hashes) { - t=(size-1)/c + 1; + *hashes=ptr; } - if( (t >> 24) != 0 ) + + if(total_size) { - return MC_ERR_INVALID_PARAMETER_VALUE; + *total_size=0; } + for(c=0;c MAX_CHUNK_SIZE) + { + return MC_ERR_ERROR_IN_SCRIPT; + } + + ptr+=shift; + + if(ptr+MC_DCT_SCRIPT_CHUNK_HASH_SIZE>ptrEnd) + { + return MC_ERR_ERROR_IN_SCRIPT; + } + + ptr+=MC_DCT_SCRIPT_CHUNK_HASH_SIZE; + + if(total_size) + { + *total_size+=size; + } + } + + return MC_ERR_NOERROR; +} + +int mc_Script::SetChunkDefHeader(const uint32_t format,int chunk_count) +{ + int err,shift; + unsigned char buf[MC_DCT_SCRIPT_IDENTIFIER_LEN+14]; + err=AddElement(); if(err) { @@ -2512,27 +2477,38 @@ int mc_Script::SetChunkDef(const uint32_t format,unsigned char* hashes,uint64_t memcpy(buf,MC_DCT_SCRIPT_MULTICHAIN_IDENTIFIER,MC_DCT_SCRIPT_IDENTIFIER_LEN); buf[MC_DCT_SCRIPT_IDENTIFIER_LEN]=MC_DCT_SCRIPT_MULTICHAIN_DATA_FORMAT_PREFIX; - buf[MC_DCT_SCRIPT_IDENTIFIER_LEN+1]=MC_SCR_DATA_FORMAT_EXTENDED_MASK; - buf[MC_DCT_SCRIPT_IDENTIFIER_LEN+2]=(unsigned char)MC_DCT_SCRIPT_EXTENDED_TYPE_DATA_FORMAT; - buf[MC_DCT_SCRIPT_IDENTIFIER_LEN+3]=(unsigned char)format; - buf[MC_DCT_SCRIPT_IDENTIFIER_LEN+4]=(unsigned char)MC_DCT_SCRIPT_EXTENDED_TYPE_CHUNK_DEF; - buf[MC_DCT_SCRIPT_IDENTIFIER_LEN+5]=(unsigned char)chunk_size_log; - mc_PutLE(buf+MC_DCT_SCRIPT_IDENTIFIER_LEN+6,&size,8); - - err=SetData(buf,MC_DCT_SCRIPT_IDENTIFIER_LEN+14); + buf[MC_DCT_SCRIPT_IDENTIFIER_LEN+1]=MC_DCT_SCRIPT_EXTENDED_TYPE_CHUNK_DEF; + buf[MC_DCT_SCRIPT_IDENTIFIER_LEN+2]=(unsigned char)format; + shift=mc_PutVarInt(buf+MC_DCT_SCRIPT_IDENTIFIER_LEN+3,11,chunk_count); + + err=SetData(buf,MC_DCT_SCRIPT_IDENTIFIER_LEN+3+shift); if(err) { return err; } - err=SetData(hashes,t*MC_DCT_SCRIPT_CHUNK_HASH_SIZE); + return MC_ERR_NOERROR; +} + +int mc_Script::SetChunkDefHash(const uint32_t format,unsigned char *hash,int size) +{ + int err,shift; + unsigned char buf[16]; + + shift=mc_PutVarInt(buf,16,size); + err=SetData(buf,shift); if(err) { return err; } + err=SetData(hash,MC_DCT_SCRIPT_CHUNK_HASH_SIZE); + if(err) + { + return err; + } - return MC_ERR_NOERROR; + return MC_ERR_NOERROR; } diff --git a/src/protocol/multichainscript.h b/src/protocol/multichainscript.h index dc1f1542..1377b44d 100644 --- a/src/protocol/multichainscript.h +++ b/src/protocol/multichainscript.h @@ -105,8 +105,9 @@ typedef struct mc_Script int GetDataFormat(uint32_t *format); int SetDataFormat(const uint32_t format); - int GetChunkDef(uint32_t *format,unsigned char** hashes,uint64_t *size,uint32_t *chunk_size_log); - int SetChunkDef(const uint32_t format,unsigned char* hashes,uint64_t size,uint32_t chunk_size_log); + int GetChunkDef(uint32_t *format,unsigned char** hashes,int *chunk_count,uint64_t *total_size); + int SetChunkDefHeader(const uint32_t format,int chunk_count); + int SetChunkDefHash(const uint32_t format,unsigned char *hash,int size); int ExtractAndDeleteDataFormat(uint32_t *format); int DeleteDuplicatesInRange(int from,int to); diff --git a/src/utils/utilwrapper.cpp b/src/utils/utilwrapper.cpp index ef8df17c..1fadfcf3 100644 --- a/src/utils/utilwrapper.cpp +++ b/src/utils/utilwrapper.cpp @@ -777,6 +777,8 @@ int mc_MultichainParams::SetGlobals() MAX_OP_RETURN_RELAY=(unsigned int)GetInt64Param("maxstdopreturnsize"); MAX_OP_RETURN_RELAY=GetArg("-datacarriersize", MAX_OP_RETURN_RELAY); MAX_BLOCK_SIZE=(unsigned int)GetInt64Param("maximumblocksize"); + MAX_CHUNK_SIZE=(unsigned int)GetInt64Param("maximumchunksize"); + MAX_CHUNK_COUNT=(unsigned int)GetInt64Param("maximumchunkcount"); DEFAULT_BLOCK_MAX_SIZE=MAX_BLOCK_SIZE; while(MAX_BLOCK_SIZE>MAX_BLOCKFILE_SIZE) { diff --git a/src/wallet/chunkdb.cpp b/src/wallet/chunkdb.cpp index 85becae5..e2e0ca32 100644 --- a/src/wallet/chunkdb.cpp +++ b/src/wallet/chunkdb.cpp @@ -189,8 +189,6 @@ int mc_ChunkDB::AddSubscription(mc_SubscriptionDBRow *subscription) } return m_Subscriptions->PutRow(subscription->m_SubscriptionID,subscription,(char*)subscription+m_ValueOffset); - -// return m_Subscriptions->Add(subscription,(char*)subscription+m_ValueOffset); } mc_SubscriptionDBRow *mc_ChunkDB::FindSubscription(const mc_TxEntity* entity) @@ -407,16 +405,28 @@ int mc_ChunkDB::Initialize(const char *name,uint32_t mode) } else { - m_Subscriptions->SetCount(m_DBStat.m_LastSubscription+1); + m_Subscriptions->SetCount(m_DBStat.m_LastSubscription+2); subscription.Zero(); AddSubscription(&subscription); + subscription.m_Entity.m_EntityType=MC_TET_AUTHOR; + subscription.m_SubscriptionID=1; + AddSubscription(&subscription); + + m_DBStat.m_LastSubscription=1; + err=m_DB->Write((char*)&m_DBStat+m_KeyOffset,m_KeySize,(char*)&m_DBStat+m_ValueOffset,m_ValueSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); if(err) { return err; } + err=m_DB->Write((char*)&subscription+m_KeyOffset,m_KeySize,(char*)&subscription+m_ValueOffset,m_ValueSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + if(err) + { + return err; + } + err=m_DB->Commit(MC_OPT_DB_DATABASE_TRANSACTIONAL); if(err) @@ -460,7 +470,7 @@ void mc_ChunkDB::Dump(const char *message) mc_ChunkDBRow dbrow; unsigned char *ptr; - int dbvalue_len,err; + int dbvalue_len,err,i; char ShortName[65]; char FileName[MC_DCT_DB_MAX_PATH]; FILE *fHan; @@ -474,9 +484,9 @@ void mc_ChunkDB::Dump(const char *message) return; } - mc_LogString(fHan,message); + mc_LogString(fHan,message); - fprintf(fHan,"Entities\n"); + fprintf(fHan,"\nDB\n"); dbrow.Zero(); ptr=(unsigned char*)m_DB->Read((char*)&dbrow+m_KeyOffset,m_KeySize,&dbvalue_len,MC_OPT_DB_DATABASE_SEEK_ON_READ,&err); if(err) @@ -497,36 +507,20 @@ void mc_ChunkDB::Dump(const char *message) } } } - -/* - for(i=0;iGetCount()) { - if(m_RawMemPools[i]) - { - if(m_RawMemPools[i]->GetCount()) - { - fprintf(fHan,"RawMemPool %d\n",m_Imports[i].m_ImportID); - mc_MemoryDumpCharSizeToFile(fHan,m_RawMemPools[i]->GetRow(0),0,m_RawMemPools[i]->GetCount()*m_Database->m_TotalSize,m_Database->m_TotalSize); - } - } - if(m_MemPools[i]) - { - if(m_MemPools[i]->GetCount()) - { - fprintf(fHan,"MemPool %d\n",m_Imports[i].m_ImportID); - mc_MemoryDumpCharSizeToFile(fHan,m_MemPools[i]->GetRow(0),0,m_MemPools[i]->GetCount()*m_Database->m_TotalSize,m_Database->m_TotalSize); - } - } + mc_MemoryDumpCharSizeToFile(fHan,m_MemPool->GetRow(0),0,m_MemPool->GetCount()*m_TotalSize,m_TotalSize); } - if(m_RawUpdatePool) + + fprintf(fHan,"\nSubscriptions\n"); + + for(i=0;iGetCount();i++) { - if(m_RawUpdatePool->GetCount()) - { - fprintf(fHan,"RawUpdatePool\n"); - mc_MemoryDumpCharSizeToFile(fHan,m_RawUpdatePool->GetRow(0),0,m_RawUpdatePool->GetCount()*m_Database->m_TotalSize,m_Database->m_TotalSize); - } + mc_MemoryDumpCharSizeToFile(fHan,m_Subscriptions->GetRow(i),0,m_TotalSize,m_TotalSize); } -*/ + fprintf(fHan,"\n<<<<<< \tChain height: %6d\t%s\n\n",mc_gState->m_Permissions->m_Block,message); fclose(fHan); } diff --git a/src/wallet/chunkdb.h b/src/wallet/chunkdb.h index ec8dd165..d44bc971 100644 --- a/src/wallet/chunkdb.h +++ b/src/wallet/chunkdb.h @@ -53,11 +53,11 @@ typedef struct mc_SubscriptionDBRow uint32_t m_Flags; // Flags passed from higher level int32_t m_SubscriptionID; // Subscription ID uint32_t m_Count; // Total chunk count + uint32_t m_TmpFlags; int32_t m_FirstFileID; // First data file ID/Reserved uint32_t m_FirstFileOffset; // First data file offset/Reserved int32_t m_LastFileID; // Last data file ID uint32_t m_LastFileSize; // Last data file size - uint32_t m_TmpFlags; uint64_t m_FullSize; // Total tx size char m_DirName[MC_DCT_DB_MAX_PATH]; // Full file name @@ -73,13 +73,14 @@ typedef struct mc_ChunkDBStat uint32_t m_Zero2; // Should be Zero mc_TxEntity m_ZeroEntity; // Zero Entity uint32_t m_InitMode; - uint32_t m_ChunkDBVersion; int32_t m_LastSubscription; - uint32_t m_Reserved1; uint32_t m_Count; // Total tx count - uint64_t m_FullSize; // Total tx size + uint32_t m_ChunkDBVersion; + uint32_t m_Reserved1; uint32_t m_Reserved2; uint32_t m_Reserved3; + uint32_t m_Reserved4; + uint64_t m_FullSize; // Total tx size void Zero(); } mc_ChunkDBStat; diff --git a/src/wallet/wallettxs.cpp b/src/wallet/wallettxs.cpp index 855b05c9..bda835b0 100644 --- a/src/wallet/wallettxs.cpp +++ b/src/wallet/wallettxs.cpp @@ -478,6 +478,13 @@ int mc_WalletTxs::Commit(mc_TxImport *import) { imp=import; } + if(mc_gState->m_Features->Chunks()) + { + if(imp->m_ImportID == 0) + { + err=m_ChunkDB->Commit(imp->m_Block+1); + } + } m_Database->Lock(1,0); if(err == MC_ERR_NOERROR) From 9f94bc8be36260aa675002172aae6caf2e9a67fb Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 22 Feb 2018 17:12:54 +0200 Subject: [PATCH 013/157] Off-chain items. Publishing using storechunk and rawdata. --- src/entities/asset.h | 10 +- src/protocol/multichainscript.cpp | 80 +++++++++++++- src/protocol/multichainscript.h | 3 +- src/protocol/multichaintx.cpp | 13 +++ src/rpc/rpclist.cpp | 1 + src/rpc/rpcrawdata.cpp | 174 ++++++++++++++++++++++-------- src/rpc/rpcserver.h | 1 + src/rpc/rpcstreams.cpp | 2 +- src/rpc/rpcutils.cpp | 2 +- src/rpc/rpcutils.h | 9 +- src/rpc/rpcwallet.cpp | 68 ++++++++++++ src/rpc/rpcwallet.h | 1 + src/rpc/rpcwalletutils.cpp | 52 +++++++++ src/wallet/chunkdb.cpp | 118 ++++++++++++++++---- src/wallet/chunkdb.h | 1 + src/wallet/wallettxs.cpp | 55 +++++++++- src/wallet/wallettxs.h | 3 +- 17 files changed, 520 insertions(+), 73 deletions(-) diff --git a/src/entities/asset.h b/src/entities/asset.h index a0b2c1e1..79518edb 100644 --- a/src/entities/asset.h +++ b/src/entities/asset.h @@ -63,10 +63,12 @@ #define MC_ENT_SPRM_UPGRADE_CHAIN_PARAMS 0x44 #define MC_ENT_SPRM_TIMESTAMP 0x81 -#define MC_ENT_SPRM_SOURCE_TXID 0x82 -#define MC_ENT_SPRM_SOURCE_VOUT 0x83 -#define MC_ENT_SPRM_CHUNK_DETAILS 0x84 -#define MC_ENT_SPRM_CHUNK_DATA 0x85 +#define MC_ENT_SPRM_CHUNK_HASH 0x82 +#define MC_ENT_SPRM_SOURCE_TXID 0x83 +#define MC_ENT_SPRM_SOURCE_VOUT 0x84 +#define MC_ENT_SPRM_CHUNK_SIZE 0x85 +#define MC_ENT_SPRM_CHUNK_DETAILS 0x86 +#define MC_ENT_SPRM_CHUNK_DATA 0x87 #define MC_ENT_FLAG_OFFSET_IS_SET 0x00000001 diff --git a/src/protocol/multichainscript.cpp b/src/protocol/multichainscript.cpp index e3fae82f..2cd2d830 100644 --- a/src/protocol/multichainscript.cpp +++ b/src/protocol/multichainscript.cpp @@ -2490,7 +2490,7 @@ int mc_Script::SetChunkDefHeader(const uint32_t format,int chunk_count) return MC_ERR_NOERROR; } -int mc_Script::SetChunkDefHash(const uint32_t format,unsigned char *hash,int size) +int mc_Script::SetChunkDefHash(unsigned char *hash,int size) { int err,shift; unsigned char buf[16]; @@ -2514,6 +2514,8 @@ int mc_Script::SetChunkDefHash(const uint32_t format,unsigned char *hash,int siz int mc_Script::ExtractAndDeleteDataFormat(uint32_t *format) { + return ExtractAndDeleteDataFormat(format,NULL,NULL,NULL); +/* int elem,err; if(format) @@ -2544,6 +2546,82 @@ int mc_Script::ExtractAndDeleteDataFormat(uint32_t *format) } return MC_ERR_NOERROR; + */ +} + +int mc_Script::ExtractAndDeleteDataFormat(uint32_t *format,unsigned char** hashes,int *chunk_count,uint64_t *total_size) +{ + int elem,err; + + if(format) + { + *format=MC_SCR_DATA_FORMAT_UNKNOWN; + } + + if(hashes) + { + *hashes=NULL; + } + + if(mc_gState->m_Features->FormattedData() == 0) + { + return MC_ERR_NOERROR; + } + + if(m_NumElements < 2) + { + return MC_ERR_NOERROR; + } + + + elem=m_NumElements-2; + + SetElement(elem); + if( (err=GetDataFormat(format)) != MC_ERR_WRONG_SCRIPT ) + { + if( (mc_gState->m_Features->OffChainData() == 0) || (GetChunkDef(NULL,NULL,NULL,NULL) == MC_ERR_WRONG_SCRIPT) ) + { + DeleteElement(elem); + } + if(err) + { + return err; + } + } + + if(mc_gState->m_Features->OffChainData() == 0) + { + return MC_ERR_NOERROR; + } + + if(m_NumElements < 2) + { + return MC_ERR_NOERROR; + } + + elem=m_NumElements-2; + + SetElement(elem); + while( (elem >= 0 ) && ((err=GetChunkDef(format,hashes,chunk_count,total_size)) == MC_ERR_NOERROR) ) + { + DeleteElement(elem); + elem--; + if(elem < 0) + { + err=MC_ERR_WRONG_SCRIPT; + } + else + { + SetElement(elem); + } + } + + if(err != MC_ERR_WRONG_SCRIPT) + { + return err; + } + + return MC_ERR_NOERROR; } int mc_Script::DeleteDuplicatesInRange(int from,int to) diff --git a/src/protocol/multichainscript.h b/src/protocol/multichainscript.h index 1377b44d..8460334f 100644 --- a/src/protocol/multichainscript.h +++ b/src/protocol/multichainscript.h @@ -107,9 +107,10 @@ typedef struct mc_Script int GetChunkDef(uint32_t *format,unsigned char** hashes,int *chunk_count,uint64_t *total_size); int SetChunkDefHeader(const uint32_t format,int chunk_count); - int SetChunkDefHash(const uint32_t format,unsigned char *hash,int size); + int SetChunkDefHash(unsigned char *hash,int size); int ExtractAndDeleteDataFormat(uint32_t *format); + int ExtractAndDeleteDataFormat(uint32_t *format,unsigned char** hashes,int *chunk_count,uint64_t *total_size); int DeleteDuplicatesInRange(int from,int to); diff --git a/src/protocol/multichaintx.cpp b/src/protocol/multichaintx.cpp index 54997fd7..17b714b9 100644 --- a/src/protocol/multichaintx.cpp +++ b/src/protocol/multichaintx.cpp @@ -561,6 +561,13 @@ void MultiChainTransaction_FillAdminPermissionsBeforeTx(const CTransaction& tx, bool MultiChainTransaction_VerifyAndDeleteDataFormatElements(string& reason) { + if(mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(NULL)) + { + reason="Error in data format script"; + return false; + } + +/* int elem,err; mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(NULL); // Format scripts are eliminated for protocol checks @@ -577,6 +584,7 @@ bool MultiChainTransaction_VerifyAndDeleteDataFormatElements(string& reason) elem=mc_gState->m_TmpScript->m_NumElements-2; + mc_gState->m_TmpScript->SetElement(elem); while( (elem >= 0 ) && ((err=mc_gState->m_TmpScript->GetChunkDef(NULL,NULL,NULL,NULL)) == MC_ERR_NOERROR) ) { mc_gState->m_TmpScript->DeleteElement(elem); @@ -585,6 +593,10 @@ bool MultiChainTransaction_VerifyAndDeleteDataFormatElements(string& reason) { err=MC_ERR_WRONG_SCRIPT; } + else + { + mc_gState->m_TmpScript->SetElement(elem); + } } if(err != MC_ERR_WRONG_SCRIPT) @@ -592,6 +604,7 @@ bool MultiChainTransaction_VerifyAndDeleteDataFormatElements(string& reason) reason="Error in data format script"; return false; } + */ return true; } diff --git a/src/rpc/rpclist.cpp b/src/rpc/rpclist.cpp index 7aadf45c..0c97698f 100644 --- a/src/rpc/rpclist.cpp +++ b/src/rpc/rpclist.cpp @@ -241,6 +241,7 @@ static const CRPCCommand vRPCCommands[] = { "wallet", "liststreamblockitems", &liststreamblockitems, false, false, false }, { "wallet", "getstreamkeysummary", &getstreamkeysummary, false, false, true }, { "wallet", "getstreampublishersummary", &getstreampublishersummary, false, false, true }, + { "wallet", "storechunk", &storechunk, false, false, true }, /* MCHN END */ { "wallet", "setaccount", &setaccount, true, false, true }, diff --git a/src/rpc/rpcrawdata.cpp b/src/rpc/rpcrawdata.cpp index 36be5970..d7619507 100644 --- a/src/rpc/rpcrawdata.cpp +++ b/src/rpc/rpcrawdata.cpp @@ -191,8 +191,12 @@ CScript RawDataScriptRawHex(Value *param,int *errorCode,string *strError) return scriptOpReturn; } -vector ParseRawFormattedData(const Value *value,uint32_t *data_format,mc_Script *lpDetailsScript,bool allow_formatted,int *errorCode,string *strError) +vector ParseRawFormattedData(const Value *value,uint32_t *data_format,mc_Script *lpDetailsScript,uint32_t in_options,uint32_t* out_options,int *errorCode,string *strError) { + if(out_options) + { + *out_options=MC_RFD_OPTION_NONE; + } vector vValue; if(value->type() == str_type) { @@ -210,7 +214,9 @@ vector ParseRawFormattedData(const Value *value,uint32_t *data_fo } else { - if(allow_formatted || (mc_gState->m_Features->FormattedData() != 0) ) + if( (in_options & MC_RFD_OPTION_INLINE) || + (mc_gState->m_Features->FormattedData() != 0) || + (mc_gState->m_Features->OffChainData() != 0) ) { if(value->type() == obj_type) { @@ -222,18 +228,6 @@ vector ParseRawFormattedData(const Value *value,uint32_t *data_fo { BOOST_FOREACH(const Pair& d, value->get_obj()) { - /* - if(d.name_ == "raw") - { - bool fIsHex; - vValue=ParseHex(d.value_.get_str().c_str(),fIsHex); - if(!fIsHex) - { - *strError=string("value in data object should be hexadecimal string"); - } - *data_format=MC_SCR_DATA_FORMAT_RAW; - } - */ if(d.name_ == "text") { if(d.value_.type() == str_type) @@ -261,10 +255,68 @@ vector ParseRawFormattedData(const Value *value,uint32_t *data_fo vValue=vector (script,script+bytes); *data_format=MC_SCR_DATA_FORMAT_UBJSON; } - if(*data_format == MC_SCR_DATA_FORMAT_UNKNOWN) + if(d.name_ == "chunks") + { + if(mc_gState->m_Features->OffChainData()) + { + if(d.value_.type() == array_type) + { + Array arr=d.value_.get_array(); + for(int i=0;i<(int)arr.size();i++) + { + if(strError->size() == 0) + { + if(arr[i].type() == str_type) + { + vector vHash; + bool fIsHex; + vHash=ParseHex(arr[i].get_str().c_str(),fIsHex); + if(!fIsHex) + { + *strError=string("Chunk hash should be hexadecimal string"); + } + else + { + if(vHash.size() != MC_CDB_CHUNK_HASH_SIZE) + { + *strError=strprintf("Chunk hash should be %d bytes long",MC_CDB_CHUNK_HASH_SIZE); + } + else + { + uint256 hash; + hash.SetHex(arr[i].get_str()); + + vValue.insert(vValue.end(),(unsigned char*)&hash,(unsigned char*)&hash+MC_CDB_CHUNK_HASH_SIZE); + } + } + } + } + } + } + else + { + *strError=string("value in data object should be array"); + } + + if(out_options) + { + *out_options |= MC_RFD_OPTION_OFFCHAIN; + } + *data_format=MC_SCR_DATA_FORMAT_UNKNOWN; + } + else + { + *errorCode=RPC_NOT_SUPPORTED; + *strError="Unsupported item data type: " + d.name_; + } + } + else { - throw JSONRPCError(RPC_NOT_SUPPORTED, "Unsupported item data type: " + d.name_); - } + if(*data_format == MC_SCR_DATA_FORMAT_UNKNOWN) + { + throw JSONRPCError(RPC_NOT_SUPPORTED, "Unsupported item data type: " + d.name_); + } + } } } } @@ -336,25 +388,13 @@ CScript RawDataScriptFormatted(Value *param,uint32_t *data_format,mc_Script *lpD BOOST_FOREACH(const Pair& d, param->get_obj()) { field_parsed=false; -/* - if(d.name_ == "data") - { - if(!missing_data) - { - *strError=string("data field can appear only once in the object"); - } - vValue=ParseRawFormattedData(&(d.value_),data_format,lpDetailsScript,errorCode,strError); - field_parsed=true; - missing_data=false; - } - */ if( (d.name_ == "text") || (d.name_ == "json") ) { if(!missing_data) { *strError=string("data object should have single key - json or text"); } - vValue=ParseRawFormattedData(param,data_format,lpDetailsScript,false,errorCode,strError); + vValue=ParseRawFormattedData(param,data_format,lpDetailsScript,MC_RFD_OPTION_NONE,NULL,errorCode,strError); field_parsed=true; missing_data=false; } @@ -948,7 +988,8 @@ CScript RawDataScriptCreateUpgrade(Value *param,mc_Script *lpDetails,mc_Script * return scriptOpReturn; } -CScript RawDataScriptPublish(Value *param,mc_EntityDetails *entity,uint32_t *data_format,mc_Script *lpDetailsScript,int *errorCode,string *strError) + +CScript RawDataScriptPublish(Value *param,mc_EntityDetails *entity,uint32_t *data_format,mc_Script *lpDetailsScript,vector* vChunkHashes,int *errorCode,string *strError) { CScript scriptOpReturn=CScript(); vector vValue; @@ -959,6 +1000,9 @@ CScript RawDataScriptPublish(Value *param,mc_EntityDetails *entity,uint32_t *dat bool field_parsed; bool missing_data=true; bool missing_key=true; + uint32_t in_options,out_options; + in_options=MC_RFD_OPTION_NONE; + out_options=MC_RFD_OPTION_NONE; vKeys.clear(); BOOST_FOREACH(const Pair& d, param->get_obj()) { @@ -1013,10 +1057,36 @@ CScript RawDataScriptPublish(Value *param,mc_EntityDetails *entity,uint32_t *dat { *strError=string("data field can appear only once in the object"); } - vValue=ParseRawFormattedData(&(d.value_),data_format,lpDetailsScript,false,errorCode,strError); + vValue=ParseRawFormattedData(&(d.value_),data_format,lpDetailsScript,MC_RFD_OPTION_NONE,&out_options,errorCode,strError); field_parsed=true; missing_data=false; } + if(d.name_ == "options") + { + if( mc_gState->m_Features->OffChainData() == 0 ) + { + *errorCode=RPC_NOT_SUPPORTED; + *strError=string("Format options are not supported by this protocol version"); + goto exitlbl; + } + if(d.value_.type() != null_type && (d.value_.type()==str_type)) + { + if(d.value_.get_str() == "offchain") + { + in_options |= MC_RFD_OPTION_OFFCHAIN; + } + else + { + *strError=string("Invalid options"); + } + } + else + { + *strError=string("Invalid options"); + } + field_parsed=true; + missing_key=false; + } if(d.name_ == "for")field_parsed=true; // if(d.name_ == "format")field_parsed=true; if(!field_parsed) @@ -1070,21 +1140,40 @@ CScript RawDataScriptPublish(Value *param,mc_EntityDetails *entity,uint32_t *dat } } - if(*data_format != MC_SCR_DATA_FORMAT_UNKNOWN) + if(in_options & MC_RFD_OPTION_OFFCHAIN) { - lpDetailsScript->Clear(); - lpDetailsScript->SetDataFormat(*data_format); + AppendOffChainFormatData(*data_format,out_options,lpDetailsScript,vValue,vChunkHashes,errorCode,strError); + if(strError->size()) + { + goto exitlbl; + } script = lpDetailsScript->GetData(0,&bytes); scriptOpReturn << vector(script, script + bytes) << OP_DROP; - } - - if(vValue.size()) - { - scriptOpReturn << OP_RETURN << vValue; + scriptOpReturn << OP_RETURN; } else { - scriptOpReturn << OP_RETURN; + if(out_options & MC_RFD_OPTION_OFFCHAIN) + { + *strError=string("chunks data type is not allowed with missing options field"); + goto exitlbl; + } + if(*data_format != MC_SCR_DATA_FORMAT_UNKNOWN) + { + lpDetailsScript->Clear(); + lpDetailsScript->SetDataFormat(*data_format); + script = lpDetailsScript->GetData(0,&bytes); + scriptOpReturn << vector(script, script + bytes) << OP_DROP; + } + + if(vValue.size()) + { + scriptOpReturn << OP_RETURN << vValue; + } + else + { + scriptOpReturn << OP_RETURN; + } } } @@ -1270,6 +1359,7 @@ CScript RawDataScriptInputCache(Value *param,mc_Script *lpDetails,int *errorCode CScript ParseRawMetadata(Value param,uint32_t allowed_objects,mc_EntityDetails *given_entity,mc_EntityDetails *found_entity) { + vector vChunkHashes; string strError=""; int errorCode=RPC_INVALID_PARAMETER; uint32_t data_format; @@ -1325,7 +1415,7 @@ CScript ParseRawMetadata(Value param,uint32_t allowed_objects,mc_EntityDetails * scriptOpReturn=RawDataScriptCreateStream(¶m,lpDetails,lpDetailsScript,&errorCode,&strError); break; case MC_DATA_API_PARAM_TYPE_PUBLISH: - scriptOpReturn=RawDataScriptPublish(¶m,&entity,&data_format,lpDetailsScript,&errorCode,&strError); + scriptOpReturn=RawDataScriptPublish(¶m,&entity,&data_format,lpDetailsScript,&vChunkHashes,&errorCode,&strError); break; case MC_DATA_API_PARAM_TYPE_CREATE_UPGRADE: scriptOpReturn=RawDataScriptCreateUpgrade(¶m,lpDetails,lpDetailsScript,&errorCode,&strError); diff --git a/src/rpc/rpcserver.h b/src/rpc/rpcserver.h index a0401fbc..44aa2c00 100644 --- a/src/rpc/rpcserver.h +++ b/src/rpc/rpcserver.h @@ -262,6 +262,7 @@ extern json_spirit::Value listblocks(const json_spirit::Array& params, bool fHel extern json_spirit::Value liststreamblockitems(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value getstreamkeysummary(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value getstreampublishersummary(const json_spirit::Array& params, bool fHelp); +extern json_spirit::Value storechunk(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value purehelpitem(const json_spirit::Array& params, bool fHelp); /* MCHN END */ diff --git a/src/rpc/rpcstreams.cpp b/src/rpc/rpcstreams.cpp index f001de36..09b85cc9 100644 --- a/src/rpc/rpcstreams.cpp +++ b/src/rpc/rpcstreams.cpp @@ -578,7 +578,7 @@ Value publishfrom(const Array& params, bool fHelp) string strError; int errorCode=RPC_INVALID_PARAMETER; - dataData=ParseRawFormattedData(&(params[3]),&data_format,lpDetailsScript,false,&errorCode,&strError); + dataData=ParseRawFormattedData(&(params[3]),&data_format,lpDetailsScript,MC_RFD_OPTION_NONE,NULL,&errorCode,&strError); if(strError.size()) { diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index d36d7f85..498d1f25 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -1572,7 +1572,7 @@ string ParseRawOutputObject(Value param,CAmount& nAmount,mc_Script *lpScript, in mc_gState->m_TmpScript->Clear(); - vector vData=ParseRawFormattedData(&(arr[i]),&data_format,mc_gState->m_TmpScript,true,&errorCode,&strError); + vector vData=ParseRawFormattedData(&(arr[i]),&data_format,mc_gState->m_TmpScript,MC_RFD_OPTION_INLINE,NULL,&errorCode,&strError); if(strError.size()) { if(eErrorCode) diff --git a/src/rpc/rpcutils.h b/src/rpc/rpcutils.h index 50d72971..b5faa09d 100644 --- a/src/rpc/rpcutils.h +++ b/src/rpc/rpcutils.h @@ -19,7 +19,7 @@ #include "json/json_spirit_value.h" #include "multichain/multichain.h" #include "utils/utilparse.h" - +#include "wallet/chunkdb.h" using namespace std; using namespace json_spirit; @@ -57,6 +57,10 @@ using namespace json_spirit; #define MC_VMM_TAKE_FIRST_FOR_FIELD 0x00000010 #define MC_VMM_OMIT_NULL 0x00000020 +#define MC_RFD_OPTION_NONE 0x00000000 +#define MC_RFD_OPTION_INLINE 0x00000001 +#define MC_RFD_OPTION_OFFCHAIN 0x00000002 + // codes for allowed_objects fields // 0x0001 - create @@ -108,11 +112,12 @@ void ParseRawAction(string action,bool& lock_it, bool& sign_it,bool& send_it); bool paramtobool(Value param); int paramtoint(Value param,bool check_for_min,int min_value,string error_message); vector ParseBlockSetIdentifier(Value blockset_identifier); -vector ParseRawFormattedData(const Value *value,uint32_t *data_format,mc_Script *lpDetailsScript,bool allow_formatted,int *errorCode,string *strError); +vector ParseRawFormattedData(const Value *value,uint32_t *data_format,mc_Script *lpDetailsScript,uint32_t in_options,uint32_t *out_options,int *errorCode,string *strError); void ParseRawDetails(const Value *value,mc_Script *lpDetails,mc_Script *lpDetailsScript,int *errorCode,string *strError); bool mc_IsJsonObjectForMerge(const Value *value,int level); Value mc_MergeValues(const Value *value1,const Value *value2,uint32_t mode,int level,int *error); Value mc_ExtractDetailsJSONObject(const unsigned char *script,uint32_t total); +void AppendOffChainFormatData(uint32_t data_format,uint32_t out_options,mc_Script *lpDetailsScript,vector& vValue,vector* vChunkHashes,int *errorCode,string *strError); #endif /* RPCMULTICHAINUTILS_H */ diff --git a/src/rpc/rpcwallet.cpp b/src/rpc/rpcwallet.cpp index aefa80c7..1d7ca853 100644 --- a/src/rpc/rpcwallet.cpp +++ b/src/rpc/rpcwallet.cpp @@ -514,7 +514,75 @@ Value listaddresses(const Array& params, bool fHelp) +Value storechunk(const Array& params, bool fHelp) +{ + int err; + + if (fHelp || params.size() != 1) + throw runtime_error("Help message not found\n"); + + if((mc_gState->m_WalletMode & MC_WMD_TXS) == 0) + { + throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported with this wallet version. To get this functionality, run \"multichaind -walletdbversion=2 -rescan\" "); + } + + vector vValue; + if(params[0].type() != str_type) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "data should be hexadecimal string"); + } + + bool fIsHex; + vValue=ParseHex(params[0].get_str().c_str(),fIsHex); + if(!fIsHex) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "data should be hexadecimal string"); + } + + if(vValue.size() == 0) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "data should be non-empty hexadecimal string"); + } + if((int)vValue.size() > MAX_CHUNK_SIZE) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "data is too long"); + } + + uint256 hash; + mc_SHA256 *hasher; + + hasher=new mc_SHA256; + hasher->Write(&vValue[0],(int)vValue.size()); + hasher->GetHash((unsigned char*)&hash); + hasher->Reset(); + hasher->Write((unsigned char*)&hash,32); + hasher->GetHash((unsigned char*)&hash); + + delete hasher; + + mc_TxEntity entity; + entity.Zero(); + entity.m_EntityType=MC_TET_AUTHOR; + + err=pwalletTxsMain->m_ChunkDB->AddChunk((unsigned char*)&hash,&entity,NULL,-1,(unsigned char*)&vValue[0],NULL,(int)vValue.size(),0,0); + + if(err) + { + switch(err) + { + case MC_ERR_FOUND: + break; + default: + throw JSONRPCError(RPC_INVALID_PARAMETER, "Internal error: couldn't store chunk"); + break; + } + } + + pwalletTxsMain->m_ChunkDB->Dump("storechunk"); + + return hash.GetHex(); +} diff --git a/src/rpc/rpcwallet.h b/src/rpc/rpcwallet.h index 615643eb..adf4cf49 100644 --- a/src/rpc/rpcwallet.h +++ b/src/rpc/rpcwallet.h @@ -20,6 +20,7 @@ #include "multichain/multichain.h" #include "wallet/wallettxs.h" +#include "wallet/chunkdb.h" #include "rpc/rpcutils.h" void SendMoneyToSeveralAddresses(const std::vector addresses, CAmount nValue, CWalletTx& wtxNew,mc_Script *dropscript,CScript scriptOpReturn,const std::vector& fromaddresses); diff --git a/src/rpc/rpcwalletutils.cpp b/src/rpc/rpcwalletutils.cpp index bbef8f11..e5faa9c4 100644 --- a/src/rpc/rpcwalletutils.cpp +++ b/src/rpc/rpcwalletutils.cpp @@ -732,3 +732,55 @@ Object TxOutEntry(const CTxOut& TxOutIn,int vout,const CTxIn& TxIn,uint256 hash, return txout_entry; } +void AppendOffChainFormatData(uint32_t data_format, + uint32_t out_options, + mc_Script *lpDetailsScript, + vector& vValue, + vector* vChunkHashes, + int *errorCode, + string *strError) +{ + if((mc_gState->m_WalletMode & MC_WMD_TXS) == 0) + { + *strError="Offchain data is not supported with this wallet version. To get this functionality, run \"multichaind -walletdbversion=2 -rescan\""; + *errorCode=RPC_NOT_SUPPORTED; + return; + } + mc_ChunkDBRow chunk_def; + lpDetailsScript->Clear(); + + int chunk_count; + int err; + mc_TxEntity entity; + entity.Zero(); + entity.m_EntityType=MC_TET_AUTHOR; + + if(out_options & MC_RFD_OPTION_OFFCHAIN) + { + chunk_count=(int)vValue.size()/MC_CDB_CHUNK_HASH_SIZE; + if(chunk_count > MAX_CHUNK_COUNT) + { + *strError="Too many chunks in the script"; + return; + } + + lpDetailsScript->SetChunkDefHeader(data_format,chunk_count); + for(int i=0;im_ChunkDB->GetChunkDef(&chunk_def,(unsigned char*)&vValue[i*MC_CDB_CHUNK_HASH_SIZE],&entity,NULL,-1); + if(err) + { + *strError="Chunk not found in this wallet"; + return; + } + lpDetailsScript->SetChunkDefHash((unsigned char*)&vValue[i*MC_CDB_CHUNK_HASH_SIZE],chunk_def.m_Size); + } + } + else + { + + } + +// err=pwalletTxsMain->m_ChunkDB->AddChunk((unsigned char*)&hash,&entity,NULL,-1,(unsigned char*)&vValue[0],NULL,(int)vValue.size(),0,0); +} + diff --git a/src/wallet/chunkdb.cpp b/src/wallet/chunkdb.cpp index e2e0ca32..c1f8786b 100644 --- a/src/wallet/chunkdb.cpp +++ b/src/wallet/chunkdb.cpp @@ -6,6 +6,7 @@ #define MC_CDB_TMP_FLAG_SHOULD_COMMIT 0x00000001 +unsigned char null_txid[MC_TDB_TXID_SIZE]; void sprintf_hex(char *hex,const unsigned char *bin,int size); /* @@ -44,6 +45,18 @@ void mc_ChunkDBRow::Zero() memset(this,0,sizeof(mc_ChunkDBRow)); } +void mc_ChunkDBRow::SwapPosBytes() +{ + unsigned char *ptr=(unsigned char *)&m_Pos; + unsigned char t; + t=ptr[0]; + ptr[0]=ptr[3]; + ptr[3]=t; + t=ptr[1]; + ptr[1]=ptr[2]; + ptr[2]=t; +} + void mc_ChunkDB::LogString(const char *message) { FILE *fHan; @@ -161,7 +174,7 @@ int mc_ChunkDB::AddSubscription(mc_SubscriptionDBRow *subscription) sprintf(dir_name,"chunks/data/stream-%s",enthex); break; case MC_TET_AUTHOR: - sprintf(dir_name,"chunks/data/author"); + sprintf(dir_name,"chunks/data/publisher"); break; case MC_TET_NONE: break; @@ -435,6 +448,7 @@ int mc_ChunkDB::Initialize(const char *name,uint32_t mode) } } + m_DBStat.m_InitMode &= MC_WMD_MODE_MASK; m_DBStat.m_InitMode |= mode; m_MemPool=new mc_Buffer; // Key - entity with m_Pos set to 0 + txid @@ -446,6 +460,8 @@ int mc_ChunkDB::Initialize(const char *name,uint32_t mode) m_TmpScript=new mc_Script; m_TmpScript->Clear(); + memset(null_txid,0,MC_TDB_TXID_SIZE); + m_Semaphore=__US_SemCreate(); if(m_Semaphore == NULL) { @@ -553,7 +569,7 @@ int mc_ScriptMatchesTxIDAndVOut(unsigned char *ptr,size_t bytes,const unsigned c } } - return MC_ERR_NOERROR; + return MC_ERR_NOT_FOUND; } int mc_ChunkDB::GetChunkDefInternal( @@ -581,7 +597,12 @@ int mc_ChunkDB::GetChunkDefInternal( { return MC_ERR_NOT_FOUND; } - + + if(mempool_row) + { + *mempool_row=-1; + } + memcpy(chunk_def->m_Hash,hash,MC_CDB_CHUNK_HASH_SIZE); chunk_def->m_SubscriptionID=subscription->m_SubscriptionID; chunk_def->m_Pos=0; @@ -655,7 +676,9 @@ int mc_ChunkDB::GetChunkDefInternal( } else { + chunk_def->SwapPosBytes(); ptr=(unsigned char*)m_DB->Read((char*)chunk_def+m_KeyOffset,m_KeySize,&value_len,0,&err); + chunk_def->SwapPosBytes(); if(err) { return err; @@ -678,7 +701,9 @@ int mc_ChunkDB::GetChunkDefInternal( if(chunk_def->m_Pos < total_items) { + chunk_def->SwapPosBytes(); ptr=(unsigned char*)m_DB->Read((char*)chunk_def+m_KeyOffset,m_KeySize,&value_len,0,&err); + chunk_def->SwapPosBytes(); if(err) { return err; @@ -694,6 +719,10 @@ int mc_ChunkDB::GetChunkDefInternal( chunk_def->m_Pos=total_items; } } + else + { + ptr=NULL; + } } err=MC_ERR_NOT_FOUND; } @@ -706,7 +735,9 @@ int mc_ChunkDB::GetChunkDefInternal( while(chunk_def->m_NextSubscriptionID) { chunk_def->m_SubscriptionID=chunk_def->m_NextSubscriptionID; + chunk_def->SwapPosBytes(); ptr=(unsigned char*)m_DB->Read((char*)chunk_def+m_KeyOffset,m_KeySize,&value_len,0,&err); + chunk_def->SwapPosBytes(); if(err) { return err; @@ -845,13 +876,16 @@ int mc_ChunkDB::AddChunkInternal( if(err == MC_ERR_NOT_FOUND) { add_null_row=1; - } - if(err != MC_ERR_NOERROR) - { - return err; + } + else + { + if(err != MC_ERR_NOERROR) + { + return err; + } } } - + chunk_def.Zero(); memcpy(chunk_def.m_Hash,hash,MC_CDB_CHUNK_HASH_SIZE); chunk_def.m_InternalFileID=-1; @@ -862,14 +896,21 @@ int mc_ChunkDB::AddChunkInternal( timestamp=mc_TimeNowAsUInt(); m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_TIMESTAMP,(unsigned char*)×tamp,sizeof(timestamp)); + m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_CHUNK_HASH,hash,MC_CDB_CHUNK_HASH_SIZE); if(txid) { - m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_SOURCE_TXID,hash,MC_CDB_CHUNK_HASH_SIZE); + m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_SOURCE_TXID,txid,MC_TDB_TXID_SIZE); if(vout >= 0) { - m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_SOURCE_TXID,(unsigned char*)&vout,sizeof(vout)); + m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_SOURCE_VOUT,(unsigned char*)&vout,sizeof(vout)); } } + else + { + m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_SOURCE_TXID,null_txid,MC_TDB_TXID_SIZE); + m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_SOURCE_VOUT,(unsigned char*)&vout,sizeof(vout)); + } + m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_CHUNK_SIZE,(unsigned char*)&chunk_size,sizeof(chunk_size)); if(details_size) { m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_CHUNK_DETAILS,details,details_size); @@ -902,7 +943,7 @@ int mc_ChunkDB::AddChunkInternal( chunk_def.m_NextSubscriptionID=subscription->m_SubscriptionID; m_MemPool->Add(&chunk_def,(char*)&chunk_def+m_ValueOffset); } - + chunk_def.m_SubscriptionID=subscription->m_SubscriptionID; chunk_def.m_Pos=total_items-on_disk_items; chunk_def.m_ItemCount=total_items+1; @@ -928,7 +969,6 @@ int mc_ChunkDB::AddChunkInternal( m_MemPool->Add(&chunk_def,(char*)&chunk_def+m_ValueOffset); - if(mempool_entity_row >= 0) { ((mc_ChunkDBRow *)m_MemPool->GetRow(mempool_entity_row))->m_ItemCount += 1; @@ -938,7 +978,7 @@ int mc_ChunkDB::AddChunkInternal( { if(add_entity_row) { - ((mc_ChunkDBRow *)m_MemPool->GetRow(mempool_entity_row))->m_NextSubscriptionID = subscription->m_SubscriptionID; + ((mc_ChunkDBRow *)m_MemPool->GetRow(mempool_last_null_row))->m_NextSubscriptionID = subscription->m_SubscriptionID; } } @@ -982,6 +1022,7 @@ unsigned char *mc_ChunkDB::GetChunkInternal(mc_ChunkDBRow *chunk_def, { unsigned char *ptr; size_t bytes_to_read; + int subscription_id; mc_SubscriptionDBRow *subscription; char FileName[MC_DCT_DB_MAX_PATH]; int FileHan; @@ -1010,11 +1051,15 @@ unsigned char *mc_ChunkDB::GetChunkInternal(mc_ChunkDBRow *chunk_def, } else { - subscription=(mc_SubscriptionDBRow *)m_Subscriptions->GetRow(chunk_def->m_SubscriptionID); + subscription_id=chunk_def->m_SubscriptionID; + if(subscription_id == 0) + { + subscription_id=chunk_def->m_NextSubscriptionID; + } + subscription=(mc_SubscriptionDBRow *)m_Subscriptions->GetRow(subscription_id); SetFileName(FileName,subscription,chunk_def->m_InternalFileID); FileHan=open(FileName,_O_BINARY | O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); - if(FileHan<=0) { return NULL; @@ -1053,7 +1098,7 @@ unsigned char *mc_ChunkDB::GetChunkInternal(mc_ChunkDBRow *chunk_def, { *bytes=bytes_to_read; } - } +} exitlbl: @@ -1090,7 +1135,6 @@ int mc_ChunkDB::AddToFile(const void* chunk, SetFileName(FileName,subscription,fileid); err=MC_ERR_NOERROR; - FileHan=open(FileName,_O_BINARY | O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); if(FileHan<=0) @@ -1140,10 +1184,15 @@ int mc_ChunkDB::CommitInternal(int block) int err; int last_file_id, last_file_offset; uint32_t size; + int value_len; + unsigned char *ptr; char msg[256]; mc_SubscriptionDBRow *subscription; mc_ChunkDBRow *chunk_def; + mc_ChunkDBRow entity_chunk_def; + + err=MC_ERR_NOERROR; Dump("Before Commit"); if(m_MemPool->GetCount() == 0) @@ -1189,7 +1238,6 @@ int mc_ChunkDB::CommitInternal(int block) chunk_def->m_InternalFileID=subscription->m_LastFileID; chunk_def->m_InternalFileOffset=subscription->m_LastFileSize; chunk_def->m_Pos+=chunk_def->m_TmpOnDiskItems; - chunk_def->m_TmpOnDiskItems=0; if(chunk_def->m_SubscriptionID == 0) { @@ -1209,11 +1257,43 @@ int mc_ChunkDB::CommitInternal(int block) chunk_def->m_InternalFileID=last_file_id; chunk_def->m_InternalFileOffset=last_file_offset; chunk_def->m_Pos+=chunk_def->m_TmpOnDiskItems; - chunk_def->m_TmpOnDiskItems=0; last_file_id=-1; } + + if(chunk_def->m_TmpOnDiskItems) + { + if(chunk_def->m_SubscriptionID) + { + entity_chunk_def.Zero(); + memcpy(&entity_chunk_def,chunk_def,sizeof(mc_ChunkDBRow)); + entity_chunk_def.m_Pos=0; + entity_chunk_def.m_TmpOnDiskItems=0; + entity_chunk_def.SwapPosBytes(); + ptr=(unsigned char*)m_DB->Read((char*)&entity_chunk_def+m_KeyOffset,m_KeySize,&value_len,0,&err); + entity_chunk_def.SwapPosBytes(); + if(err) + { + goto exitlbl; + } + if(ptr) + { + memcpy((char*)&entity_chunk_def+m_ValueOffset,ptr,m_ValueSize); + entity_chunk_def.m_ItemCount=chunk_def->m_ItemCount; + entity_chunk_def.SwapPosBytes(); + err=m_DB->Write((char*)&entity_chunk_def+m_KeyOffset,m_KeySize,(char*)&entity_chunk_def+m_ValueOffset,m_ValueSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + entity_chunk_def.SwapPosBytes(); + if(err) + { + goto exitlbl; + } + } + } + } + chunk_def->m_TmpOnDiskItems=0; + chunk_def->SwapPosBytes(); err=m_DB->Write((char*)chunk_def+m_KeyOffset,m_KeySize,(char*)chunk_def+m_ValueOffset,m_ValueSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + chunk_def->SwapPosBytes(); if(err) { goto exitlbl; diff --git a/src/wallet/chunkdb.h b/src/wallet/chunkdb.h index d44bc971..0a7897dc 100644 --- a/src/wallet/chunkdb.h +++ b/src/wallet/chunkdb.h @@ -104,6 +104,7 @@ typedef struct mc_ChunkDBRow int32_t m_NextSubscriptionID; // Next Subscription ID for this hash void Zero(); + void SwapPosBytes(); } mc_ChunkDBRow; diff --git a/src/wallet/wallettxs.cpp b/src/wallet/wallettxs.cpp index bda835b0..fa534439 100644 --- a/src/wallet/wallettxs.cpp +++ b/src/wallet/wallettxs.cpp @@ -209,6 +209,7 @@ void mc_WalletTxs::Zero() m_Database=NULL; m_ChunkDB=NULL; m_lpWallet=NULL; + m_ChunkBuffer=NULL; for(i=0;im_DBStat.m_Block,m_Database->m_DBStat.m_Block); } + + m_ChunkBuffer=(unsigned char*)mc_New(MAX_CHUNK_SIZE); return err; } @@ -304,6 +307,10 @@ int mc_WalletTxs::Destroy() delete m_ChunkDB; } + if(m_ChunkBuffer) + { + mc_Delete(m_ChunkBuffer); + } Zero(); return MC_ERR_NOERROR; @@ -2276,7 +2283,13 @@ int mc_WalletTxs::AddTx(mc_TxImport *import,const CWalletTx& tx,int block,CDiskT } else { - mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(NULL); + unsigned char *chunk_hashes; + unsigned char *chunk_found; + int chunk_count,chunk_err; + int chunk_size,chunk_shift; + size_t chunk_bytes; + + mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(NULL,&chunk_hashes,&chunk_count,NULL); if(mc_gState->m_TmpScript->GetNumElements() >= 3) // 2 OP_DROPs + OP_RETURN - item key { mc_gState->m_TmpScript->DeleteDuplicatesInRange(1,mc_gState->m_TmpScript->GetNumElements()-1); @@ -2291,6 +2304,46 @@ int mc_WalletTxs::AddTx(mc_TxImport *import,const CWalletTx& tx,int block,CDiskT entity.m_EntityType=MC_TET_STREAM | MC_TET_CHAINPOS; if(imp->FindEntity(&entity) >= 0) { + mc_ChunkDBRow chunk_def; + mc_TxEntity chunk_entity; + chunk_entity.Zero(); + memcpy(entity.m_EntityID,short_txid,MC_AST_SHORT_TXID_SIZE); + entity.m_EntityType=MC_TET_STREAM; + if(chunk_hashes) + { + for(int chunk=0;chunkGetChunkDef(&chunk_def,chunk_hashes,&entity,(unsigned char*)&hash,i) != MC_ERR_NOERROR) + { + if(m_ChunkDB->GetChunkDef(&chunk_def,chunk_hashes,NULL,NULL,-1) == MC_ERR_NOERROR) + { + chunk_found=m_ChunkDB->GetChunk(&chunk_def,0,-1,&chunk_bytes); + if(chunk_found) + { + memcpy(m_ChunkBuffer,chunk_found,chunk_size); + chunk_err=m_ChunkDB->AddChunk(chunk_hashes,&entity,(unsigned char*)&hash,i,m_ChunkBuffer,NULL,chunk_size,0,0); + if(chunk_err) + { + err=chunk_err; + goto exitlbl; + } + } + else + { + err=MC_ERR_CORRUPTED; + goto exitlbl; + } + } + else + { + // Feeding async chunk retriever here + } + } + chunk_hashes+=MC_CDB_CHUNK_HASH_SIZE; + } + } // if(imp->m_TmpEntities->Seek(&entity) < 0) { extension.Zero(); diff --git a/src/wallet/wallettxs.h b/src/wallet/wallettxs.h index 54f26f0f..edd1bd1a 100644 --- a/src/wallet/wallettxs.h +++ b/src/wallet/wallettxs.h @@ -54,7 +54,8 @@ typedef struct mc_WalletTxs std::map m_UnconfirmedSends; std::vector m_UnconfirmedSendsHashes; std::map vAvailableCoins; - + + unsigned char* m_ChunkBuffer; mc_WalletTxs() { Zero(); From aefbde4a27411758359af516a4be7259a197cd0d Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 25 Feb 2018 12:49:17 +0200 Subject: [PATCH 014/157] Fixed change calculation when issuing asset and fee --- src/wallet/walletcoins.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/wallet/walletcoins.cpp b/src/wallet/walletcoins.cpp index 0a3d8219..ac718714 100644 --- a/src/wallet/walletcoins.cpp +++ b/src/wallet/walletcoins.cpp @@ -1152,7 +1152,9 @@ bool CalculateChangeAmounts(CWallet *lpWallet, CTxOut txout; uint256 hash=out.GetHashAndTxOut(txout); out_i=out.i; - tmp_amounts->Clear(); + tmp_amounts->Clear(); + allowed=expected_required; + required=expected_required; if(ParseMultichainTxOutToBuffer(hash,txout,tmp_amounts,lpScript,&allowed,&required,mapSpecialEntity,strError)) { for(int i=0;iGetCount();i++) @@ -2328,7 +2330,7 @@ bool CreateAssetGroupingTransaction(CWallet *lpWallet, const vector Date: Mon, 26 Feb 2018 10:55:07 +0200 Subject: [PATCH 015/157] Publishing off-chain data with raw txs, without storechunk --- src/chainparams/state.h | 5 ++++- src/rpc/rpcrawdata.cpp | 4 ++-- src/rpc/rpcutils.h | 2 +- src/rpc/rpcwallet.cpp | 5 ++++- src/rpc/rpcwalletutils.cpp | 46 ++++++++++++++++++++++++++++++++++++-- src/utils/declare.h | 2 ++ src/utils/utilwrapper.cpp | 10 +++++++++ src/wallet/chunkdb.cpp | 10 ++++++++- 8 files changed, 76 insertions(+), 8 deletions(-) diff --git a/src/chainparams/state.h b/src/chainparams/state.h index 7fba20ee..a2c7003c 100644 --- a/src/chainparams/state.h +++ b/src/chainparams/state.h @@ -172,6 +172,7 @@ typedef struct mc_TmpBuffers mc_Buffer *m_RpcABNoMapBuffer1; mc_Buffer *m_RpcABNoMapBuffer2; mc_Buffer *m_RpcEntityRows; + mc_SHA256 *m_RpcHasher1; void Init() { @@ -189,7 +190,8 @@ typedef struct mc_TmpBuffers m_RpcABNoMapBuffer2=new mc_Buffer; mc_InitABufferDefault(m_RpcABNoMapBuffer2); m_RpcEntityRows=new mc_Buffer; - m_RpcEntityRows->Initialize(MC_TDB_ENTITY_KEY_SIZE,MC_TDB_ROW_SIZE,MC_BUF_MODE_DEFAULT); + m_RpcEntityRows->Initialize(MC_TDB_ENTITY_KEY_SIZE,MC_TDB_ROW_SIZE,MC_BUF_MODE_DEFAULT); + m_RpcHasher1=new mc_SHA256(); } void Destroy() @@ -204,6 +206,7 @@ typedef struct mc_TmpBuffers delete m_RpcABNoMapBuffer1; delete m_RpcABNoMapBuffer2; delete m_RpcEntityRows; + delete m_RpcHasher1; } } mc_TmpBuffers; diff --git a/src/rpc/rpcrawdata.cpp b/src/rpc/rpcrawdata.cpp index d7619507..b36be30e 100644 --- a/src/rpc/rpcrawdata.cpp +++ b/src/rpc/rpcrawdata.cpp @@ -989,7 +989,7 @@ CScript RawDataScriptCreateUpgrade(Value *param,mc_Script *lpDetails,mc_Script * } -CScript RawDataScriptPublish(Value *param,mc_EntityDetails *entity,uint32_t *data_format,mc_Script *lpDetailsScript,vector* vChunkHashes,int *errorCode,string *strError) +CScript RawDataScriptPublish(Value *param,mc_EntityDetails *entity,uint32_t *data_format,mc_Script *lpDetailsScript,vector* vChunkHashes,int *errorCode,string *strError) { CScript scriptOpReturn=CScript(); vector vValue; @@ -1359,7 +1359,7 @@ CScript RawDataScriptInputCache(Value *param,mc_Script *lpDetails,int *errorCode CScript ParseRawMetadata(Value param,uint32_t allowed_objects,mc_EntityDetails *given_entity,mc_EntityDetails *found_entity) { - vector vChunkHashes; + vector vChunkHashes; string strError=""; int errorCode=RPC_INVALID_PARAMETER; uint32_t data_format; diff --git a/src/rpc/rpcutils.h b/src/rpc/rpcutils.h index b5faa09d..a30519b7 100644 --- a/src/rpc/rpcutils.h +++ b/src/rpc/rpcutils.h @@ -117,7 +117,7 @@ void ParseRawDetails(const Value *value,mc_Script *lpDetails,mc_Script *lpDetail bool mc_IsJsonObjectForMerge(const Value *value,int level); Value mc_MergeValues(const Value *value1,const Value *value2,uint32_t mode,int level,int *error); Value mc_ExtractDetailsJSONObject(const unsigned char *script,uint32_t total); -void AppendOffChainFormatData(uint32_t data_format,uint32_t out_options,mc_Script *lpDetailsScript,vector& vValue,vector* vChunkHashes,int *errorCode,string *strError); +void AppendOffChainFormatData(uint32_t data_format,uint32_t out_options,mc_Script *lpDetailsScript,vector& vValue,vector* vChunkHashes,int *errorCode,string *strError); #endif /* RPCMULTICHAINUTILS_H */ diff --git a/src/rpc/rpcwallet.cpp b/src/rpc/rpcwallet.cpp index 1d7ca853..1176e015 100644 --- a/src/rpc/rpcwallet.cpp +++ b/src/rpc/rpcwallet.cpp @@ -550,9 +550,12 @@ Value storechunk(const Array& params, bool fHelp) } uint256 hash; + mc_gState->m_TmpBuffers->m_RpcHasher1->DoubleHash(&vValue[0],(int)vValue.size(),&hash); +/* mc_SHA256 *hasher; hasher=new mc_SHA256; + hasher->Reset(); hasher->Write(&vValue[0],(int)vValue.size()); hasher->GetHash((unsigned char*)&hash); hasher->Reset(); @@ -560,7 +563,7 @@ Value storechunk(const Array& params, bool fHelp) hasher->GetHash((unsigned char*)&hash); delete hasher; - +*/ mc_TxEntity entity; entity.Zero(); entity.m_EntityType=MC_TET_AUTHOR; diff --git a/src/rpc/rpcwalletutils.cpp b/src/rpc/rpcwalletutils.cpp index e5faa9c4..d688e67d 100644 --- a/src/rpc/rpcwalletutils.cpp +++ b/src/rpc/rpcwalletutils.cpp @@ -736,7 +736,7 @@ void AppendOffChainFormatData(uint32_t data_format, uint32_t out_options, mc_Script *lpDetailsScript, vector& vValue, - vector* vChunkHashes, + vector* vChunkHashes, int *errorCode, string *strError) { @@ -750,7 +750,9 @@ void AppendOffChainFormatData(uint32_t data_format, lpDetailsScript->Clear(); int chunk_count; + int tail_size,size; int err; + uint256 hash; mc_TxEntity entity; entity.Zero(); entity.m_EntityType=MC_TET_AUTHOR; @@ -774,11 +776,51 @@ void AppendOffChainFormatData(uint32_t data_format, return; } lpDetailsScript->SetChunkDefHash((unsigned char*)&vValue[i*MC_CDB_CHUNK_HASH_SIZE],chunk_def.m_Size); + if(vChunkHashes) + { + vChunkHashes->push_back(*(uint256*)&vValue[i*MC_CDB_CHUNK_HASH_SIZE]); + } } } else { - + chunk_count=0; + tail_size=0; + if(vValue.size()) + { + chunk_count=((int)vValue.size()-1)/MAX_CHUNK_SIZE+1; + tail_size=(int)vValue.size()-(chunk_count-1)*MAX_CHUNK_SIZE; + lpDetailsScript->SetChunkDefHeader(data_format,chunk_count); + for(int i=0;im_TmpBuffers->m_RpcHasher1->DoubleHash((unsigned char*)&vValue[i*MAX_CHUNK_SIZE],size,&hash); + + err=pwalletTxsMain->m_ChunkDB->AddChunk((unsigned char*)&hash,&entity,NULL,-1,(unsigned char*)&vValue[i*MAX_CHUNK_SIZE],NULL,size,0,0); + if(err) + { + switch(err) + { + case MC_ERR_FOUND: + break; + default: + *strError="Internal error: couldn't store chunk"; + return; + } + } + + pwalletTxsMain->m_ChunkDB->Dump("storechunk"); + lpDetailsScript->SetChunkDefHash((unsigned char*)&hash,size); + if(vChunkHashes) + { + vChunkHashes->push_back(*(uint256*)&hash); + } + } + } } // err=pwalletTxsMain->m_ChunkDB->AddChunk((unsigned char*)&hash,&entity,NULL,-1,(unsigned char*)&vValue[0],NULL,(int)vValue.size(),0,0); diff --git a/src/utils/declare.h b/src/utils/declare.h index 5dafb493..58e3b393 100644 --- a/src/utils/declare.h +++ b/src/utils/declare.h @@ -138,6 +138,8 @@ typedef struct mc_SHA256 void Write(const void *lpData,int size); void GetHash(unsigned char *hash); + void DoubleHash(const void *lpData,int size,void *hash); + mc_SHA256() { Init(); diff --git a/src/utils/utilwrapper.cpp b/src/utils/utilwrapper.cpp index 1fadfcf3..75b790bb 100644 --- a/src/utils/utilwrapper.cpp +++ b/src/utils/utilwrapper.cpp @@ -865,6 +865,16 @@ void mc_SHA256::GetHash(unsigned char *hash) } } +void mc_SHA256::DoubleHash(const void *lpData,int size,void *hash) +{ + Reset(); + Write(lpData,size); + GetHash((unsigned char *)hash); + Reset(); + Write((unsigned char *)hash,32); + GetHash((unsigned char *)hash); +} + int mc_MultichainParams::Import(const char *name,const char *source_address) { diff --git a/src/wallet/chunkdb.cpp b/src/wallet/chunkdb.cpp index c1f8786b..a87ff841 100644 --- a/src/wallet/chunkdb.cpp +++ b/src/wallet/chunkdb.cpp @@ -801,6 +801,10 @@ int mc_ChunkDB::AddChunkInternal( uint32_t timestamp; size_t bytes; const unsigned char *ptr; + char chunk_hex[65]; + char msg[256]; + char enthex[65]; + mc_ChunkDBRow chunk_def; mc_ChunkDBRow entity_chunk_def; @@ -981,7 +985,11 @@ int mc_ChunkDB::AddChunkInternal( ((mc_ChunkDBRow *)m_MemPool->GetRow(mempool_last_null_row))->m_NextSubscriptionID = subscription->m_SubscriptionID; } } - + + sprintf_hex(enthex,entity->m_EntityID,MC_TDB_ENTITY_ID_SIZE); + sprintf_hex(chunk_hex,hash,MC_CDB_CHUNK_HASH_SIZE); + sprintf(msg,"New Chunk %s, size %d, flags %08X, entity (%08X, %s)",chunk_hex,chunk_size,flags,entity->m_EntityType,enthex); + LogString(msg); return MC_ERR_NOERROR; } From 16361d19375ee93a07586a89c66b66aadadff867 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 26 Feb 2018 19:06:56 +0200 Subject: [PATCH 016/157] Off-chain items, output stream APIs --- src/protocol/multichainscript.cpp | 4 +- src/rpc/rpcutils.cpp | 94 ++++++++++++++++++++++++++++++- src/rpc/rpcutils.h | 10 ++++ src/rpc/rpcwalletutils.cpp | 12 +++- src/wallet/wallettxs.cpp | 14 ++--- 5 files changed, 120 insertions(+), 14 deletions(-) diff --git a/src/protocol/multichainscript.cpp b/src/protocol/multichainscript.cpp index 2cd2d830..3f367bd5 100644 --- a/src/protocol/multichainscript.cpp +++ b/src/protocol/multichainscript.cpp @@ -2350,12 +2350,12 @@ int mc_Script::GetChunkDef(uint32_t *format,unsigned char** hashes,int *chunk_co unsigned char *ptrEnd; unsigned char f; int c,count,shift,size; - +/* if(format) { *format=MC_SCR_DATA_FORMAT_UNKNOWN; } - +*/ if(m_CurrentElement<0) { return MC_ERR_INVALID_PARAMETER_VALUE; diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index 498d1f25..667a90c5 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -897,13 +897,93 @@ int mc_IsUTF8(const unsigned char *elem,size_t elem_size) return 1; } -Value OpReturnFormatEntry(const unsigned char *elem,size_t elem_size,uint256 txid, int vout, uint32_t format, string *format_text_out) +uint32_t GetFormattedData(mc_Script *lpScript,const unsigned char **elem,size_t *elem_size,unsigned char* hashes,int chunk_count) +{ + uint32_t status; + mc_ChunkDBRow chunk_def; + int size,shift; + unsigned char *ptr; + unsigned char *ptrEnd; + + status=MC_OST_UNKNOWN; + + *elem = lpScript->GetData(lpScript->GetNumElements()-1,elem_size); + if(hashes == NULL) + { + return MC_OST_ON_CHAIN; + } + if((mc_gState->m_WalletMode & MC_WMD_TXS) == 0) + { + return status; + } + if(chunk_count > 1) + { + return status; + } + + ptr=hashes; + ptrEnd=ptr+MC_CDB_CHUNK_HASH_SIZE+16; + + size=(int)mc_GetVarInt(ptr,ptrEnd-ptr,-1,&shift); + + if(size<0) + { + status |= MC_OST_ERROR; + return status; + } + + if(size > MAX_CHUNK_SIZE) + { + status |= MC_OST_ERROR; + return status; + } + + ptr+=shift; + + if(pwalletTxsMain->m_ChunkDB->GetChunkDef(&chunk_def,ptr,NULL,NULL,-1) == MC_ERR_NOERROR) + { + *elem=pwalletTxsMain->m_ChunkDB->GetChunk(&chunk_def,0,-1,elem_size); + if(*elem) + { + status=MC_OST_RETRIEVED; + } + else + { + status |= MC_OST_ERROR; + } + } + + return status; +} + + +Value OpReturnFormatEntry(const unsigned char *elem,size_t elem_size,uint256 txid, int vout, uint32_t format, string *format_text_out,uint32_t status) { string metadata=""; Object metadata_object; Value metadata_value; + bool available; + string status_str; int err; - if( ((int)elem_size <= GetArg("-maxshowndata",MAX_OP_RETURN_SHOWN)) || (txid == 0) ) + + available=true; + status_str=""; + switch(status & MC_OST_STATUS_MASK) + { + case MC_OST_ON_CHAIN: + status_str="on-chain"; + break; + case MC_OST_RETRIEVED: + status_str="retrieved"; + break; + case MC_OST_UNKNOWN: + status_str="unknown"; + available=false; + break; + } + + + if( (((int)elem_size <= GetArg("-maxshowndata",MAX_OP_RETURN_SHOWN)) || (txid == 0)) && available && ((status & MC_OST_MULTIPLE) == 0) ) { if(format_text_out) { @@ -954,9 +1034,19 @@ Value OpReturnFormatEntry(const unsigned char *elem,size_t elem_size,uint256 txi metadata_object.push_back(Pair("vout", vout)); metadata_object.push_back(Pair("format", OpReturnFormatToText(format))); metadata_object.push_back(Pair("size", (int)elem_size)); + if(status != MC_OST_UNDEFINED) + { + metadata_object.push_back(Pair("available", available)); + metadata_object.push_back(Pair("status", status_str)); + } return metadata_object; } +Value OpReturnFormatEntry(const unsigned char *elem,size_t elem_size,uint256 txid, int vout, uint32_t format, string *format_text_out) +{ + return OpReturnFormatEntry(elem,elem_size,txid,vout,format,format_text_out,MC_OST_UNDEFINED); +} + Value OpReturnFormatEntry(const unsigned char *elem,size_t elem_size,uint256 txid, int vout, uint32_t format) { Value format_item_value; diff --git a/src/rpc/rpcutils.h b/src/rpc/rpcutils.h index a30519b7..48e7854b 100644 --- a/src/rpc/rpcutils.h +++ b/src/rpc/rpcutils.h @@ -61,6 +61,14 @@ using namespace json_spirit; #define MC_RFD_OPTION_INLINE 0x00000001 #define MC_RFD_OPTION_OFFCHAIN 0x00000002 +#define MC_OST_UNDEFINED 0x00000000 +#define MC_OST_UNKNOWN 0x00000001 +#define MC_OST_ON_CHAIN 0x00000002 +#define MC_OST_RETRIEVED 0x00000004 +#define MC_OST_STATUS_MASK 0x0000FFFF +#define MC_OST_ERROR 0x00010000 +#define MC_OST_MULTIPLE 0x00020000 + // codes for allowed_objects fields // 0x0001 - create @@ -93,7 +101,9 @@ Array PerOutputDataEntries(const CTxOut& txout,mc_Script *lpScript,uint256 txid, Array PermissionEntries(const CTxOut& txout,mc_Script *lpScript,bool fLong); Object StreamEntry(const unsigned char *txid,uint32_t output_level); Object UpgradeEntry(const unsigned char *txid); +uint32_t GetFormattedData(mc_Script *lpScript,const unsigned char **elem,size_t *elem_size,unsigned char* hashes,int chunk_count); Value OpReturnEntry(const unsigned char *elem,size_t elem_size,uint256 txid, int vout); +Value OpReturnFormatEntry(const unsigned char *elem,size_t elem_size,uint256 txid, int vout, uint32_t format, string *format_text_out,uint32_t status); Value OpReturnFormatEntry(const unsigned char *elem,size_t elem_size,uint256 txid, int vout, uint32_t format, string *format_text_out); Value OpReturnFormatEntry(const unsigned char *elem,size_t elem_size,uint256 txid, int vout, uint32_t format); Value DataItemEntry(const CTransaction& tx,int n,set & already_seen,uint32_t stream_output_level); diff --git a/src/rpc/rpcwalletutils.cpp b/src/rpc/rpcwalletutils.cpp index d688e67d..0f5289b6 100644 --- a/src/rpc/rpcwalletutils.cpp +++ b/src/rpc/rpcwalletutils.cpp @@ -442,6 +442,9 @@ Object StreamItemEntry(const CWalletTx& wtx,int first_output,const unsigned char int item_key_size; Value item_value; uint32_t format; + unsigned char *chunk_hashes; + int chunk_count; + uint32_t retrieve_status; Value format_item_value; string format_text_str; int start_from=first_output; @@ -470,7 +473,9 @@ Object StreamItemEntry(const CWalletTx& wtx,int first_output,const unsigned char { if(mc_gState->m_TmpScript->GetNumElements()) // 2 OP_DROPs + OP_RETURN - item key { - mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(&format); + mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(&format,&chunk_hashes,&chunk_count,NULL); +// chunk_hashes=NULL; +// mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(&format); unsigned char short_txid[MC_AST_SHORT_TXID_SIZE]; mc_gState->m_TmpScript->SetElement(0); @@ -511,9 +516,10 @@ Object StreamItemEntry(const CWalletTx& wtx,int first_output,const unsigned char size_t elem_size; const unsigned char *elem; - elem = mc_gState->m_TmpScript->GetData(mc_gState->m_TmpScript->GetNumElements()-1,&elem_size); +// elem = mc_gState->m_TmpScript->GetData(mc_gState->m_TmpScript->GetNumElements()-1,&elem_size); + retrieve_status = GetFormattedData(mc_gState->m_TmpScript,&elem,&elem_size,chunk_hashes,chunk_count); item_value=OpReturnEntry(elem,elem_size,wtx.GetHash(),j); - format_item_value=OpReturnFormatEntry(elem,elem_size,wtx.GetHash(),j,format,&format_text_str); + format_item_value=OpReturnFormatEntry(elem,elem_size,wtx.GetHash(),j,format,&format_text_str,retrieve_status); } } } diff --git a/src/wallet/wallettxs.cpp b/src/wallet/wallettxs.cpp index fa534439..43dec4fb 100644 --- a/src/wallet/wallettxs.cpp +++ b/src/wallet/wallettxs.cpp @@ -2304,18 +2304,18 @@ int mc_WalletTxs::AddTx(mc_TxImport *import,const CWalletTx& tx,int block,CDiskT entity.m_EntityType=MC_TET_STREAM | MC_TET_CHAINPOS; if(imp->FindEntity(&entity) >= 0) { - mc_ChunkDBRow chunk_def; - mc_TxEntity chunk_entity; - chunk_entity.Zero(); - memcpy(entity.m_EntityID,short_txid,MC_AST_SHORT_TXID_SIZE); - entity.m_EntityType=MC_TET_STREAM; if(chunk_hashes) { + mc_ChunkDBRow chunk_def; + mc_TxEntity chunk_entity; + chunk_entity.Zero(); + memcpy(chunk_entity.m_EntityID,short_txid,MC_AST_SHORT_TXID_SIZE); + chunk_entity.m_EntityType=MC_TET_STREAM; for(int chunk=0;chunkGetChunkDef(&chunk_def,chunk_hashes,&entity,(unsigned char*)&hash,i) != MC_ERR_NOERROR) + if(m_ChunkDB->GetChunkDef(&chunk_def,chunk_hashes,&chunk_entity,(unsigned char*)&hash,i) != MC_ERR_NOERROR) { if(m_ChunkDB->GetChunkDef(&chunk_def,chunk_hashes,NULL,NULL,-1) == MC_ERR_NOERROR) { @@ -2323,7 +2323,7 @@ int mc_WalletTxs::AddTx(mc_TxImport *import,const CWalletTx& tx,int block,CDiskT if(chunk_found) { memcpy(m_ChunkBuffer,chunk_found,chunk_size); - chunk_err=m_ChunkDB->AddChunk(chunk_hashes,&entity,(unsigned char*)&hash,i,m_ChunkBuffer,NULL,chunk_size,0,0); + chunk_err=m_ChunkDB->AddChunk(chunk_hashes,&chunk_entity,(unsigned char*)&hash,i,m_ChunkBuffer,NULL,chunk_size,0,0); if(chunk_err) { err=chunk_err; From 3a24dd62cee57aba4464373a2b6a77f4a4405179 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 27 Feb 2018 13:19:54 +0200 Subject: [PATCH 017/157] Off-chain items, multi-chunk items --- src/chainparams/state.h | 3 ++ src/protocol/multichainscript.cpp | 4 +- src/protocol/multichainscript.h | 4 +- src/rpc/rpcutils.cpp | 78 ++++++++++++++++++++----------- src/rpc/rpcutils.h | 3 +- src/rpc/rpcwalletutils.cpp | 5 +- src/wallet/chunkdb.cpp | 8 ++++ 7 files changed, 71 insertions(+), 34 deletions(-) diff --git a/src/chainparams/state.h b/src/chainparams/state.h index a2c7003c..a564a823 100644 --- a/src/chainparams/state.h +++ b/src/chainparams/state.h @@ -173,6 +173,7 @@ typedef struct mc_TmpBuffers mc_Buffer *m_RpcABNoMapBuffer2; mc_Buffer *m_RpcEntityRows; mc_SHA256 *m_RpcHasher1; + mc_Script *m_RpcChunkScript1; void Init() { @@ -192,6 +193,7 @@ typedef struct mc_TmpBuffers m_RpcEntityRows=new mc_Buffer; m_RpcEntityRows->Initialize(MC_TDB_ENTITY_KEY_SIZE,MC_TDB_ROW_SIZE,MC_BUF_MODE_DEFAULT); m_RpcHasher1=new mc_SHA256(); + m_RpcChunkScript1=new mc_Script(); } void Destroy() @@ -207,6 +209,7 @@ typedef struct mc_TmpBuffers delete m_RpcABNoMapBuffer2; delete m_RpcEntityRows; delete m_RpcHasher1; + delete m_RpcChunkScript1; } } mc_TmpBuffers; diff --git a/src/protocol/multichainscript.cpp b/src/protocol/multichainscript.cpp index 3f367bd5..dd99b7d5 100644 --- a/src/protocol/multichainscript.cpp +++ b/src/protocol/multichainscript.cpp @@ -2344,7 +2344,7 @@ int mc_Script::SetDataFormat(const uint32_t format) return MC_ERR_NOERROR; } -int mc_Script::GetChunkDef(uint32_t *format,unsigned char** hashes,int *chunk_count,uint64_t *total_size) +int mc_Script::GetChunkDef(uint32_t *format,unsigned char** hashes,int *chunk_count,int64_t *total_size) { unsigned char *ptr; unsigned char *ptrEnd; @@ -2549,7 +2549,7 @@ int mc_Script::ExtractAndDeleteDataFormat(uint32_t *format) */ } -int mc_Script::ExtractAndDeleteDataFormat(uint32_t *format,unsigned char** hashes,int *chunk_count,uint64_t *total_size) +int mc_Script::ExtractAndDeleteDataFormat(uint32_t *format,unsigned char** hashes,int *chunk_count,int64_t *total_size) { int elem,err; diff --git a/src/protocol/multichainscript.h b/src/protocol/multichainscript.h index 8460334f..9f5156a8 100644 --- a/src/protocol/multichainscript.h +++ b/src/protocol/multichainscript.h @@ -105,12 +105,12 @@ typedef struct mc_Script int GetDataFormat(uint32_t *format); int SetDataFormat(const uint32_t format); - int GetChunkDef(uint32_t *format,unsigned char** hashes,int *chunk_count,uint64_t *total_size); + int GetChunkDef(uint32_t *format,unsigned char** hashes,int *chunk_count,int64_t *total_size); int SetChunkDefHeader(const uint32_t format,int chunk_count); int SetChunkDefHash(unsigned char *hash,int size); int ExtractAndDeleteDataFormat(uint32_t *format); - int ExtractAndDeleteDataFormat(uint32_t *format,unsigned char** hashes,int *chunk_count,uint64_t *total_size); + int ExtractAndDeleteDataFormat(uint32_t *format,unsigned char** hashes,int *chunk_count,int64_t *total_size); int DeleteDuplicatesInRange(int from,int to); diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index 667a90c5..1320872c 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -897,13 +897,20 @@ int mc_IsUTF8(const unsigned char *elem,size_t elem_size) return 1; } -uint32_t GetFormattedData(mc_Script *lpScript,const unsigned char **elem,size_t *elem_size,unsigned char* hashes,int chunk_count) +uint32_t GetFormattedData(mc_Script *lpScript,const unsigned char **elem,size_t *elem_size,unsigned char* hashes,int chunk_count,int64_t total_size) { uint32_t status; mc_ChunkDBRow chunk_def; - int size,shift; + int size,shift,chunk; unsigned char *ptr; unsigned char *ptrEnd; + bool use_tmp_buf=false; + + if( (chunk_count >1) && ( total_size <= GetArg("-maxshowndata",MAX_OP_RETURN_SHOWN)) ) + { + use_tmp_buf=true; + } + status=MC_OST_UNKNOWN; @@ -916,41 +923,60 @@ uint32_t GetFormattedData(mc_Script *lpScript,const unsigned char **elem,size_t { return status; } - if(chunk_count > 1) + + if(use_tmp_buf) { - return status; - } + mc_gState->m_TmpBuffers->m_RpcChunkScript1->Clear(); + mc_gState->m_TmpBuffers->m_RpcChunkScript1->AddElement(); + } ptr=hashes; ptrEnd=ptr+MC_CDB_CHUNK_HASH_SIZE+16; - - size=(int)mc_GetVarInt(ptr,ptrEnd-ptr,-1,&shift); - - if(size<0) + for(chunk=0;chunk MAX_CHUNK_SIZE) - { - status |= MC_OST_ERROR; - return status; - } + if(size > MAX_CHUNK_SIZE) + { + status |= MC_OST_ERROR; + return status; + } - ptr+=shift; - - if(pwalletTxsMain->m_ChunkDB->GetChunkDef(&chunk_def,ptr,NULL,NULL,-1) == MC_ERR_NOERROR) - { - *elem=pwalletTxsMain->m_ChunkDB->GetChunk(&chunk_def,0,-1,elem_size); - if(*elem) + ptr+=shift; + if(pwalletTxsMain->m_ChunkDB->GetChunkDef(&chunk_def,ptr,NULL,NULL,-1) == MC_ERR_NOERROR) { - status=MC_OST_RETRIEVED; + *elem=pwalletTxsMain->m_ChunkDB->GetChunk(&chunk_def,0,-1,elem_size); + if(*elem) + { + status=MC_OST_RETRIEVED; + if(use_tmp_buf) + { + mc_gState->m_TmpBuffers->m_RpcChunkScript1->SetData(*elem,*elem_size); + } + } + else + { + status = MC_OST_UNKNOWN | MC_OST_ERROR; + return status; + } } else { - status |= MC_OST_ERROR; + status=MC_OST_UNKNOWN; + return status; } + ptr+=MC_CDB_CHUNK_HASH_SIZE; + } + + if(use_tmp_buf) + { + *elem = mc_gState->m_TmpBuffers->m_RpcChunkScript1->GetData(0,elem_size); } return status; @@ -983,7 +1009,7 @@ Value OpReturnFormatEntry(const unsigned char *elem,size_t elem_size,uint256 txi } - if( (((int)elem_size <= GetArg("-maxshowndata",MAX_OP_RETURN_SHOWN)) || (txid == 0)) && available && ((status & MC_OST_MULTIPLE) == 0) ) + if( (((int)elem_size <= GetArg("-maxshowndata",MAX_OP_RETURN_SHOWN)) || (txid == 0)) && available && ((status & MC_OST_ERROR) == 0) ) { if(format_text_out) { diff --git a/src/rpc/rpcutils.h b/src/rpc/rpcutils.h index 48e7854b..090fb00e 100644 --- a/src/rpc/rpcutils.h +++ b/src/rpc/rpcutils.h @@ -67,7 +67,6 @@ using namespace json_spirit; #define MC_OST_RETRIEVED 0x00000004 #define MC_OST_STATUS_MASK 0x0000FFFF #define MC_OST_ERROR 0x00010000 -#define MC_OST_MULTIPLE 0x00020000 // codes for allowed_objects fields @@ -101,7 +100,7 @@ Array PerOutputDataEntries(const CTxOut& txout,mc_Script *lpScript,uint256 txid, Array PermissionEntries(const CTxOut& txout,mc_Script *lpScript,bool fLong); Object StreamEntry(const unsigned char *txid,uint32_t output_level); Object UpgradeEntry(const unsigned char *txid); -uint32_t GetFormattedData(mc_Script *lpScript,const unsigned char **elem,size_t *elem_size,unsigned char* hashes,int chunk_count); +uint32_t GetFormattedData(mc_Script *lpScript,const unsigned char **elem,size_t *elem_size,unsigned char* hashes,int chunk_count,int64_t total_size); Value OpReturnEntry(const unsigned char *elem,size_t elem_size,uint256 txid, int vout); Value OpReturnFormatEntry(const unsigned char *elem,size_t elem_size,uint256 txid, int vout, uint32_t format, string *format_text_out,uint32_t status); Value OpReturnFormatEntry(const unsigned char *elem,size_t elem_size,uint256 txid, int vout, uint32_t format, string *format_text_out); diff --git a/src/rpc/rpcwalletutils.cpp b/src/rpc/rpcwalletutils.cpp index 0f5289b6..2da9aa32 100644 --- a/src/rpc/rpcwalletutils.cpp +++ b/src/rpc/rpcwalletutils.cpp @@ -444,6 +444,7 @@ Object StreamItemEntry(const CWalletTx& wtx,int first_output,const unsigned char uint32_t format; unsigned char *chunk_hashes; int chunk_count; + int64_t total_chunk_size; uint32_t retrieve_status; Value format_item_value; string format_text_str; @@ -473,7 +474,7 @@ Object StreamItemEntry(const CWalletTx& wtx,int first_output,const unsigned char { if(mc_gState->m_TmpScript->GetNumElements()) // 2 OP_DROPs + OP_RETURN - item key { - mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(&format,&chunk_hashes,&chunk_count,NULL); + mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(&format,&chunk_hashes,&chunk_count,&total_chunk_size); // chunk_hashes=NULL; // mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(&format); @@ -517,7 +518,7 @@ Object StreamItemEntry(const CWalletTx& wtx,int first_output,const unsigned char const unsigned char *elem; // elem = mc_gState->m_TmpScript->GetData(mc_gState->m_TmpScript->GetNumElements()-1,&elem_size); - retrieve_status = GetFormattedData(mc_gState->m_TmpScript,&elem,&elem_size,chunk_hashes,chunk_count); + retrieve_status = GetFormattedData(mc_gState->m_TmpScript,&elem,&elem_size,chunk_hashes,chunk_count,total_chunk_size); item_value=OpReturnEntry(elem,elem_size,wtx.GetHash(),j); format_item_value=OpReturnFormatEntry(elem,elem_size,wtx.GetHash(),j,format,&format_text_str,retrieve_status); } diff --git a/src/wallet/chunkdb.cpp b/src/wallet/chunkdb.cpp index a87ff841..1ad4e8de 100644 --- a/src/wallet/chunkdb.cpp +++ b/src/wallet/chunkdb.cpp @@ -530,6 +530,13 @@ void mc_ChunkDB::Dump(const char *message) mc_MemoryDumpCharSizeToFile(fHan,m_MemPool->GetRow(0),0,m_MemPool->GetCount()*m_TotalSize,m_TotalSize); } + fprintf(fHan,"\nData\n"); + if(m_ChunkData->m_Size) + { + mc_MemoryDumpCharSizeToFile(fHan,m_ChunkData->m_lpData,0,m_ChunkData->m_Size,m_TotalSize); + } + + fprintf(fHan,"\nSubscriptions\n"); for(i=0;iGetCount();i++) @@ -930,6 +937,7 @@ int mc_ChunkDB::AddChunkInternal( ptr=m_TmpScript->GetData(0,&bytes); + m_ChunkData->SetElement(m_ChunkData->m_NumElements-1); m_ChunkData->AddElement(); err=m_ChunkData->SetData(ptr,bytes); if(err) From 949b6dde792315b86283b0440166bacb0f3f280b Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 27 Feb 2018 15:32:34 +0200 Subject: [PATCH 018/157] Off-chain items, reporting APIs --- src/protocol/multichainscript.cpp | 8 +++++ src/protocol/multichaintx.cpp | 39 ----------------------- src/rpc/rpcassets.cpp | 25 +++++++++------ src/rpc/rpcrawtransaction.cpp | 21 +++++++----- src/rpc/rpcutils.cpp | 53 +++++++++++++++++++++---------- src/rpc/rpcutils.h | 4 +-- src/rpc/rpcwallettxs.cpp | 20 ++++++++---- src/rpc/rpcwalletutils.cpp | 11 +++---- 8 files changed, 92 insertions(+), 89 deletions(-) diff --git a/src/protocol/multichainscript.cpp b/src/protocol/multichainscript.cpp index dd99b7d5..cc088529 100644 --- a/src/protocol/multichainscript.cpp +++ b/src/protocol/multichainscript.cpp @@ -2618,6 +2618,14 @@ int mc_Script::ExtractAndDeleteDataFormat(uint32_t *format,unsigned char** hashe if(err != MC_ERR_WRONG_SCRIPT) { + if(hashes) + { + *hashes=NULL; + } + if(chunk_count) + { + *chunk_count=0; + } return err; } diff --git a/src/protocol/multichaintx.cpp b/src/protocol/multichaintx.cpp index 17b714b9..40ecad71 100644 --- a/src/protocol/multichaintx.cpp +++ b/src/protocol/multichaintx.cpp @@ -567,45 +567,6 @@ bool MultiChainTransaction_VerifyAndDeleteDataFormatElements(string& reason) return false; } -/* - int elem,err; - - mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(NULL); // Format scripts are eliminated for protocol checks - - if(mc_gState->m_Features->OffChainData() == 0) - { - return true; - } - - if(mc_gState->m_TmpScript->m_NumElements < 2) - { - return true; - } - - elem=mc_gState->m_TmpScript->m_NumElements-2; - - mc_gState->m_TmpScript->SetElement(elem); - while( (elem >= 0 ) && ((err=mc_gState->m_TmpScript->GetChunkDef(NULL,NULL,NULL,NULL)) == MC_ERR_NOERROR) ) - { - mc_gState->m_TmpScript->DeleteElement(elem); - elem--; - if(elem < 0) - { - err=MC_ERR_WRONG_SCRIPT; - } - else - { - mc_gState->m_TmpScript->SetElement(elem); - } - } - - if(err != MC_ERR_WRONG_SCRIPT) - { - reason="Error in data format script"; - return false; - } - */ - return true; } diff --git a/src/rpc/rpcassets.cpp b/src/rpc/rpcassets.cpp index 9ddd5c04..94b86ebc 100644 --- a/src/rpc/rpcassets.cpp +++ b/src/rpc/rpcassets.cpp @@ -1636,9 +1636,13 @@ Object ListAssetTransactions(const CWalletTx& wtx, mc_EntityDetails *entity, boo unsigned char bufEmptyAssetRef[MC_AST_ASSET_QUANTITY_OFFSET]; uint32_t new_entity_type; set streams_already_seen; - Array aMetaData; +// Array aMetaData; Array aItems; uint32_t format; + unsigned char *chunk_hashes; + int chunk_count; + int64_t total_chunk_size,out_size; + uint32_t retrieve_status; Array aFormatMetaData; vector aFormatMetaDataPerOutput; // string format_text_str; @@ -1767,17 +1771,17 @@ Object ListAssetTransactions(const CWalletTx& wtx, mc_EntityDetails *entity, boo lpScript->Clear(); lpScript->SetScript((unsigned char*)(&pc2[0]),(size_t)(script2.end()-pc2),MC_SCR_TYPE_SCRIPTPUBKEY); - lpScript->ExtractAndDeleteDataFormat(&format); - size_t elem_size; +// lpScript->ExtractAndDeleteDataFormat(&format); + lpScript->ExtractAndDeleteDataFormat(&format,&chunk_hashes,&chunk_count,&total_chunk_size); const unsigned char *elem; if(lpScript->GetNumElements()<=1) { if(lpScript->GetNumElements()==1) { - elem = lpScript->GetData(lpScript->GetNumElements()-1,&elem_size); -// aMetaData.push_back(OpReturnEntry(elem,elem_size,wtx.GetHash(),i)); - Value metadata=OpReturnFormatEntry(elem,elem_size,wtx.GetHash(),i,format,NULL); +// elem = lpScript->GetData(lpScript->GetNumElements()-1,&elem_size); + retrieve_status = GetFormattedData(lpScript,&elem,&out_size,chunk_hashes,chunk_count,total_chunk_size); + Value metadata=OpReturnFormatEntry(elem,out_size,wtx.GetHash(),i,format,NULL,retrieve_status); aFormatMetaData.push_back(metadata); aFormatMetaDataPerOutput[i].push_back(metadata); } @@ -1786,11 +1790,12 @@ Object ListAssetTransactions(const CWalletTx& wtx, mc_EntityDetails *entity, boo { if(mc_gState->m_Compatibility & MC_VCM_1_0) { - elem = lpScript->GetData(lpScript->GetNumElements()-1,&elem_size); - if(elem_size) +// elem = lpScript->GetData(lpScript->GetNumElements()-1,&elem_size); + retrieve_status = GetFormattedData(lpScript,&elem,&out_size,chunk_hashes,chunk_count,total_chunk_size); + if(out_size) { - aMetaData.push_back(OpReturnEntry(elem,elem_size,wtx.GetHash(),i)); - aFormatMetaData.push_back(OpReturnFormatEntry(elem,elem_size,wtx.GetHash(),i,format,NULL)); +// aMetaData.push_back(OpReturnEntry(elem,elem_size,wtx.GetHash(),i)); + aFormatMetaData.push_back(OpReturnFormatEntry(elem,out_size,wtx.GetHash(),i,format,NULL,retrieve_status)); } } diff --git a/src/rpc/rpcrawtransaction.cpp b/src/rpc/rpcrawtransaction.cpp index 0735a6e8..67c03362 100644 --- a/src/rpc/rpcrawtransaction.cpp +++ b/src/rpc/rpcrawtransaction.cpp @@ -125,6 +125,10 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, Object& entry) new_entity_type=MC_ENT_TYPE_NONE; set streams_already_seen; uint32_t format; + unsigned char *chunk_hashes; + int chunk_count; + int64_t total_chunk_size,out_size; + uint32_t retrieve_status; Array aFormatMetaData; Array aFullFormatMetaData; @@ -157,8 +161,8 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, Object& entry) if(lpScript->IsOpReturnScript()) { -// int e=mc_gState->m_TmpScript->GetNumElements()-1; - lpScript->ExtractAndDeleteDataFormat(&format); +// lpScript->ExtractAndDeleteDataFormat(&format); + lpScript->ExtractAndDeleteDataFormat(&format,&chunk_hashes,&chunk_count,&total_chunk_size); lpScript->SetElement(0); err=lpScript->GetNewEntityType(&new_entity_type,&asset_update,details_script,&details_script_size); @@ -201,7 +205,6 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, Object& entry) } } - size_t elem_size; const unsigned char *elem; int cs_err,cs_offset,cs_new_offset,cs_size,cs_vin; unsigned char *cs_script; @@ -210,9 +213,10 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, Object& entry) { if(lpScript->GetNumElements()==1) { - elem = lpScript->GetData(lpScript->GetNumElements()-1,&elem_size); +// elem = lpScript->GetData(lpScript->GetNumElements()-1,&elem_size); + retrieve_status = GetFormattedData(lpScript,&elem,&out_size,chunk_hashes,chunk_count,total_chunk_size); // vdata.push_back(OpReturnEntry(elem,elem_size,tx.GetHash(),i)); - aFormatMetaData.push_back(OpReturnFormatEntry(elem,elem_size,tx.GetHash(),i,format,NULL)); + aFormatMetaData.push_back(OpReturnFormatEntry(elem,out_size,tx.GetHash(),i,format,NULL,retrieve_status)); if(mc_gState->m_Compatibility & MC_VCM_1_0) { aFullFormatMetaData.push_back(aFormatMetaData[0]); @@ -223,11 +227,12 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, Object& entry) { if(mc_gState->m_Compatibility & MC_VCM_1_0) { - elem = lpScript->GetData(lpScript->GetNumElements()-1,&elem_size); - if(elem_size) +// elem = lpScript->GetData(lpScript->GetNumElements()-1,&elem_size); + retrieve_status = GetFormattedData(lpScript,&elem,&out_size,chunk_hashes,chunk_count,total_chunk_size); + if(out_size) { // vdata.push_back(OpReturnEntry(elem,elem_size,tx.GetHash(),i)); - aFullFormatMetaData.push_back(OpReturnFormatEntry(elem,elem_size,tx.GetHash(),i,format,NULL)); + aFullFormatMetaData.push_back(OpReturnFormatEntry(elem,out_size,tx.GetHash(),i,format,NULL,retrieve_status)); } } lpScript->SetElement(0); diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index 1320872c..4a5788cc 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -897,24 +897,29 @@ int mc_IsUTF8(const unsigned char *elem,size_t elem_size) return 1; } -uint32_t GetFormattedData(mc_Script *lpScript,const unsigned char **elem,size_t *elem_size,unsigned char* hashes,int chunk_count,int64_t total_size) +uint32_t GetFormattedData(mc_Script *lpScript,const unsigned char **elem,int64_t *out_size,unsigned char* hashes,int chunk_count,int64_t total_size) { uint32_t status; mc_ChunkDBRow chunk_def; int size,shift,chunk; unsigned char *ptr; unsigned char *ptrEnd; - bool use_tmp_buf=false; - - if( (chunk_count >1) && ( total_size <= GetArg("-maxshowndata",MAX_OP_RETURN_SHOWN)) ) + bool use_tmp_buf=false; + size_t elem_size; + + if(chunk_count > 1) { - use_tmp_buf=true; + if(total_size <= GetArg("-maxshowndata",MAX_OP_RETURN_SHOWN)) + { + use_tmp_buf=true; + } } status=MC_OST_UNKNOWN; - *elem = lpScript->GetData(lpScript->GetNumElements()-1,elem_size); + *elem = lpScript->GetData(lpScript->GetNumElements()-1,&elem_size); + *out_size=elem_size; if(hashes == NULL) { return MC_OST_ON_CHAIN; @@ -951,13 +956,13 @@ uint32_t GetFormattedData(mc_Script *lpScript,const unsigned char **elem,size_t ptr+=shift; if(pwalletTxsMain->m_ChunkDB->GetChunkDef(&chunk_def,ptr,NULL,NULL,-1) == MC_ERR_NOERROR) { - *elem=pwalletTxsMain->m_ChunkDB->GetChunk(&chunk_def,0,-1,elem_size); + *elem=pwalletTxsMain->m_ChunkDB->GetChunk(&chunk_def,0,-1,&elem_size); if(*elem) { status=MC_OST_RETRIEVED; if(use_tmp_buf) { - mc_gState->m_TmpBuffers->m_RpcChunkScript1->SetData(*elem,*elem_size); + mc_gState->m_TmpBuffers->m_RpcChunkScript1->SetData(*elem,elem_size); } } else @@ -976,14 +981,23 @@ uint32_t GetFormattedData(mc_Script *lpScript,const unsigned char **elem,size_t if(use_tmp_buf) { - *elem = mc_gState->m_TmpBuffers->m_RpcChunkScript1->GetData(0,elem_size); + *elem = mc_gState->m_TmpBuffers->m_RpcChunkScript1->GetData(0,&elem_size); + } + + if(chunk_count > 1) + { + *out_size=total_size; + } + else + { + *out_size=elem_size; } return status; } -Value OpReturnFormatEntry(const unsigned char *elem,size_t elem_size,uint256 txid, int vout, uint32_t format, string *format_text_out,uint32_t status) +Value OpReturnFormatEntry(const unsigned char *elem,int64_t elem_size,uint256 txid, int vout, uint32_t format, string *format_text_out,uint32_t status) { string metadata=""; Object metadata_object; @@ -1059,7 +1073,7 @@ Value OpReturnFormatEntry(const unsigned char *elem,size_t elem_size,uint256 txi metadata_object.push_back(Pair("txid", txid.ToString())); metadata_object.push_back(Pair("vout", vout)); metadata_object.push_back(Pair("format", OpReturnFormatToText(format))); - metadata_object.push_back(Pair("size", (int)elem_size)); + metadata_object.push_back(Pair("size", elem_size)); if(status != MC_OST_UNDEFINED) { metadata_object.push_back(Pair("available", available)); @@ -1096,8 +1110,12 @@ Value DataItemEntry(const CTransaction& tx,int n,set & already_seen,uin const unsigned char *ptr; unsigned char item_key[MC_ENT_MAX_ITEM_KEY_SIZE+1]; int item_key_size; - Value item_value; +// Value item_value; uint32_t format; + unsigned char *chunk_hashes; + int chunk_count; + int64_t total_chunk_size,out_size; + uint32_t retrieve_status; Value format_item_value; string format_text_str; @@ -1121,7 +1139,8 @@ Value DataItemEntry(const CTransaction& tx,int n,set & already_seen,uin return Value::null; } - mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(&format); +// mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(&format); + mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(&format,&chunk_hashes,&chunk_count,&total_chunk_size); unsigned char short_txid[MC_AST_SHORT_TXID_SIZE]; mc_gState->m_TmpScript->SetElement(0); @@ -1157,12 +1176,12 @@ Value DataItemEntry(const CTransaction& tx,int n,set & already_seen,uin keys.push_back(string(item_key,item_key+item_key_size)); } - size_t elem_size; const unsigned char *elem; - elem = mc_gState->m_TmpScript->GetData(mc_gState->m_TmpScript->GetNumElements()-1,&elem_size); - item_value=OpReturnEntry(elem,elem_size,tx.GetHash(),n); - format_item_value=OpReturnFormatEntry(elem,elem_size,tx.GetHash(),n,format,&format_text_str); +// elem = mc_gState->m_TmpScript->GetData(mc_gState->m_TmpScript->GetNumElements()-1,&elem_size); + retrieve_status = GetFormattedData(mc_gState->m_TmpScript,&elem,&out_size,chunk_hashes,chunk_count,total_chunk_size); +// item_value=OpReturnEntry(elem,elem_size,tx.GetHash(),n); + format_item_value=OpReturnFormatEntry(elem,out_size,tx.GetHash(),n,format,&format_text_str,retrieve_status); already_seen.insert(hash); diff --git a/src/rpc/rpcutils.h b/src/rpc/rpcutils.h index 090fb00e..61b715f6 100644 --- a/src/rpc/rpcutils.h +++ b/src/rpc/rpcutils.h @@ -100,9 +100,9 @@ Array PerOutputDataEntries(const CTxOut& txout,mc_Script *lpScript,uint256 txid, Array PermissionEntries(const CTxOut& txout,mc_Script *lpScript,bool fLong); Object StreamEntry(const unsigned char *txid,uint32_t output_level); Object UpgradeEntry(const unsigned char *txid); -uint32_t GetFormattedData(mc_Script *lpScript,const unsigned char **elem,size_t *elem_size,unsigned char* hashes,int chunk_count,int64_t total_size); +uint32_t GetFormattedData(mc_Script *lpScript,const unsigned char **elem,int64_t *out_size,unsigned char* hashes,int chunk_count,int64_t total_size); Value OpReturnEntry(const unsigned char *elem,size_t elem_size,uint256 txid, int vout); -Value OpReturnFormatEntry(const unsigned char *elem,size_t elem_size,uint256 txid, int vout, uint32_t format, string *format_text_out,uint32_t status); +Value OpReturnFormatEntry(const unsigned char *elem,int64_t elem_size,uint256 txid, int vout, uint32_t format, string *format_text_out,uint32_t status); Value OpReturnFormatEntry(const unsigned char *elem,size_t elem_size,uint256 txid, int vout, uint32_t format, string *format_text_out); Value OpReturnFormatEntry(const unsigned char *elem,size_t elem_size,uint256 txid, int vout, uint32_t format); Value DataItemEntry(const CTransaction& tx,int n,set & already_seen,uint32_t stream_output_level); diff --git a/src/rpc/rpcwallettxs.cpp b/src/rpc/rpcwallettxs.cpp index a20507bd..c184a775 100644 --- a/src/rpc/rpcwallettxs.cpp +++ b/src/rpc/rpcwallettxs.cpp @@ -177,6 +177,10 @@ Object ListWalletTransactions(const CWalletTx& wtx, bool fLong, const isminefilt Array aMetaData; Array aItems; uint32_t format; + unsigned char *chunk_hashes; + int chunk_count; + int64_t total_chunk_size,out_size; + uint32_t retrieve_status; Array aFormatMetaData; vector aFormatMetaDataPerOutput; @@ -248,18 +252,19 @@ Object ListWalletTransactions(const CWalletTx& wtx, bool fLong, const isminefilt lpScript->Clear(); lpScript->SetScript((unsigned char*)(&pc2[0]),(size_t)(script2.end()-pc2),MC_SCR_TYPE_SCRIPTPUBKEY); - lpScript->ExtractAndDeleteDataFormat(&format); +// lpScript->ExtractAndDeleteDataFormat(&format); + lpScript->ExtractAndDeleteDataFormat(&format,&chunk_hashes,&chunk_count,&total_chunk_size); - size_t elem_size; const unsigned char *elem; if(lpScript->GetNumElements()<=1) { if(lpScript->GetNumElements()==1) { - elem = lpScript->GetData(lpScript->GetNumElements()-1,&elem_size); +// elem = lpScript->GetData(lpScript->GetNumElements()-1,&elem_size); // aMetaData.push_back(OpReturnEntry(elem,elem_size,wtx.GetHash(),i)); - Value metadata=OpReturnFormatEntry(elem,elem_size,wtx.GetHash(),i,format,NULL); + retrieve_status = GetFormattedData(lpScript,&elem,&out_size,chunk_hashes,chunk_count,total_chunk_size); + Value metadata=OpReturnFormatEntry(elem,out_size,wtx.GetHash(),i,format,NULL,retrieve_status); aFormatMetaData.push_back(metadata); aFormatMetaDataPerOutput[i].push_back(metadata); } @@ -268,11 +273,12 @@ Object ListWalletTransactions(const CWalletTx& wtx, bool fLong, const isminefilt { if(mc_gState->m_Compatibility & MC_VCM_1_0) { - elem = lpScript->GetData(lpScript->GetNumElements()-1,&elem_size); - if(elem_size) +// elem = lpScript->GetData(lpScript->GetNumElements()-1,&elem_size); + retrieve_status = GetFormattedData(lpScript,&elem,&out_size,chunk_hashes,chunk_count,total_chunk_size); + if(out_size) { // aMetaData.push_back(OpReturnEntry(elem,elem_size,wtx.GetHash(),i)); - aFormatMetaData.push_back(OpReturnFormatEntry(elem,elem_size,wtx.GetHash(),i,format,NULL)); + aFormatMetaData.push_back(OpReturnFormatEntry(elem,out_size,wtx.GetHash(),i,format,NULL,retrieve_status)); } } diff --git a/src/rpc/rpcwalletutils.cpp b/src/rpc/rpcwalletutils.cpp index 2da9aa32..c00822cb 100644 --- a/src/rpc/rpcwalletutils.cpp +++ b/src/rpc/rpcwalletutils.cpp @@ -440,11 +440,11 @@ Object StreamItemEntry(const CWalletTx& wtx,int first_output,const unsigned char const unsigned char *ptr; unsigned char item_key[MC_ENT_MAX_ITEM_KEY_SIZE+1]; int item_key_size; - Value item_value; +// Value item_value; uint32_t format; unsigned char *chunk_hashes; int chunk_count; - int64_t total_chunk_size; + int64_t total_chunk_size,out_size; uint32_t retrieve_status; Value format_item_value; string format_text_str; @@ -514,13 +514,12 @@ Object StreamItemEntry(const CWalletTx& wtx,int first_output,const unsigned char } } - size_t elem_size; const unsigned char *elem; // elem = mc_gState->m_TmpScript->GetData(mc_gState->m_TmpScript->GetNumElements()-1,&elem_size); - retrieve_status = GetFormattedData(mc_gState->m_TmpScript,&elem,&elem_size,chunk_hashes,chunk_count,total_chunk_size); - item_value=OpReturnEntry(elem,elem_size,wtx.GetHash(),j); - format_item_value=OpReturnFormatEntry(elem,elem_size,wtx.GetHash(),j,format,&format_text_str,retrieve_status); + retrieve_status = GetFormattedData(mc_gState->m_TmpScript,&elem,&out_size,chunk_hashes,chunk_count,total_chunk_size); +// item_value=OpReturnEntry(elem,elem_size,wtx.GetHash(),j); + format_item_value=OpReturnFormatEntry(elem,out_size,wtx.GetHash(),j,format,&format_text_str,retrieve_status); } } } From 46c26f43017701a4230c0183cc017d72e286409f Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 27 Feb 2018 17:49:26 +0200 Subject: [PATCH 019/157] Off-chain items, gettxoutdata --- src/rpc/rpcutils.cpp | 126 ++++++++++++++++++++++++++++++++++++++---- src/rpc/rpcutils.h | 2 + src/rpc/rpcwallet.cpp | 51 +++++++++++------ 3 files changed, 152 insertions(+), 27 deletions(-) diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index 4a5788cc..9bc7cf53 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -897,6 +897,73 @@ int mc_IsUTF8(const unsigned char *elem,size_t elem_size) return 1; } +const unsigned char *GetChunkDataInRange(int64_t *out_size,unsigned char* hashes,int chunk_count,int64_t start,int64_t count) +{ + mc_ChunkDBRow chunk_def; + int size,shift,chunk; + unsigned char *ptr; + unsigned char *ptrEnd; + size_t elem_size; + int64_t total_size=0; + unsigned char *elem; + int64_t read_from,read_size; + + mc_gState->m_TmpBuffers->m_RpcChunkScript1->Clear(); + mc_gState->m_TmpBuffers->m_RpcChunkScript1->AddElement(); + + *out_size=0; + + ptr=hashes; + ptrEnd=ptr+MC_CDB_CHUNK_HASH_SIZE+16; + for(chunk=0;chunk MAX_CHUNK_SIZE) + { + return NULL; + } + + + ptr+=shift; + if(pwalletTxsMain->m_ChunkDB->GetChunkDef(&chunk_def,ptr,NULL,NULL,-1) == MC_ERR_NOERROR) + { + read_from=0; + read_size=chunk_def.m_Size; + if( (total_size+read_size > start) && (total_size < start+count) ) + { + if(total_size < start) + { + read_from=start-read_size; + } + if(total_size+read_size > start+count) + { + read_size=start+count-total_size; + } + elem=pwalletTxsMain->m_ChunkDB->GetChunk(&chunk_def,0,-1,&elem_size); + if(elem) + { + mc_gState->m_TmpBuffers->m_RpcChunkScript1->SetData(elem+read_from,read_size); + *out_size+=read_size; + } + } + total_size+=chunk_def.m_Size; + } + else + { + return NULL; + } + ptr+=MC_CDB_CHUNK_HASH_SIZE; + } + + return mc_gState->m_TmpBuffers->m_RpcChunkScript1->GetData(0,&elem_size); +} + uint32_t GetFormattedData(mc_Script *lpScript,const unsigned char **elem,int64_t *out_size,unsigned char* hashes,int chunk_count,int64_t total_size) { uint32_t status; @@ -905,6 +972,7 @@ uint32_t GetFormattedData(mc_Script *lpScript,const unsigned char **elem,int64_t unsigned char *ptr; unsigned char *ptrEnd; bool use_tmp_buf=false; + bool skip_read=false; size_t elem_size; if(chunk_count > 1) @@ -913,6 +981,10 @@ uint32_t GetFormattedData(mc_Script *lpScript,const unsigned char **elem,int64_t { use_tmp_buf=true; } + else + { + skip_read=true; + } } @@ -956,19 +1028,22 @@ uint32_t GetFormattedData(mc_Script *lpScript,const unsigned char **elem,int64_t ptr+=shift; if(pwalletTxsMain->m_ChunkDB->GetChunkDef(&chunk_def,ptr,NULL,NULL,-1) == MC_ERR_NOERROR) { - *elem=pwalletTxsMain->m_ChunkDB->GetChunk(&chunk_def,0,-1,&elem_size); - if(*elem) + status=MC_OST_RETRIEVED; + if(!skip_read) { - status=MC_OST_RETRIEVED; - if(use_tmp_buf) + *elem=pwalletTxsMain->m_ChunkDB->GetChunk(&chunk_def,0,-1,&elem_size); + if(*elem) { - mc_gState->m_TmpBuffers->m_RpcChunkScript1->SetData(*elem,elem_size); + if(use_tmp_buf) + { + mc_gState->m_TmpBuffers->m_RpcChunkScript1->SetData(*elem,elem_size); + } + } + else + { + status = MC_OST_UNKNOWN | MC_OST_ERROR; + return status; } - } - else - { - status = MC_OST_UNKNOWN | MC_OST_ERROR; - return status; } } else @@ -983,6 +1058,13 @@ uint32_t GetFormattedData(mc_Script *lpScript,const unsigned char **elem,int64_t { *elem = mc_gState->m_TmpBuffers->m_RpcChunkScript1->GetData(0,&elem_size); } + else + { + if(chunk_count > 1) + { + *elem=NULL; + } + } if(chunk_count > 1) { @@ -1023,7 +1105,7 @@ Value OpReturnFormatEntry(const unsigned char *elem,int64_t elem_size,uint256 tx } - if( (((int)elem_size <= GetArg("-maxshowndata",MAX_OP_RETURN_SHOWN)) || (txid == 0)) && available && ((status & MC_OST_ERROR) == 0) ) + if( (((int)elem_size <= GetArg("-maxshowndata",MAX_OP_RETURN_SHOWN)) || (txid == 0)) && available && ((status & MC_OST_ERROR) == 0) && (elem != NULL) ) { if(format_text_out) { @@ -3682,6 +3764,28 @@ int paramtoint(Value param,bool check_for_min,int min_value,string error_message return result; } +int64_t paramtoint64(Value param,bool check_for_min,int64_t min_value,string error_message) +{ + int64_t result; + + if(param.type() != int_type) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, error_message); + } + + result=param.get_int64(); + if(check_for_min) + { + if(result < min_value) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, error_message); + } + } + + return result; +} + + bool mc_IsJsonObjectForMerge(const Value *value,int level) { if(value->type() != obj_type) diff --git a/src/rpc/rpcutils.h b/src/rpc/rpcutils.h index 61b715f6..9be1afb9 100644 --- a/src/rpc/rpcutils.h +++ b/src/rpc/rpcutils.h @@ -100,6 +100,7 @@ Array PerOutputDataEntries(const CTxOut& txout,mc_Script *lpScript,uint256 txid, Array PermissionEntries(const CTxOut& txout,mc_Script *lpScript,bool fLong); Object StreamEntry(const unsigned char *txid,uint32_t output_level); Object UpgradeEntry(const unsigned char *txid); +const unsigned char *GetChunkDataInRange(int64_t *out_size,unsigned char* hashes,int chunk_count,int64_t start,int64_t count); uint32_t GetFormattedData(mc_Script *lpScript,const unsigned char **elem,int64_t *out_size,unsigned char* hashes,int chunk_count,int64_t total_size); Value OpReturnEntry(const unsigned char *elem,size_t elem_size,uint256 txid, int vout); Value OpReturnFormatEntry(const unsigned char *elem,int64_t elem_size,uint256 txid, int vout, uint32_t format, string *format_text_out,uint32_t status); @@ -120,6 +121,7 @@ Array AssetArrayFromAmounts(mc_Buffer *asset_amounts,int issue_asset_id,uint256 void ParseRawAction(string action,bool& lock_it, bool& sign_it,bool& send_it); bool paramtobool(Value param); int paramtoint(Value param,bool check_for_min,int min_value,string error_message); +int64_t paramtoint64(Value param,bool check_for_min,int64_t min_value,string error_message); vector ParseBlockSetIdentifier(Value blockset_identifier); vector ParseRawFormattedData(const Value *value,uint32_t *data_format,mc_Script *lpDetailsScript,uint32_t in_options,uint32_t *out_options,int *errorCode,string *strError); void ParseRawDetails(const Value *value,mc_Script *lpDetails,mc_Script *lpDetailsScript,int *errorCode,string *strError); diff --git a/src/rpc/rpcwallet.cpp b/src/rpc/rpcwallet.cpp index 1176e015..08b57707 100644 --- a/src/rpc/rpcwallet.cpp +++ b/src/rpc/rpcwallet.cpp @@ -642,7 +642,10 @@ Value gettxoutdata(const Array& params, bool fHelp) mc_gState->m_TmpScript->SetScript((unsigned char*)(&pc1[0]),(size_t)(script1.end()-pc1),MC_SCR_TYPE_SCRIPTPUBKEY); uint32_t format; - string metadata=""; + unsigned char *chunk_hashes; + int chunk_count; + int64_t total_chunk_size,out_size; + uint32_t retrieve_status; size_t elem_size; const unsigned char *elem; @@ -669,6 +672,7 @@ Value gettxoutdata(const Array& params, bool fHelp) } elem=ptr; elem_size=size; + out_size=elem_size; } } if(elem == NULL) @@ -678,38 +682,49 @@ Value gettxoutdata(const Array& params, bool fHelp) } else { - mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(&format); - elem = mc_gState->m_TmpScript->GetData(mc_gState->m_TmpScript->GetNumElements()-1,&elem_size); +// mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(&format); + mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(&format,&chunk_hashes,&chunk_count,&total_chunk_size); + retrieve_status = GetFormattedData(mc_gState->m_TmpScript,&elem,&out_size,chunk_hashes,chunk_count,total_chunk_size); + elem_size=(size_t)out_size; + switch(retrieve_status & MC_OST_STATUS_MASK) + { + case MC_OST_ON_CHAIN: + case MC_OST_RETRIEVED: + break; + default: + throw JSONRPCError(RPC_OUTPUT_NOT_FOUND, "Data for this output is not available"); + } +// elem = mc_gState->m_TmpScript->GetData(mc_gState->m_TmpScript->GetNumElements()-1,&elem_size); } - int count,start; - count=elem_size; + int64_t count,start; + count=out_size; start=0; if (params.size() > 2) { - count=paramtoint(params[2],true,0,"Invalid count"); + count=paramtoint64(params[2],true,0,"Invalid count"); } if (params.size() > 3) { - start=paramtoint(params[3],false,0,"Invalid start"); + start=paramtoint64(params[3],false,0,"Invalid start"); } if(start < 0) { - start=elem_size+start; + start=out_size+start; if(start<0) { start=0; } } - if(start > (int)elem_size) + if(start > out_size) { start=elem_size; } - if(start+count > (int)elem_size) + if(start+count > out_size) { count=elem_size-start; } @@ -720,20 +735,24 @@ Value gettxoutdata(const Array& params, bool fHelp) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid start, must be 0 for text and JSON data"); } - if(count != (int)elem_size) + if(count != out_size) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid count, must include all text or JSON data"); } } -/* - if(mc_gState->m_Compatibility & MC_VCM_1_0) + if(count > 0x4000000) { - if( format == MC_SCR_DATA_FORMAT_UNKNOWN ) + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid count, must be below 64MB"); + } + + if(chunk_count > 1) + { + if(elem == NULL) { - return HexStr(elem+start,elem+start+count); + elem=GetChunkDataInRange(&out_size,chunk_hashes,chunk_count,start,count); } } -*/ + return OpReturnFormatEntry(elem+start,count,0,0,format,NULL); } From fcb56186605b9f62b8d7e679f3c20582b715dc97 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 27 Feb 2018 18:21:24 +0200 Subject: [PATCH 020/157] Off-chain data: publish --- src/rpc/rpcstreams.cpp | 105 +++++++++++++++++++++++++++++------------ 1 file changed, 75 insertions(+), 30 deletions(-) diff --git a/src/rpc/rpcstreams.cpp b/src/rpc/rpcstreams.cpp index 09b85cc9..c34dc0e2 100644 --- a/src/rpc/rpcstreams.cpp +++ b/src/rpc/rpcstreams.cpp @@ -488,7 +488,7 @@ Value publish(const Array& params, bool fHelp) Value publishfrom(const Array& params, bool fHelp) { - if (fHelp || params.size() != 4) + if (fHelp || params.size() < 4 || params.size() > 5) throw runtime_error("Help message not found\n"); mc_Script *lpScript=mc_gState->m_TmpBuffers->m_RpcScript3; @@ -501,6 +501,11 @@ Value publishfrom(const Array& params, bool fHelp) // Wallet comments CWalletTx wtx; + uint32_t in_options,out_options; + + in_options=MC_RFD_OPTION_NONE; + out_options=MC_RFD_OPTION_NONE; + vector addresses; vector fromaddresses; @@ -559,7 +564,27 @@ Value publishfrom(const Array& params, bool fHelp) throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid item-key-string: *"); } } - + + if(params.size() > 4 ) + { + if(params[4].type() != str_type) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid options"); + } + if( mc_gState->m_Features->OffChainData() == 0 ) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Format options are not supported by this protocol version"); + } + if(params[4].get_str() == "offchain") + { + in_options |= MC_RFD_OPTION_OFFCHAIN; + } + else + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid options"); + } + } + if(keys.size() > 1) { if( mc_gState->m_Features->MultipleStreamKeys() == 0 ) @@ -578,58 +603,78 @@ Value publishfrom(const Array& params, bool fHelp) string strError; int errorCode=RPC_INVALID_PARAMETER; - dataData=ParseRawFormattedData(&(params[3]),&data_format,lpDetailsScript,MC_RFD_OPTION_NONE,NULL,&errorCode,&strError); + vector vChunkHashes; + + dataData=ParseRawFormattedData(&(params[3]),&data_format,lpDetailsScript,MC_RFD_OPTION_NONE,&out_options,&errorCode,&strError); if(strError.size()) { throw JSONRPCError(errorCode, strError); } + size_t elem_size; + const unsigned char *elem; + CScript scriptOpReturn=CScript(); + lpDetailsScript->Clear(); lpDetailsScript->SetEntity(stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET); for(int k=0;k<(int)keys.size();k++) { lpDetailsScript->SetItemKey((unsigned char*)keys[k].get_str().c_str(),keys[k].get_str().size()); } - if( data_format != MC_SCR_DATA_FORMAT_UNKNOWN ) - { - lpDetailsScript->SetDataFormat(data_format); - } - - lpDetailsScript->AddElement(); - if(dataData.size()) + + if( (in_options & MC_RFD_OPTION_OFFCHAIN) == 0) { - lpDetailsScript->SetData(&dataData[0],dataData.size()); + if( data_format != MC_SCR_DATA_FORMAT_UNKNOWN ) + { + lpDetailsScript->SetDataFormat(data_format); + } } - - size_t elem_size; - const unsigned char *elem; - CScript scriptOpReturn=CScript(); for(int e=0;eGetNumElements();e++) { elem = lpDetailsScript->GetData(e,&elem_size); - if(e == (lpDetailsScript->GetNumElements() - 1) ) + if(elem_size > 0) { - if(elem_size > 0) - { - scriptOpReturn << OP_RETURN << vector(elem, elem + elem_size); - } - else - { - scriptOpReturn << OP_RETURN; - } + scriptOpReturn << vector(elem, elem + elem_size) << OP_DROP; + } + } + + lpDetailsScript->Clear(); + if(in_options & MC_RFD_OPTION_OFFCHAIN) + { + AppendOffChainFormatData(data_format,out_options,lpDetailsScript,dataData,&vChunkHashes,&errorCode,&strError); + if(strError.size()) + { + throw JSONRPCError(errorCode, strError); + } + elem = lpDetailsScript->GetData(0,&elem_size); + scriptOpReturn << vector(elem, elem + elem_size) << OP_DROP; + scriptOpReturn << OP_RETURN; + } + else + { + if(out_options & MC_RFD_OPTION_OFFCHAIN) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "chunks data type is not allowed with missing options field"); + } + lpDetailsScript->AddElement(); + if(dataData.size()) + { + lpDetailsScript->SetData(&dataData[0],dataData.size()); + } + elem = lpDetailsScript->GetData(0,&elem_size); + if(elem_size > 0) + { + scriptOpReturn << OP_RETURN << vector(elem, elem + elem_size); } else { - if(elem_size > 0) - { - scriptOpReturn << vector(elem, elem + elem_size) << OP_DROP; - } + scriptOpReturn << OP_RETURN; } - } - + } + lpScript->Clear(); EnsureWalletIsUnlocked(); From 697b55c0905b1a1f94d4761bcfbc78ddcba3c759 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 27 Feb 2018 18:35:37 +0200 Subject: [PATCH 021/157] minimum0data-fee, not implemented --- src/chainparams/paramlist.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/chainparams/paramlist.h b/src/chainparams/paramlist.h index e7c0a8ba..4bbcd5f1 100644 --- a/src/chainparams/paramlist.h +++ b/src/chainparams/paramlist.h @@ -184,8 +184,12 @@ static const mc_OneMultichainParam MultichainParamArray[] = "Minimum native currency per output (anti-dust), in raw units.\nIf set to -1, this is calculated from minimum-relay-fee."}, { "maximumperoutput" , "maximum-per-output" , MC_PRM_INT64 | MC_PRM_USER | MC_PRM_CLONE , -1, 100000000000000, 0,1000000000000000000, 0.0, 10001, 0, "-mc-maximumperoutput", - "minimumrelayfee","", + "minimumdatafee","", "Maximum native currency per output, in raw units."}, + { "minimumdatafee" , "minimum-data-fee" , + MC_PRM_INT64 | MC_PRM_USER | MC_PRM_CLONE , -1, 0, 0,1000000000, 0.0, 20003, 0, "-mc-minimumrelayfee", + "minimumrelayfee","", + "Minimum fee for publishing off-chain data items, per 1000 bytes, in raw units of native currency."}, { "minimumrelayfee" , "minimum-relay-fee" , MC_PRM_INT64 | MC_PRM_USER | MC_PRM_CLONE , -1, 0, 0,1000000000, 0.0, 10001, 0, "-mc-minimumrelayfee", "nativecurrencymultiple","", From 78189acf747c68a5e92c575998349b10419aa499 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 27 Feb 2018 18:49:09 +0200 Subject: [PATCH 022/157] Off-chain data, commit on exit --- src/wallet/wallettxs.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/wallet/wallettxs.cpp b/src/wallet/wallettxs.cpp index 43dec4fb..a0ca8431 100644 --- a/src/wallet/wallettxs.cpp +++ b/src/wallet/wallettxs.cpp @@ -297,6 +297,7 @@ int mc_WalletTxs::SetMode(uint32_t mode, uint32_t mask) int mc_WalletTxs::Destroy() { + if(m_Database) { delete m_Database; @@ -304,6 +305,7 @@ int mc_WalletTxs::Destroy() if(m_ChunkDB) { + m_ChunkDB->Commit(-1); delete m_ChunkDB; } From 9e9c171edc8bd4c783875d2dfc4f51ef6318a725 Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 28 Feb 2018 08:49:18 +0200 Subject: [PATCH 023/157] minimum-offchain-fee, not implemented --- src/chainparams/paramlist.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/chainparams/paramlist.h b/src/chainparams/paramlist.h index 4bbcd5f1..0e0c71e0 100644 --- a/src/chainparams/paramlist.h +++ b/src/chainparams/paramlist.h @@ -184,10 +184,10 @@ static const mc_OneMultichainParam MultichainParamArray[] = "Minimum native currency per output (anti-dust), in raw units.\nIf set to -1, this is calculated from minimum-relay-fee."}, { "maximumperoutput" , "maximum-per-output" , MC_PRM_INT64 | MC_PRM_USER | MC_PRM_CLONE , -1, 100000000000000, 0,1000000000000000000, 0.0, 10001, 0, "-mc-maximumperoutput", - "minimumdatafee","", + "minimumoffchainfee","", "Maximum native currency per output, in raw units."}, - { "minimumdatafee" , "minimum-data-fee" , - MC_PRM_INT64 | MC_PRM_USER | MC_PRM_CLONE , -1, 0, 0,1000000000, 0.0, 20003, 0, "-mc-minimumrelayfee", + { "minimumoffchainfee" , "minimum-offchain-fee" , + MC_PRM_INT64 | MC_PRM_USER | MC_PRM_CLONE , -1, 0, 0,1000000000, 0.0, 20003, 0, "-mc-minimumoffchainfee", "minimumrelayfee","", "Minimum fee for publishing off-chain data items, per 1000 bytes, in raw units of native currency."}, { "minimumrelayfee" , "minimum-relay-fee" , From 0f28ab0bf10059688f373496e6cf10d8bcee53fb Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 28 Feb 2018 09:23:17 +0200 Subject: [PATCH 024/157] Off-chain items, help messages --- src/rpc/rpchelp.cpp | 44 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/src/rpc/rpchelp.cpp b/src/rpc/rpchelp.cpp index f4ca707e..b99ca5f5 100644 --- a/src/rpc/rpchelp.cpp +++ b/src/rpc/rpchelp.cpp @@ -2949,7 +2949,7 @@ void mc_InitRPCHelpMap13() )); mapHelpStrings.insert(std::make_pair("publish", - "publish \"stream-identifier\" \"key\"|keys \"data-hex\"|data-obj \n" + "publish \"stream-identifier\" \"key\"|keys \"data-hex\"|data-obj \"options\" \n" "\nPublishes stream item\n" + HelpRequiringPassphraseWrapper() + "\nArguments:\n" @@ -2968,6 +2968,12 @@ void mc_InitRPCHelpMap13() " {\n" " \"text\" : \"data-text\" (string, required) Data string\n" " }\n" + " or\n" + "3. data-chunks (object, required) Offchain chunk hashes\n" + " {\n" + " \"chunks\" : chunk-hashes (array, required) Array of chunk hashes created by storechunk\n" + " }\n" + "4. \"options\" (string, optional) Should be \"offchain\" or omitted\n" "\nResult:\n" "\"transactionid\" (string) The transaction id.\n" "\nExamples:\n" @@ -2976,7 +2982,7 @@ void mc_InitRPCHelpMap13() )); mapHelpStrings.insert(std::make_pair("publishfrom", - "publishfrom \"from-address\" \"stream-identifier\" \"key\"|keys \"data-hex\"|data-obj \n" + "publishfrom \"from-address\" \"stream-identifier\" \"key\"|keys \"data-hex\"|data-obj \"options\" \n" "\nPublishes stream item from specific address\n" + HelpRequiringPassphraseWrapper() + "\nArguments:\n" @@ -2996,6 +3002,12 @@ void mc_InitRPCHelpMap13() " {\n" " \"text\" : \"data-text\" (string, required) Data string\n" " }\n" + " or\n" + "4. data-chunks (object, required) Offchain chunk hashes\n" + " {\n" + " \"chunks\" : chunk-hashes (array, required) Array of chunk hashes created by storechunk\n" + " }\n" + "5. \"options\" (string, optional) Should be \"offchain\" or omitted\n" "\nResult:\n" "\"transactionid\" (string) The transaction id.\n" "\nExamples:\n" @@ -3737,6 +3749,7 @@ void mc_InitRPCHelpMap16() "publish-new-stream-item (object, required) A json object with stream item\n" " {\n" " \"for\" : \"stream-identifier\" (string, required) Stream identifier - one of the following: stream txid, stream reference, stream name.\n" + " \"options\" : \"options\" (string, optional) Should be \"offchain\" or omitted\n" " \"key\" : \"key\" (string, optional, default: \"\") Item key\n" " or\n" " \"keys\" : keys (array, optional) Item keys, array of strings\n" @@ -3747,10 +3760,15 @@ void mc_InitRPCHelpMap16() " \"json\" : data-json (object, required) Valid JSON string\n" " }\n" " or\n" - " \"data\" : (object, required) JSON data object\n" + " \"data\" : (object, required) Text data object\n" " {\n" " \"text\" : \"data-text\" (string, required) Data string\n" " }\n" + " or\n" + " \"data\" : (object, required) Offchain chunk hashes\n" + " {\n" + " \"chunks\" : chunk-hashes (array, required) Array of chunk hashes created by storechunk\n" + " }\n" " }\n" " or\n" "create-new-upgrade (object, required) A json object with new upgrade details\n" @@ -3796,6 +3814,7 @@ void mc_InitRPCHelpMap16() "publish-new-stream-item (object, required) A json object with stream item\n" " {\n" " \"for\" : \"stream-identifier\" (string, required) Stream identifier - one of the following: stream txid, stream reference, stream name.\n" + " \"options\" : \"options\" (string, optional) Should be \"offchain\" or omitted\n" " \"key\" : \"key\" (string, optional, default: \"\") Item key\n" " or\n" " \"keys\" : keys (array, optional) Item keys, array of strings\n" @@ -3810,6 +3829,11 @@ void mc_InitRPCHelpMap16() " {\n" " \"text\" : \"data-text\" (string, required) Data string\n" " }\n" + " or\n" + " \"data\" : (object, required) Text chunk hashes\n" + " {\n" + " \"chunks\" : chunk-hashes (array, required) Array of chunk hashes created by storechunk\n" + " }\n" " }\n" )); @@ -3921,7 +3945,19 @@ void mc_InitRPCHelpMap17() + HelpExampleRpc("liststreampublisheritems", "\"test-stream\", \"1M72Sfpbz1BPpXFHz9m3CdqATR44Jvaydd\", \"jsonobjectmerge,ignore,recursive\"") )); - + mapHelpStrings.insert(std::make_pair("storechunk", + "storechunk \"data-hex\" \n" + "\nStores chunk of data in local wallet. Returns hash of the data, which can be used later when publishing offchain stream items.\n" + "\nArguments:\n" + "1. \"data-hex\" (string, required) The hex string of the data chunk\n" + "\nResult:\n" + "\"hex\" (string) The chunk hash in hex\n" + "\nExamples:\n" + + HelpExampleCli("storechunk", "\"f4c3dd510dd55761015c9d96bff7793b0d501dd6f01a959fd7\"") + + HelpExampleRpc("storechunk", "\"f4c3dd510dd55761015c9d96bff7793b0d501dd6f01a959fd7\"") + )); + + mapHelpStrings.insert(std::make_pair("AAAAAAA", "" )); From 86d9740c4b5cb8cbad8507c59fcc7799e52ab871 Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 28 Feb 2018 11:32:09 +0200 Subject: [PATCH 025/157] Fixed start/count calcualtion in gettxoutdata --- src/rpc/rpcwallet.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rpc/rpcwallet.cpp b/src/rpc/rpcwallet.cpp index 08b57707..70868163 100644 --- a/src/rpc/rpcwallet.cpp +++ b/src/rpc/rpcwallet.cpp @@ -722,11 +722,11 @@ Value gettxoutdata(const Array& params, bool fHelp) if(start > out_size) { - start=elem_size; + start=out_size; } if(start+count > out_size) { - count=elem_size-start; + count=out_size-start; } if( (format == MC_SCR_DATA_FORMAT_UBJSON) || (format == MC_SCR_DATA_FORMAT_UTF8) ) From 340af044bb0d95044596e032c408a8a27aa20eaa Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 20 Mar 2018 10:58:42 +0200 Subject: [PATCH 026/157] Off-chain items tweaks --- src/chainparams/paramlist.h | 2 +- src/rpc/rpclist.cpp | 2 +- src/rpc/rpcrawdata.cpp | 7 +++++-- src/rpc/rpcutils.cpp | 1 + 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/chainparams/paramlist.h b/src/chainparams/paramlist.h index 0e0c71e0..8ff21da9 100644 --- a/src/chainparams/paramlist.h +++ b/src/chainparams/paramlist.h @@ -37,7 +37,7 @@ static const mc_OneMultichainParam MultichainParamArray[] = { "maximumchunksize" , "maximum-chunk-size" , MC_PRM_UINT32 | MC_PRM_USER | MC_PRM_CLONE , -1, 16777216, 256, 67108864, 0.0, 20003, 0, "-mc-maximumchunksize", "maximumchunkcount","", - "Maximum chunk size in bytes."}, + "Maximum chunk size for off-chain items in bytes."}, { "maximumchunkcount" , "maximum-chunk-count" , MC_PRM_UINT32 | MC_PRM_USER | MC_PRM_CLONE , -1, 128, 16, 1024, 0.0, 20003, 0, "-mc-maximumchunkcount", "timingupgrademingap","", diff --git a/src/rpc/rpclist.cpp b/src/rpc/rpclist.cpp index 0c97698f..ecae1836 100644 --- a/src/rpc/rpclist.cpp +++ b/src/rpc/rpclist.cpp @@ -241,7 +241,7 @@ static const CRPCCommand vRPCCommands[] = { "wallet", "liststreamblockitems", &liststreamblockitems, false, false, false }, { "wallet", "getstreamkeysummary", &getstreamkeysummary, false, false, true }, { "wallet", "getstreampublishersummary", &getstreampublishersummary, false, false, true }, - { "wallet", "storechunk", &storechunk, false, false, true }, + { "hidden", "storechunk", &storechunk, false, false, true }, /* MCHN END */ { "wallet", "setaccount", &setaccount, true, false, true }, diff --git a/src/rpc/rpcrawdata.cpp b/src/rpc/rpcrawdata.cpp index b36be30e..fc4493d0 100644 --- a/src/rpc/rpcrawdata.cpp +++ b/src/rpc/rpcrawdata.cpp @@ -1077,12 +1077,15 @@ CScript RawDataScriptPublish(Value *param,mc_EntityDetails *entity,uint32_t *dat } else { - *strError=string("Invalid options"); + if(d.value_.get_str().size()) + { + *strError=string("Stream item options must be offchain or empty"); + } } } else { - *strError=string("Invalid options"); + *strError=string("Stream item options must be offchain or empty"); } field_parsed=true; missing_key=false; diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index 9bc7cf53..b347141e 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -1049,6 +1049,7 @@ uint32_t GetFormattedData(mc_Script *lpScript,const unsigned char **elem,int64_t else { status=MC_OST_UNKNOWN; + *out_size=total_size; return status; } ptr+=MC_CDB_CHUNK_HASH_SIZE; From d8615ac089cff5c051c5bcc70f38ae0e0c1333b4 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 20 Mar 2018 17:09:10 +0200 Subject: [PATCH 027/157] RPC binary cache --- src/Makefile.am | 1 + src/rpc/rpccache.cpp | 96 ++++++++++++++++ src/rpc/rpchelp.cpp | 74 +++++++++++++ src/rpc/rpclist.cpp | 4 + src/rpc/rpcrawdata.cpp | 217 +++++++++++++++++++++++++------------ src/rpc/rpcserver.h | 3 + src/rpc/rpcutils.cpp | 39 +++++++ src/rpc/rpcutils.h | 3 + src/rpc/rpcwalletutils.cpp | 72 +++++++++++- 9 files changed, 435 insertions(+), 74 deletions(-) create mode 100644 src/rpc/rpccache.cpp diff --git a/src/Makefile.am b/src/Makefile.am index 4c5da512..90ae7135 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -191,6 +191,7 @@ libbitcoin_server_a_SOURCES = \ json/json_spirit_ubjson.cpp \ rpc/rpcrawdata.cpp \ rpc/rpcutils.cpp \ + rpc/rpccache.cpp \ rpc/rpchelp.cpp \ rpc/rpcblockchain.cpp \ rpc/rpcmining.cpp \ diff --git a/src/rpc/rpccache.cpp b/src/rpc/rpccache.cpp new file mode 100644 index 00000000..a9ca7d4e --- /dev/null +++ b/src/rpc/rpccache.cpp @@ -0,0 +1,96 @@ +// Copyright (c) 2014-2017 Coin Sciences Ltd +// MultiChain code distributed under the GPLv3 license, see COPYING file. + +#include "rpc/rpcutils.h" + + +Value createbinarycache(const Array& params, bool fHelp) +{ + if (fHelp || params.size() != 0) + throw runtime_error("Help message not found\n"); + + unsigned char dest[8]; + string str; + int fHan; + + while(true) + { + GetRandBytes((unsigned char*)dest, 8); + str=EncodeBase58((unsigned char*)(&dest[0]),(unsigned char*)(&dest[0])+8); + fHan=mc_BinaryCacheFile(str,0); + if(fHan > 0) + { + close(fHan); + } + else + { + return str; + } + } + + return Value::null; +} + +Value appendbinarycache(const Array& params, bool fHelp) +{ + vector vValue; + int64_t size; + + if (fHelp || params.size() != 2) + throw runtime_error("Help message not found\n"); + + if(params[0].type() != str_type) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "cache identifier should be string"); + } + + if(params[1].type() != str_type) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "data should be hexadecimal string"); + } + + bool fIsHex; + vValue=ParseHex(params[1].get_str().c_str(),fIsHex); + if(!fIsHex) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "data should be hexadecimal string"); + } + + int fHan=mc_BinaryCacheFile(params[0].get_str(),1); + if(fHan <= 0) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Binary cache item with this identifier not found"); + } + + size=lseek64(fHan,0,SEEK_END); + + if(vValue.size()) + { + if(write(fHan,&vValue[0],vValue.size()) != (int)vValue.size()) + { + close(fHan); + throw JSONRPCError(RPC_INTERNAL_ERROR, "Cannot store binary cache item"); + } + } + + close(fHan); + + size+=vValue.size(); + + return size; +} + +Value clearbinarycache(const Array& params, bool fHelp) +{ + if (fHelp || params.size() != 1) + throw runtime_error("Help message not found\n"); + + if(params[0].type() != str_type) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "cache identifier should be string"); + } + + mc_RemoveBinaryCacheFile(params[0].get_str()); + + return Value::null; +} diff --git a/src/rpc/rpchelp.cpp b/src/rpc/rpchelp.cpp index b99ca5f5..7a125db1 100644 --- a/src/rpc/rpchelp.cpp +++ b/src/rpc/rpchelp.cpp @@ -2969,10 +2969,16 @@ void mc_InitRPCHelpMap13() " \"text\" : \"data-text\" (string, required) Data string\n" " }\n" " or\n" + "3. data-cached (object, required) Binary raw data created with appendbinarycache\n" + " {\n" + " \"cache\" : \"identifier\" (string, required) Binary cache identifier\n" + " }\n" +/* "3. data-chunks (object, required) Offchain chunk hashes\n" " {\n" " \"chunks\" : chunk-hashes (array, required) Array of chunk hashes created by storechunk\n" " }\n" + */ "4. \"options\" (string, optional) Should be \"offchain\" or omitted\n" "\nResult:\n" "\"transactionid\" (string) The transaction id.\n" @@ -3003,10 +3009,16 @@ void mc_InitRPCHelpMap13() " \"text\" : \"data-text\" (string, required) Data string\n" " }\n" " or\n" + "4. data-cached (object, required) Binary raw data created with appendbinarycache\n" + " {\n" + " \"cache\" : \"identifier\" (string, required) Binary cache identifier\n" + " }\n" +/* "4. data-chunks (object, required) Offchain chunk hashes\n" " {\n" " \"chunks\" : chunk-hashes (array, required) Array of chunk hashes created by storechunk\n" " }\n" + */ "5. \"options\" (string, optional) Should be \"offchain\" or omitted\n" "\nResult:\n" "\"transactionid\" (string) The transaction id.\n" @@ -3710,6 +3722,11 @@ void mc_InitRPCHelpMap16() " \"text\" : \"data-text\" (string, required) Data string\n" " }\n" " or\n" + "data-cached (object, required) Binary raw data created with appendbinarycache\n" + " {\n" + " \"cache\" : \"identifier\" (string, required) Binary cache identifier\n" + " }\n" + " or\n" "issue-details (object, required) A json object with issue metadata\n" " {\n" " \"create\" : \"asset\" (string, required) asset\n" @@ -3765,10 +3782,16 @@ void mc_InitRPCHelpMap16() " \"text\" : \"data-text\" (string, required) Data string\n" " }\n" " or\n" + " \"data\" (object, required) Binary raw data created with appendbinarycache\n" + " {\n" + " \"cache\" : \"identifier\" (string, required) Binary cache identifier\n" + " }\n" +/* " \"data\" : (object, required) Offchain chunk hashes\n" " {\n" " \"chunks\" : chunk-hashes (array, required) Array of chunk hashes created by storechunk\n" " }\n" + */ " }\n" " or\n" "create-new-upgrade (object, required) A json object with new upgrade details\n" @@ -3811,6 +3834,11 @@ void mc_InitRPCHelpMap16() " \"text\" : \"data-text\" (string, required) Data string\n" " }\n" " or\n" + "data-cached (object, required) Binary raw data created with appendbinarycache\n" + " {\n" + " \"cache\" : \"identifier\" (string, required) Binary cache identifier\n" + " }\n" + " or\n" "publish-new-stream-item (object, required) A json object with stream item\n" " {\n" " \"for\" : \"stream-identifier\" (string, required) Stream identifier - one of the following: stream txid, stream reference, stream name.\n" @@ -3830,10 +3858,16 @@ void mc_InitRPCHelpMap16() " \"text\" : \"data-text\" (string, required) Data string\n" " }\n" " or\n" + " \"data\" : (object, required) Binary raw data created with appendbinarycache\n" + " {\n" + " \"cache\" : \"identifier\" (string, required) Binary cache identifier\n" + " }\n" +/* " \"data\" : (object, required) Text chunk hashes\n" " {\n" " \"chunks\" : chunk-hashes (array, required) Array of chunk hashes created by storechunk\n" " }\n" + */ " }\n" )); @@ -3891,6 +3925,11 @@ void mc_InitRPCHelpMap16() " {\n" " \"text\" : \"data-text\" (string, required) Data string\n" " }\n" + " or\n" + " data-cached (object, required) Binary raw data created with appendbinarycache\n" + " {\n" + " \"cache\" : \"identifier\" (string, required) Binary cache identifier\n" + " }\n" " }\n" " }\n" @@ -3957,6 +3996,41 @@ void mc_InitRPCHelpMap17() + HelpExampleRpc("storechunk", "\"f4c3dd510dd55761015c9d96bff7793b0d501dd6f01a959fd7\"") )); + mapHelpStrings.insert(std::make_pair("createbinarycache", + "createbinarycache \n" + "\nReturns random string, which can be used as binary cache item identifier\n" + "\nArguments:\n" + "\nResult:\n" + "\"identifier\" (string) Binary cache item identifier\n" + "\nExamples:\n" + + HelpExampleCli("createbinarycache","") + + HelpExampleRpc("createbinarycache","") + )); + + mapHelpStrings.insert(std::make_pair("appendbinarycache", + "appendbinarycache \"identifier\" \"data-hex\" \n" + "\nAppends data to binary cache.\n" + "\nArguments:\n" + "1. \"identifier\" (string, required) Binary cache item identifier\n" + "2. \"data-hex\" (string, required) The hex string to be added to binary cache item\n" + "\nResult:\n" + "size (numeric) Size of the binary cache item\n" + "\nExamples:\n" + + HelpExampleCli("appendbinarycache", "\"TjnVWwHYEg4\" \"f4c3dd510dd55761015c9d96bff7793b0d501dd6f01a959fd7\"") + + HelpExampleRpc("appendbinarycache", "\"TjnVWwHYEg4\",\"f4c3dd510dd55761015c9d96bff7793b0d501dd6f01a959fd7\"") + )); + + mapHelpStrings.insert(std::make_pair("clearbinarycache", + "clearbinarycache \"identifier\" \n" + "\nClear binary cache item\n" + "\nArguments:\n" + "1. \"identifier\" (string, required) Binary cache item identifier, \"*\" - to clear all items\n" + "\nResult:\n" + "\nExamples:\n" + + HelpExampleCli("clearbinarycache", "\"TjnVWwHYEg4\"") + + HelpExampleRpc("clearbinarycache", "\"TjnVWwHYEg4\"") + )); + mapHelpStrings.insert(std::make_pair("AAAAAAA", "" diff --git a/src/rpc/rpclist.cpp b/src/rpc/rpclist.cpp index ecae1836..afce9b9d 100644 --- a/src/rpc/rpclist.cpp +++ b/src/rpc/rpclist.cpp @@ -130,6 +130,10 @@ static const CRPCCommand vRPCCommands[] = { "util", "verifymessage", &verifymessage, true, false, false }, { "util", "estimatefee", &estimatefee, true, true, false }, { "util", "estimatepriority", &estimatepriority, true, true, false }, + + { "util", "createbinarycache", &createbinarycache, true, false, false }, + { "util", "appendbinarycache", &appendbinarycache, true, false, false }, + { "util", "clearbinarycache", &clearbinarycache, true, false, false }, /* Not shown in help */ { "hidden", "invalidateblock", &invalidateblock, true, true, false }, diff --git a/src/rpc/rpcrawdata.cpp b/src/rpc/rpcrawdata.cpp index fc4493d0..42892717 100644 --- a/src/rpc/rpcrawdata.cpp +++ b/src/rpc/rpcrawdata.cpp @@ -87,7 +87,7 @@ uint32_t ParseRawDataParamType(Value *param,mc_EntityDetails *given_entity,mc_En goto exitlbl; } } - if( (d.name_ == "text") || (d.name_ == "json") ) + if( (d.name_ == "text") || (d.name_ == "json") || (d.name_ == "cache") ) { if( mc_gState->m_Features->FormattedData() == 0 ) { @@ -255,68 +255,139 @@ vector ParseRawFormattedData(const Value *value,uint32_t *data_fo vValue=vector (script,script+bytes); *data_format=MC_SCR_DATA_FORMAT_UBJSON; } - if(d.name_ == "chunks") + if(d.name_ == "cache") { - if(mc_gState->m_Features->OffChainData()) + if(d.value_.type() == str_type) { - if(d.value_.type() == array_type) + vValue=vector (d.value_.get_str().begin(),d.value_.get_str().end()); + vValue.push_back(0); + if(in_options & MC_RFD_OPTION_OFFCHAIN) { - Array arr=d.value_.get_array(); - for(int i=0;i<(int)arr.size();i++) + if(out_options) + { + *out_options |= MC_RFD_OPTION_CACHE; + } + } + else + { + int fHan=mc_BinaryCacheFile((char*)&vValue[0],0); + if(fHan <= 0) + { + *strError="Binary cache item with this identifier not found"; + } + int64_t total_size=0; + if(strError->size() == 0) + { + total_size=lseek64(fHan,0,SEEK_END); + if(lseek64(fHan,0,SEEK_SET) != 0) + { + *strError="Cannot read binary cache item"; + *errorCode=RPC_INTERNAL_ERROR; + close(fHan); + } + } + if(strError->size() == 0) + { + if(total_size > MAX_OP_RETURN_RELAY) + { + *strError="Cannot read binary cache item"; + *errorCode=RPC_NOT_SUPPORTED; + close(fHan); + } + } + if(strError->size() == 0) { - if(strError->size() == 0) + if(total_size) { - if(arr[i].type() == str_type) + mc_gState->m_TmpBuffers->m_RpcChunkScript1->Clear(); + mc_gState->m_TmpBuffers->m_RpcChunkScript1->Resize(total_size,1); + unsigned char* ptr=mc_gState->m_TmpBuffers->m_RpcChunkScript1->m_lpData; + if(read(fHan,ptr,total_size) != total_size) { - vector vHash; - bool fIsHex; - vHash=ParseHex(arr[i].get_str().c_str(),fIsHex); - if(!fIsHex) - { - *strError=string("Chunk hash should be hexadecimal string"); - } - else + *errorCode=RPC_INTERNAL_ERROR; + *strError="Cannot read binary cache item"; + } + close(fHan); + vValue=vector (ptr,ptr+total_size); + } + else + { + vValue.clear(); + } + } + } + } + else + { + *strError=string("cache identifier in data object should be string"); + } + *data_format=MC_SCR_DATA_FORMAT_UNKNOWN; + } + else + { + if(d.name_ == "chunks") + { + if(mc_gState->m_Features->OffChainData()) + { + if(d.value_.type() == array_type) + { + Array arr=d.value_.get_array(); + for(int i=0;i<(int)arr.size();i++) + { + if(strError->size() == 0) + { + if(arr[i].type() == str_type) { - if(vHash.size() != MC_CDB_CHUNK_HASH_SIZE) + vector vHash; + bool fIsHex; + vHash=ParseHex(arr[i].get_str().c_str(),fIsHex); + if(!fIsHex) { - *strError=strprintf("Chunk hash should be %d bytes long",MC_CDB_CHUNK_HASH_SIZE); + *strError=string("Chunk hash should be hexadecimal string"); } else - { - uint256 hash; - hash.SetHex(arr[i].get_str()); - - vValue.insert(vValue.end(),(unsigned char*)&hash,(unsigned char*)&hash+MC_CDB_CHUNK_HASH_SIZE); - } - } - } + { + if(vHash.size() != MC_CDB_CHUNK_HASH_SIZE) + { + *strError=strprintf("Chunk hash should be %d bytes long",MC_CDB_CHUNK_HASH_SIZE); + } + else + { + uint256 hash; + hash.SetHex(arr[i].get_str()); + + vValue.insert(vValue.end(),(unsigned char*)&hash,(unsigned char*)&hash+MC_CDB_CHUNK_HASH_SIZE); + } + } + } + } } } + else + { + *strError=string("value in data object should be array"); + } + + if(out_options) + { + *out_options |= MC_RFD_OPTION_OFFCHAIN; + } + *data_format=MC_SCR_DATA_FORMAT_UNKNOWN; } else { - *strError=string("value in data object should be array"); + *errorCode=RPC_NOT_SUPPORTED; + *strError="Unsupported item data type: " + d.name_; } - - if(out_options) - { - *out_options |= MC_RFD_OPTION_OFFCHAIN; - } - *data_format=MC_SCR_DATA_FORMAT_UNKNOWN; } else { - *errorCode=RPC_NOT_SUPPORTED; - *strError="Unsupported item data type: " + d.name_; + if(*data_format == MC_SCR_DATA_FORMAT_UNKNOWN) + { + throw JSONRPCError(RPC_NOT_SUPPORTED, "Unsupported item data type: " + d.name_); + } } } - else - { - if(*data_format == MC_SCR_DATA_FORMAT_UNKNOWN) - { - throw JSONRPCError(RPC_NOT_SUPPORTED, "Unsupported item data type: " + d.name_); - } - } } } } @@ -388,7 +459,7 @@ CScript RawDataScriptFormatted(Value *param,uint32_t *data_format,mc_Script *lpD BOOST_FOREACH(const Pair& d, param->get_obj()) { field_parsed=false; - if( (d.name_ == "text") || (d.name_ == "json") ) + if( (d.name_ == "text") || (d.name_ == "json") || (d.name_ == "cache") ) { if(!missing_data) { @@ -1004,6 +1075,39 @@ CScript RawDataScriptPublish(Value *param,mc_EntityDetails *entity,uint32_t *dat in_options=MC_RFD_OPTION_NONE; out_options=MC_RFD_OPTION_NONE; vKeys.clear(); + BOOST_FOREACH(const Pair& d, param->get_obj()) + { + if(d.name_ == "options") + { + if( mc_gState->m_Features->OffChainData() == 0 ) + { + *errorCode=RPC_NOT_SUPPORTED; + *strError=string("Format options are not supported by this protocol version"); + goto exitlbl; + } + if(d.value_.type() != null_type && (d.value_.type()==str_type)) + { + if(d.value_.get_str() == "offchain") + { + in_options |= MC_RFD_OPTION_OFFCHAIN; + } + else + { + if(d.value_.get_str().size()) + { + *strError=string("Stream item options must be offchain or empty"); + } + } + } + else + { + *strError=string("Stream item options must be offchain or empty"); + } + field_parsed=true; + missing_key=false; + } + } + BOOST_FOREACH(const Pair& d, param->get_obj()) { field_parsed=false; @@ -1057,38 +1161,13 @@ CScript RawDataScriptPublish(Value *param,mc_EntityDetails *entity,uint32_t *dat { *strError=string("data field can appear only once in the object"); } - vValue=ParseRawFormattedData(&(d.value_),data_format,lpDetailsScript,MC_RFD_OPTION_NONE,&out_options,errorCode,strError); + vValue=ParseRawFormattedData(&(d.value_),data_format,lpDetailsScript,in_options,&out_options,errorCode,strError); field_parsed=true; missing_data=false; } if(d.name_ == "options") { - if( mc_gState->m_Features->OffChainData() == 0 ) - { - *errorCode=RPC_NOT_SUPPORTED; - *strError=string("Format options are not supported by this protocol version"); - goto exitlbl; - } - if(d.value_.type() != null_type && (d.value_.type()==str_type)) - { - if(d.value_.get_str() == "offchain") - { - in_options |= MC_RFD_OPTION_OFFCHAIN; - } - else - { - if(d.value_.get_str().size()) - { - *strError=string("Stream item options must be offchain or empty"); - } - } - } - else - { - *strError=string("Stream item options must be offchain or empty"); - } field_parsed=true; - missing_key=false; } if(d.name_ == "for")field_parsed=true; // if(d.name_ == "format")field_parsed=true; diff --git a/src/rpc/rpcserver.h b/src/rpc/rpcserver.h index 44aa2c00..d8e8a077 100644 --- a/src/rpc/rpcserver.h +++ b/src/rpc/rpcserver.h @@ -194,6 +194,9 @@ extern json_spirit::Value sendtoaddress(const json_spirit::Array& params, bool f /* MCHN START */ extern json_spirit::Value createkeypairs(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value getaddresses(const json_spirit::Array& params, bool fHelp); +extern json_spirit::Value createbinarycache(const json_spirit::Array& params, bool fHelp); +extern json_spirit::Value appendbinarycache(const json_spirit::Array& params, bool fHelp); +extern json_spirit::Value clearbinarycache(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value combineunspent(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value grantcmd(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value revokecmd(const json_spirit::Array& params, bool fHelp); diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index b347141e..f1283645 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -3964,3 +3964,42 @@ Value mc_MergeValues(const Value *value1,const Value *value2,uint32_t mode,int l return result; } + +int mc_BinaryCacheFile(string id,int mode) +{ + char dir_name[MC_DCT_DB_MAX_PATH]; + char file_name[MC_DCT_DB_MAX_PATH]; + int flags; + + + string str_file_name=strprintf("cache/%s",id); + + mc_GetFullFileName(mc_gState->m_Params->NetworkName(),"cache","",MC_FOM_RELATIVE_TO_DATADIR | MC_FOM_CREATE_DIR,dir_name); + mc_CreateDir(dir_name); + mc_GetFullFileName(mc_gState->m_Params->NetworkName(),str_file_name.c_str(),"",MC_FOM_RELATIVE_TO_DATADIR,file_name); + + flags=O_RDONLY; + if(mode) + { + flags=O_CREAT | O_RDWR; + } + return open(file_name,_O_BINARY | flags, S_IRUSR | S_IWUSR); +} + +void mc_RemoveBinaryCacheFile(string id) +{ + char file_name[MC_DCT_DB_MAX_PATH]; + + if(id == "*") + { + mc_RemoveDir(mc_gState->m_Params->NetworkName(),"cache"); + return; + } + + string str_file_name=strprintf("cache/%s",id); + + mc_GetFullFileName(mc_gState->m_Params->NetworkName(),str_file_name.c_str(),"",MC_FOM_RELATIVE_TO_DATADIR,file_name); + + unlink(file_name); +} + diff --git a/src/rpc/rpcutils.h b/src/rpc/rpcutils.h index 9be1afb9..f64d1abe 100644 --- a/src/rpc/rpcutils.h +++ b/src/rpc/rpcutils.h @@ -60,6 +60,7 @@ using namespace json_spirit; #define MC_RFD_OPTION_NONE 0x00000000 #define MC_RFD_OPTION_INLINE 0x00000001 #define MC_RFD_OPTION_OFFCHAIN 0x00000002 +#define MC_RFD_OPTION_CACHE 0x00000004 #define MC_OST_UNDEFINED 0x00000000 #define MC_OST_UNKNOWN 0x00000001 @@ -129,6 +130,8 @@ bool mc_IsJsonObjectForMerge(const Value *value,int level); Value mc_MergeValues(const Value *value1,const Value *value2,uint32_t mode,int level,int *error); Value mc_ExtractDetailsJSONObject(const unsigned char *script,uint32_t total); void AppendOffChainFormatData(uint32_t data_format,uint32_t out_options,mc_Script *lpDetailsScript,vector& vValue,vector* vChunkHashes,int *errorCode,string *strError); +int mc_BinaryCacheFile(string id,int mode); +void mc_RemoveBinaryCacheFile(string id); #endif /* RPCMULTICHAINUTILS_H */ diff --git a/src/rpc/rpcwalletutils.cpp b/src/rpc/rpcwalletutils.cpp index c00822cb..c19bc575 100644 --- a/src/rpc/rpcwalletutils.cpp +++ b/src/rpc/rpcwalletutils.cpp @@ -756,8 +756,11 @@ void AppendOffChainFormatData(uint32_t data_format, lpDetailsScript->Clear(); int chunk_count; - int tail_size,size; + int tail_size,size,max_chunk_size; + int64_t total_size; + int fHan; int err; + unsigned char *ptr; uint256 hash; mc_TxEntity entity; entity.Zero(); @@ -792,10 +795,45 @@ void AppendOffChainFormatData(uint32_t data_format, { chunk_count=0; tail_size=0; + ptr=NULL; + fHan=-1; if(vValue.size()) { - chunk_count=((int)vValue.size()-1)/MAX_CHUNK_SIZE+1; - tail_size=(int)vValue.size()-(chunk_count-1)*MAX_CHUNK_SIZE; + if(out_options & MC_RFD_OPTION_CACHE) + { + fHan=mc_BinaryCacheFile((char*)&vValue[0],0); + if(fHan <= 0) + { + *strError="Binary cache item with this identifier not found"; + return; + } + total_size=lseek64(fHan,0,SEEK_END); + if(lseek64(fHan,0,SEEK_SET) != 0) + { + *strError="Cannot read binary cache item"; + close(fHan); + return; + } + if(total_size) + { + chunk_count=(int)((total_size-1)/MAX_CHUNK_SIZE)+1; + } + tail_size=(int)(total_size-(int64_t)(chunk_count-1)*MAX_CHUNK_SIZE); + } + else + { + chunk_count=((int)vValue.size()-1)/MAX_CHUNK_SIZE+1; + tail_size=(int)vValue.size()-(chunk_count-1)*MAX_CHUNK_SIZE; + } + max_chunk_size=tail_size; + if(chunk_count > 1) + { + max_chunk_size=MAX_CHUNK_SIZE; + } + mc_gState->m_TmpBuffers->m_RpcChunkScript1->Clear(); + mc_gState->m_TmpBuffers->m_RpcChunkScript1->Resize(max_chunk_size,1); + ptr=mc_gState->m_TmpBuffers->m_RpcChunkScript1->m_lpData; + lpDetailsScript->SetChunkDefHeader(data_format,chunk_count); for(int i=0;im_TmpBuffers->m_RpcHasher1->DoubleHash((unsigned char*)&vValue[i*MAX_CHUNK_SIZE],size,&hash); + if(fHan > 0) + { + if(read(fHan,ptr,size) != size) + { + *errorCode=RPC_INTERNAL_ERROR; + *strError="Cannot read binary cache item"; + close(fHan); + return; + } + } + else + { + ptr=(unsigned char*)&vValue[i*MAX_CHUNK_SIZE]; + } + + mc_gState->m_TmpBuffers->m_RpcHasher1->DoubleHash(ptr,size,&hash); - err=pwalletTxsMain->m_ChunkDB->AddChunk((unsigned char*)&hash,&entity,NULL,-1,(unsigned char*)&vValue[i*MAX_CHUNK_SIZE],NULL,size,0,0); + err=pwalletTxsMain->m_ChunkDB->AddChunk((unsigned char*)&hash,&entity,NULL,-1,ptr,NULL,size,0,0); if(err) { switch(err) @@ -815,6 +868,10 @@ void AppendOffChainFormatData(uint32_t data_format, break; default: *strError="Internal error: couldn't store chunk"; + if(fHan > 0) + { + close(fHan); + } return; } } @@ -827,6 +884,11 @@ void AppendOffChainFormatData(uint32_t data_format, } } } + if(fHan > 0) + { + close(fHan); + fHan=0; + } } // err=pwalletTxsMain->m_ChunkDB->AddChunk((unsigned char*)&hash,&entity,NULL,-1,(unsigned char*)&vValue[0],NULL,(int)vValue.size(),0,0); From 3848fb90d62714bbf608a879dc4ccabdbf998f79 Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 22 Mar 2018 09:30:49 +0200 Subject: [PATCH 028/157] Offchain data, salt length and API tweaks --- src/protocol/multichainscript.cpp | 20 ++++-- src/rpc/rpcrawdata.cpp | 1 - src/rpc/rpcrawtransaction.cpp | 6 +- src/rpc/rpcutils.cpp | 114 +++++++++++++++++++++++------- src/rpc/rpcutils.h | 14 +++- src/rpc/rpcwallet.cpp | 15 ++-- 6 files changed, 127 insertions(+), 43 deletions(-) diff --git a/src/protocol/multichainscript.cpp b/src/protocol/multichainscript.cpp index cc088529..9e3500d8 100644 --- a/src/protocol/multichainscript.cpp +++ b/src/protocol/multichainscript.cpp @@ -2348,7 +2348,7 @@ int mc_Script::GetChunkDef(uint32_t *format,unsigned char** hashes,int *chunk_co { unsigned char *ptr; unsigned char *ptrEnd; - unsigned char f; + unsigned char f,s; int c,count,shift,size; /* if(format) @@ -2391,7 +2391,7 @@ int mc_Script::GetChunkDef(uint32_t *format,unsigned char** hashes,int *chunk_co ptr++; - if(ptr+2 > ptrEnd) + if(ptr+3 > ptrEnd) { return MC_ERR_ERROR_IN_SCRIPT; } @@ -2402,7 +2402,16 @@ int mc_Script::GetChunkDef(uint32_t *format,unsigned char** hashes,int *chunk_co } ptr++; - + + s=(uint32_t)(*ptr); + + if(s != 0) + { + return MC_ERR_ERROR_IN_SCRIPT; // Salt length should be 0 + } + + ptr++; + count=(int)mc_GetVarInt(ptr,ptrEnd-ptr,-1,&shift); if(count<0) @@ -2479,9 +2488,10 @@ int mc_Script::SetChunkDefHeader(const uint32_t format,int chunk_count) buf[MC_DCT_SCRIPT_IDENTIFIER_LEN]=MC_DCT_SCRIPT_MULTICHAIN_DATA_FORMAT_PREFIX; buf[MC_DCT_SCRIPT_IDENTIFIER_LEN+1]=MC_DCT_SCRIPT_EXTENDED_TYPE_CHUNK_DEF; buf[MC_DCT_SCRIPT_IDENTIFIER_LEN+2]=(unsigned char)format; - shift=mc_PutVarInt(buf+MC_DCT_SCRIPT_IDENTIFIER_LEN+3,11,chunk_count); + buf[MC_DCT_SCRIPT_IDENTIFIER_LEN+3]=0; // Salt length + shift=mc_PutVarInt(buf+MC_DCT_SCRIPT_IDENTIFIER_LEN+4,11,chunk_count); - err=SetData(buf,MC_DCT_SCRIPT_IDENTIFIER_LEN+3+shift); + err=SetData(buf,MC_DCT_SCRIPT_IDENTIFIER_LEN+4+shift); if(err) { return err; diff --git a/src/rpc/rpcrawdata.cpp b/src/rpc/rpcrawdata.cpp index 42892717..5a00ff21 100644 --- a/src/rpc/rpcrawdata.cpp +++ b/src/rpc/rpcrawdata.cpp @@ -1104,7 +1104,6 @@ CScript RawDataScriptPublish(Value *param,mc_EntityDetails *entity,uint32_t *dat *strError=string("Stream item options must be offchain or empty"); } field_parsed=true; - missing_key=false; } } diff --git a/src/rpc/rpcrawtransaction.cpp b/src/rpc/rpcrawtransaction.cpp index 67c03362..f30bb8eb 100644 --- a/src/rpc/rpcrawtransaction.cpp +++ b/src/rpc/rpcrawtransaction.cpp @@ -216,7 +216,7 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, Object& entry) // elem = lpScript->GetData(lpScript->GetNumElements()-1,&elem_size); retrieve_status = GetFormattedData(lpScript,&elem,&out_size,chunk_hashes,chunk_count,total_chunk_size); // vdata.push_back(OpReturnEntry(elem,elem_size,tx.GetHash(),i)); - aFormatMetaData.push_back(OpReturnFormatEntry(elem,out_size,tx.GetHash(),i,format,NULL,retrieve_status)); + aFormatMetaData.push_back(OpReturnFormatEntry(elem,out_size,tx.GetHash(),i,format,NULL,retrieve_status | MC_OST_CONTROL_NO_DATA)); if(mc_gState->m_Compatibility & MC_VCM_1_0) { aFullFormatMetaData.push_back(aFormatMetaData[0]); @@ -232,7 +232,7 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, Object& entry) if(out_size) { // vdata.push_back(OpReturnEntry(elem,elem_size,tx.GetHash(),i)); - aFullFormatMetaData.push_back(OpReturnFormatEntry(elem,out_size,tx.GetHash(),i,format,NULL,retrieve_status)); + aFullFormatMetaData.push_back(OpReturnFormatEntry(elem,out_size,tx.GetHash(),i,format,NULL,retrieve_status | MC_OST_CONTROL_NO_DATA)); } } lpScript->SetElement(0); @@ -350,7 +350,7 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, Object& entry) } Array items; - Value data_item_entry=DataItemEntry(tx,i,streams_already_seen, 0x03); + Value data_item_entry=DataItemEntry(tx,i,streams_already_seen, 0x0103); if(!data_item_entry.is_null()) { items.push_back(data_item_entry); diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index f1283645..432a36bc 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -988,8 +988,6 @@ uint32_t GetFormattedData(mc_Script *lpScript,const unsigned char **elem,int64_t } - status=MC_OST_UNKNOWN; - *elem = lpScript->GetData(lpScript->GetNumElements()-1,&elem_size); *out_size=elem_size; if(hashes == NULL) @@ -998,7 +996,7 @@ uint32_t GetFormattedData(mc_Script *lpScript,const unsigned char **elem,int64_t } if((mc_gState->m_WalletMode & MC_WMD_TXS) == 0) { - return status; + return MC_OST_OFF_CHAIN | MC_OST_ERROR_NOT_SUPPORTED; } if(use_tmp_buf) @@ -1007,6 +1005,8 @@ uint32_t GetFormattedData(mc_Script *lpScript,const unsigned char **elem,int64_t mc_gState->m_TmpBuffers->m_RpcChunkScript1->AddElement(); } + status=MC_OST_OFF_CHAIN; + ptr=hashes; ptrEnd=ptr+MC_CDB_CHUNK_HASH_SIZE+16; for(chunk=0;chunk MAX_CHUNK_SIZE) { - status |= MC_OST_ERROR; + status |= MC_OST_ERROR_SCRIPT; return status; } ptr+=shift; if(pwalletTxsMain->m_ChunkDB->GetChunkDef(&chunk_def,ptr,NULL,NULL,-1) == MC_ERR_NOERROR) { - status=MC_OST_RETRIEVED; + if(size != (int)chunk_def.m_Size) + { + status = MC_OST_OFF_CHAIN | MC_OST_RETRIEVED | MC_OST_ERROR_WRONG_SIZES; + return status; + } + status |= MC_OST_RETRIEVED; if(!skip_read) { *elem=pwalletTxsMain->m_ChunkDB->GetChunk(&chunk_def,0,-1,&elem_size); @@ -1041,14 +1046,14 @@ uint32_t GetFormattedData(mc_Script *lpScript,const unsigned char **elem,int64_t } else { - status = MC_OST_UNKNOWN | MC_OST_ERROR; + status = MC_OST_OFF_CHAIN | MC_OST_ERROR_CORRUPTED; return status; } } } else { - status=MC_OST_UNKNOWN; + status=MC_OST_OFF_CHAIN; *out_size=total_size; return status; } @@ -1079,34 +1084,77 @@ uint32_t GetFormattedData(mc_Script *lpScript,const unsigned char **elem,int64_t return status; } +string OffChainError(uint32_t status,int *errorCode) +{ + string error_str=""; + switch(status & MC_OST_ERROR_MASK) + { + case MC_OST_ERROR_SCRIPT: + error_str="Error in script"; + *errorCode=RPC_VERIFY_ERROR; + break; + case MC_OST_ERROR_WRONG_SIZES: + error_str="Chunk sizes don't match output script"; + *errorCode=RPC_VERIFY_ERROR; + break; + case MC_OST_ERROR_CORRUPTED: + error_str="Internal error"; + *errorCode=RPC_INTERNAL_ERROR; + break; + case MC_OST_ERROR_NOT_SUPPORTED: + error_str="Not supported"; + *errorCode=RPC_NOT_SUPPORTED; + break; + } + return error_str; +} Value OpReturnFormatEntry(const unsigned char *elem,int64_t elem_size,uint256 txid, int vout, uint32_t format, string *format_text_out,uint32_t status) { string metadata=""; Object metadata_object; Value metadata_value; - bool available; - string status_str; + bool available,offchain; + string status_str,error_str; + int errorCode; int err; - available=true; + available=false; + offchain=true; status_str=""; - switch(status & MC_OST_STATUS_MASK) + + if( status == MC_OST_UNDEFINED ) { - case MC_OST_ON_CHAIN: - status_str="on-chain"; - break; - case MC_OST_RETRIEVED: + available=true; + } + + if( (status & MC_OST_STORAGE_MASK) == MC_OST_ON_CHAIN ) + { + status_str="on-chain"; + available=true; + offchain=false; + } + + if( (status & MC_OST_STORAGE_MASK) == MC_OST_OFF_CHAIN ) + { + if( (status & MC_OST_STATUS_MASK) == MC_OST_RETRIEVED ) + { status_str="retrieved"; - break; - case MC_OST_UNKNOWN: - status_str="unknown"; + available=true; + } + + if( status & MC_OST_CONTROL_NO_DATA ) + { available=false; - break; - } - + } + } - if( (((int)elem_size <= GetArg("-maxshowndata",MAX_OP_RETURN_SHOWN)) || (txid == 0)) && available && ((status & MC_OST_ERROR) == 0) && (elem != NULL) ) + if(status & MC_OST_ERROR_MASK) + { + error_str=OffChainError(status,&errorCode); + } + + if( (((int)elem_size <= GetArg("-maxshowndata",MAX_OP_RETURN_SHOWN)) || (txid == 0)) && available && ((status & MC_OST_ERROR_MASK) == 0) && (elem != NULL) ) { if(format_text_out) { @@ -1157,10 +1205,18 @@ Value OpReturnFormatEntry(const unsigned char *elem,int64_t elem_size,uint256 tx metadata_object.push_back(Pair("vout", vout)); metadata_object.push_back(Pair("format", OpReturnFormatToText(format))); metadata_object.push_back(Pair("size", elem_size)); - if(status != MC_OST_UNDEFINED) + metadata_object.push_back(Pair("offchain", offchain)); + if( ( status & MC_OST_CONTROL_NO_DATA ) == 0) { - metadata_object.push_back(Pair("available", available)); - metadata_object.push_back(Pair("status", status_str)); + if( (status & MC_OST_STORAGE_MASK) == MC_OST_OFF_CHAIN ) + { + if(status & MC_OST_ERROR_MASK) + { + metadata_object.push_back(Pair("error", error_str)); + } + metadata_object.push_back(Pair("available", available)); + metadata_object.push_back(Pair("status", status_str)); + } } return metadata_object; } @@ -1186,6 +1242,8 @@ Value OpReturnFormatEntry(const unsigned char *elem,size_t elem_size,uint256 txi Value DataItemEntry(const CTransaction& tx,int n,set & already_seen,uint32_t stream_output_level) { + // 0x0100 No offchain data + Object entry; Array publishers; set publishers_set; @@ -1264,6 +1322,10 @@ Value DataItemEntry(const CTransaction& tx,int n,set & already_seen,uin // elem = mc_gState->m_TmpScript->GetData(mc_gState->m_TmpScript->GetNumElements()-1,&elem_size); retrieve_status = GetFormattedData(mc_gState->m_TmpScript,&elem,&out_size,chunk_hashes,chunk_count,total_chunk_size); // item_value=OpReturnEntry(elem,elem_size,tx.GetHash(),n); + if(stream_output_level & 0x0100) + { + retrieve_status |= MC_OST_CONTROL_NO_DATA; + } format_item_value=OpReturnFormatEntry(elem,out_size,tx.GetHash(),n,format,&format_text_str,retrieve_status); already_seen.insert(hash); diff --git a/src/rpc/rpcutils.h b/src/rpc/rpcutils.h index f64d1abe..02cc7178 100644 --- a/src/rpc/rpcutils.h +++ b/src/rpc/rpcutils.h @@ -65,9 +65,16 @@ using namespace json_spirit; #define MC_OST_UNDEFINED 0x00000000 #define MC_OST_UNKNOWN 0x00000001 #define MC_OST_ON_CHAIN 0x00000002 -#define MC_OST_RETRIEVED 0x00000004 -#define MC_OST_STATUS_MASK 0x0000FFFF -#define MC_OST_ERROR 0x00010000 +#define MC_OST_OFF_CHAIN 0x00000003 +#define MC_OST_STORAGE_MASK 0x000000FF +#define MC_OST_RETRIEVED 0x00000100 +#define MC_OST_STATUS_MASK 0x0000FF00 +#define MC_OST_ERROR_SCRIPT 0x00010000 +#define MC_OST_ERROR_WRONG_SIZES 0x00020000 +#define MC_OST_ERROR_CORRUPTED 0x00030000 +#define MC_OST_ERROR_NOT_SUPPORTED 0x00040000 +#define MC_OST_ERROR_MASK 0x00FF0000 +#define MC_OST_CONTROL_NO_DATA 0x01000000 // codes for allowed_objects fields @@ -132,6 +139,7 @@ Value mc_ExtractDetailsJSONObject(const unsigned char *script,uint32_t total); void AppendOffChainFormatData(uint32_t data_format,uint32_t out_options,mc_Script *lpDetailsScript,vector& vValue,vector* vChunkHashes,int *errorCode,string *strError); int mc_BinaryCacheFile(string id,int mode); void mc_RemoveBinaryCacheFile(string id); +string OffChainError(uint32_t status,int *errorCode); #endif /* RPCMULTICHAINUTILS_H */ diff --git a/src/rpc/rpcwallet.cpp b/src/rpc/rpcwallet.cpp index 70868163..b42a6546 100644 --- a/src/rpc/rpcwallet.cpp +++ b/src/rpc/rpcwallet.cpp @@ -648,6 +648,8 @@ Value gettxoutdata(const Array& params, bool fHelp) uint32_t retrieve_status; size_t elem_size; const unsigned char *elem; + string error_str; + int errorCode; if(mc_gState->m_TmpScript->IsOpReturnScript() == 0) { @@ -685,13 +687,16 @@ Value gettxoutdata(const Array& params, bool fHelp) // mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(&format); mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(&format,&chunk_hashes,&chunk_count,&total_chunk_size); retrieve_status = GetFormattedData(mc_gState->m_TmpScript,&elem,&out_size,chunk_hashes,chunk_count,total_chunk_size); + if(retrieve_status & MC_OST_ERROR_MASK) + { + error_str=OffChainError(retrieve_status,&errorCode); + throw JSONRPCError(errorCode, error_str); + } + elem_size=(size_t)out_size; - switch(retrieve_status & MC_OST_STATUS_MASK) + if( ( (retrieve_status & MC_OST_STATUS_MASK) != MC_OST_RETRIEVED ) && + ( (retrieve_status & MC_OST_STORAGE_MASK) != MC_OST_ON_CHAIN ) ) { - case MC_OST_ON_CHAIN: - case MC_OST_RETRIEVED: - break; - default: throw JSONRPCError(RPC_OUTPUT_NOT_FOUND, "Data for this output is not available"); } // elem = mc_gState->m_TmpScript->GetData(mc_gState->m_TmpScript->GetNumElements()-1,&elem_size); From 7d4973fdae7e3686325b8f412f99f8ce41829df1 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 26 Mar 2018 15:19:33 +0300 Subject: [PATCH 029/157] Offchain APIs tweaks --- src/rpc/rpccache.cpp | 20 ++++++++++++++++++-- src/rpc/rpchelp.cpp | 8 ++++---- src/rpc/rpclist.cpp | 2 +- src/rpc/rpcserver.h | 2 +- src/rpc/rpcstreams.cpp | 17 ++++++++++------- src/rpc/rpcutils.cpp | 28 +++++++++++++++------------- src/rpc/rpcwalletutils.cpp | 4 +++- src/utils/declare.h | 1 + src/utils/utility.cpp | 11 +++++++++++ src/wallet/chunkdb.cpp | 30 +++++++++++++----------------- src/wallet/wallettxdb.cpp | 11 ----------- 11 files changed, 77 insertions(+), 57 deletions(-) diff --git a/src/rpc/rpccache.cpp b/src/rpc/rpccache.cpp index a9ca7d4e..42824e25 100644 --- a/src/rpc/rpccache.cpp +++ b/src/rpc/rpccache.cpp @@ -24,9 +24,19 @@ Value createbinarycache(const Array& params, bool fHelp) } else { + fHan=mc_BinaryCacheFile(str,1); + if(fHan > 0) + { + close(fHan); + } + else + { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Cannot store binary cache item"); + } return str; } } + return Value::null; } @@ -56,7 +66,7 @@ Value appendbinarycache(const Array& params, bool fHelp) throw JSONRPCError(RPC_INVALID_PARAMETER, "data should be hexadecimal string"); } - int fHan=mc_BinaryCacheFile(params[0].get_str(),1); + int fHan=mc_BinaryCacheFile(params[0].get_str(),2); if(fHan <= 0) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Binary cache item with this identifier not found"); @@ -80,7 +90,7 @@ Value appendbinarycache(const Array& params, bool fHelp) return size; } -Value clearbinarycache(const Array& params, bool fHelp) +Value deletebinarycache(const Array& params, bool fHelp) { if (fHelp || params.size() != 1) throw runtime_error("Help message not found\n"); @@ -90,6 +100,12 @@ Value clearbinarycache(const Array& params, bool fHelp) throw JSONRPCError(RPC_INVALID_PARAMETER, "cache identifier should be string"); } + int fHan=mc_BinaryCacheFile(params[0].get_str(),0); + if(fHan <= 0) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Binary cache item with this identifier not found"); + } + mc_RemoveBinaryCacheFile(params[0].get_str()); return Value::null; diff --git a/src/rpc/rpchelp.cpp b/src/rpc/rpchelp.cpp index 7a125db1..2e4c0d0e 100644 --- a/src/rpc/rpchelp.cpp +++ b/src/rpc/rpchelp.cpp @@ -4020,15 +4020,15 @@ void mc_InitRPCHelpMap17() + HelpExampleRpc("appendbinarycache", "\"TjnVWwHYEg4\",\"f4c3dd510dd55761015c9d96bff7793b0d501dd6f01a959fd7\"") )); - mapHelpStrings.insert(std::make_pair("clearbinarycache", - "clearbinarycache \"identifier\" \n" + mapHelpStrings.insert(std::make_pair("deletebinarycache", + "deletebinarycache \"identifier\" \n" "\nClear binary cache item\n" "\nArguments:\n" "1. \"identifier\" (string, required) Binary cache item identifier, \"*\" - to clear all items\n" "\nResult:\n" "\nExamples:\n" - + HelpExampleCli("clearbinarycache", "\"TjnVWwHYEg4\"") - + HelpExampleRpc("clearbinarycache", "\"TjnVWwHYEg4\"") + + HelpExampleCli("deletebinarycache", "\"TjnVWwHYEg4\"") + + HelpExampleRpc("deletebinarycache", "\"TjnVWwHYEg4\"") )); diff --git a/src/rpc/rpclist.cpp b/src/rpc/rpclist.cpp index afce9b9d..a6079782 100644 --- a/src/rpc/rpclist.cpp +++ b/src/rpc/rpclist.cpp @@ -133,7 +133,7 @@ static const CRPCCommand vRPCCommands[] = { "util", "createbinarycache", &createbinarycache, true, false, false }, { "util", "appendbinarycache", &appendbinarycache, true, false, false }, - { "util", "clearbinarycache", &clearbinarycache, true, false, false }, + { "util", "deletebinarycache", &deletebinarycache, true, false, false }, /* Not shown in help */ { "hidden", "invalidateblock", &invalidateblock, true, true, false }, diff --git a/src/rpc/rpcserver.h b/src/rpc/rpcserver.h index d8e8a077..b69345c7 100644 --- a/src/rpc/rpcserver.h +++ b/src/rpc/rpcserver.h @@ -196,7 +196,7 @@ extern json_spirit::Value createkeypairs(const json_spirit::Array& params, bool extern json_spirit::Value getaddresses(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value createbinarycache(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value appendbinarycache(const json_spirit::Array& params, bool fHelp); -extern json_spirit::Value clearbinarycache(const json_spirit::Array& params, bool fHelp); +extern json_spirit::Value deletebinarycache(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value combineunspent(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value grantcmd(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value revokecmd(const json_spirit::Array& params, bool fHelp); diff --git a/src/rpc/rpcstreams.cpp b/src/rpc/rpcstreams.cpp index c34dc0e2..0a46a8a9 100644 --- a/src/rpc/rpcstreams.cpp +++ b/src/rpc/rpcstreams.cpp @@ -569,19 +569,22 @@ Value publishfrom(const Array& params, bool fHelp) { if(params[4].type() != str_type) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid options"); + throw JSONRPCError(RPC_INVALID_PARAMETER, "Stream item options must be offchain or empty"); } if( mc_gState->m_Features->OffChainData() == 0 ) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Format options are not supported by this protocol version"); } - if(params[4].get_str() == "offchain") + if(params[4].get_str().size()) { - in_options |= MC_RFD_OPTION_OFFCHAIN; - } - else - { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid options"); + if(params[4].get_str() == "offchain") + { + in_options |= MC_RFD_OPTION_OFFCHAIN; + } + else + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Stream item options must be offchain or empty"); + } } } diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index 432a36bc..023b31f6 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -1114,13 +1114,13 @@ Value OpReturnFormatEntry(const unsigned char *elem,int64_t elem_size,uint256 tx string metadata=""; Object metadata_object; Value metadata_value; - bool available,offchain; + bool available;//,offchain; string status_str,error_str; int errorCode; int err; available=false; - offchain=true; +// offchain=true; status_str=""; if( status == MC_OST_UNDEFINED ) @@ -1132,7 +1132,7 @@ Value OpReturnFormatEntry(const unsigned char *elem,int64_t elem_size,uint256 tx { status_str="on-chain"; available=true; - offchain=false; +// offchain=false; } if( (status & MC_OST_STORAGE_MASK) == MC_OST_OFF_CHAIN ) @@ -1205,18 +1205,15 @@ Value OpReturnFormatEntry(const unsigned char *elem,int64_t elem_size,uint256 tx metadata_object.push_back(Pair("vout", vout)); metadata_object.push_back(Pair("format", OpReturnFormatToText(format))); metadata_object.push_back(Pair("size", elem_size)); - metadata_object.push_back(Pair("offchain", offchain)); +// metadata_object.push_back(Pair("offchain", offchain)); if( ( status & MC_OST_CONTROL_NO_DATA ) == 0) { - if( (status & MC_OST_STORAGE_MASK) == MC_OST_OFF_CHAIN ) + if(status & MC_OST_ERROR_MASK) { - if(status & MC_OST_ERROR_MASK) - { - metadata_object.push_back(Pair("error", error_str)); - } - metadata_object.push_back(Pair("available", available)); - metadata_object.push_back(Pair("status", status_str)); + metadata_object.push_back(Pair("error", error_str)); } + metadata_object.push_back(Pair("available", available)); +// metadata_object.push_back(Pair("status", status_str)); } return metadata_object; } @@ -1369,6 +1366,7 @@ Value DataItemEntry(const CTransaction& tx,int n,set & already_seen,uin entry.push_back(Pair("key", keys[0])); } entry.push_back(Pair("data", format_item_value)); + entry.push_back(Pair("offchain", (retrieve_status & MC_OST_STORAGE_MASK) == MC_OST_OFF_CHAIN)); return entry; } @@ -4041,9 +4039,13 @@ int mc_BinaryCacheFile(string id,int mode) mc_GetFullFileName(mc_gState->m_Params->NetworkName(),str_file_name.c_str(),"",MC_FOM_RELATIVE_TO_DATADIR,file_name); flags=O_RDONLY; - if(mode) + if(mode & 1) + { + flags=O_CREAT; + } + if(mode & 2) { - flags=O_CREAT | O_RDWR; + flags=O_RDWR; } return open(file_name,_O_BINARY | flags, S_IRUSR | S_IWUSR); } diff --git a/src/rpc/rpcwalletutils.cpp b/src/rpc/rpcwalletutils.cpp index c19bc575..73790595 100644 --- a/src/rpc/rpcwalletutils.cpp +++ b/src/rpc/rpcwalletutils.cpp @@ -445,7 +445,7 @@ Object StreamItemEntry(const CWalletTx& wtx,int first_output,const unsigned char unsigned char *chunk_hashes; int chunk_count; int64_t total_chunk_size,out_size; - uint32_t retrieve_status; + uint32_t retrieve_status=0; Value format_item_value; string format_text_str; int start_from=first_output; @@ -603,6 +603,8 @@ Object StreamItemEntry(const CWalletTx& wtx,int first_output,const unsigned char entry.push_back(Pair("key", keys[0])); } entry.push_back(Pair("data", format_item_value)); + entry.push_back(Pair("offchain", (retrieve_status & MC_OST_STORAGE_MASK) == MC_OST_OFF_CHAIN)); + if(verbose) { WalletTxToJSON(wtx, entry, true, stream_output); diff --git a/src/utils/declare.h b/src/utils/declare.h index 58e3b393..9dcae8dd 100644 --- a/src/utils/declare.h +++ b/src/utils/declare.h @@ -280,6 +280,7 @@ uint64_t __US_ThreadID(); const char* __US_UserHomeDir(); char * __US_FullPath(const char* path, char *full_path, int len); void __US_FlushFile(int FileHan); +void sprintf_hex(char *hex,const unsigned char *bin,int size); diff --git a/src/utils/utility.cpp b/src/utils/utility.cpp index 58e21107..31c5334b 100644 --- a/src/utils/utility.cpp +++ b/src/utils/utility.cpp @@ -1876,3 +1876,14 @@ void mc_AdjustStartAndCount(int *count,int *start,int size) } } +void sprintf_hex(char *hex,const unsigned char *bin,int size) +{ + int i; + for(i=0;im_EntityType,enthex); LogString(msg); + + if(entity->m_EntityType == MC_TET_AUTHOR) + { + CommitInternal(-2); + } + return MC_ERR_NOERROR; } @@ -1210,11 +1203,11 @@ int mc_ChunkDB::CommitInternal(int block) err=MC_ERR_NOERROR; - Dump("Before Commit"); if(m_MemPool->GetCount() == 0) { goto exitlbl; } + Dump("Before Commit"); last_file_id=-1; last_file_offset=0; @@ -1355,8 +1348,11 @@ int mc_ChunkDB::CommitInternal(int block) } else { - sprintf(msg,"NewBlock %d, Chunks: %d,",block,m_MemPool->GetCount()); - LogString(msg); + if(block >= -1) + { + sprintf(msg,"NewBlock %d, Chunks: %d,",block,m_MemPool->GetCount()); + LogString(msg); + } m_MemPool->Clear(); m_ChunkData->Clear(); } diff --git a/src/wallet/wallettxdb.cpp b/src/wallet/wallettxdb.cpp index 77c720f4..16fb6087 100644 --- a/src/wallet/wallettxdb.cpp +++ b/src/wallet/wallettxdb.cpp @@ -6,17 +6,6 @@ #define MC_TDB_MAX_TXS_FILE_SIZE 0x8000000 // Maximal data file size, 128MB -void sprintf_hex(char *hex,const unsigned char *bin,int size) -{ - int i; - for(i=0;i Date: Mon, 26 Mar 2018 18:46:00 +0300 Subject: [PATCH 030/157] Offchain items, collector db code --- src/Makefile.am | 2 + src/rpc/rpcrawtransaction.cpp | 2 +- src/wallet/chunkcollector.cpp | 466 ++++++++++++++++++++++++++++++++++ src/wallet/chunkcollector.h | 108 ++++++++ src/wallet/wallettxs.cpp | 26 +- src/wallet/wallettxs.h | 2 + 6 files changed, 604 insertions(+), 2 deletions(-) create mode 100644 src/wallet/chunkcollector.cpp create mode 100644 src/wallet/chunkcollector.h diff --git a/src/Makefile.am b/src/Makefile.am index 90ae7135..e909746b 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -261,6 +261,7 @@ multichain_libbitcoin_multichain_a_SOURCES = \ utils/dbwrapper.cpp \ wallet/wallettxdb.cpp \ wallet/chunkdb.cpp \ + wallet/chunkcollector.cpp \ permissions/permission.cpp \ entities/asset.cpp @@ -525,6 +526,7 @@ libbitcoinconsensus_la_SOURCES = \ utils/dbwrapper.cpp \ wallet/wallettxdb.cpp \ wallet/chunkdb.cpp \ + wallet/chunkcollector.cpp \ permissions/permission.cpp \ entities/asset.cpp \ structs/hash.cpp \ diff --git a/src/rpc/rpcrawtransaction.cpp b/src/rpc/rpcrawtransaction.cpp index f30bb8eb..5df11dd1 100644 --- a/src/rpc/rpcrawtransaction.cpp +++ b/src/rpc/rpcrawtransaction.cpp @@ -117,7 +117,7 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, Object& entry) unsigned char details_script[MC_ENT_MAX_SCRIPT_SIZE]; unsigned char short_txid[MC_AST_SHORT_TXID_SIZE]; char asset_name[MC_ENT_MAX_NAME_SIZE+1]; - int multiple; + int multiple=1; int details_script_size=0; int err; bool detals_script_found=false; diff --git a/src/wallet/chunkcollector.cpp b/src/wallet/chunkcollector.cpp new file mode 100644 index 00000000..1c32a56d --- /dev/null +++ b/src/wallet/chunkcollector.cpp @@ -0,0 +1,466 @@ +// Copyright (c) 2014-2017 Coin Sciences Ltd +// MultiChain code distributed under the GPLv3 license, see COPYING file. + +#include "multichain/multichain.h" +#include "wallet/chunkcollector.h" + +void mc_ChunkCollectorRow::Zero() +{ + memset(this,0, sizeof(mc_ChunkCollectorRow)); +} + +void mc_ChunkCollector::Zero() +{ + m_DB=NULL; + m_ChunkDB=NULL; + m_KeyOffset=0; + m_KeySize=MC_CDB_CHUNK_HASH_SIZE+MC_TDB_TXID_SIZE+sizeof(mc_TxEntity)+4; + m_ValueOffset=m_KeySize; + m_ValueSize=8; + m_ValueDBSize=4; + m_TotalSize=m_KeySize+m_ValueSize; + m_TotalDBSize=m_KeySize+m_ValueDBSize; + m_Name[0]=0; + m_DBName[0]=0; + + m_MarkPool=NULL; + m_MemPool=NULL; + m_MemPoolNext=NULL; + m_MemPool1=NULL; + m_MemPool2=NULL; + + m_Semaphore=NULL; + m_LockedBy=0; + + m_InitMode=0; +} + +int mc_ChunkCollector::Destroy() +{ + if(m_DB) + { + m_DB->Close(); + delete m_DB; + m_DB=NULL; + } + + if(m_MarkPool) + { + delete m_MarkPool; + } + + if(m_MemPool1) + { + delete m_MemPool1; + } + + if(m_MemPool2) + { + delete m_MemPool2; + } + + if(m_Semaphore) + { + __US_SemDestroy(m_Semaphore); + } + + + Zero(); + return MC_ERR_NOERROR; +} + +int mc_ChunkCollector::Lock(int write_mode,int allow_secondary) +{ + uint64_t this_thread; + this_thread=__US_ThreadID(); + + if(this_thread == m_LockedBy) + { + return allow_secondary; + } + + __US_SemWait(m_Semaphore); + m_LockedBy=this_thread; + + return 0; +} + +void mc_ChunkCollector::UnLock() +{ + m_LockedBy=0; + __US_SemPost(m_Semaphore); +} + +int mc_ChunkCollector::Lock() +{ + return Lock(1,0); +} + +void mc_ChunkCollector::Dump(const char *message) +{ + int i; + + if((m_InitMode & MC_WMD_DEBUG) == 0) + { + return; + } + mc_ChunkCollectorRow dbrow; + + unsigned char *ptr; + int dbvalue_len,err; + char ShortName[65]; + char FileName[MC_DCT_DB_MAX_PATH]; + FILE *fHan; + + sprintf(ShortName,"chunks/collect"); + mc_GetFullFileName(m_Name,ShortName,".dmp",MC_FOM_RELATIVE_TO_DATADIR | MC_FOM_CREATE_DIR,FileName); + + fHan=fopen(FileName,"a"); + if(fHan == NULL) + { + return; + } + + mc_LogString(fHan,message); + + fprintf(fHan,"\nDB\n"); + dbrow.Zero(); + ptr=(unsigned char*)m_DB->Read((char*)&dbrow+m_KeyOffset,m_KeySize,&dbvalue_len,MC_OPT_DB_DATABASE_SEEK_ON_READ,&err); + if(err) + { + return; + } + + if(ptr) + { + memcpy((char*)&dbrow+m_ValueOffset,ptr,m_ValueDBSize); + while(ptr) + { + mc_MemoryDumpCharSizeToFile(fHan,(char*)&dbrow+m_KeyOffset,0,m_TotalDBSize,56); + ptr=(unsigned char*)m_DB->MoveNext(&err); + if(ptr) + { + memcpy((char*)&dbrow+m_KeyOffset,ptr,m_TotalDBSize); + } + } + } + + fprintf(fHan,"\nMempool\n"); + for(i=0;iGetCount();i++) + { + mc_MemoryDumpCharSizeToFile(fHan,m_MemPool->GetRow(i),0,m_TotalSize,56); + } + + fprintf(fHan,"\n<<<<<< \tChain height: %6d\t%s\n\n",mc_gState->m_Permissions->m_Block,message); + fclose(fHan); +} + +int mc_ChunkCollector::Initialize(mc_ChunkDB *chunk_db,const char *name,uint32_t mode) +{ + int err,value_len; + mc_ChunkCollectorRow collect_row; + mc_ChunkDBRow chunk_def; + unsigned char *ptr; + + strcpy(m_Name,name); + + m_DB=new mc_Database; + + mc_GetFullFileName(name,"chunks/collect",".db",MC_FOM_RELATIVE_TO_DATADIR | MC_FOM_CREATE_DIR,m_DBName); + + m_DB->SetOption("KeySize",0,m_KeySize); + m_DB->SetOption("ValueSize",0,m_ValueDBSize); + + + err=m_DB->Open(m_DBName,MC_OPT_DB_DATABASE_CREATE_IF_MISSING | MC_OPT_DB_DATABASE_TRANSACTIONAL | MC_OPT_DB_DATABASE_LEVELDB); + + if(err) + { + return err; + } + + m_InitMode=mode; + m_ChunkDB=chunk_db; + + + m_MarkPool=new mc_Buffer; + err=m_MarkPool->Initialize(m_KeySize,m_TotalSize,MC_BUF_MODE_DEFAULT); + m_MemPool1=new mc_Buffer; + err=m_MemPool1->Initialize(m_KeySize,m_TotalSize,MC_BUF_MODE_MAP); + m_MemPool2=new mc_Buffer; + err=m_MemPool2->Initialize(m_KeySize,m_TotalSize,MC_BUF_MODE_MAP); + + m_MemPool=m_MemPool1; + + collect_row.Zero(); + + ptr=(unsigned char*)m_DB->Read((char*)&collect_row+m_KeyOffset,m_KeySize,&value_len,MC_OPT_DB_DATABASE_SEEK_ON_READ,&err); + if(err) + { + return err; + } + + if(ptr) + { + ptr=(unsigned char*)m_DB->MoveNext(&err); + if(ptr) + { + memcpy((char*)&collect_row,ptr,m_TotalDBSize); + } + while(ptr) + { + ptr=(unsigned char*)m_DB->MoveNext(&err); + if(err) + { + return MC_ERR_CORRUPTED; + } + collect_row.m_Flags |= MC_CCF_INSERTED; + if(m_ChunkDB->GetChunkDef(&chunk_def,collect_row.m_Hash,NULL,NULL,-1) == MC_ERR_NOERROR) + { + collect_row.m_Flags |= MC_CCF_DELETED; + } + m_MemPool->Add(&collect_row); + if(ptr) + { + memcpy((char*)&collect_row,ptr,m_TotalDBSize); + } + } + } + else + { + err=m_DB->Write((char*)&collect_row+m_KeyOffset,m_KeySize,(char*)&collect_row+m_ValueOffset,m_ValueSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + if(err) + { + return err; + } + err=m_DB->Commit(MC_OPT_DB_DATABASE_TRANSACTIONAL); + if(err) + { + return err; + } + } + + Dump("Initialize"); + + return err; +} + +int mc_ChunkCollector::InsertChunk( // Adds chunk to mempool + const unsigned char *hash, // Chunk hash (before chopping) + const mc_TxEntity *entity, // Parent entity + const unsigned char *txid, + const int vout, + const uint32_t chunk_size) +{ + int err; + + Lock(); + err=InsertChunkInternal(hash,entity,txid,vout,chunk_size); + UnLock(); + + return err; +} + +int mc_ChunkCollector::InsertChunkInternal( + const unsigned char *hash, + const mc_TxEntity *entity, + const unsigned char *txid, + const int vout, + const uint32_t chunk_size) +{ + mc_ChunkCollectorRow collect_row; + int mprow; + + collect_row.Zero(); + memcpy(collect_row.m_Hash,hash,MC_CDB_CHUNK_HASH_SIZE); + memcpy(&collect_row.m_Entity,entity,sizeof(mc_TxEntity)); + memcpy(collect_row.m_TxID,txid,MC_TDB_TXID_SIZE); + collect_row.m_Vout=vout; + collect_row.m_Size=chunk_size; + collect_row.m_Flags=MC_CCF_NEW; + + mprow=m_MemPool->Seek(&collect_row); + if(mprow<0) + { + m_MemPool->Add(&collect_row); + } + + return MC_ERR_NOERROR; +} + +int mc_ChunkCollector::MarkAndClear(uint32_t flag, int unmark) +{ + int i,mprow; + + mc_ChunkCollectorRow *row; + Lock(); + + for(i=0;iGetCount();i++) + { + mprow=m_MemPool->Seek(m_MarkPool->GetRow(i)); + if(mprow >= 0) + { + row=(mc_ChunkCollectorRow *)m_MemPool->GetRow(mprow); + if(unmark) + { + row->m_Flags &= ~flag; + } + else + { + row->m_Flags |= flag; + } + } + } + + m_MarkPool->Clear(); + UnLock(); + + return MC_ERR_NOERROR; +} + +int mc_ChunkCollector::CopyFlags() +{ + int i,mprow; + + mc_ChunkCollectorRow *row; + mc_ChunkCollectorRow *mark_row; + Lock(); + + for(i=0;iGetCount();i++) + { + mark_row=(mc_ChunkCollectorRow *)m_MarkPool->GetRow(i); + mprow=m_MemPool->Seek(mark_row); + if(mprow >= 0) + { + row=(mc_ChunkCollectorRow *)m_MemPool->GetRow(mprow); + row->m_Flags=mark_row->m_Flags; + } + } + + m_MarkPool->Clear(); + UnLock(); + + return MC_ERR_NOERROR; +} + +int mc_ChunkCollector::FillMarkPoolByHash(const unsigned char *hash) +{ + int i; + mc_ChunkCollectorRow *row; + + Lock(); + + m_MarkPool->Clear(); + + for(i=0;iGetCount();i++) + { + row=(mc_ChunkCollectorRow *)m_MemPool->GetRow(i); + if(memcmp(row->m_Hash,hash,MC_CDB_CHUNK_HASH_SIZE) == 0) + { + m_MarkPool->Add(row); + } + } + + UnLock(); + + return MC_ERR_NOERROR; +} + +int mc_ChunkCollector::FillMarkPoolByFlag(uint32_t flag, uint32_t not_flag) +{ + int i; + mc_ChunkCollectorRow *row; + + Lock(); + + m_MarkPool->Clear(); + + for(i=0;iGetCount();i++) + { + row=(mc_ChunkCollectorRow *)m_MemPool->GetRow(i); + if(row->m_Flags & flag) + { + if( (row->m_Flags & not_flag) == 0) + { + m_MarkPool->Add(row); + } + } + } + + UnLock(); + + return MC_ERR_NOERROR; +} + +int mc_ChunkCollector::Commit() +{ + int err; + + Lock(); + err=CommitInternal(); + UnLock(); + + return err; +} + +int mc_ChunkCollector::CommitInternal() +{ + int i; + mc_ChunkCollectorRow *row; + int err,commit_required; + + err=MC_ERR_NOERROR; + + if(m_MemPool == m_MemPool1) + { + m_MemPoolNext=m_MemPool2; + } + else + { + m_MemPoolNext=m_MemPool1; + } + + m_MemPoolNext->Clear(); + + commit_required=0; + + for(i=0;iGetCount();i++) + { + row=(mc_ChunkCollectorRow *)m_MemPool->GetRow(i); + + if(row->m_Flags & MC_CCF_DELETED) + { + if(row->m_Flags & MC_CCF_INSERTED) + { + commit_required=1; + m_DB->Delete((char*)row+m_KeyOffset,m_KeySize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + } + } + else + { + if( (row->m_Flags & MC_CCF_INSERTED) == 0 ) + { + commit_required=1; + m_DB->Write((char*)row+m_KeyOffset,m_KeySize,(char*)row+m_ValueOffset,m_ValueDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + } + m_MemPoolNext->Add(row); + } + } + + if(commit_required) + { + err=m_DB->Commit(MC_OPT_DB_DATABASE_TRANSACTIONAL); + if(err) + { + return err; + } + } + + m_MemPool->Clear(); + m_MemPool=m_MemPoolNext; + m_MemPoolNext->Clear(); + + Dump("Commit"); + + return err; +} diff --git a/src/wallet/chunkcollector.h b/src/wallet/chunkcollector.h new file mode 100644 index 00000000..3d829667 --- /dev/null +++ b/src/wallet/chunkcollector.h @@ -0,0 +1,108 @@ +// Copyright (c) 2014-2017 Coin Sciences Ltd +// MultiChain code distributed under the GPLv3 license, see COPYING file. + +#ifndef MULTICHAIN_CHUNKCOLLECTOR_H +#define MULTICHAIN_CHUNKCOLLECTOR_H + +#include "utils/declare.h" +#include "wallet/chunkdb.h" +#include "wallet/wallettxdb.h" + +#define MC_CCF_NONE 0x00000000 +#define MC_CCF_NEW 0x00000000 +#define MC_CCF_INSERTED 0x00000001 +#define MC_CCF_DELETED 0x00000002 +#define MC_CCF_ALL 0xFFFFFFFF + + +typedef struct mc_ChunkCollectorRow +{ + unsigned char m_Hash[MC_CDB_CHUNK_HASH_SIZE]; // Chunk hash + mc_TxEntity m_Entity; + unsigned char m_TxID[MC_TDB_TXID_SIZE]; + int m_Vout; + uint32_t m_Size; + uint32_t m_Flags; + + void Zero(); +} mc_ChunkCollectorRow; + + +typedef struct mc_ChunkCollector +{ + mc_Database *m_DB; // Database object + mc_ChunkDB *m_ChunkDB; + uint32_t m_KeyOffset; + uint32_t m_KeySize; + uint32_t m_ValueOffset; + uint32_t m_ValueSize; + uint32_t m_TotalSize; + uint32_t m_ValueDBSize; + uint32_t m_TotalDBSize; + + + char m_Name[MC_PRM_NETWORK_NAME_MAX_SIZE+1]; // Chain name + char m_DBName[MC_DCT_DB_MAX_PATH]; // Full database name + + mc_Buffer *m_MarkPool; + mc_Buffer *m_MemPool; + mc_Buffer *m_MemPoolNext; + mc_Buffer *m_MemPool1; + mc_Buffer *m_MemPool2; + + void *m_Semaphore; // mc_TxDB object semaphore + uint64_t m_LockedBy; // ID of the thread locking it + + uint32_t m_InitMode; + + mc_ChunkCollector() + { + Zero(); + } + + ~mc_ChunkCollector() + { + Destroy(); + } + + int Initialize( // Initialization + mc_ChunkDB *chunk_db, + const char *name, // Chain name + uint32_t mode); // Unused + + int InsertChunk( // Adds chunk to mempool + const unsigned char *hash, // Chunk hash (before chopping) + const mc_TxEntity *entity, // Parent entity + const unsigned char *txid, + const int vout, + const uint32_t chunk_size); + + int InsertChunkInternal( + const unsigned char *hash, + const mc_TxEntity *entity, + const unsigned char *txid, + const int vout, + const uint32_t chunk_size); + + int MarkAndClear(uint32_t flag, int unmark); + int CopyFlags(); + int FillMarkPoolByHash(const unsigned char *hash); + int FillMarkPoolByFlag(uint32_t flag, uint32_t not_flag); + + int Commit(); + int CommitInternal(); + + void Zero(); + int Destroy(); + void Dump(const char *message); + + void LogString(const char *message); + + + int Lock(); + int Lock(int write_mode, int allow_secondary); + void UnLock(); +} mc_ChunkCollector; + +#endif /* MULTICHAIN_CHUNKCOLLECTOR_H */ + diff --git a/src/wallet/wallettxs.cpp b/src/wallet/wallettxs.cpp index a0ca8431..e6f1bb12 100644 --- a/src/wallet/wallettxs.cpp +++ b/src/wallet/wallettxs.cpp @@ -208,6 +208,7 @@ void mc_WalletTxs::Zero() int i; m_Database=NULL; m_ChunkDB=NULL; + m_ChunkCollector=NULL; m_lpWallet=NULL; m_ChunkBuffer=NULL; for(i=0;iInitialize(m_ChunkDB,name,mode); + if(err) + { + return err; + } + m_Database=new mc_TxDB; m_Mode=mode; err=m_Database->Initialize(name,mode); @@ -302,6 +311,12 @@ int mc_WalletTxs::Destroy() { delete m_Database; } + + if(m_ChunkCollector) + { + m_ChunkCollector->Commit(); + delete m_ChunkCollector; + } if(m_ChunkDB) { @@ -491,7 +506,15 @@ int mc_WalletTxs::Commit(mc_TxImport *import) { if(imp->m_ImportID == 0) { - err=m_ChunkDB->Commit(imp->m_Block+1); + if(m_ChunkCollector) + { + err=m_ChunkCollector->Commit(); + } + + if(err == MC_ERR_NOERROR) + { + err=m_ChunkDB->Commit(imp->m_Block+1); + } } } m_Database->Lock(1,0); @@ -2340,6 +2363,7 @@ int mc_WalletTxs::AddTx(mc_TxImport *import,const CWalletTx& tx,int block,CDiskT } else { + m_ChunkCollector->InsertChunk(chunk_hashes,&chunk_entity,(unsigned char*)&hash,i,chunk_size); // Feeding async chunk retriever here } } diff --git a/src/wallet/wallettxs.h b/src/wallet/wallettxs.h index edd1bd1a..fb662754 100644 --- a/src/wallet/wallettxs.h +++ b/src/wallet/wallettxs.h @@ -12,6 +12,7 @@ #include "multichain/multichain.h" #include "wallet/wallettxdb.h" #include "wallet/chunkdb.h" +#include "wallet/chunkcollector.h" #define MC_TDB_MAX_OP_RETURN_SIZE 256 @@ -48,6 +49,7 @@ typedef struct mc_WalletTxs { mc_TxDB *m_Database; mc_ChunkDB *m_ChunkDB; + mc_ChunkCollector *m_ChunkCollector; CWallet *m_lpWallet; uint32_t m_Mode; std::map m_UTXOs[MC_TDB_MAX_IMPORTS]; From 817596ae12cc2fe6aa0eee21126aa64e857f8207 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 2 Apr 2018 09:00:33 +0300 Subject: [PATCH 031/157] Offchain items minor fixes --- src/rpc/rpcutils.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index 023b31f6..40e4094c 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -970,7 +970,6 @@ uint32_t GetFormattedData(mc_Script *lpScript,const unsigned char **elem,int64_t mc_ChunkDBRow chunk_def; int size,shift,chunk; unsigned char *ptr; - unsigned char *ptrEnd; bool use_tmp_buf=false; bool skip_read=false; size_t elem_size; @@ -1008,10 +1007,10 @@ uint32_t GetFormattedData(mc_Script *lpScript,const unsigned char **elem,int64_t status=MC_OST_OFF_CHAIN; ptr=hashes; - ptrEnd=ptr+MC_CDB_CHUNK_HASH_SIZE+16; + for(chunk=0;chunkm_ChunkDB->GetChunk(&chunk_def,0,-1,&elem_size); @@ -1060,6 +1058,8 @@ uint32_t GetFormattedData(mc_Script *lpScript,const unsigned char **elem,int64_t ptr+=MC_CDB_CHUNK_HASH_SIZE; } + status |= MC_OST_RETRIEVED; + if(use_tmp_buf) { *elem = mc_gState->m_TmpBuffers->m_RpcChunkScript1->GetData(0,&elem_size); From 229005ef9ca7e26bf26533e449b24d95a25891e7 Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 5 Apr 2018 17:51:03 +0300 Subject: [PATCH 032/157] Relay code, not debugged --- src/Makefile.am | 1 + src/core/init.cpp | 16 + src/core/init.h | 2 + src/protocol/relay.cpp | 1093 ++++++++++++++++++++++++++++++++++++++++ src/protocol/relay.h | 173 +++++++ 5 files changed, 1285 insertions(+) create mode 100644 src/protocol/relay.cpp create mode 100644 src/protocol/relay.h diff --git a/src/Makefile.am b/src/Makefile.am index e909746b..102b99aa 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -180,6 +180,7 @@ libbitcoin_server_a_SOURCES = \ protocol/multichaintx.cpp \ protocol/multichainblock.cpp \ custom/custom_server.cpp \ + protocol/relay.cpp \ protocol/handshake.cpp \ chain/merkleblock.cpp \ miner/miner.cpp \ diff --git a/src/core/init.cpp b/src/core/init.cpp index 4e73de8c..4b78606d 100644 --- a/src/core/init.cpp +++ b/src/core/init.cpp @@ -36,6 +36,8 @@ #include "structs/base58.h" #include "multichain/multichain.h" #include "wallet/wallettxs.h" +#include "protocol/relay.h" + std::string BurnAddress(const std::vector& vchVersion); std::string SetBannedTxs(std::string txlist); std::string SetLockedBlock(std::string hash); @@ -63,6 +65,7 @@ using namespace std; CWallet* pwalletMain = NULL; mc_WalletTxs* pwalletTxsMain = NULL; #endif +mc_RelayManager* pRelayManager = NULL; bool fFeeEstimatesInitialized = false; #ifdef WIN32 @@ -216,6 +219,11 @@ void Shutdown() delete pwalletTxsMain; pwalletTxsMain=NULL; } + if(pRelayManager) + { + delete pRelayManager; + pRelayManager=NULL; + } /* MCHN END */ #endif globalVerifyHandle.reset(); @@ -1854,6 +1862,8 @@ bool AppInit2(boost::thread_group& threadGroup,int OutputPipe) return InitError(_("Failed to listen on any port. Use -listen=0 if you want this.")); } /* MCHN START */ + pRelayManager=new mc_RelayManager; + int max_ips=64; uint32_t all_ips[64]; int found_ips=1; @@ -1861,6 +1871,12 @@ bool AppInit2(boost::thread_group& threadGroup,int OutputPipe) { found_ips=mc_FindIPv4ServerAddress(all_ips,max_ips); } + else + { + all_ips[0]=mc_gState->m_IPv4Address; + } + pRelayManager->SetMyIPs(all_ips,found_ips); + if(!GetBoolArg("-shortoutput", false)) { if(fListen && !GetBoolArg("-offline",false)) diff --git a/src/core/init.h b/src/core/init.h index 0a0cfde9..f0ee4bd6 100644 --- a/src/core/init.h +++ b/src/core/init.h @@ -11,6 +11,7 @@ class CWallet; struct mc_WalletTxs; +struct mc_RelayManager; namespace boost { @@ -19,6 +20,7 @@ class thread_group; extern CWallet* pwalletMain; extern mc_WalletTxs* pwalletTxsMain; +extern mc_RelayManager* pRelayManager; void StartShutdown(); diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp new file mode 100644 index 00000000..dd9fdf3b --- /dev/null +++ b/src/protocol/relay.cpp @@ -0,0 +1,1093 @@ +// Copyright (c) 2014-2017 Coin Sciences Ltd +// MultiChain code distributed under the GPLv3 license, see COPYING file. + +#include "protocol/relay.h" + +void mc_Limiter::Zero() +{ + memset(this,0,sizeof(mc_Limiter)); +} + +int mc_Limiter::Intitialize(int seconds,int measures) +{ + Zero(); + + if(seconds > MC_LIM_MAX_SECONDS) + { + return MC_ERR_INVALID_PARAMETER_VALUE; + } + if(measures > MC_LIM_MAX_MEASURES) + { + return MC_ERR_INVALID_PARAMETER_VALUE; + } + m_SecondCount=seconds; + m_MeasureCount=measures; + m_Time=mc_TimeNowAsUInt(); + + return MC_ERR_NOERROR; +} + +int mc_Limiter::SetLimit(int meausure, int64_t limit) +{ + if( (meausure > MC_LIM_MAX_MEASURES) || (meausure <0) ) + { + return MC_ERR_INVALID_PARAMETER_VALUE; + } + m_Limits[meausure]=limit*m_SecondCount; + return MC_ERR_NOERROR; +} + +void mc_Limiter::CheckTime() +{ + CheckTime(mc_TimeNowAsUInt()); +} + +void mc_Limiter::CheckTime(uint32_t time_now) +{ + if(m_SecondCount == 0) + { + return; + } + if(time_now == m_Time) + { + return; + } + if(time_now >= m_Time + m_SecondCount) + { + memset(m_Totals,0,MC_LIM_MAX_MEASURES); + memset(m_Measures,0,MC_LIM_MAX_MEASURES*MC_LIM_MAX_SECONDS); + } + + for(uint32_t t=m_Time;t m_Limits[m]) + { + return 1; + } + } + + return 0; +} + +void mc_Limiter::Increment() +{ + if(m_SecondCount == 0) + { + return; + } + + int p=(m_Time+1)%m_SecondCount; + for(int m=0;mm_Address=pto->kAddrLocal; + } + else + { + node_address->m_Address=CKeyID(0); + } + + node_address->m_NetAddresses.clear(); + + if(action & MC_PRA_MY_ORIGIN_NT_ADDRESS) + { + node_address->m_NetAddresses.push_back(CAddress(pto->addrLocal)); + + pto_address_local=0; + if(pto->addrLocal.IsIPv4()) + { + pto_address_local=(pto->addrLocal.GetByte(3)<<24)+(pto->addrLocal.GetByte(2)<<16)+(pto->addrLocal.GetByte(1)<<8)+pto->addrLocal.GetByte(0); + addr.s_addr=pto_address_local; + node_address->m_NetAddresses.push_back(CAddress(CService(addr,GetListenPort()))); + } + } + + + for(int i=0;im_NetAddresses.push_back(CAddress(CService(addr,GetListenPort()))); + } + } +} + +void mc_RelayManager::InitNodeAddress(mc_NodeFullAddress *node_address,CKeyID& mc_address, vector& net_addresses) +{ + node_address->m_Address=mc_address; + node_address->m_NetAddresses=net_addresses; +} + +void mc_RelayManager::MsgTypeSettings(uint32_t msg_type,int latency,int seconds,int64_t serves_per_second,int64_t bytes_per_second) +{ + mc_Limiter limiter; + + map::iterator itlat = m_Latency.find(msg_type); + if (itlat == m_Latency.end()) + { + m_Latency.insert(make_pair(msg_type,latency)); + } + else + { + itlat->second=latency; + } + + limiter.Intitialize(seconds,2); + limiter.SetLimit(0,serves_per_second); + limiter.SetLimit(1,bytes_per_second); + + map::iterator itlim = m_Limiters.find(msg_type); + if (itlim == m_Limiters.end()) + { + m_Limiters.insert(make_pair(msg_type,limiter)); + } + else + { + itlim->second=limiter; + } +} + +void mc_RelayManager::SetDefaults() +{ + MsgTypeSettings(MC_RMT_NONE , 0,10,1000,100*1024*1024); + MsgTypeSettings(MC_RMT_GLOBAL_PING ,10,10, 100, 1*1024*1024); + MsgTypeSettings(MC_RMT_GLOBAL_PONG , 0,10, 100, 1*1024*1024); + MsgTypeSettings(MC_RMT_GLOBAL_REJECT , 0,10,1000, 1*1024*1024); + MsgTypeSettings(MC_RMT_GLOBAL_BUSY , 0,10,1000, 1*1024*1024); + MsgTypeSettings(MC_RMT_CHUNK_QUERY ,10,10, 100, 1*1024*1024); + MsgTypeSettings(MC_RMT_CHUNK_QUERY_HIT,30,10, 100, 1*1024*1024); + MsgTypeSettings(MC_RMT_CHUNK_REQUEST ,30,10, 100, 1*1024*1024); + MsgTypeSettings(MC_RMT_CHUNK_RESPONSE , 0,10, 100,100*1024*1024); +} + +void mc_RelayManager::CheckTime() +{ + uint32_t time_now=mc_TimeNowAsUInt(); + if(time_now == m_LastTime) + { + return; + } + + for(map::iterator it = m_RelayRecords.begin(); it != m_RelayRecords.end();) + { + if(it->second.m_Timestamp < m_LastTime) + { + m_RelayRecords.erase(it++); + } + else + { + it++; + } + } +} + +void mc_RelayManager::SetRelayRecord(CNode *pto,CNode *pfrom,uint32_t msg_type,int64_t nonce) +{ + map::iterator itlat = m_Latency.find(msg_type); + if (itlat == m_Latency.end()) + { + return; + } + if(itlat->second <= 0 ) + { + return; + } + + NodeId pto_id=0; + if(pto) + { + pto_id=pto->GetId(); + } + const mc_RelayRecordKey key=mc_RelayRecordKey(nonce,pto_id); + mc_RelayRecordValue value; + value.m_NodeFrom=0; + if(pfrom) + { + value.m_NodeFrom=pfrom->GetId(); + } + value.m_MsgType=msg_type; + value.m_Timestamp=m_LastTime+itlat->second; + + map::iterator it = m_RelayRecords.find(key); + if (it == m_RelayRecords.end()) + { + m_RelayRecords.insert(make_pair(key,value)); + } + else + { + it->second=value; + } +} + +int mc_RelayManager::GetRelayRecord(CNode *pfrom,int64_t nonce,uint32_t* msg_type,CNode **pto) +{ + NodeId pfrom_id,pto_id; + + pfrom_id=0; + if(pfrom) + { + pfrom_id=pfrom->GetId(); + } + const mc_RelayRecordKey key=mc_RelayRecordKey(nonce,pfrom_id); + map::iterator it = m_RelayRecords.find(key); + if (it == m_RelayRecords.end()) + { + return MC_ERR_NOT_FOUND; + } + + if(msg_type) + { + *msg_type=it->second.m_MsgType; + } + + + if(pto) + { + pto_id=it->second.m_NodeFrom; + + if(pto_id) + { + LOCK(cs_vNodes); + BOOST_FOREACH(CNode* pnode, vNodes) + { + if(pnode->GetId() == pto_id) + { + *pto=pnode; + return MC_ERR_NOERROR; + } + } + } + else + { + return MC_ERR_NOERROR; + } + } + else + { + return MC_ERR_NOERROR; + } + + return MC_ERR_NOT_ALLOWED; +} + +void mc_RelayManager::PushRelay(CNode* pto, + uint32_t msg_type, + int64_t nonce_to_send, + int64_t response_to_nonce, + mc_NodeFullAddress *origin_to_relay, + mc_NodeFullAddress *destination_to_relay, + uint32_t flags, + vector& payload, + uint256 message_hash_to_relay, + vector& sigScripts_to_relay, + vector& path, + CNode* pfrom, + uint32_t action) +{ + vector vOriginAddress; + vector vDestinationAddress; + vector sigScripts; + vectorvSigScript; + CScript sigScript; + uint256 message_hash; + int64_t nonce; + mc_NodeFullAddress origin_to_send; + mc_NodeFullAddress *origin; + mc_NodeFullAddress destination_to_send; + mc_NodeFullAddress *destination; + + nonce=nonce_to_send; + + if(action & MC_PRA_GENERATE_NONCE) + { + GetRandBytes((unsigned char*)&nonce, sizeof(nonce)); + } + + if(action & (MC_PRA_MY_ORIGIN_MC_ADDRESS | MC_PRA_MY_ORIGIN_NT_ADDRESS) ) + { + InitNodeAddress(&origin_to_send,pto,action); + origin=&origin_to_send; + } + else + { + if(origin_to_relay) + { + origin=origin_to_relay; + } + else + { + origin=&origin_to_send; + } + } + + destination=&destination_to_send; + if(action & MC_PRA_USE_DESTINATION_ADDRESS) + { + if(destination_to_relay) + { + destination=destination_to_relay; + } + } + + if(MCP_ANYONE_CAN_CONNECT == 0) + { + vOriginAddress.resize(sizeof(CKeyID)); + memcpy(&vOriginAddress[0],&(origin->m_Address),sizeof(CKeyID)); + if(destination->m_Address != 0) + { + vDestinationAddress.resize(sizeof(CKeyID)); + memcpy(&vOriginAddress[0],&(origin->m_Address),sizeof(CKeyID)); + } + } + + message_hash=message_hash_to_relay; + + if(action & MC_PRA_CALCULATE_MESSAGE_HASH) + { + CHashWriter ssHash(SER_GETHASH, 0); + ssHash << nonce; + ssHash << msg_type; + ssHash << vOriginAddress; + ssHash << origin->m_NetAddresses; + ssHash << vDestinationAddress; + ssHash << destination->m_NetAddresses; + ssHash << response_to_nonce; + ssHash << flags; + ssHash << payload; + message_hash=ssHash.GetHash(); + } + + if( (action & (MC_PRA_SIGN | MC_PRA_SIGN_WITH_HANDSHAKE_ADDRESS)) && (MCP_ANYONE_CAN_CONNECT == 0) ) + { + CHashWriter ssSig(SER_GETHASH, 0); + ssSig << message_hash; + ssSig << vector((unsigned char*)&response_to_nonce, (unsigned char*)&response_to_nonce+sizeof(response_to_nonce)); + uint256 signed_hash=ssSig.GetHash(); + CKey key; + CKeyID keyID; + CPubKey pkey; + + keyID=origin->m_Address; + if(action & MC_PRA_SIGN_WITH_HANDSHAKE_ADDRESS) + { + if(MCP_ANYONE_CAN_CONNECT == 0) + { + keyID=pto->kAddrLocal; + } + } + + if(pwalletMain->GetKey(keyID, key)) + { + pkey=key.GetPubKey(); + vector vchSig; + sigScript.clear(); + if (key.Sign(signed_hash, vchSig)) + { + vchSig.push_back(0x00); + sigScript << vchSig; + sigScript << ToByteVector(pkey); + } + else + { + LogPrintf("PushRelay(): Internal error: Cannot sign\n"); + } + } + else + { + LogPrintf("PushRelay(): Internal error: Cannot find key for signature\n"); + } + sigScripts.push_back(sigScript); + } + + for(unsigned int i=0;iPushMessage("relay", + msg_type, + nonce, + vOriginAddress, + origin->m_NetAddresses, + vDestinationAddress, + destination->m_NetAddresses, + response_to_nonce, + flags, + payload, + message_hash, + path, + sigScripts); + + SetRelayRecord(pto,NULL,msg_type,nonce); + SetRelayRecord(pto,pfrom,msg_type,nonce); +} + +bool mc_RelayManager::ProcessRelay( CNode* pfrom, + CDataStream& vRecv, + CValidationState &state, + uint32_t verify_flags_in) +{ + uint32_t msg_type_in; + uint32_t verify_flags; + int64_t nonce_received; + int64_t response_to_nonce; + vector vOriginMCAddressIn; + vector vDestinationMCAddressIn; + vector vOriginNetAddressIn; + vector vDestinationNetAddressIn; + vector vPayloadIn; + vector vSigScripts; + uint256 message_hash,message_hash_in; + uint32_t flags_in; + CKeyID origin_mc_address; + CKeyID destination_mc_address; + vector vchSigOut; + vector vchPubKey; + CPubKey pubkey; + vector path; + mc_NodeFullAddress origin_to_relay; + mc_NodeFullAddress destination_to_relay; + vector vPayloadResponse; + vector vSigScriptsResponse; + vector vPathResponse; + CNode *pto_stored; + uint32_t msg_type_stored; + + msg_type_stored=MC_RMT_NONE; + pto_stored=NULL; + + verify_flags=verify_flags_in; + vRecv >> msg_type_in; + switch(msg_type_in) + { + case MC_RMT_GLOBAL_PING: + break; + case MC_RMT_GLOBAL_PONG: + verify_flags |= MC_VRA_ORIGIN_MC_ADDRESS | MC_VRA_ORIGIN_NT_ADDRESS | MC_VRA_RESPONSE_TO_NONCE | MC_VRA_MESSAGE_HASH | MC_VRA_SIGNATURE_ORIGIN; + break; + case MC_RMT_GLOBAL_REJECT: + case MC_RMT_GLOBAL_BUSY: + verify_flags |= MC_VRA_ORIGIN_MC_ADDRESS | MC_VRA_RESPONSE_TO_NONCE | MC_VRA_MESSAGE_HASH | MC_VRA_SIGNATURE_ORIGIN; + break; + case MC_RMT_CHUNK_QUERY: + verify_flags |= MC_VRA_ORIGIN_MC_ADDRESS | MC_VRA_MESSAGE_HASH; + break; + case MC_RMT_CHUNK_QUERY_HIT: + verify_flags |= MC_VRA_ORIGIN_MC_ADDRESS | MC_VRA_ORIGIN_NT_ADDRESS | MC_VRA_DESTINATION_MC_ADDRESS | MC_VRA_RESPONSE_TO_NONCE | MC_VRA_MESSAGE_HASH | + MC_VRA_SIGNATURE_ORIGIN; + case MC_RMT_CHUNK_REQUEST: + case MC_RMT_CHUNK_RESPONSE: + verify_flags |= MC_VRA_ORIGIN_MC_ADDRESS | MC_VRA_ORIGIN_NT_ADDRESS | MC_VRA_DESTINATION_MC_ADDRESS | MC_VRA_RESPONSE_TO_NONCE | MC_VRA_MESSAGE_HASH | + MC_VRA_SIGNATURE_ORIGIN | MC_VRA_SINGLE_HOP; + break; + default: + if(verify_flags & MC_VRA_MESSAGE_TYPE) + { + LogPrintf("ProcessRelay() : Unsupported relay message type %d\n",msg_type_in); + return false; + } + break; + } + + CheckTime(); + + vRecv >> nonce_received; + + if(verify_flags & MC_VRA_ALREADY_PROCESSED) + { + if(GetRelayRecord(NULL,nonce_received,NULL,NULL) == MC_ERR_NOERROR) + { + return false; + } + } + + vRecv >> vOriginMCAddressIn; + vRecv >> vOriginNetAddressIn; + vRecv >> vDestinationMCAddressIn; + vRecv >> vDestinationNetAddressIn; + vRecv >> response_to_nonce; + vRecv >> flags_in; + vRecv >> vPayloadIn; + vRecv >> message_hash_in; + vRecv >> path; + vRecv >> vSigScripts; + + + if(vOriginMCAddressIn.size() == sizeof(uint160)) + { + origin_mc_address=CKeyID(*(uint160*)&vOriginMCAddressIn[0]); + } + else + { + if(verify_flags & MC_VRA_ORIGIN_MC_ADDRESS) + { + if(MCP_ANYONE_CAN_CONNECT == 0) + { + return state.DoS(100, error("ProcessRelay() : Bad origin address"),REJECT_INVALID, "bad-origin-address"); + } + } + } + + if(verify_flags & MC_VRA_ORIGIN_NT_ADDRESS) + { + if(vOriginNetAddressIn.size() == 0) + { + return state.DoS(100, error("ProcessRelay() : No origin network address"),REJECT_INVALID, "bad-origin-net-address"); + } + } + + InitNodeAddress(&origin_to_relay,origin_mc_address,vOriginNetAddressIn); + + if(vDestinationMCAddressIn.size() == sizeof(uint160)) + { + destination_mc_address=CKeyID(*(uint160*)&vDestinationMCAddressIn[0]); + } + else + { + if(verify_flags & MC_VRA_DESTINATION_MC_ADDRESS) + { + if(MCP_ANYONE_CAN_CONNECT == 0) + { + return state.DoS(100, error("ProcessRelay() : Bad destination address"),REJECT_INVALID, "bad-destination-address"); + } + } + } + + if(verify_flags & MC_VRA_DESTINATION_NT_ADDRESS) + { + if(vDestinationNetAddressIn.size() == 0) + { + return state.DoS(100, error("ProcessRelay() : No destination network address"),REJECT_INVALID, "bad-destination-net-address"); + } + } + + InitNodeAddress(&destination_to_relay,destination_mc_address,vDestinationNetAddressIn); + + map::iterator itlim_all = m_Limiters.find(MC_RMT_NONE); + map::iterator itlim_msg = m_Limiters.find(msg_type_in); + + if(itlim_all != m_Limiters.end()) + { + itlim_all->second.SetEvent(1,vRecv.size()); + if( verify_flags & MC_VRA_LIMIT_ALL ) + { + if(itlim_all->second.Disallowed(m_LastTime)) + { + PushRelay(pfrom,MC_RMT_GLOBAL_BUSY,0,nonce_received,NULL,NULL,0,vPayloadResponse,0,vSigScriptsResponse,vPathResponse,NULL, + MC_PRA_GENERATE_NONCE | MC_PRA_MY_ORIGIN_MC_ADDRESS | MC_PRA_CALCULATE_MESSAGE_HASH | MC_PRA_SIGN | MC_PRA_SIGN_WITH_HANDSHAKE_ADDRESS); + return false; + } + } + } + + if(itlim_msg != m_Limiters.end()) + { + itlim_msg->second.SetEvent(1,vRecv.size()); + if( verify_flags & MC_VRA_LIMIT_MSG_TYPE ) + { + if(itlim_msg->second.Disallowed(m_LastTime)) + { + PushRelay(pfrom,MC_RMT_GLOBAL_BUSY,0,nonce_received,NULL,NULL,0,vPayloadResponse,0,vSigScriptsResponse,vPathResponse,NULL, + MC_PRA_GENERATE_NONCE | MC_PRA_MY_ORIGIN_MC_ADDRESS | MC_PRA_CALCULATE_MESSAGE_HASH | MC_PRA_SIGN | MC_PRA_SIGN_WITH_HANDSHAKE_ADDRESS); + return false; + } + } + } + + + if( verify_flags & MC_VRA_RESPONSE_TO_NONCE ) + { + if(response_to_nonce == 0) + { + return state.DoS(100, error("ProcessRelay() : Missing response_to_nonce"),REJECT_INVALID, "bad-nonce"); + } + if(GetRelayRecord(pfrom,response_to_nonce,&msg_type_stored,&pto_stored)) + { + LogPrintf("ProcessRelay() : Response without request from peer %d\n",pfrom->GetId()); + return false; + } + } + + if( verify_flags & (MC_VRA_MESSAGE_HASH | MC_VRA_SIGNATURE_ORIGIN | MC_VRA_SIGNATURE_ALL) ) + { + CHashWriter ssHash(SER_GETHASH, 0); + ssHash << nonce_received; + ssHash << msg_type_in; + ssHash << vOriginMCAddressIn; + ssHash << vOriginNetAddressIn; + ssHash << vDestinationMCAddressIn; + ssHash << vDestinationNetAddressIn; + ssHash << response_to_nonce; + ssHash << flags_in; + ssHash << vPayloadIn; + message_hash=ssHash.GetHash(); + if(message_hash != message_hash_in) + { + return state.DoS(100, error("ProcessRelay() : Payload hash mismatch"),REJECT_INVALID, "bad-payload-hash"); + } + } + + message_hash=message_hash_in; + + if( verify_flags & MC_VRA_SIGNATURE_ORIGIN ) + { + if(vSigScripts.size() < 1) + { + return state.DoS(100, error("ProcessRelay() : Missing sigScript"),REJECT_INVALID, "bad-sigscript"); + } + if(vSigScripts[0].size()) + { + CScript scriptSig=vSigScripts[0]; + + opcodetype opcode; + + CScript::const_iterator pc = scriptSig.begin(); + + if (!scriptSig.GetOp(pc, opcode, vchSigOut)) + { + return state.DoS(100, error("ProcessRelay() : Cannot extract signature from sigScript"),REJECT_INVALID, "bad-sigscript-signature"); + } + + vchSigOut.resize(vchSigOut.size()-1); + if (!scriptSig.GetOp(pc, opcode, vchPubKey)) + { + return state.DoS(100, error("ProcessRelay() : Cannot extract pubkey from sigScript"),REJECT_INVALID, "bad-sigscript-pubkey"); + } + + CPubKey pubKeyOut(vchPubKey); + if (!pubKeyOut.IsValid()) + { + return state.DoS(100, error("ProcessRelay() : Invalid pubkey"),REJECT_INVALID, "bad-sigscript-pubkey"); + } + + pubkey=pubKeyOut; + + CHashWriter ss(SER_GETHASH, 0); + ss << vector((unsigned char*)&message_hash, (unsigned char*)&message_hash+32); + ss << vector((unsigned char*)&response_to_nonce, (unsigned char*)&response_to_nonce+sizeof(response_to_nonce)); + uint256 signed_hash=ss.GetHash(); + + if(!pubkey.Verify(signed_hash,vchSigOut)) + { + return state.DoS(100, error("ProcessRelay() : Wrong signature"),REJECT_INVALID, "bad-signature"); + } + } + else + { + return state.DoS(100, error("ProcessRelay() : Empty sigScript"),REJECT_INVALID, "bad-sigscript"); + } + } + + if(itlim_all != m_Limiters.end()) + { + itlim_all->second.Increment(); + } + + if(itlim_msg != m_Limiters.end()) + { + itlim_msg->second.Increment(); + } + + return true; +} + +/* +bool ProcessMultichainRelay(CNode* pfrom, CDataStream& vRecv, CValidationState &state, uint32_t verify_flags_in) +{ + uint32_t msg_type_in; + uint32_t verify_flags; + int64_t nonce_received; + int64_t response_to_nonce; + vector vOriginMCAddressIn; + vector vDestinationMCAddressIn; + vector vOriginNetAddressIn; + vector vDestinationNetAddressIn; + vector vPayloadIn; + vector vSigScript; + uint256 payload_hash,payload_hash_in; + uint32_t flags_in; + CKeyID origin_mc_address; + CKeyID destination_mc_address; + bool destination_cannot_be_empty; + vector vchSigOut; + vector vchPubKey; + CPubKey pubkey; + + vRecv >> msg_type_in; + + vRecv >> nonce_received; + vRecv >> vOriginMCAddressIn; + vRecv >> vOriginNetAddressIn; + vRecv >> vDestinationMCAddressIn; + vRecv >> vDestinationNetAddressIn; + vRecv >> flags_in; + vRecv >> response_to_nonce; + vRecv >> payload_hash_in; + vRecv >> vSigScript; + vRecv >> vPayloadIn; + + if(verify_flags & MC_VRA_MESSAGE_TYPE) + { + switch(msg_type_in) + { + case MC_RMT_CHUNK_QUERY: + case MC_RMT_CHUNK_QUERY_HIT: + case MC_RMT_CHUNK_REQUEST: + case MC_RMT_CHUNK_RESPONSE: + break; + default: + LogPrintf("ProcessMultichainRelay() : Unsupported relay message type %d\n",msg_type_in); + return false; + } + } + + if(verify_flags & MC_VRA_ORIGIN_ADDRESS) + { + if(vOriginNetAddressIn.size() == 0) + { + return state.DoS(100, error("ProcessMultichainRelay() : No origin network address"),REJECT_INVALID, "bad-origin-net-address"); + } + if(MCP_ANYONE_CAN_CONNECT == 0) + { + if(vOriginMCAddressIn.size() != sizeof(uint160)) + { + return state.DoS(100, error("ProcessMultichainRelay() : Bad origin address"),REJECT_INVALID, "bad-origin-address"); + } + origin_mc_address=CKeyID(*(uint160*)&vOriginMCAddressIn[0]); + } + } + + if(verify_flags & MC_VRA_DESTINATION_ADDRESS) + { + destination_cannot_be_empty=false; + switch(msg_type_in) + { + case MC_RMT_CHUNK_QUERY_HIT: + case MC_RMT_CHUNK_RESPONSE: + destination_cannot_be_empty=true; + break; + } + if(destination_cannot_be_empty) + { + if( ((vDestinationMCAddressIn.size()) == 0) && (MCP_ANYONE_CAN_CONNECT == 0) ) + { + return state.DoS(100, error("ProcessMultichainRelay() : Empty destination address"),REJECT_INVALID, "bad-destination-address"); + } + if(vOriginNetAddressIn.size() == 0) + { + return state.DoS(100, error("ProcessMultichainRelay() : No destination network address"),REJECT_INVALID, "bad-destination-net-address"); + } + } + if(vDestinationMCAddressIn.size()) + { + if(MCP_ANYONE_CAN_CONNECT == 0) + { + if(vDestinationMCAddressIn.size() != sizeof(uint160)) + { + return state.DoS(100, error("ProcessMultichainRelay() : Bad destination address"),REJECT_INVALID, "bad-destination-address"); + } + destination_mc_address=CKeyID(*(uint160*)&vDestinationMCAddressIn[0]); + } + } + } + + if( verify_flags & MC_VRA_SIGSCRIPT) + { + if(vSigScript.size()) + { + CScript scriptSig((unsigned char*)&vSigScript[0],(unsigned char*)&vSigScript[0]+vSigScript.size()); + + opcodetype opcode; + + CScript::const_iterator pc = scriptSig.begin(); + + if (!scriptSig.GetOp(pc, opcode, vchSigOut)) + { + return state.DoS(100, error("ProcessMultichainRelay() : Cannot extract signature from sigScript"),REJECT_INVALID, "bad-sigscript-signature"); + } + + vchSigOut.resize(vchSigOut.size()-1); + if (!scriptSig.GetOp(pc, opcode, vchPubKey)) + { + return state.DoS(100, error("ProcessMultichainRelay() : Cannot extract pubkey from sigScript"),REJECT_INVALID, "bad-sigscript-pubkey"); + } + + CPubKey pubKeyOut(vchPubKey); + if (!pubKeyOut.IsValid()) + { + return state.DoS(100, error("ProcessMultichainRelay() : Invalid pubkey"),REJECT_INVALID, "bad-sigscript-pubkey"); + } + + pubkey=pubKeyOut; + } + else + { + switch(msg_type_in) + { + case MC_RMT_CHUNK_QUERY: + break; + default: + return state.DoS(100, error("ProcessMultichainRelay() : Missing sigScript"),REJECT_INVALID, "bad-sigscript"); + } + } + } + + if( verify_flags & MC_VRA_RESPONSE_TO_NONCE ) + { + if(response_to_nonce == 0) + { + switch(msg_type_in) + { + case MC_RMT_CHUNK_QUERY: + break; + default: + return state.DoS(100, error("ProcessMultichainRelay() : Missing response_to_nonce"),REJECT_INVALID, "bad-nonce"); + } + } + } + + if( verify_flags & (MC_VRA_PAYLOAD_HASH | MC_VRA_SIGNATURE) ) + { + CHashWriter ssHash(SER_GETHASH, 0); + ssHash << vPayloadIn; + payload_hash=ssHash.GetHash(); + + if(payload_hash != payload_hash_in) + { + return state.DoS(100, error("ProcessMultichainRelay() : Bad payload hash"),REJECT_INVALID, "bad-payload-hash"); + } + } + + if( verify_flags & MC_VRA_SIGNATURE ) + { + if(vSigScript.size()) + { + CHashWriter ss(SER_GETHASH, 0); + ss << vector((unsigned char*)&payload_hash, (unsigned char*)&payload_hash+32); + ss << vector((unsigned char*)&response_to_nonce, (unsigned char*)&response_to_nonce+sizeof(response_to_nonce)); + uint256 signed_hash=ss.GetHash(); + + if(!pubkey.Verify(signed_hash,vchSigOut)) + { + return state.DoS(100, error("ProcessMultichainRelay() : Wrong signature"),REJECT_INVALID, "bad-signature"); + } + } + } + + return true; +} +*/ +void PushMultiChainRelay(CNode* pto, uint32_t msg_type,vector& path,vector& path_to_follow,vector& payload) +{ + pto->PushMessage("relay", msg_type, path, path_to_follow, payload); +} + + +void PushMultiChainRelay(CNode* pto, uint32_t msg_type,vector& path,vector& path_to_follow,unsigned char *payload,size_t size) +{ + vector vPayload= vector((unsigned char *)payload,(unsigned char *)payload+size); + PushMultiChainRelay(pto,msg_type,path,path_to_follow,vPayload); +} + +bool MultichainRelayResponse(uint32_t msg_type_in,vector& path,vector& vPayloadIn, + uint32_t* msg_type_response,vector& vPayloadResponse, + uint32_t* msg_type_relay,vector& vPayloadRelay) +{ + switch(msg_type_in) + { + case MC_RMT_GLOBAL_PING: + *msg_type_response=MC_RMT_GLOBAL_PONG; + vPayloadResponse=vPayloadIn; + if(path.size() < 4) + { + *msg_type_relay=MC_RMT_GLOBAL_PING; + vPayloadRelay=vPayloadIn; + } + break; + case MC_RMT_GLOBAL_PONG: + string str_path=""; + for(int i=(int)path.size()-1;i>=0;i--) + { + str_path+="->" + path[i].ToString(); + } + LogPrintf("PONG : %s\n",str_path.c_str()); + break; + } + return true; +} + +bool ProcessMultichainRelay(CNode* pfrom, CDataStream& vRecv, CValidationState &state) +{ + uint32_t msg_type_in,msg_type_response,msg_type_relay; + size_t common_size; + CNode* pto; + + vector path; + vector path_to_follow; + vector path_response; + vector path_to_follow_response; + vector vPayloadIn; + vector vPayloadResponse; + vector vPayloadRelay; + + vRecv >> msg_type_in; + vRecv >> path; + vRecv >> path_to_follow; + vRecv >> vPayloadIn; + + msg_type_response=MC_RMT_NONE; + msg_type_relay=MC_RMT_NONE; + pto=NULL; + + common_size=path_to_follow.size(); + if(common_size > path.size()) + { + common_size=path.size(); + if(pfrom->addrLocal != (CService)path_to_follow[common_size]) + { + return state.DoS(100, error("ProcessMultichainRelay() : Bad path"),REJECT_INVALID, "bad-relay-path"); + } + } + + path.push_back(pfrom->addr); + common_size++; + + if(common_size < path_to_follow.size()) + { + LOCK(cs_vNodes); + BOOST_FOREACH(CNode* pnode, vNodes) + { + if(pnode->addr == path_to_follow[common_size]) + { + pto=pnode; + } + } + if(pto == NULL) + { + LogPrintf("Request to relay to unconnected peer %s\n",path_to_follow[common_size].ToString().c_str()); + return false; + } + } + + if(pto) + { + PushMultiChainRelay(pto, msg_type_in,path,path_to_follow,vPayloadIn); + return true; + } + + if(pto == NULL) + { + if(MultichainRelayResponse(msg_type_in,path,vPayloadIn,&msg_type_response,vPayloadResponse,&msg_type_relay,vPayloadRelay)) + { + if(msg_type_relay) + { + BOOST_FOREACH(CNode* pnode, vNodes) + { + int path_index=-1; + for(unsigned int i=0;iaddr == path[i]) + { + path_index=i; + } + } + } + if(path_index < 0) + { + PushMultiChainRelay(pnode, msg_type_relay,path,path_to_follow,vPayloadRelay); + } + } + } + if(msg_type_response) + { + for(int i=(int)path.size()-1;i>=0;i--) + { + path_to_follow_response.push_back(path[i]); + PushMultiChainRelay(pfrom, msg_type_response,path_response,path_to_follow_response,vPayloadResponse); + } + } + } + else + { + return false; + } + } + + return true; +} + + diff --git a/src/protocol/relay.h b/src/protocol/relay.h new file mode 100644 index 00000000..62fdb099 --- /dev/null +++ b/src/protocol/relay.h @@ -0,0 +1,173 @@ +// Copyright (c) 2014-2017 Coin Sciences Ltd +// MultiChain code distributed under the GPLv3 license, see COPYING file. + + +#ifndef RELAY_H +#define RELAY_H + +#include "core/init.h" +#include "core/main.h" +#include "utils/util.h" +#include "wallet/wallet.h" +#include "multichain/multichain.h" +#include "keys/pubkey.h" +#include "keys/key.h" +#include "net/net.h" + +#define MC_PRA_NONE 0x00000000 +#define MC_PRA_MY_ORIGIN_MC_ADDRESS 0x00000001 +#define MC_PRA_MY_ORIGIN_NT_ADDRESS 0x00000002 +#define MC_PRA_USE_DESTINATION_ADDRESS 0x00000004 +#define MC_PRA_CALCULATE_MESSAGE_HASH 0x00000008 +#define MC_PRA_SIGN 0x00000010 +#define MC_PRA_SIGN_WITH_HANDSHAKE_ADDRESS 0x00000020 +#define MC_PRA_GENERATE_NONCE 0x00000040 + +#define MC_VRA_NONE 0x00000000 +#define MC_VRA_MESSAGE_TYPE 0x00000001 +#define MC_VRA_ORIGIN_MC_ADDRESS 0x00000002 +#define MC_VRA_ORIGIN_NT_ADDRESS 0x00000004 +#define MC_VRA_DESTINATION_MC_ADDRESS 0x00000008 +#define MC_VRA_DESTINATION_NT_ADDRESS 0x00000010 +#define MC_VRA_RESPONSE_TO_NONCE 0x00000020 +#define MC_VRA_MESSAGE_HASH 0x00000040 +#define MC_VRA_SIGSCRIPT 0x00000080 +#define MC_VRA_SIGNATURE_ORIGIN 0x00000100 +#define MC_VRA_SIGNATURE_ALL 0x00000200 +#define MC_VRA_LIMIT_ALL 0x00000400 +#define MC_VRA_LIMIT_MSG_TYPE 0x00000800 +#define MC_VRA_SINGLE_HOP 0x00001000 +#define MC_VRA_ALREADY_PROCESSED 0x00002000 + +#define MC_RMT_NONE 0 +#define MC_RMT_GLOBAL_PING 0x00000001 +#define MC_RMT_GLOBAL_PONG 0x00000002 +#define MC_RMT_GLOBAL_REJECT 0x00000003 +#define MC_RMT_GLOBAL_BUSY 0x00000004 +#define MC_RMT_CHUNK_QUERY 0x00000101 +#define MC_RMT_CHUNK_QUERY_HIT 0x00000102 +#define MC_RMT_CHUNK_REQUEST 0x00000103 +#define MC_RMT_CHUNK_RESPONSE 0x00000104 + + + + +using namespace std; + +#define MC_LIM_MAX_SECONDS 60 +#define MC_LIM_MAX_MEASURES 4 + + +typedef struct mc_Limiter +{ + int m_SecondCount; + int m_MeasureCount; + uint32_t m_Time; + int64_t m_Limits[MC_LIM_MAX_MEASURES]; + int64_t m_Totals[MC_LIM_MAX_MEASURES]; + int64_t m_Event[MC_LIM_MAX_MEASURES]; + int64_t m_Measures[MC_LIM_MAX_MEASURES*MC_LIM_MAX_SECONDS]; + void Zero(); + int Intitialize(int seconds,int measures); + int SetLimit(int meausure,int64_t limit); + + void CheckTime(); + void CheckTime(uint32_t t); + + void SetEvent(int64_t m1); + void SetEvent(int64_t m1,int64_t m2); + void SetEvent(int64_t m1,int64_t m2,int64_t m3); + void SetEvent(int64_t m1,int64_t m2,int64_t m3,int64_t m4); + + int Disallowed(); + int Disallowed(uint32_t t); + void Increment(); + +} mc_Limiter; + + +typedef struct mc_NodeFullAddress +{ + CKeyID m_Address; + vector m_NetAddresses; + + void Zero(); + + mc_NodeFullAddress() + { + Zero(); + } + +} mc_NodeFullAddress; + +typedef struct mc_RelayRecordKey +{ + int64_t m_Nonce; + NodeId m_NodeTo; + + mc_RelayRecordKey(int64_t nonce,NodeId node) + { + m_Nonce=nonce; + m_NodeTo=node; + } + + friend bool operator<(const mc_RelayRecordKey& a, const mc_RelayRecordKey& b) + { + return (a.m_Nonce < b.m_Nonce || (a.m_Nonce == b.m_Nonce && a.m_NodeTo < b.m_NodeTo)); + } + +} mc_RelayRecordKey; + +typedef struct mc_RelayRecordValue +{ + uint32_t m_MsgType; + NodeId m_NodeFrom; + uint32_t m_Timestamp; + +} mc_RelayRecordValue; + +typedef struct mc_RelayManager +{ + uint32_t m_MyIPs[64]; + int m_MyIPCount; + uint32_t m_LastTime; + + map m_Latency; + map m_Limiters; + map m_RelayRecords; + + void SetDefaults(); + void SetMyIPs(uint32_t *ips,int ip_count); + void MsgTypeSettings(uint32_t msg_type,int latency,int seconds,int64_t serves_per_second,int64_t bytes_per_second); + void InitNodeAddress(mc_NodeFullAddress* node_address,CNode* pto,uint32_t action); + void InitNodeAddress(mc_NodeFullAddress* node_address,CKeyID& mc_address, vector& net_addresses); + void CheckTime(); + void SetRelayRecord(CNode *pto,CNode *pfrom,uint32_t msg_type,int64_t nonce); + int GetRelayRecord(CNode *pfrom,int64_t nonce,uint32_t *msg_type,CNode **pto); + + void PushRelay ( CNode* pto, + uint32_t msg_type, + int64_t nonce_to_send, + int64_t response_to_nonce, + mc_NodeFullAddress *origin, + mc_NodeFullAddress *destination, + uint32_t flags, + vector& payload, + uint256 message_hash_to_relay, + vector& sigScripts_to_relay, + vector& path, + CNode* pfrom, + uint32_t action); + + bool ProcessRelay ( CNode* pfrom, + CDataStream& vRecv, + CValidationState &state, + uint32_t verify_flags_in); + +} mc_RelayManager; + + + + +#endif /* RELAY_H */ + From 9fd695e0bb827f724f04aa9653dac614736b624f Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 11 Apr 2018 17:39:50 +0300 Subject: [PATCH 033/157] Offchain messaging code, not debugged --- src/core/main.cpp | 19 +- src/protocol/relay.cpp | 924 +++++++++++++++++++++++++++++++---------- src/protocol/relay.h | 162 ++++++-- 3 files changed, 844 insertions(+), 261 deletions(-) diff --git a/src/core/main.cpp b/src/core/main.cpp index 2ce353e2..d223c476 100644 --- a/src/core/main.cpp +++ b/src/core/main.cpp @@ -30,10 +30,11 @@ #include "multichain/multichain.h" #include "wallet/wallettxs.h" #include "script/script.h" +#include "protocol/relay.h" extern mc_WalletTxs* pwalletTxsMain; - +extern mc_RelayManager* pRelayManager; /* MCHN END */ @@ -61,6 +62,7 @@ bool ReplayMemPool(CTxMemPool& pool, int from,bool accept); bool VerifyBlockSignature(CBlock *block,bool force); bool VerifyBlockMiner(CBlock *block,CBlockIndex* pindexNew); bool CheckBlockPermissions(const CBlock& block,CBlockIndex* prev_block,unsigned char *lpMinerAddress); +bool ProcessMultichainRelay(CNode* pfrom, CDataStream& vRecv, CValidationState &state); bool ProcessMultichainVerack(CNode* pfrom, CDataStream& vRecv,bool fIsVerackack,bool *disconnect_flag); bool PushMultiChainVerack(CNode* pfrom, bool fIsVerackack); bool MultichainNode_CanConnect(CNode *pnode); @@ -5777,6 +5779,21 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, return false; } + else if (strCommand == "relay") + { + CValidationState state; + if(pRelayManager) + { + if(!pRelayManager->ProcessRelay(pfrom,vRecv,state,MC_VRA_DEFAULT)) + { + int nDos = 0; + if (state.IsInvalid(nDos) && nDos > 0) + { + Misbehaving(pfrom->GetId(), nDos); + } + } + } + } else if (strCommand == "verack") { diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index dd9fdf3b..e3a6dcac 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -3,6 +3,32 @@ #include "protocol/relay.h" +bool MultichainRelayResponse(uint32_t msg_type_stored, CNode *pto_stored, + uint32_t msg_type_in, uint32_t flags, vector& vPayloadIn,vector& vAddrIn, + uint32_t* msg_type_response,uint32_t *flags_response,vector& vPayloadResponse,vector& vAddrResponse, + uint32_t* msg_type_relay,uint32_t *flags_relay,vector& vPayloadRelay,vector& vAddrRelay,string& strError) +{ + strError=""; + switch(msg_type_in) + { + case MC_RMT_GLOBAL_PING: + if(msg_type_stored) + { + strError="Invalid response message type"; + goto exitlbl; + } + break; + } + +exitlbl: + + if(strError.size()) + { + return false; + } + return true; +} + void mc_Limiter::Zero() { memset(this,0,sizeof(mc_Limiter)); @@ -219,17 +245,73 @@ void mc_RelayManager::MsgTypeSettings(uint32_t msg_type,int latency,int seconds, } } +int64_t mc_RelayManager::AggregateNonce(uint32_t timestamp,uint32_t nonce) +{ + return ((int64_t)nonce<<32)+(int64_t)timestamp; +} + +void mc_RelayManager::Zero() +{ + m_Semaphore=NULL; + m_LockedBy=0; +} + +void mc_RelayManager::Destroy() +{ + if(m_Semaphore) + { + __US_SemDestroy(m_Semaphore); + } + + Zero(); +} + +int mc_RelayManager::Intialize() +{ + m_Semaphore=__US_SemCreate(); + return MC_ERR_NOERROR; +} + +int mc_RelayManager::Lock(int write_mode,int allow_secondary) +{ + uint64_t this_thread; + this_thread=__US_ThreadID(); + + if(this_thread == m_LockedBy) + { + return allow_secondary; + } + + __US_SemWait(m_Semaphore); + m_LockedBy=this_thread; + + return 0; +} + +void mc_RelayManager::UnLock() +{ + m_LockedBy=0; + __US_SemPost(m_Semaphore); +} + +int mc_RelayManager::Lock() +{ + return Lock(1,0); +} + void mc_RelayManager::SetDefaults() { MsgTypeSettings(MC_RMT_NONE , 0,10,1000,100*1024*1024); MsgTypeSettings(MC_RMT_GLOBAL_PING ,10,10, 100, 1*1024*1024); MsgTypeSettings(MC_RMT_GLOBAL_PONG , 0,10, 100, 1*1024*1024); MsgTypeSettings(MC_RMT_GLOBAL_REJECT , 0,10,1000, 1*1024*1024); - MsgTypeSettings(MC_RMT_GLOBAL_BUSY , 0,10,1000, 1*1024*1024); MsgTypeSettings(MC_RMT_CHUNK_QUERY ,10,10, 100, 1*1024*1024); MsgTypeSettings(MC_RMT_CHUNK_QUERY_HIT,30,10, 100, 1*1024*1024); MsgTypeSettings(MC_RMT_CHUNK_REQUEST ,30,10, 100, 1*1024*1024); MsgTypeSettings(MC_RMT_CHUNK_RESPONSE , 0,10, 100,100*1024*1024); + + m_MinTimeShift=180; + m_MaxTimeShift=180; } void mc_RelayManager::CheckTime() @@ -253,7 +335,7 @@ void mc_RelayManager::CheckTime() } } -void mc_RelayManager::SetRelayRecord(CNode *pto,CNode *pfrom,uint32_t msg_type,int64_t nonce) +void mc_RelayManager::SetRelayRecord(CNode *pto,CNode *pfrom,uint32_t msg_type,uint32_t timestamp,uint32_t nonce) { map::iterator itlat = m_Latency.find(msg_type); if (itlat == m_Latency.end()) @@ -270,7 +352,7 @@ void mc_RelayManager::SetRelayRecord(CNode *pto,CNode *pfrom,uint32_t msg_type,i { pto_id=pto->GetId(); } - const mc_RelayRecordKey key=mc_RelayRecordKey(nonce,pto_id); + const mc_RelayRecordKey key=mc_RelayRecordKey(timestamp,nonce,pto_id); mc_RelayRecordValue value; value.m_NodeFrom=0; if(pfrom) @@ -291,7 +373,7 @@ void mc_RelayManager::SetRelayRecord(CNode *pto,CNode *pfrom,uint32_t msg_type,i } } -int mc_RelayManager::GetRelayRecord(CNode *pfrom,int64_t nonce,uint32_t* msg_type,CNode **pto) +int mc_RelayManager::GetRelayRecord(CNode *pfrom,uint32_t timestamp,uint32_t nonce,uint32_t* msg_type,CNode **pto) { NodeId pfrom_id,pto_id; @@ -300,7 +382,7 @@ int mc_RelayManager::GetRelayRecord(CNode *pfrom,int64_t nonce,uint32_t* msg_typ { pfrom_id=pfrom->GetId(); } - const mc_RelayRecordKey key=mc_RelayRecordKey(nonce,pfrom_id); + const mc_RelayRecordKey key=mc_RelayRecordKey(timestamp,nonce,pfrom_id); map::iterator it = m_RelayRecords.find(key); if (it == m_RelayRecords.end()) { @@ -342,111 +424,75 @@ int mc_RelayManager::GetRelayRecord(CNode *pfrom,int64_t nonce,uint32_t* msg_typ return MC_ERR_NOT_ALLOWED; } -void mc_RelayManager::PushRelay(CNode* pto, +uint32_t mc_RelayManager::GenerateNonce() +{ + uint32_t nonce; + + GetRandBytes((unsigned char*)&nonce, sizeof(nonce)); + + return nonce; +} + +int64_t mc_RelayManager::PushRelay(CNode* pto, + uint32_t msg_format, + unsigned char hop_count, uint32_t msg_type, - int64_t nonce_to_send, - int64_t response_to_nonce, - mc_NodeFullAddress *origin_to_relay, - mc_NodeFullAddress *destination_to_relay, + uint32_t timestamp_to_send, + uint32_t nonce_to_send, + uint32_t timestamp_to_respond, + uint32_t nonce_to_respond, uint32_t flags, vector& payload, - uint256 message_hash_to_relay, vector& sigScripts_to_relay, - vector& path, CNode* pfrom, uint32_t action) { vector vOriginAddress; vector vDestinationAddress; vector sigScripts; - vectorvSigScript; CScript sigScript; uint256 message_hash; - int64_t nonce; - mc_NodeFullAddress origin_to_send; - mc_NodeFullAddress *origin; - mc_NodeFullAddress destination_to_send; - mc_NodeFullAddress *destination; + uint32_t timestamp; + uint32_t nonce; nonce=nonce_to_send; + timestamp=timestamp_to_send; - if(action & MC_PRA_GENERATE_NONCE) - { - GetRandBytes((unsigned char*)&nonce, sizeof(nonce)); - } - - if(action & (MC_PRA_MY_ORIGIN_MC_ADDRESS | MC_PRA_MY_ORIGIN_NT_ADDRESS) ) - { - InitNodeAddress(&origin_to_send,pto,action); - origin=&origin_to_send; - } - else + if(action & MC_PRA_GENERATE_TIMESTAMP) { - if(origin_to_relay) - { - origin=origin_to_relay; - } - else - { - origin=&origin_to_send; - } + timestamp=mc_TimeNowAsUInt(); } - destination=&destination_to_send; - if(action & MC_PRA_USE_DESTINATION_ADDRESS) + if(action & MC_PRA_GENERATE_NONCE) { - if(destination_to_relay) - { - destination=destination_to_relay; - } + nonce=GenerateNonce(); } - if(MCP_ANYONE_CAN_CONNECT == 0) - { - vOriginAddress.resize(sizeof(CKeyID)); - memcpy(&vOriginAddress[0],&(origin->m_Address),sizeof(CKeyID)); - if(destination->m_Address != 0) - { - vDestinationAddress.resize(sizeof(CKeyID)); - memcpy(&vOriginAddress[0],&(origin->m_Address),sizeof(CKeyID)); - } - } - - message_hash=message_hash_to_relay; + int64_t aggr_nonce=AggregateNonce(timestamp,nonce); - if(action & MC_PRA_CALCULATE_MESSAGE_HASH) + if( (action & MC_PRA_SIGN_WITH_HANDSHAKE_ADDRESS) && (MCP_ANYONE_CAN_CONNECT == 0) ) { CHashWriter ssHash(SER_GETHASH, 0); + ssHash << msg_type; + ssHash << timestamp; ssHash << nonce; - ssHash << msg_type; - ssHash << vOriginAddress; - ssHash << origin->m_NetAddresses; - ssHash << vDestinationAddress; - ssHash << destination->m_NetAddresses; - ssHash << response_to_nonce; + ssHash << timestamp_to_respond; + ssHash << nonce_to_respond; ssHash << flags; ssHash << payload; - message_hash=ssHash.GetHash(); - } - - if( (action & (MC_PRA_SIGN | MC_PRA_SIGN_WITH_HANDSHAKE_ADDRESS)) && (MCP_ANYONE_CAN_CONNECT == 0) ) - { + + message_hash=ssHash.GetHash(); + CHashWriter ssSig(SER_GETHASH, 0); + ssSig << message_hash; - ssSig << vector((unsigned char*)&response_to_nonce, (unsigned char*)&response_to_nonce+sizeof(response_to_nonce)); + ssSig << vector((unsigned char*)&aggr_nonce, (unsigned char*)&aggr_nonce+sizeof(aggr_nonce)); uint256 signed_hash=ssSig.GetHash(); CKey key; CKeyID keyID; CPubKey pkey; - keyID=origin->m_Address; - if(action & MC_PRA_SIGN_WITH_HANDSHAKE_ADDRESS) - { - if(MCP_ANYONE_CAN_CONNECT == 0) - { - keyID=pto->kAddrLocal; - } - } + keyID=pto->kAddrLocal; if(pwalletMain->GetKey(keyID, key)) { @@ -477,21 +523,24 @@ void mc_RelayManager::PushRelay(CNode* pto, } pto->PushMessage("relay", + msg_format, + hop_count, msg_type, + timestamp, nonce, - vOriginAddress, - origin->m_NetAddresses, - vDestinationAddress, - destination->m_NetAddresses, - response_to_nonce, + timestamp_to_respond, + nonce_to_respond, flags, payload, - message_hash, - path, sigScripts); - SetRelayRecord(pto,NULL,msg_type,nonce); - SetRelayRecord(pto,pfrom,msg_type,nonce); + SetRelayRecord(pto,NULL,msg_type,timestamp,nonce); + if(pfrom) + { + SetRelayRecord(pto,pfrom,msg_type,timestamp,nonce); + } + + return aggr_nonce; } bool mc_RelayManager::ProcessRelay( CNode* pfrom, @@ -501,56 +550,70 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, { uint32_t msg_type_in; uint32_t verify_flags; - int64_t nonce_received; - int64_t response_to_nonce; - vector vOriginMCAddressIn; - vector vDestinationMCAddressIn; - vector vOriginNetAddressIn; - vector vDestinationNetAddressIn; + uint32_t timestamp_received; + uint32_t nonce_received; + uint32_t timestamp_to_respond; + uint32_t nonce_to_respond; vector vPayloadIn; vector vSigScripts; - uint256 message_hash,message_hash_in; - uint32_t flags_in; - CKeyID origin_mc_address; - CKeyID destination_mc_address; + vector vSigScriptsEmpty; + uint256 message_hash; + uint32_t flags_in,flags_response,flags_relay; vector vchSigOut; vector vchPubKey; CPubKey pubkey; vector path; - mc_NodeFullAddress origin_to_relay; - mc_NodeFullAddress destination_to_relay; - vector vPayloadResponse; - vector vSigScriptsResponse; - vector vPathResponse; CNode *pto_stored; uint32_t msg_type_stored; + uint32_t msg_format; + unsigned char hop_count; + uint32_t msg_type_response,msg_type_relay; + uint32_t *msg_type_relay_ptr; + uint32_t *msg_type_response_ptr; + vector vPayloadResponse; + vector vPayloadRelay; + vector vAddrIn; + vector vAddrResponse; + vector vAddrRelay; + string strError; msg_type_stored=MC_RMT_NONE; + msg_type_response=MC_RMT_NONE; + msg_type_relay=MC_RMT_NONE; pto_stored=NULL; verify_flags=verify_flags_in; + vRecv >> msg_format; + if(msg_format != 0) + { + LogPrintf("ProcessRelay() : Unsupported message format %08X\n",msg_format); + return false; + } + + vRecv >> hop_count; vRecv >> msg_type_in; switch(msg_type_in) { case MC_RMT_GLOBAL_PING: + verify_flags |= MC_VRA_IS_NOT_RESPONSE; break; case MC_RMT_GLOBAL_PONG: - verify_flags |= MC_VRA_ORIGIN_MC_ADDRESS | MC_VRA_ORIGIN_NT_ADDRESS | MC_VRA_RESPONSE_TO_NONCE | MC_VRA_MESSAGE_HASH | MC_VRA_SIGNATURE_ORIGIN; + verify_flags |= MC_VRA_IS_RESPONSE | MC_VRA_SIGNATURE_ORIGIN; break; case MC_RMT_GLOBAL_REJECT: - case MC_RMT_GLOBAL_BUSY: - verify_flags |= MC_VRA_ORIGIN_MC_ADDRESS | MC_VRA_RESPONSE_TO_NONCE | MC_VRA_MESSAGE_HASH | MC_VRA_SIGNATURE_ORIGIN; + verify_flags |= MC_VRA_IS_RESPONSE | MC_VRA_SIGNATURE_ORIGIN; break; case MC_RMT_CHUNK_QUERY: - verify_flags |= MC_VRA_ORIGIN_MC_ADDRESS | MC_VRA_MESSAGE_HASH; + verify_flags |= MC_VRA_IS_NOT_RESPONSE; break; case MC_RMT_CHUNK_QUERY_HIT: - verify_flags |= MC_VRA_ORIGIN_MC_ADDRESS | MC_VRA_ORIGIN_NT_ADDRESS | MC_VRA_DESTINATION_MC_ADDRESS | MC_VRA_RESPONSE_TO_NONCE | MC_VRA_MESSAGE_HASH | - MC_VRA_SIGNATURE_ORIGIN; + verify_flags |= MC_VRA_IS_RESPONSE | MC_VRA_SIGNATURE_ORIGIN | MC_VRA_PROCESS_ONCE; + break; case MC_RMT_CHUNK_REQUEST: + verify_flags |= MC_VRA_IS_RESPONSE | MC_VRA_DOUBLE_HOP | MC_VRA_SIGNATURE_ORIGIN; + break; case MC_RMT_CHUNK_RESPONSE: - verify_flags |= MC_VRA_ORIGIN_MC_ADDRESS | MC_VRA_ORIGIN_NT_ADDRESS | MC_VRA_DESTINATION_MC_ADDRESS | MC_VRA_RESPONSE_TO_NONCE | MC_VRA_MESSAGE_HASH | - MC_VRA_SIGNATURE_ORIGIN | MC_VRA_SINGLE_HOP; + verify_flags |= MC_VRA_IS_RESPONSE | MC_VRA_DOUBLE_HOP | MC_VRA_SIGNATURE_ORIGIN | MC_VRA_PROCESS_ONCE; break; default: if(verify_flags & MC_VRA_MESSAGE_TYPE) @@ -561,79 +624,100 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, break; } - CheckTime(); - - vRecv >> nonce_received; + if(verify_flags & MC_VRA_SINGLE_HOP) + { + if(hop_count) + { + LogPrintf("ProcessRelay() : Unsupported hop count %d for msg type %d\n",hop_count,msg_type_in); + return false; + } + } - if(verify_flags & MC_VRA_ALREADY_PROCESSED) + if(verify_flags & MC_VRA_DOUBLE_HOP) { - if(GetRelayRecord(NULL,nonce_received,NULL,NULL) == MC_ERR_NOERROR) + if(hop_count > 1) { - return false; + LogPrintf("ProcessRelay() : Unsupported hop count %d for msg type %d\n",hop_count,msg_type_in); + return false; } } + + CheckTime(); - vRecv >> vOriginMCAddressIn; - vRecv >> vOriginNetAddressIn; - vRecv >> vDestinationMCAddressIn; - vRecv >> vDestinationNetAddressIn; - vRecv >> response_to_nonce; - vRecv >> flags_in; - vRecv >> vPayloadIn; - vRecv >> message_hash_in; - vRecv >> path; - vRecv >> vSigScripts; + vRecv >> timestamp_received; + + if(verify_flags & MC_VRA_TIMESTAMP) + { + if(timestamp_received+m_MinTimeShift < m_LastTime) + { + LogPrintf("ProcessRelay() : Timestamp too far in the past: %d\n",timestamp_received); + return false; + } + if(timestamp_received > m_LastTime + m_MaxTimeShift) + { + LogPrintf("ProcessRelay() : Timestamp too far in the future: %d\n",timestamp_received); + return false; + } + } + vRecv >> nonce_received; + msg_type_relay_ptr=&msg_type_relay; + msg_type_response_ptr=&msg_type_response; - if(vOriginMCAddressIn.size() == sizeof(uint160)) + if( verify_flags & (MC_VRA_PROCESS_ONCE | MC_VRA_BROADCAST_ONCE)) { - origin_mc_address=CKeyID(*(uint160*)&vOriginMCAddressIn[0]); - } - else - { - if(verify_flags & MC_VRA_ORIGIN_MC_ADDRESS) + if(GetRelayRecord(NULL,timestamp_received,nonce_received,NULL,NULL) == MC_ERR_NOERROR) { - if(MCP_ANYONE_CAN_CONNECT == 0) + if(verify_flags & MC_VRA_PROCESS_ONCE) { - return state.DoS(100, error("ProcessRelay() : Bad origin address"),REJECT_INVALID, "bad-origin-address"); - } + return false; + } + if(verify_flags & MC_VRA_BROADCAST_ONCE) + { + msg_type_relay_ptr=NULL; + } } } - if(verify_flags & MC_VRA_ORIGIN_NT_ADDRESS) + + vRecv >> timestamp_to_respond; + vRecv >> nonce_to_respond; + + if( verify_flags & MC_VRA_IS_NOT_RESPONSE ) { - if(vOriginNetAddressIn.size() == 0) + if( (timestamp_to_respond != 0) || (nonce_to_respond != 0) ) { - return state.DoS(100, error("ProcessRelay() : No origin network address"),REJECT_INVALID, "bad-origin-net-address"); - } - } + return state.DoS(100, error("ProcessRelay() : This message should not be response"),REJECT_INVALID, "bad-nonce"); + } + } - InitNodeAddress(&origin_to_relay,origin_mc_address,vOriginNetAddressIn); + int64_t aggr_nonce=AggregateNonce(timestamp_to_respond,nonce_to_respond); - if(vDestinationMCAddressIn.size() == sizeof(uint160)) + if( verify_flags & MC_VRA_IS_RESPONSE ) { - destination_mc_address=CKeyID(*(uint160*)&vDestinationMCAddressIn[0]); - } - else - { - if(verify_flags & MC_VRA_DESTINATION_MC_ADDRESS) + if( timestamp_to_respond == 0 ) { - if(MCP_ANYONE_CAN_CONNECT == 0) - { - return state.DoS(100, error("ProcessRelay() : Bad destination address"),REJECT_INVALID, "bad-destination-address"); - } + return state.DoS(100, error("ProcessRelay() : This message should be response"),REJECT_INVALID, "bad-nonce"); + } + + if(GetRelayRecord(pfrom,timestamp_to_respond,nonce_to_respond,&msg_type_stored,&pto_stored)) + { + LogPrintf("ProcessRelay() : Response without request from peer %d\n",pfrom->GetId()); + return false; } } - if(verify_flags & MC_VRA_DESTINATION_NT_ADDRESS) + if(timestamp_to_respond) { - if(vDestinationNetAddressIn.size() == 0) + if(pto_stored) { - return state.DoS(100, error("ProcessRelay() : No destination network address"),REJECT_INVALID, "bad-destination-net-address"); - } - } - - InitNodeAddress(&destination_to_relay,destination_mc_address,vDestinationNetAddressIn); + msg_type_response_ptr=NULL; + } + else + { + msg_type_relay_ptr=NULL; + } + } map::iterator itlim_all = m_Limiters.find(MC_RMT_NONE); map::iterator itlim_msg = m_Limiters.find(msg_type_in); @@ -645,8 +729,6 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, { if(itlim_all->second.Disallowed(m_LastTime)) { - PushRelay(pfrom,MC_RMT_GLOBAL_BUSY,0,nonce_received,NULL,NULL,0,vPayloadResponse,0,vSigScriptsResponse,vPathResponse,NULL, - MC_PRA_GENERATE_NONCE | MC_PRA_MY_ORIGIN_MC_ADDRESS | MC_PRA_CALCULATE_MESSAGE_HASH | MC_PRA_SIGN | MC_PRA_SIGN_WITH_HANDSHAKE_ADDRESS); return false; } } @@ -659,94 +741,76 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, { if(itlim_msg->second.Disallowed(m_LastTime)) { - PushRelay(pfrom,MC_RMT_GLOBAL_BUSY,0,nonce_received,NULL,NULL,0,vPayloadResponse,0,vSigScriptsResponse,vPathResponse,NULL, - MC_PRA_GENERATE_NONCE | MC_PRA_MY_ORIGIN_MC_ADDRESS | MC_PRA_CALCULATE_MESSAGE_HASH | MC_PRA_SIGN | MC_PRA_SIGN_WITH_HANDSHAKE_ADDRESS); return false; } } } - - if( verify_flags & MC_VRA_RESPONSE_TO_NONCE ) - { - if(response_to_nonce == 0) - { - return state.DoS(100, error("ProcessRelay() : Missing response_to_nonce"),REJECT_INVALID, "bad-nonce"); - } - if(GetRelayRecord(pfrom,response_to_nonce,&msg_type_stored,&pto_stored)) - { - LogPrintf("ProcessRelay() : Response without request from peer %d\n",pfrom->GetId()); - return false; - } - } - - if( verify_flags & (MC_VRA_MESSAGE_HASH | MC_VRA_SIGNATURE_ORIGIN | MC_VRA_SIGNATURE_ALL) ) + vRecv >> flags_in; + vRecv >> vPayloadIn; + vRecv >> vSigScripts; + + if( verify_flags & MC_VRA_SIGNATURES ) { CHashWriter ssHash(SER_GETHASH, 0); + ssHash << msg_type_in; + ssHash << timestamp_received; ssHash << nonce_received; - ssHash << msg_type_in; - ssHash << vOriginMCAddressIn; - ssHash << vOriginNetAddressIn; - ssHash << vDestinationMCAddressIn; - ssHash << vDestinationNetAddressIn; - ssHash << response_to_nonce; + ssHash << timestamp_to_respond; + ssHash << nonce_to_respond; ssHash << flags_in; ssHash << vPayloadIn; + message_hash=ssHash.GetHash(); - if(message_hash != message_hash_in) - { - return state.DoS(100, error("ProcessRelay() : Payload hash mismatch"),REJECT_INVALID, "bad-payload-hash"); - } - } - - message_hash=message_hash_in; - if( verify_flags & MC_VRA_SIGNATURE_ORIGIN ) - { - if(vSigScripts.size() < 1) - { - return state.DoS(100, error("ProcessRelay() : Missing sigScript"),REJECT_INVALID, "bad-sigscript"); - } - if(vSigScripts[0].size()) + if( verify_flags & MC_VRA_SIGNATURE_ORIGIN ) { - CScript scriptSig=vSigScripts[0]; + if(vSigScripts.size() < 1) + { + return state.DoS(100, error("ProcessRelay() : Missing sigScript"),REJECT_INVALID, "bad-sigscript"); + } + if(vSigScripts[0].size()) + { + CScript scriptSig=vSigScripts[0]; - opcodetype opcode; + opcodetype opcode; - CScript::const_iterator pc = scriptSig.begin(); + CScript::const_iterator pc = scriptSig.begin(); - if (!scriptSig.GetOp(pc, opcode, vchSigOut)) - { - return state.DoS(100, error("ProcessRelay() : Cannot extract signature from sigScript"),REJECT_INVALID, "bad-sigscript-signature"); - } + if (!scriptSig.GetOp(pc, opcode, vchSigOut)) + { + return state.DoS(100, error("ProcessRelay() : Cannot extract signature from sigScript"),REJECT_INVALID, "bad-sigscript-signature"); + } - vchSigOut.resize(vchSigOut.size()-1); - if (!scriptSig.GetOp(pc, opcode, vchPubKey)) - { - return state.DoS(100, error("ProcessRelay() : Cannot extract pubkey from sigScript"),REJECT_INVALID, "bad-sigscript-pubkey"); - } + vchSigOut.resize(vchSigOut.size()-1); + if (!scriptSig.GetOp(pc, opcode, vchPubKey)) + { + return state.DoS(100, error("ProcessRelay() : Cannot extract pubkey from sigScript"),REJECT_INVALID, "bad-sigscript-pubkey"); + } - CPubKey pubKeyOut(vchPubKey); - if (!pubKeyOut.IsValid()) - { - return state.DoS(100, error("ProcessRelay() : Invalid pubkey"),REJECT_INVALID, "bad-sigscript-pubkey"); - } + CPubKey pubKeyOut(vchPubKey); + if (!pubKeyOut.IsValid()) + { + return state.DoS(100, error("ProcessRelay() : Invalid pubkey"),REJECT_INVALID, "bad-sigscript-pubkey"); + } - pubkey=pubKeyOut; - - CHashWriter ss(SER_GETHASH, 0); - ss << vector((unsigned char*)&message_hash, (unsigned char*)&message_hash+32); - ss << vector((unsigned char*)&response_to_nonce, (unsigned char*)&response_to_nonce+sizeof(response_to_nonce)); - uint256 signed_hash=ss.GetHash(); + pubkey=pubKeyOut; - if(!pubkey.Verify(signed_hash,vchSigOut)) + CHashWriter ss(SER_GETHASH, 0); + + ss << vector((unsigned char*)&message_hash, (unsigned char*)&message_hash+32); + ss << vector((unsigned char*)&aggr_nonce, (unsigned char*)&aggr_nonce+sizeof(aggr_nonce)); + uint256 signed_hash=ss.GetHash(); + + if(!pubkey.Verify(signed_hash,vchSigOut)) + { + return state.DoS(100, error("ProcessRelay() : Wrong signature"),REJECT_INVALID, "bad-signature"); + } + } + else { - return state.DoS(100, error("ProcessRelay() : Wrong signature"),REJECT_INVALID, "bad-signature"); - } - } - else - { - return state.DoS(100, error("ProcessRelay() : Empty sigScript"),REJECT_INVALID, "bad-sigscript"); + return state.DoS(100, error("ProcessRelay() : Empty sigScript"),REJECT_INVALID, "bad-sigscript"); + } } } @@ -759,10 +823,260 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, { itlim_msg->second.Increment(); } + + if(pto_stored) + { + PushRelay(pto_stored,msg_format,hop_count+1,msg_type_in,timestamp_received,nonce_received,timestamp_to_respond,nonce_to_respond,flags_in, + vPayloadIn,vSigScripts,pfrom,MC_PRA_NONE); + } + else + { + if( (msg_type_relay_ptr != NULL) || (msg_type_response_ptr != NULL) ) + { + if(MultichainRelayResponse(msg_type_stored,pto_stored, + msg_type_in,flags_in,vPayloadIn,vAddrIn, + msg_type_response_ptr,&flags_response,vPayloadResponse,vAddrResponse, + msg_type_relay_ptr,&flags_relay,vPayloadRelay,vAddrRelay,strError)) + { + if(msg_type_response_ptr && *msg_type_response_ptr) + { + if(*msg_type_response_ptr != MC_RMT_ADD_RESPONSE) + { + PushRelay(pfrom,0,0,*msg_type_response_ptr,m_LastTime,0,timestamp_received,nonce_received,flags_response, + vPayloadResponse,vSigScriptsEmpty,NULL,MC_PRA_GENERATE_NONCE); + } + else + { + map::iterator itreq = m_Requests.find(AggregateNonce(timestamp_to_respond,nonce_to_respond)); + if(itreq != m_Requests.end()) + { + Lock(); + AddResponse(itreq->second.m_Nonce,pfrom,NULL,hop_count,AggregateNonce(timestamp_received,nonce_received),msg_type_in,flags_in,vPayloadIn,MC_RST_SUCCESS); + UnLock(); + } + } + } + if(msg_type_relay_ptr && *msg_type_relay_ptr) + { + LOCK(cs_vNodes); + BOOST_FOREACH(CNode* pnode, vNodes) + { + if(pnode != pfrom) + { + PushRelay(pnode,msg_format,hop_count+1,*msg_type_relay_ptr,timestamp_received,nonce_received,timestamp_to_respond,nonce_to_respond,flags_relay, + vPayloadRelay,vSigScriptsEmpty,pfrom,MC_PRA_NONE); + } + } + } + } + else + { + LogPrintf("ProcessRelay() : Error processing request %08X from peer %d: %s\n",msg_type_in,pfrom->GetId(),strError.c_str()); + return false; + } + } + } return true; } +/* +typedef struct mc_RelayResponse +{ + int64_t m_Nonce; + uint32_t m_MsgType; + uint32_t m_Flags; + CNode *m_NodeFrom; + int m_HopCount; + mc_NodeFullAddress m_Source; + uint32_t m_LastTryTimestamp; + vector m_Payload; + vector m_Requests; + + void Zero(); +} mc_RelayResponse; + +typedef struct mc_RelayRequest +{ + int64_t m_Nonce; + uint32_t m_MsgType; + uint32_t m_Flags; + int64_t m_ParentNonce; + int m_ParentResponseID; + uint32_t m_LastTryTimestamp; + int m_TryCount; + uint32_t m_Status; + vector m_Payload; + vector m_Responses; + + void Zero(); +} mc_RelayRequest; +*/ + +void mc_RelayResponse::Zero() +{ + m_Nonce=0; + m_MsgType=MC_RMT_NONE; + m_Flags=0; + m_NodeFrom=NULL; + m_HopCount=0; + m_Source.Zero(); + m_LastTryTimestamp=0; + m_Status=MC_RST_NONE; + m_Payload.clear(); + m_Requests.clear(); +} + +void mc_RelayRequest::Zero() +{ + m_Nonce=0; + m_MsgType=MC_RMT_NONE; + m_Flags=0; + m_NodeTo=NULL; + m_ParentNonce=0; + m_ParentResponseID=-1; + m_LastTryTimestamp=0; + m_TryCount=0; + m_Status=MC_RST_NONE; + m_Payload.clear(); + m_Responses.clear(); +} + +int mc_RelayManager::AddRequest(int64_t parent_nonce,int parent_response_id,CNode *pto,int64_t nonce,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status) +{ + int err=MC_ERR_NOERROR; + + Lock(); + map::iterator itreq_this = m_Requests.find(nonce); + if(itreq_this == m_Requests.end()) + { + mc_RelayRequest request; + + request.m_Nonce=nonce; + request.m_MsgType=msg_type; + request.m_Flags=flags; + request.m_NodeTo=pto; + request.m_ParentNonce=parent_nonce; + request.m_ParentResponseID=parent_response_id; + request.m_LastTryTimestamp=0; + request.m_TryCount=0; + request.m_Status=status; + request.m_Payload=payload; + request.m_Responses.clear(); + + if(parent_nonce) + { + map::iterator itreq = m_Requests.find(parent_nonce); + if(itreq != m_Requests.end()) + { + if(parent_response_id < (int)(itreq->second.m_Responses.size())) + { + itreq->second.m_Responses[parent_response_id].m_Requests.push_back(request); + } + } + else + { + err=MC_ERR_NOT_FOUND; + } + } + + m_Requests.insert(make_pair(nonce,request)); + } + else + { + err=MC_ERR_FOUND; + } + + UnLock(); + return err; +} + +int mc_RelayManager::AddResponse(int64_t request,CNode *pfrom,mc_NodeFullAddress *source,int hop_count,int64_t nonce,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status) +{ + if(request == 0) + { + return MC_ERR_NOERROR; + } + + mc_RelayResponse response; + response.m_Nonce=nonce; + response.m_MsgType=msg_type; + response.m_Flags=flags; + response.m_NodeFrom=pfrom; + response.m_HopCount=hop_count; + response.m_Source=*source; + response.m_LastTryTimestamp=0; + response.m_Status=status; + response.m_Payload=payload; + response.m_Requests.clear(); + + if(request) + { + map::iterator itreq = m_Requests.find(request); + if(itreq == m_Requests.end()) + { + itreq->second.m_Responses.push_back(response); + if(status & MC_RST_SUCCESS) + { + itreq->second.m_Status |= MC_RST_SUCCESS; + } + else + { + return MC_ERR_NOT_FOUND; + } + } + } + + return MC_ERR_NOERROR; +} + +int mc_RelayManager::DeleteRequest(int64_t request) +{ + int err=MC_ERR_NOERROR; + + Lock(); + map::iterator itreq = m_Requests.find(request); + if(itreq != m_Requests.end()) + { + m_Requests.erase(itreq); + } + else + { + err= MC_ERR_NOT_FOUND; + } + + UnLock(); + return err; +} + + +int64_t mc_RelayManager::SendRequest(CNode* pto,uint32_t msg_type,uint32_t flags,vector & payload) +{ + uint32_t timestamp=mc_TimeNowAsUInt(); + uint32_t nonce=GenerateNonce(); + int64_t aggr_nonce=AggregateNonce(timestamp,nonce); + + vector vSigScriptsEmpty; + + { + LOCK(cs_vNodes); + BOOST_FOREACH(CNode* pnode, vNodes) + { + if( (pto == NULL) && (pnode == pto) ) + { + PushRelay(NULL,0,0,msg_type,timestamp,nonce,0,0,flags,payload,vSigScriptsEmpty,NULL,0); + } + } + } + + if(AddRequest(0,0,pto,aggr_nonce,msg_type,flags,payload,MC_RST_NONE) != MC_ERR_NOERROR) + { + return 0; + } + + return aggr_nonce; +} + /* bool ProcessMultichainRelay(CNode* pfrom, CDataStream& vRecv, CValidationState &state, uint32_t verify_flags_in) { @@ -1090,4 +1404,164 @@ bool ProcessMultichainRelay(CNode* pfrom, CDataStream& vRecv, CValidationState & return true; } +/* +void mc_RelayManager::PushRelay(CNode* pto, + uint32_t msg_format, + unsigned char hop_count, + uint32_t msg_type, + uint32_t timestamp_to_send, + uint32_t nonce_to_send, + uint32_t timestamp_to_respond, + uint32_t nonce_to_respond, + uint32_t flags, + vector& payload, + vector& sigScripts_to_relay, + CNode* pfrom, + uint32_t action) +{ + vector vOriginAddress; + vector vDestinationAddress; + vector sigScripts; + vectorvSigScript; + CScript sigScript; + uint256 message_hash; + uint32_t timestamp; + uint32_t nonce; + mc_NodeFullAddress origin_to_send; + mc_NodeFullAddress *origin; + mc_NodeFullAddress destination_to_send; + mc_NodeFullAddress *destination; + + nonce=nonce_to_send; + timestamp=timestamp_to_send; + + if(action & MC_PRA_GENERATE_TIMSTAMP) + { + timestamp=mc_TimeNowAsUInt(); + } + + if(action & MC_PRA_GENERATE_NONCE) + { + GetRandBytes((unsigned char*)&nonce, sizeof(nonce)); + } + if(action & (MC_PRA_MY_ORIGIN_MC_ADDRESS | MC_PRA_MY_ORIGIN_NT_ADDRESS) ) + { + InitNodeAddress(&origin_to_send,pto,action); + origin=&origin_to_send; + } + else + { + if(origin_to_relay) + { + origin=origin_to_relay; + } + else + { + origin=&origin_to_send; + } + } + + destination=&destination_to_send; + if(action & MC_PRA_USE_DESTINATION_ADDRESS) + { + if(destination_to_relay) + { + destination=destination_to_relay; + } + } + + if(MCP_ANYONE_CAN_CONNECT == 0) + { + vOriginAddress.resize(sizeof(CKeyID)); + memcpy(&vOriginAddress[0],&(origin->m_Address),sizeof(CKeyID)); + if(destination->m_Address != 0) + { + vDestinationAddress.resize(sizeof(CKeyID)); + memcpy(&vOriginAddress[0],&(origin->m_Address),sizeof(CKeyID)); + } + } + + message_hash=message_hash_to_relay; + + if(action & MC_PRA_CALCULATE_MESSAGE_HASH) + { + CHashWriter ssHash(SER_GETHASH, 0); + ssHash << nonce; + ssHash << msg_type; + ssHash << vOriginAddress; + ssHash << origin->m_NetAddresses; + ssHash << vDestinationAddress; + ssHash << destination->m_NetAddresses; + ssHash << response_to_nonce; + ssHash << flags; + ssHash << payload; + message_hash=ssHash.GetHash(); + } + + if( (action & (MC_PRA_SIGN | MC_PRA_SIGN_WITH_HANDSHAKE_ADDRESS)) && (MCP_ANYONE_CAN_CONNECT == 0) ) + { + CHashWriter ssSig(SER_GETHASH, 0); + ssSig << message_hash; + ssSig << vector((unsigned char*)&response_to_nonce, (unsigned char*)&response_to_nonce+sizeof(response_to_nonce)); + uint256 signed_hash=ssSig.GetHash(); + CKey key; + CKeyID keyID; + CPubKey pkey; + + keyID=origin->m_Address; + if(action & MC_PRA_SIGN_WITH_HANDSHAKE_ADDRESS) + { + if(MCP_ANYONE_CAN_CONNECT == 0) + { + keyID=pto->kAddrLocal; + } + } + + if(pwalletMain->GetKey(keyID, key)) + { + pkey=key.GetPubKey(); + vector vchSig; + sigScript.clear(); + if (key.Sign(signed_hash, vchSig)) + { + vchSig.push_back(0x00); + sigScript << vchSig; + sigScript << ToByteVector(pkey); + } + else + { + LogPrintf("PushRelay(): Internal error: Cannot sign\n"); + } + } + else + { + LogPrintf("PushRelay(): Internal error: Cannot find key for signature\n"); + } + sigScripts.push_back(sigScript); + } + + for(unsigned int i=0;iPushMessage("relay", + msg_type, + nonce, + vOriginAddress, + origin->m_NetAddresses, + vDestinationAddress, + destination->m_NetAddresses, + response_to_nonce, + flags, + payload, + message_hash, + path, + sigScripts); + + SetRelayRecord(pto,NULL,msg_type,nonce); + SetRelayRecord(pto,pfrom,msg_type,nonce); +} + +*/ \ No newline at end of file diff --git a/src/protocol/relay.h b/src/protocol/relay.h index 62fdb099..dfe32e04 100644 --- a/src/protocol/relay.h +++ b/src/protocol/relay.h @@ -18,44 +18,58 @@ #define MC_PRA_MY_ORIGIN_MC_ADDRESS 0x00000001 #define MC_PRA_MY_ORIGIN_NT_ADDRESS 0x00000002 #define MC_PRA_USE_DESTINATION_ADDRESS 0x00000004 -#define MC_PRA_CALCULATE_MESSAGE_HASH 0x00000008 -#define MC_PRA_SIGN 0x00000010 #define MC_PRA_SIGN_WITH_HANDSHAKE_ADDRESS 0x00000020 -#define MC_PRA_GENERATE_NONCE 0x00000040 +#define MC_PRA_GENERATE_TIMESTAMP 0x00000040 +#define MC_PRA_GENERATE_NONCE 0x00000080 #define MC_VRA_NONE 0x00000000 -#define MC_VRA_MESSAGE_TYPE 0x00000001 -#define MC_VRA_ORIGIN_MC_ADDRESS 0x00000002 -#define MC_VRA_ORIGIN_NT_ADDRESS 0x00000004 -#define MC_VRA_DESTINATION_MC_ADDRESS 0x00000008 -#define MC_VRA_DESTINATION_NT_ADDRESS 0x00000010 -#define MC_VRA_RESPONSE_TO_NONCE 0x00000020 -#define MC_VRA_MESSAGE_HASH 0x00000040 +#define MC_VRA_IS_RESPONSE 0x00000002 +#define MC_VRA_IS_NOT_RESPONSE 0x00000004 #define MC_VRA_SIGSCRIPT 0x00000080 #define MC_VRA_SIGNATURE_ORIGIN 0x00000100 #define MC_VRA_SIGNATURE_ALL 0x00000200 -#define MC_VRA_LIMIT_ALL 0x00000400 -#define MC_VRA_LIMIT_MSG_TYPE 0x00000800 -#define MC_VRA_SINGLE_HOP 0x00001000 -#define MC_VRA_ALREADY_PROCESSED 0x00002000 +#define MC_VRA_MESSAGE_TYPE 0x00010000 +#define MC_VRA_LIMIT_ALL 0x00020000 +#define MC_VRA_LIMIT_MSG_TYPE 0x00040000 +#define MC_VRA_SINGLE_HOP 0x00080000 +#define MC_VRA_DOUBLE_HOP 0x00100000 +#define MC_VRA_SIGNATURES 0x00200000 +#define MC_VRA_PROCESS_ONCE 0x00400000 +#define MC_VRA_BROADCAST_ONCE 0x00800000 +#define MC_VRA_SINGLE_HOP_BROADCAST 0x01000000 +#define MC_VRA_TIMESTAMP 0x02000000 + +#define MC_VRA_DEFAULT 0x03970000 #define MC_RMT_NONE 0 -#define MC_RMT_GLOBAL_PING 0x00000001 -#define MC_RMT_GLOBAL_PONG 0x00000002 -#define MC_RMT_GLOBAL_REJECT 0x00000003 -#define MC_RMT_GLOBAL_BUSY 0x00000004 +#define MC_RMT_GLOBAL_REJECT 0x00000001 +#define MC_RMT_GLOBAL_PING 0x00000002 +#define MC_RMT_GLOBAL_PONG 0x00000003 #define MC_RMT_CHUNK_QUERY 0x00000101 #define MC_RMT_CHUNK_QUERY_HIT 0x00000102 #define MC_RMT_CHUNK_REQUEST 0x00000103 #define MC_RMT_CHUNK_RESPONSE 0x00000104 +#define MC_RMT_ADD_RESPONSE 0x01000001 +#define MC_LIM_MAX_SECONDS 60 +#define MC_LIM_MAX_MEASURES 4 - +#define MC_RST_NONE 0x00000000 +#define MC_RST_DELETED 0x00000001 +#define MC_RST_SUCCESS 0x00000002 +#define MC_RST_PERMANENT_FAILURE 0x00000004 +#define MC_RST_TEMPORARY_FAILURE 0x00000008 using namespace std; -#define MC_LIM_MAX_SECONDS 60 -#define MC_LIM_MAX_MEASURES 4 +bool MultichainRelayResponse(uint32_t msg_type_stored, CNode *pto_stored, + uint32_t msg_type_in, uint32_t flags, vector& vPayloadIn,vector& vAddrIn, + uint32_t* msg_type_response,uint32_t *flags_response,vector& vPayloadResponse,vector& vAddrResponse, + uint32_t* msg_type_relay,uint32_t *flags_relay,vector& vPayloadRelay,vector& vAddrRelay, + string& strError); + + + typedef struct mc_Limiter @@ -102,18 +116,22 @@ typedef struct mc_NodeFullAddress typedef struct mc_RelayRecordKey { - int64_t m_Nonce; + uint32_t m_TimeStamp; + uint32_t m_Nonce; NodeId m_NodeTo; - mc_RelayRecordKey(int64_t nonce,NodeId node) + mc_RelayRecordKey(uint32_t timestamp,uint32_t nonce,NodeId node) { + m_TimeStamp=timestamp; m_Nonce=nonce; m_NodeTo=node; } friend bool operator<(const mc_RelayRecordKey& a, const mc_RelayRecordKey& b) { - return (a.m_Nonce < b.m_Nonce || (a.m_Nonce == b.m_Nonce && a.m_NodeTo < b.m_NodeTo)); + return ((a.m_TimeStamp < b.m_TimeStamp) || + (a.m_TimeStamp == b.m_TimeStamp && a.m_Nonce < b.m_Nonce) || + (a.m_TimeStamp == b.m_TimeStamp && a.m_Nonce == b.m_Nonce && a.m_NodeTo < b.m_NodeTo)); } } mc_RelayRecordKey; @@ -126,36 +144,105 @@ typedef struct mc_RelayRecordValue } mc_RelayRecordValue; +struct mc_RelayRequest; + +typedef struct mc_RelayResponse +{ + int64_t m_Nonce; + uint32_t m_MsgType; + uint32_t m_Flags; + CNode *m_NodeFrom; + int m_HopCount; + mc_NodeFullAddress m_Source; + uint32_t m_LastTryTimestamp; + uint32_t m_Status; + vector m_Payload; + vector m_Requests; + + mc_RelayResponse() + { + Zero(); + } + + void Zero(); +} mc_RelayResponse; + +typedef struct mc_RelayRequest +{ + int64_t m_Nonce; + uint32_t m_MsgType; + uint32_t m_Flags; + CNode *m_NodeTo; + int64_t m_ParentNonce; + int m_ParentResponseID; + uint32_t m_LastTryTimestamp; + int m_TryCount; + uint32_t m_Status; + vector m_Payload; + vector m_Responses; + + mc_RelayRequest() + { + Zero(); + } + + void Zero(); +} mc_RelayRequest; + typedef struct mc_RelayManager { uint32_t m_MyIPs[64]; int m_MyIPCount; - uint32_t m_LastTime; + uint32_t m_LastTime; + uint32_t m_MinTimeShift; + uint32_t m_MaxTimeShift; + void *m_Semaphore; + uint64_t m_LockedBy; + + mc_RelayManager() + { + Zero(); + }; + + ~mc_RelayManager() + { + Destroy(); + }; map m_Latency; map m_Limiters; map m_RelayRecords; + map m_Requests; + + void Zero(); + void Destroy(); + int Lock(); + int Lock(int write_mode, int allow_secondary); + void UnLock(); + int Intialize(); + int64_t AggregateNonce(uint32_t timestamp,uint32_t nonce); + uint32_t GenerateNonce(); void SetDefaults(); void SetMyIPs(uint32_t *ips,int ip_count); void MsgTypeSettings(uint32_t msg_type,int latency,int seconds,int64_t serves_per_second,int64_t bytes_per_second); void InitNodeAddress(mc_NodeFullAddress* node_address,CNode* pto,uint32_t action); void InitNodeAddress(mc_NodeFullAddress* node_address,CKeyID& mc_address, vector& net_addresses); void CheckTime(); - void SetRelayRecord(CNode *pto,CNode *pfrom,uint32_t msg_type,int64_t nonce); - int GetRelayRecord(CNode *pfrom,int64_t nonce,uint32_t *msg_type,CNode **pto); + void SetRelayRecord(CNode *pto,CNode *pfrom,uint32_t msg_type,uint32_t timestamp,uint32_t nonce); + int GetRelayRecord(CNode *pfrom,uint32_t timestamp,uint32_t nonce,uint32_t *msg_type,CNode **pto); - void PushRelay ( CNode* pto, + int64_t PushRelay ( CNode* pto, + uint32_t msg_format, + unsigned char hop_count, uint32_t msg_type, - int64_t nonce_to_send, - int64_t response_to_nonce, - mc_NodeFullAddress *origin, - mc_NodeFullAddress *destination, + uint32_t timestamp_to_send, + uint32_t nonce_to_send, + uint32_t timestamp_to_respond, + uint32_t nonce_to_respond, uint32_t flags, vector& payload, - uint256 message_hash_to_relay, vector& sigScripts_to_relay, - vector& path, CNode* pfrom, uint32_t action); @@ -164,7 +251,12 @@ typedef struct mc_RelayManager CValidationState &state, uint32_t verify_flags_in); -} mc_RelayManager; + int AddRequest(int64_t parent_nonce,int parent_response_id,CNode *pto,int64_t nonce,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status); + int AddResponse(int64_t request,CNode *pfrom,mc_NodeFullAddress *source,int hop_count,int64_t nonce,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status); + int DeleteRequest(int64_t request); + + int64_t SendRequest(CNode* pto,uint32_t msg_type,uint32_t flags,vector & payload); +} mc_RelayManager; From b457f64881951878aae60a3e3e9232327960630a Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 11 Apr 2018 17:40:13 +0300 Subject: [PATCH 034/157] -addnodeonly runtime parameter --- src/net/net.cpp | 17 ++++++++++++++--- src/rpc/rpcnet.cpp | 21 +++++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/net/net.cpp b/src/net/net.cpp index b2af3c2c..e1b5b288 100644 --- a/src/net/net.cpp +++ b/src/net/net.cpp @@ -643,6 +643,7 @@ int CNetMessage::readHeader(const char *pch, unsigned int nBytes) nHdrPos += nCopy; // mc_Dump("HEAD",pch,nCopy); +// mc_Dump("HEAD",pch,nBytes); // if header incomplete, exit if (nHdrPos < 24) @@ -1475,9 +1476,12 @@ void ThreadOpenConnections() } /* MCHN START */ - if (addrConnect.IsValid()) + if(!GetBoolArg("-addnodeonly",false)) { - OpenNetworkConnection(addrConnect, &grant); + if (addrConnect.IsValid()) + { + OpenNetworkConnection(addrConnect, &grant); + } } /* MCHN END */ } @@ -1551,7 +1555,14 @@ void ThreadOpenAddedConnections() OpenNetworkConnection(CAddress(vserv[i % vserv.size()]), &grant); MilliSleep(500); } - MilliSleep(120000); // Retry every 2 minutes + if(GetBoolArg("-addnodeonly",false)) + { + MilliSleep(1000); // Retry every 1 second + } + else + { + MilliSleep(120000); // Retry every 2 minutes + } } } diff --git a/src/rpc/rpcnet.cpp b/src/rpc/rpcnet.cpp index 6dcbd768..21477127 100644 --- a/src/rpc/rpcnet.cpp +++ b/src/rpc/rpcnet.cpp @@ -25,6 +25,7 @@ using namespace json_spirit; using namespace std; +void PushMultiChainRelay(CNode* pto, uint32_t msg_type,vector& path,vector& path_to_follow,vector& payload); Value getconnectioncount(const Array& params, bool fHelp) { @@ -174,6 +175,19 @@ Value addnode(const Array& params, bool fHelp) if (it == vAddedNodes.end()) throw JSONRPCError(RPC_CLIENT_NODE_NOT_ADDED, "Error: Node has not been added."); vAddedNodes.erase(it); + if(GetBoolArg("-addnodeonly",false)) + { + LOCK(cs_vNodes); + CAddress addrNode=CAddress(CService(strNode.c_str(),Params().GetDefaultPort(),0)); + BOOST_FOREACH(CNode* pnode, vNodes) + { + if (pnode->addr == addrNode) + { + pnode->fDisconnect=true; + } + } + } + } return Value::null; @@ -254,6 +268,13 @@ Value getaddednodeinfo(const Array& params, bool fHelp) fFound = true; fConnected = true; node.push_back(Pair("connected", pnode->fInbound ? "inbound" : "outbound")); + + vectorpath; + vectorpath_to_follow; + vector payload; + +// PushMultiChainRelay(pnode, MC_RMT_GLOBAL_PING,path,path_to_follow,payload); + break; } if (!fFound) From ef83057b44836d47d84178ddffb6ffa5c3abb3f0 Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 15 Apr 2018 19:22:14 +0300 Subject: [PATCH 035/157] Offchain messaging, one hop findaddress --- src/core/init.cpp | 2 + src/core/main.cpp | 2 +- src/protocol/relay.cpp | 323 ++++++++++++++++++++++++++++++++++------- src/protocol/relay.h | 27 +++- src/rpc/rpccache.cpp | 215 +++++++++++++++++++++++++++ src/rpc/rpcclient.cpp | 2 + src/rpc/rpclist.cpp | 1 + src/rpc/rpcserver.h | 1 + 8 files changed, 515 insertions(+), 58 deletions(-) diff --git a/src/core/init.cpp b/src/core/init.cpp index 4b78606d..a122efef 100644 --- a/src/core/init.cpp +++ b/src/core/init.cpp @@ -2382,6 +2382,8 @@ bool AppInit2(boost::thread_group& threadGroup,int OutputPipe) if (!strErrors.str().empty()) return InitError(strErrors.str()); + pRelayManager->Initialize(); + // RandAddSeedPerfmon(); //// debug print diff --git a/src/core/main.cpp b/src/core/main.cpp index d223c476..346a3e01 100644 --- a/src/core/main.cpp +++ b/src/core/main.cpp @@ -5779,7 +5779,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, return false; } - else if (strCommand == "relay") + else if (strCommand == "offchain") { CValidationState state; if(pRelayManager) diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index e3a6dcac..e53b8dc6 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -2,21 +2,139 @@ // MultiChain code distributed under the GPLv3 license, see COPYING file. #include "protocol/relay.h" +#include "structs/base58.h" + +bool mc_RelayProcess_Address_Query(unsigned char *ptrStart,unsigned char *ptrEnd,vector* payload_response,vector* payload_relay,string& strError) +{ + unsigned char *ptr; + unsigned char *ptrOut; + unsigned char buf[16]; + int shift; + CKey key; + + ptr=ptrStart; + while(ptrGetKey(*(CKeyID*)ptr, key)) + { + if(payload_response) + { + shift=mc_PutVarInt(buf,16,pRelayManager->m_MyAddress.m_NetAddresses.size()); + payload_response->resize(1+sizeof(CKeyID)+1+shift+sizeof(CAddress)*pRelayManager->m_MyAddress.m_NetAddresses.size()); + ptrOut=&(*payload_response)[0]; + + *ptrOut=MC_RDT_MC_ADDRESS; + ptrOut++; + *(CKeyID*)ptrOut=pRelayManager->m_MyAddress.m_Address; + ptrOut+=sizeof(CKeyID); + + *ptrOut=MC_RDT_NET_ADDRESS; + ptrOut++; + memcpy(ptrOut,buf,shift); + ptrOut+=shift; + for(int i=0;i<(int)pRelayManager->m_MyAddress.m_NetAddresses.size();i++) + { + memcpy(ptrOut,&(pRelayManager->m_MyAddress.m_NetAddresses[i]),sizeof(CAddress)); + ptrOut+=sizeof(CAddress); + } + } + } + else + { + if(payload_relay) + { + payload_relay->resize(ptrEnd-ptrStart); + memcpy(&(*payload_relay)[0],ptrStart,ptrEnd-ptrStart); + } + } + + ptr+=sizeof(CKeyID); + break; + default: + strError=strprintf("Unsupported request format (%d, %d)",MC_RMT_MC_ADDRESS_QUERY,*ptr); + return false; + } + } + + return true; +} bool MultichainRelayResponse(uint32_t msg_type_stored, CNode *pto_stored, uint32_t msg_type_in, uint32_t flags, vector& vPayloadIn,vector& vAddrIn, uint32_t* msg_type_response,uint32_t *flags_response,vector& vPayloadResponse,vector& vAddrResponse, uint32_t* msg_type_relay,uint32_t *flags_relay,vector& vPayloadRelay,vector& vAddrRelay,string& strError) { + unsigned char *ptr; + unsigned char *ptrEnd; + vector *payload_relay_ptr=NULL; + vector *payload_response_ptr=NULL; + + if(msg_type_response) + { + payload_response_ptr=&vPayloadResponse; + } + + if(msg_type_relay) + { + payload_relay_ptr=&vPayloadRelay; + } + + ptr=&vPayloadIn[0]; + ptrEnd=ptr+vPayloadIn.size(); + strError=""; switch(msg_type_in) { - case MC_RMT_GLOBAL_PING: + case MC_RMT_MC_ADDRESS_QUERY: if(msg_type_stored) { - strError="Invalid response message type"; + strError=strprintf("Unexpected response message type (%d,%d)",msg_type_stored,msg_type_in);; + goto exitlbl; + } + if(payload_response_ptr == NULL) + { + if(payload_relay_ptr) + { + vPayloadRelay=vPayloadIn; + *msg_type_relay=msg_type_in; + } + } + else + { + if(mc_RelayProcess_Address_Query(ptr,ptrEnd,payload_response_ptr,payload_relay_ptr,strError)) + { + if(payload_response_ptr && (payload_response_ptr->size() != 0)) + { + *msg_type_response=MC_RMT_NODE_DETAILS; + } + if(payload_relay_ptr && (payload_relay_ptr->size() != 0)) + { + *msg_type_relay=MC_RMT_MC_ADDRESS_QUERY; + } + } + } + break; + case MC_RMT_NODE_DETAILS: + if(msg_type_stored != MC_RMT_MC_ADDRESS_QUERY) + { + strError=strprintf("Unexpected response message type (%d,%d)",msg_type_stored,msg_type_in);; goto exitlbl; + } + if(payload_response_ptr) + { + *msg_type_response=MC_RMT_ADD_RESPONSE; } + break; } @@ -26,6 +144,7 @@ bool MultichainRelayResponse(uint32_t msg_type_stored, CNode *pto_stored, { return false; } + return true; } @@ -34,7 +153,7 @@ void mc_Limiter::Zero() memset(this,0,sizeof(mc_Limiter)); } -int mc_Limiter::Intitialize(int seconds,int measures) +int mc_Limiter::Initialize(int seconds,int measures) { Zero(); @@ -174,26 +293,60 @@ void mc_RelayManager::InitNodeAddress(mc_NodeFullAddress *node_address,CNode* pt { uint32_t pto_address_local; in_addr addr; + CKey key; + CPubKey pkey; + bool key_found=false; if(action & MC_PRA_MY_ORIGIN_MC_ADDRESS) { - node_address->m_Address=pto->kAddrLocal; + if(pto) + { + node_address->m_Address=pto->kAddrLocal; + } } else { - node_address->m_Address=CKeyID(0); + if(pto == NULL) + { + if(mapArgs.count("-handshakelocal")) + { + CBitcoinAddress address(mapArgs["-handshakelocal"]); + if (address.IsValid()) + { + CTxDestination dst=address.Get(); + CKeyID *lpKeyID=boost::get (&dst); + if(lpKeyID) + { + if(pwalletMain->GetKey(*lpKeyID, key)) + { + node_address->m_Address=*lpKeyID; + key_found=true; + } + } + } + } + + if(!key_found) + { + if(!pwalletMain->GetKeyFromAddressBook(pkey,MC_PTP_CONNECT)) + { + pkey=pwalletMain->vchDefaultKey; + } + node_address->m_Address=pkey.GetID(); + } + } } node_address->m_NetAddresses.clear(); + pto_address_local=0; if(action & MC_PRA_MY_ORIGIN_NT_ADDRESS) { node_address->m_NetAddresses.push_back(CAddress(pto->addrLocal)); - pto_address_local=0; if(pto->addrLocal.IsIPv4()) { - pto_address_local=(pto->addrLocal.GetByte(3)<<24)+(pto->addrLocal.GetByte(2)<<16)+(pto->addrLocal.GetByte(1)<<8)+pto->addrLocal.GetByte(0); + pto_address_local=(pto->addrLocal.GetByte(0)<<24)+(pto->addrLocal.GetByte(1)<<16)+(pto->addrLocal.GetByte(2)<<8)+pto->addrLocal.GetByte(3); addr.s_addr=pto_address_local; node_address->m_NetAddresses.push_back(CAddress(CService(addr,GetListenPort()))); } @@ -230,7 +383,7 @@ void mc_RelayManager::MsgTypeSettings(uint32_t msg_type,int latency,int seconds, itlat->second=latency; } - limiter.Intitialize(seconds,2); + limiter.Initialize(seconds,2); limiter.SetLimit(0,serves_per_second); limiter.SetLimit(1,bytes_per_second); @@ -266,9 +419,11 @@ void mc_RelayManager::Destroy() Zero(); } -int mc_RelayManager::Intialize() +int mc_RelayManager::Initialize() { m_Semaphore=__US_SemCreate(); + InitNodeAddress(&m_MyAddress,NULL,MC_PRA_NONE); + SetDefaults(); return MC_ERR_NOERROR; } @@ -301,17 +456,21 @@ int mc_RelayManager::Lock() void mc_RelayManager::SetDefaults() { - MsgTypeSettings(MC_RMT_NONE , 0,10,1000,100*1024*1024); - MsgTypeSettings(MC_RMT_GLOBAL_PING ,10,10, 100, 1*1024*1024); - MsgTypeSettings(MC_RMT_GLOBAL_PONG , 0,10, 100, 1*1024*1024); - MsgTypeSettings(MC_RMT_GLOBAL_REJECT , 0,10,1000, 1*1024*1024); - MsgTypeSettings(MC_RMT_CHUNK_QUERY ,10,10, 100, 1*1024*1024); - MsgTypeSettings(MC_RMT_CHUNK_QUERY_HIT,30,10, 100, 1*1024*1024); - MsgTypeSettings(MC_RMT_CHUNK_REQUEST ,30,10, 100, 1*1024*1024); - MsgTypeSettings(MC_RMT_CHUNK_RESPONSE , 0,10, 100,100*1024*1024); + MsgTypeSettings(MC_RMT_NONE , 0,10,1000,100*1024*1024); + MsgTypeSettings(MC_RMT_MC_ADDRESS_QUERY,10,10, 100, 1*1024*1024); + MsgTypeSettings(MC_RMT_NODE_DETAILS , 0,10, 100, 1*1024*1024); + MsgTypeSettings(MC_RMT_REJECT , 0,10,1000, 1*1024*1024); + MsgTypeSettings(MC_RMT_CHUNK_QUERY ,10,10, 100, 1*1024*1024); + MsgTypeSettings(MC_RMT_CHUNK_QUERY_HIT ,30,10, 100, 1*1024*1024); + MsgTypeSettings(MC_RMT_CHUNK_REQUEST ,30,10, 100, 1*1024*1024); + MsgTypeSettings(MC_RMT_CHUNK_RESPONSE , 0,10, 100,100*1024*1024); + MsgTypeSettings(MC_RMT_ERROR_IN_MESSAGE,30,10,1000, 1*1024*1024); + MsgTypeSettings(MC_RMT_NEW_REQUEST ,30,10,1000, 1*1024*1024); + m_MinTimeShift=180; m_MaxTimeShift=180; + m_MaxResponses=16; } void mc_RelayManager::CheckTime() @@ -333,6 +492,7 @@ void mc_RelayManager::CheckTime() it++; } } + m_LastTime=time_now; } void mc_RelayManager::SetRelayRecord(CNode *pto,CNode *pfrom,uint32_t msg_type,uint32_t timestamp,uint32_t nonce) @@ -361,6 +521,7 @@ void mc_RelayManager::SetRelayRecord(CNode *pto,CNode *pfrom,uint32_t msg_type,u } value.m_MsgType=msg_type; value.m_Timestamp=m_LastTime+itlat->second; + value.m_Count=1; map::iterator it = m_RelayRecords.find(key); if (it == m_RelayRecords.end()) @@ -369,8 +530,11 @@ void mc_RelayManager::SetRelayRecord(CNode *pto,CNode *pfrom,uint32_t msg_type,u } else { + value.m_Timestamp=(it->second).m_Timestamp; + value.m_Count=(it->second).m_Count; it->second=value; } + printf("setrr: %d, ts: %u, nc: %u, mt: %d\n",pto_id,value.m_Timestamp,nonce,msg_type); } int mc_RelayManager::GetRelayRecord(CNode *pfrom,uint32_t timestamp,uint32_t nonce,uint32_t* msg_type,CNode **pto) @@ -382,6 +546,7 @@ int mc_RelayManager::GetRelayRecord(CNode *pfrom,uint32_t timestamp,uint32_t non { pfrom_id=pfrom->GetId(); } + printf("getrr: %d, ts: %u, nc: %u\n",pfrom_id,timestamp,nonce); const mc_RelayRecordKey key=mc_RelayRecordKey(timestamp,nonce,pfrom_id); map::iterator it = m_RelayRecords.find(key); if (it == m_RelayRecords.end()) @@ -389,6 +554,11 @@ int mc_RelayManager::GetRelayRecord(CNode *pfrom,uint32_t timestamp,uint32_t non return MC_ERR_NOT_FOUND; } + if(it->second.m_MsgType == MC_RMT_ERROR_IN_MESSAGE) + { + return MC_ERR_ERROR_IN_SCRIPT; + } + if(msg_type) { *msg_type=it->second.m_MsgType; @@ -418,6 +588,11 @@ int mc_RelayManager::GetRelayRecord(CNode *pfrom,uint32_t timestamp,uint32_t non } else { + it->second.m_Count+=1; + if(it->second.m_Count > m_MaxResponses) + { + return MC_ERR_NOT_ALLOWED; + } return MC_ERR_NOERROR; } @@ -449,6 +624,7 @@ int64_t mc_RelayManager::PushRelay(CNode* pto, { vector vOriginAddress; vector vDestinationAddress; + vector vHops; vector sigScripts; CScript sigScript; uint256 message_hash; @@ -522,9 +698,15 @@ int64_t mc_RelayManager::PushRelay(CNode* pto, sigScripts.push_back(sigScripts_to_relay[i]); } - pto->PushMessage("relay", + if(pfrom) + { + vHops.push_back((int32_t)pfrom->GetId()); + } + + printf("send: %d, to: %d, from: %d, size: %d, ts: %u, nc: %u\n",msg_type,pto->GetId(),pfrom ? pfrom->GetId() : 0,(int)payload.size(),timestamp,nonce); + pto->PushMessage("offchain", msg_format, - hop_count, + vHops, msg_type, timestamp, nonce, @@ -534,11 +716,8 @@ int64_t mc_RelayManager::PushRelay(CNode* pto, payload, sigScripts); - SetRelayRecord(pto,NULL,msg_type,timestamp,nonce); - if(pfrom) - { - SetRelayRecord(pto,pfrom,msg_type,timestamp,nonce); - } +// SetRelayRecord(pto,NULL,msg_type,timestamp,nonce); + SetRelayRecord(pto,pfrom,msg_type,timestamp,nonce); return aggr_nonce; } @@ -557,6 +736,7 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, vector vPayloadIn; vector vSigScripts; vector vSigScriptsEmpty; + vector vHops; uint256 message_hash; uint32_t flags_in,flags_response,flags_relay; vector vchSigOut; @@ -590,17 +770,18 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, return false; } - vRecv >> hop_count; + vRecv >> vHops; + hop_count=(int)vHops.size(); vRecv >> msg_type_in; switch(msg_type_in) { - case MC_RMT_GLOBAL_PING: + case MC_RMT_MC_ADDRESS_QUERY: verify_flags |= MC_VRA_IS_NOT_RESPONSE; break; - case MC_RMT_GLOBAL_PONG: + case MC_RMT_NODE_DETAILS: verify_flags |= MC_VRA_IS_RESPONSE | MC_VRA_SIGNATURE_ORIGIN; break; - case MC_RMT_GLOBAL_REJECT: + case MC_RMT_REJECT: verify_flags |= MC_VRA_IS_RESPONSE | MC_VRA_SIGNATURE_ORIGIN; break; case MC_RMT_CHUNK_QUERY: @@ -643,7 +824,7 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, } CheckTime(); - + vRecv >> timestamp_received; if(verify_flags & MC_VRA_TIMESTAMP) @@ -661,21 +842,30 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, } vRecv >> nonce_received; + + msg_type_relay_ptr=&msg_type_relay; msg_type_response_ptr=&msg_type_response; if( verify_flags & (MC_VRA_PROCESS_ONCE | MC_VRA_BROADCAST_ONCE)) { - if(GetRelayRecord(NULL,timestamp_received,nonce_received,NULL,NULL) == MC_ERR_NOERROR) + switch(GetRelayRecord(NULL,timestamp_received,nonce_received,NULL,NULL)) { - if(verify_flags & MC_VRA_PROCESS_ONCE) - { - return false; - } - if(verify_flags & MC_VRA_BROADCAST_ONCE) - { - msg_type_relay_ptr=NULL; - } + case MC_ERR_NOERROR: + if(verify_flags & MC_VRA_PROCESS_ONCE) + { + return false; + } + if(verify_flags & MC_VRA_BROADCAST_ONCE) + { + msg_type_relay_ptr=NULL; + } + break; + case MC_ERR_ERROR_IN_SCRIPT: // We already processed this message, it has errors + return false; + case MC_ERR_NOT_ALLOWED: + LogPrintf("ProcessRelay() : Processing this message is not allowed by current limits or requesting peer was disconnected\n"); + return false; } } @@ -707,6 +897,8 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, } } + SetRelayRecord(NULL,pfrom,MC_RMT_ERROR_IN_MESSAGE,timestamp_received,nonce_received); + if(timestamp_to_respond) { if(pto_stored) @@ -750,6 +942,8 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, vRecv >> vPayloadIn; vRecv >> vSigScripts; + printf("recv: %d, from: %d, to: %d, size: %d, ts: %u, nc: %u\n",msg_type_in,pfrom->GetId(),pto_stored ? pto_stored->GetId() : 0,(int)vPayloadIn.size(),timestamp_received,nonce_received); + if( verify_flags & MC_VRA_SIGNATURES ) { CHashWriter ssHash(SER_GETHASH, 0); @@ -851,9 +1045,14 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, if(itreq != m_Requests.end()) { Lock(); - AddResponse(itreq->second.m_Nonce,pfrom,NULL,hop_count,AggregateNonce(timestamp_received,nonce_received),msg_type_in,flags_in,vPayloadIn,MC_RST_SUCCESS); + AddResponse(itreq->second.m_Nonce,pfrom,vHops.size() ? vHops[0] : 0,hop_count,AggregateNonce(timestamp_received,nonce_received),msg_type_in,flags_in,vPayloadIn,MC_RST_SUCCESS); UnLock(); } + else + { + LogPrintf("ProcessRelay() : Response without stored request from peer %d\n",pfrom->GetId()); + return false; + } } } if(msg_type_relay_ptr && *msg_type_relay_ptr) @@ -877,6 +1076,15 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, } } + if(timestamp_to_respond) + { + SetRelayRecord(NULL,pfrom,msg_type_in,timestamp_received,nonce_received); + } + else + { + SetRelayRecord(NULL,pfrom,MC_RMT_NEW_REQUEST,timestamp_received,nonce_received); + } + return true; } @@ -920,7 +1128,7 @@ void mc_RelayResponse::Zero() m_Flags=0; m_NodeFrom=NULL; m_HopCount=0; - m_Source.Zero(); + m_Source=0; m_LastTryTimestamp=0; m_Status=MC_RST_NONE; m_Payload.clear(); @@ -980,6 +1188,7 @@ int mc_RelayManager::AddRequest(int64_t parent_nonce,int parent_response_id,CNod } } + printf("rqst: %lu, mt: %d, node: %d, size: %d, pr: %lu\n",nonce,msg_type,pto ? pto->GetId() : 0,(int)payload.size(),parent_nonce); m_Requests.insert(make_pair(nonce,request)); } else @@ -991,8 +1200,9 @@ int mc_RelayManager::AddRequest(int64_t parent_nonce,int parent_response_id,CNod return err; } -int mc_RelayManager::AddResponse(int64_t request,CNode *pfrom,mc_NodeFullAddress *source,int hop_count,int64_t nonce,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status) +int mc_RelayManager::AddResponse(int64_t request,CNode *pfrom,int32_t source,int hop_count,int64_t nonce,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status) { + printf("resp: %lu, mt: %d, node: %d, size: %d, rn: %lu\n",request,msg_type,pfrom ? pfrom->GetId() : 0,(int)payload.size(),nonce); if(request == 0) { return MC_ERR_NOERROR; @@ -1004,7 +1214,7 @@ int mc_RelayManager::AddResponse(int64_t request,CNode *pfrom,mc_NodeFullAddress response.m_Flags=flags; response.m_NodeFrom=pfrom; response.m_HopCount=hop_count; - response.m_Source=*source; + response.m_Source=source; response.m_LastTryTimestamp=0; response.m_Status=status; response.m_Payload=payload; @@ -1013,17 +1223,17 @@ int mc_RelayManager::AddResponse(int64_t request,CNode *pfrom,mc_NodeFullAddress if(request) { map::iterator itreq = m_Requests.find(request); - if(itreq == m_Requests.end()) + if(itreq != m_Requests.end()) { itreq->second.m_Responses.push_back(response); if(status & MC_RST_SUCCESS) { itreq->second.m_Status |= MC_RST_SUCCESS; } - else - { - return MC_ERR_NOT_FOUND; - } + } + else + { + return MC_ERR_NOT_FOUND; } } @@ -1049,6 +1259,19 @@ int mc_RelayManager::DeleteRequest(int64_t request) return err; } +mc_RelayRequest *mc_RelayManager::FindRequest(int64_t request) +{ + Lock(); + map::iterator itreq = m_Requests.find(request); + if(itreq != m_Requests.end()) + { + return &(itreq->second); + } + + UnLock(); + return NULL; +} + int64_t mc_RelayManager::SendRequest(CNode* pto,uint32_t msg_type,uint32_t flags,vector & payload) { @@ -1062,9 +1285,9 @@ int64_t mc_RelayManager::SendRequest(CNode* pto,uint32_t msg_type,uint32_t flags LOCK(cs_vNodes); BOOST_FOREACH(CNode* pnode, vNodes) { - if( (pto == NULL) && (pnode == pto) ) + if( (pto == NULL) || (pnode == pto) ) { - PushRelay(NULL,0,0,msg_type,timestamp,nonce,0,0,flags,payload,vSigScriptsEmpty,NULL,0); + PushRelay(pnode,0,0,msg_type,timestamp,nonce,0,0,flags,payload,vSigScriptsEmpty,NULL,0); } } } @@ -1279,6 +1502,7 @@ bool MultichainRelayResponse(uint32_t msg_type_in,vector& path,vector< uint32_t* msg_type_response,vector& vPayloadResponse, uint32_t* msg_type_relay,vector& vPayloadRelay) { +/* switch(msg_type_in) { case MC_RMT_GLOBAL_PING: @@ -1299,6 +1523,7 @@ bool MultichainRelayResponse(uint32_t msg_type_in,vector& path,vector< LogPrintf("PONG : %s\n",str_path.c_str()); break; } + */ return true; } diff --git a/src/protocol/relay.h b/src/protocol/relay.h index dfe32e04..0dcbb01d 100644 --- a/src/protocol/relay.h +++ b/src/protocol/relay.h @@ -42,14 +42,20 @@ #define MC_VRA_DEFAULT 0x03970000 #define MC_RMT_NONE 0 -#define MC_RMT_GLOBAL_REJECT 0x00000001 -#define MC_RMT_GLOBAL_PING 0x00000002 -#define MC_RMT_GLOBAL_PONG 0x00000003 +#define MC_RMT_REJECT 0x00000001 +#define MC_RMT_MC_ADDRESS_QUERY 0x00000002 +#define MC_RMT_NODE_DETAILS 0x00000003 #define MC_RMT_CHUNK_QUERY 0x00000101 #define MC_RMT_CHUNK_QUERY_HIT 0x00000102 #define MC_RMT_CHUNK_REQUEST 0x00000103 #define MC_RMT_CHUNK_RESPONSE 0x00000104 #define MC_RMT_ADD_RESPONSE 0x01000001 +#define MC_RMT_ERROR_IN_MESSAGE 0x01000002 +#define MC_RMT_NEW_REQUEST 0x01000003 + +#define MC_RDT_UNKNOWN 0 +#define MC_RDT_MC_ADDRESS 1 +#define MC_RDT_NET_ADDRESS 2 #define MC_LIM_MAX_SECONDS 60 #define MC_LIM_MAX_MEASURES 4 @@ -82,7 +88,7 @@ typedef struct mc_Limiter int64_t m_Event[MC_LIM_MAX_MEASURES]; int64_t m_Measures[MC_LIM_MAX_MEASURES*MC_LIM_MAX_SECONDS]; void Zero(); - int Intitialize(int seconds,int measures); + int Initialize(int seconds,int measures); int SetLimit(int meausure,int64_t limit); void CheckTime(); @@ -141,6 +147,7 @@ typedef struct mc_RelayRecordValue uint32_t m_MsgType; NodeId m_NodeFrom; uint32_t m_Timestamp; + int m_Count; } mc_RelayRecordValue; @@ -153,7 +160,7 @@ typedef struct mc_RelayResponse uint32_t m_Flags; CNode *m_NodeFrom; int m_HopCount; - mc_NodeFullAddress m_Source; + int32_t m_Source; uint32_t m_LastTryTimestamp; uint32_t m_Status; vector m_Payload; @@ -192,13 +199,15 @@ typedef struct mc_RelayRequest typedef struct mc_RelayManager { uint32_t m_MyIPs[64]; + int m_MaxResponses; int m_MyIPCount; uint32_t m_LastTime; uint32_t m_MinTimeShift; uint32_t m_MaxTimeShift; void *m_Semaphore; uint64_t m_LockedBy; - + mc_NodeFullAddress m_MyAddress; + mc_RelayManager() { Zero(); @@ -219,7 +228,7 @@ typedef struct mc_RelayManager int Lock(); int Lock(int write_mode, int allow_secondary); void UnLock(); - int Intialize(); + int Initialize(); int64_t AggregateNonce(uint32_t timestamp,uint32_t nonce); uint32_t GenerateNonce(); @@ -252,8 +261,10 @@ typedef struct mc_RelayManager uint32_t verify_flags_in); int AddRequest(int64_t parent_nonce,int parent_response_id,CNode *pto,int64_t nonce,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status); - int AddResponse(int64_t request,CNode *pfrom,mc_NodeFullAddress *source,int hop_count,int64_t nonce,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status); + int AddResponse(int64_t request,CNode *pfrom,int32_t source,int hop_count,int64_t nonce,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status); int DeleteRequest(int64_t request); + mc_RelayRequest *FindRequest(int64_t request); + int64_t SendRequest(CNode* pto,uint32_t msg_type,uint32_t flags,vector & payload); } mc_RelayManager; diff --git a/src/rpc/rpccache.cpp b/src/rpc/rpccache.cpp index 42824e25..03a6f6da 100644 --- a/src/rpc/rpccache.cpp +++ b/src/rpc/rpccache.cpp @@ -2,6 +2,8 @@ // MultiChain code distributed under the GPLv3 license, see COPYING file. #include "rpc/rpcutils.h" +#include "protocol/relay.h" +#include "net/net.h" Value createbinarycache(const Array& params, bool fHelp) @@ -110,3 +112,216 @@ Value deletebinarycache(const Array& params, bool fHelp) return Value::null; } + + +Value offchain(const Array& params, bool fHelp) +{ + if (fHelp || params.size() < 2 || params.size() > 4) + throw runtime_error("Help message not found\n"); + + uint32_t request_type=MC_RMT_NONE; + int timeout=5; + vector payload; + int64_t request_id; + CNode *pto=NULL; + string request_str=""; + mc_RelayRequest *request; + mc_RelayResponse *response; + Object res; + bool res_found=false; + + if(params[0].type() == str_type) + { + if(params[0].get_str() == "findaddress") + { + request_type=MC_RMT_MC_ADDRESS_QUERY; + request_str="query for address "; + } + } + + if(request_type == MC_RMT_NONE) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid request type"); + } + + if(params[1].type() != obj_type) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid request datails, should be object"); + } + + switch(request_type) + { + case MC_RMT_MC_ADDRESS_QUERY: + string addr_to_find=""; + BOOST_FOREACH(const Pair& d, params[1].get_obj()) + { + if(d.name_ == "address") + { + if(d.value_.type() ==str_type) + { + addr_to_find=d.value_.get_str(); + request_str+=addr_to_find; + } + } + } + + CBitcoinAddress address(addr_to_find); + CKeyID keyID; + if (!address.GetKeyID(keyID)) + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + + payload.resize(1+sizeof(CKeyID)); + payload[0]=MC_RDT_MC_ADDRESS; + memcpy(&payload[1],&keyID,sizeof(CKeyID)); + break; + } + + if(params.size() > 2) + { + if(params[2].type() != int_type) + { + timeout=params[2].get_int(); + if( (timeout <= 0) || (timeout > 15 ) ) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid timeout"); + } + } + else + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid timeout"); + } + } + + { + LOCK(cs_vNodes); + if(params.size() > 3) + { + if(params[3].type() != str_type) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid destination"); + } + + BOOST_FOREACH(CNode* pnode, vNodes) + { + CNodeStats stats; + pnode->copyStats(stats); + if( (params[3].get_str() == stats.addrName) || + (params[3].get_str() == CBitcoinAddress(stats.kAddrRemote).ToString()) ) + { + pto=pnode; + } + } + } + + if(pto) + { + request_str="Sending " + request_str + strprintf(" to node %d",pto->GetId()); + } + else + { + request_str="Broadcasting " + request_str; + } + + request_id=pRelayManager->SendRequest(pto,request_type,0,payload); + request_str+=strprintf(". Request ID: %lu",request_id); + printf("%s\n",request_str.c_str()); + } + + uint32_t time_now; + uint32_t time_stop; + + time_now=mc_TimeNowAsUInt(); + time_stop=time_now+timeout; + res_found=false; + + while(time_nowFindRequest(request_id); + if(request) + { + if(request->m_Responses.size()) + { + switch(request_type) + { + case MC_RMT_MC_ADDRESS_QUERY: + mc_NodeFullAddress node_addr; + int count,shift; + bool take_it=true; + unsigned char *ptr; + unsigned char *ptrEnd; + for(int r=0;r<(int)request->m_Responses.size();r++) + { + if(!res_found) + { + response=&(request->m_Responses[r]); + ptr=&(response->m_Payload[0]); + ptrEnd=ptr+response->m_Payload.size(); + while( (ptr (ptrEnd-ptr)) + { + take_it=false; + } + else + { + node_addr.m_Address=*(CKeyID*)ptr; + ptr+=sizeof(CKeyID); + } + break; + case MC_RDT_NET_ADDRESS: + ptr++; + count=(int)mc_GetVarInt(ptr,ptrEnd-ptr,-1,&shift); + ptr+=shift; + if(count*(int)sizeof(CAddress) > (ptrEnd-ptr)) + { + take_it=false; + } + else + { + for(int a;aUnLock(); + if(res_found) + { + pRelayManager->DeleteRequest(request_id); + return res; + } + } + __US_Sleep(100); + time_now=mc_TimeNowAsUInt(); + } + + return Value::null; +} diff --git a/src/rpc/rpcclient.cpp b/src/rpc/rpcclient.cpp index 55980f04..ecb76abd 100644 --- a/src/rpc/rpcclient.cpp +++ b/src/rpc/rpcclient.cpp @@ -36,6 +36,8 @@ static const CRPCConvertParam vRPCConvertParams[] = { "send", 1 }, /* MCHN START */ // { "setruntimeparam", 1 }, + { "offchain", 1 }, + { "offchain", 2 }, { "createkeypairs", 0 }, { "combineunspent", 1 }, { "combineunspent", 2 }, diff --git a/src/rpc/rpclist.cpp b/src/rpc/rpclist.cpp index a6079782..6ab5e9b2 100644 --- a/src/rpc/rpclist.cpp +++ b/src/rpc/rpclist.cpp @@ -121,6 +121,7 @@ static const CRPCCommand vRPCCommands[] = { "rawtransactions", "appendrawchange", &appendrawchange, false, false, true }, { "hidden", "appendrawmetadata", &appendrawmetadata, false, false, true }, { "rawtransactions", "appendrawdata", &appendrawmetadata, false, false, true }, + { "hidden", "offchain", &offchain, false, true, true }, /* MCHN END */ /* Utility functions */ diff --git a/src/rpc/rpcserver.h b/src/rpc/rpcserver.h index b69345c7..c830f181 100644 --- a/src/rpc/rpcserver.h +++ b/src/rpc/rpcserver.h @@ -192,6 +192,7 @@ extern json_spirit::Value getaccount(const json_spirit::Array& params, bool fHel extern json_spirit::Value getaddressesbyaccount(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value sendtoaddress(const json_spirit::Array& params, bool fHelp); /* MCHN START */ +extern json_spirit::Value offchain(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value createkeypairs(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value getaddresses(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value createbinarycache(const json_spirit::Array& params, bool fHelp); From 61eb138005ef037339e414f6ba4df61b28d49820 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 16 Apr 2018 17:39:19 +0300 Subject: [PATCH 036/157] Offhcain messaging, multihop findaddress --- src/protocol/relay.cpp | 31 ++++++++++++++++++++----------- src/protocol/relay.h | 2 +- src/rpc/rpccache.cpp | 8 ++++---- 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index e53b8dc6..69273368 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -346,7 +346,7 @@ void mc_RelayManager::InitNodeAddress(mc_NodeFullAddress *node_address,CNode* pt if(pto->addrLocal.IsIPv4()) { - pto_address_local=(pto->addrLocal.GetByte(0)<<24)+(pto->addrLocal.GetByte(1)<<16)+(pto->addrLocal.GetByte(2)<<8)+pto->addrLocal.GetByte(3); + pto_address_local=(pto->addrLocal.GetByte(3)<<24)+(pto->addrLocal.GetByte(2)<<16)+(pto->addrLocal.GetByte(1)<<8)+pto->addrLocal.GetByte(0); addr.s_addr=pto_address_local; node_address->m_NetAddresses.push_back(CAddress(CService(addr,GetListenPort()))); } @@ -357,7 +357,7 @@ void mc_RelayManager::InitNodeAddress(mc_NodeFullAddress *node_address,CNode* pt { if(m_MyIPs[i] != pto_address_local) { - addr.s_addr=m_MyIPs[i]; + addr.s_addr=htonl(m_MyIPs[i]); node_address->m_NetAddresses.push_back(CAddress(CService(addr,GetListenPort()))); } } @@ -610,7 +610,7 @@ uint32_t mc_RelayManager::GenerateNonce() int64_t mc_RelayManager::PushRelay(CNode* pto, uint32_t msg_format, - unsigned char hop_count, + vector &vHops, uint32_t msg_type, uint32_t timestamp_to_send, uint32_t nonce_to_send, @@ -623,8 +623,7 @@ int64_t mc_RelayManager::PushRelay(CNode* pto, uint32_t action) { vector vOriginAddress; - vector vDestinationAddress; - vector vHops; + vector vDestinationAddress; vector sigScripts; CScript sigScript; uint256 message_hash; @@ -703,7 +702,7 @@ int64_t mc_RelayManager::PushRelay(CNode* pto, vHops.push_back((int32_t)pfrom->GetId()); } - printf("send: %d, to: %d, from: %d, size: %d, ts: %u, nc: %u\n",msg_type,pto->GetId(),pfrom ? pfrom->GetId() : 0,(int)payload.size(),timestamp,nonce); + printf("send: %d, to: %d, from: %d, hc: %d, size: %d, ts: %u, nc: %u\n",msg_type,pto->GetId(),pfrom ? pfrom->GetId() : 0,(int)vHops.size(),(int)payload.size(),timestamp,nonce); pto->PushMessage("offchain", msg_format, vHops, @@ -737,6 +736,7 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, vector vSigScripts; vector vSigScriptsEmpty; vector vHops; + vector vEmptyHops; uint256 message_hash; uint32_t flags_in,flags_response,flags_relay; vector vchSigOut; @@ -869,6 +869,14 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, } } + if(verify_flags & MC_VRA_DOUBLE_HOP) + { + if(hop_count) + { + msg_type_relay_ptr=NULL; + } + } + vRecv >> timestamp_to_respond; vRecv >> nonce_to_respond; @@ -942,7 +950,7 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, vRecv >> vPayloadIn; vRecv >> vSigScripts; - printf("recv: %d, from: %d, to: %d, size: %d, ts: %u, nc: %u\n",msg_type_in,pfrom->GetId(),pto_stored ? pto_stored->GetId() : 0,(int)vPayloadIn.size(),timestamp_received,nonce_received); + printf("recv: %d, from: %d, to: %d, hc: %d, size: %d, ts: %u, nc: %u\n",msg_type_in,pfrom->GetId(),pto_stored ? pto_stored->GetId() : 0,hop_count,(int)vPayloadIn.size(),timestamp_received,nonce_received); if( verify_flags & MC_VRA_SIGNATURES ) { @@ -1020,7 +1028,7 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, if(pto_stored) { - PushRelay(pto_stored,msg_format,hop_count+1,msg_type_in,timestamp_received,nonce_received,timestamp_to_respond,nonce_to_respond,flags_in, + PushRelay(pto_stored,msg_format,vHops,msg_type_in,timestamp_received,nonce_received,timestamp_to_respond,nonce_to_respond,flags_in, vPayloadIn,vSigScripts,pfrom,MC_PRA_NONE); } else @@ -1036,7 +1044,7 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, { if(*msg_type_response_ptr != MC_RMT_ADD_RESPONSE) { - PushRelay(pfrom,0,0,*msg_type_response_ptr,m_LastTime,0,timestamp_received,nonce_received,flags_response, + PushRelay(pfrom,0,vEmptyHops,*msg_type_response_ptr,m_LastTime,0,timestamp_received,nonce_received,flags_response, vPayloadResponse,vSigScriptsEmpty,NULL,MC_PRA_GENERATE_NONCE); } else @@ -1062,7 +1070,7 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, { if(pnode != pfrom) { - PushRelay(pnode,msg_format,hop_count+1,*msg_type_relay_ptr,timestamp_received,nonce_received,timestamp_to_respond,nonce_to_respond,flags_relay, + PushRelay(pnode,msg_format,vHops,*msg_type_relay_ptr,timestamp_received,nonce_received,timestamp_to_respond,nonce_to_respond,flags_relay, vPayloadRelay,vSigScriptsEmpty,pfrom,MC_PRA_NONE); } } @@ -1278,6 +1286,7 @@ int64_t mc_RelayManager::SendRequest(CNode* pto,uint32_t msg_type,uint32_t flags uint32_t timestamp=mc_TimeNowAsUInt(); uint32_t nonce=GenerateNonce(); int64_t aggr_nonce=AggregateNonce(timestamp,nonce); + vector vEmptyHops; vector vSigScriptsEmpty; @@ -1287,7 +1296,7 @@ int64_t mc_RelayManager::SendRequest(CNode* pto,uint32_t msg_type,uint32_t flags { if( (pto == NULL) || (pnode == pto) ) { - PushRelay(pnode,0,0,msg_type,timestamp,nonce,0,0,flags,payload,vSigScriptsEmpty,NULL,0); + PushRelay(pnode,0,vEmptyHops,msg_type,timestamp,nonce,0,0,flags,payload,vSigScriptsEmpty,NULL,0); } } } diff --git a/src/protocol/relay.h b/src/protocol/relay.h index 0dcbb01d..093cc5de 100644 --- a/src/protocol/relay.h +++ b/src/protocol/relay.h @@ -243,7 +243,7 @@ typedef struct mc_RelayManager int64_t PushRelay ( CNode* pto, uint32_t msg_format, - unsigned char hop_count, + vector &vHops, uint32_t msg_type, uint32_t timestamp_to_send, uint32_t nonce_to_send, diff --git a/src/rpc/rpccache.cpp b/src/rpc/rpccache.cpp index 03a6f6da..23df60bd 100644 --- a/src/rpc/rpccache.cpp +++ b/src/rpc/rpccache.cpp @@ -223,8 +223,8 @@ Value offchain(const Array& params, bool fHelp) } request_id=pRelayManager->SendRequest(pto,request_type,0,payload); - request_str+=strprintf(". Request ID: %lu",request_id); - printf("%s\n",request_str.c_str()); + printf("%s",request_str.c_str()); + printf(". Request ID: %lu\n",request_id); } uint32_t time_now; @@ -282,7 +282,7 @@ Value offchain(const Array& params, bool fHelp) } else { - for(int a;a Date: Wed, 18 Apr 2018 15:35:39 +0300 Subject: [PATCH 037/157] Chunkc collection, not debugged --- src/chainparams/state.h | 3 + src/protocol/relay.cpp | 1612 +++++++++++++++++++-------------- src/protocol/relay.h | 18 +- src/rpc/rpccache.cpp | 1 + src/wallet/chunkcollector.cpp | 44 +- src/wallet/chunkcollector.h | 38 +- 6 files changed, 1026 insertions(+), 690 deletions(-) diff --git a/src/chainparams/state.h b/src/chainparams/state.h index a564a823..3bb58d2c 100644 --- a/src/chainparams/state.h +++ b/src/chainparams/state.h @@ -174,6 +174,7 @@ typedef struct mc_TmpBuffers mc_Buffer *m_RpcEntityRows; mc_SHA256 *m_RpcHasher1; mc_Script *m_RpcChunkScript1; + mc_Script *m_RelayTmpBuffer; void Init() { @@ -194,6 +195,7 @@ typedef struct mc_TmpBuffers m_RpcEntityRows->Initialize(MC_TDB_ENTITY_KEY_SIZE,MC_TDB_ROW_SIZE,MC_BUF_MODE_DEFAULT); m_RpcHasher1=new mc_SHA256(); m_RpcChunkScript1=new mc_Script(); + m_RelayTmpBuffer=new mc_Script(); } void Destroy() @@ -210,6 +212,7 @@ typedef struct mc_TmpBuffers delete m_RpcEntityRows; delete m_RpcHasher1; delete m_RpcChunkScript1; + delete m_RelayTmpBuffer; } } mc_TmpBuffers; diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index 69273368..0f4679a4 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -3,179 +3,882 @@ #include "protocol/relay.h" #include "structs/base58.h" +#include "wallet/chunkdb.h" +#include "wallet/chunkcollector.h" +#include "wallet/wallettxs.h" -bool mc_RelayProcess_Address_Query(unsigned char *ptrStart,unsigned char *ptrEnd,vector* payload_response,vector* payload_relay,string& strError) +uint32_t MultichainNextChunkQueryAttempt(uint32_t attempts) +{ + if(attempts < 4)return 0; + if(attempts < 8)return 60; + if(attempts < 12)return 3600; + if(attempts < 16)return 86400; + if(attempts < 20)return 86400*30; + if(attempts < 24)return 86400*365; + return 86400*365*10; +} + +typedef struct CRelayResponsePair +{ + int64_t request_id; + int response_id; + + friend bool operator<(const CRelayResponsePair& a, const CRelayResponsePair& b) + { + return ((a.request_id < b.request_id) || + (a.request_id == b.request_id && a.response_id < b.response_id)); + } + +} CRelayResponsePair; + +typedef struct CRelayRequestPairs +{ + map m_Pairs; +} CRelayRequestPairs; + +int MultichainProcessChunkResponse(const CRelayResponsePair *response_pair,map * request_pairs,mc_ChunkCollector* collector) { + mc_RelayRequest *request; + mc_RelayResponse *response; + request=pRelayManager->FindRequest(response_pair->request_id); + if(request == NULL) + { + return false; + } + + response=&(request->m_Responses[response_pair->response_id]); + unsigned char *ptr; + unsigned char *ptrEnd; + unsigned char *ptrStart; + int shift,count,size; + int shiftOut,countOut,sizeOut; + int chunk_err; + mc_ChunkEntityKey *chunk; unsigned char *ptrOut; - unsigned char buf[16]; - int shift; - CKey key; + bool result=false; + mc_ChunkCollectorRow *collect_row; + + uint32_t total_size=0; + ptrStart=&(request->m_Payload[0]); + + size=sizeof(mc_ChunkEntityKey); + shift=0; + count=0; ptr=ptrStart; + ptrEnd=ptr+request->m_Payload.size(); + while(ptrGetKey(*(CKeyID*)ptr, key)) + count=(int)mc_GetVarInt(ptr,ptrEnd-ptr,-1,&shift); + ptr+=shift; + if(count*size != (ptrEnd-ptr)) { - if(payload_response) - { - shift=mc_PutVarInt(buf,16,pRelayManager->m_MyAddress.m_NetAddresses.size()); - payload_response->resize(1+sizeof(CKeyID)+1+shift+sizeof(CAddress)*pRelayManager->m_MyAddress.m_NetAddresses.size()); - ptrOut=&(*payload_response)[0]; - - *ptrOut=MC_RDT_MC_ADDRESS; - ptrOut++; - *(CKeyID*)ptrOut=pRelayManager->m_MyAddress.m_Address; - ptrOut+=sizeof(CKeyID); - - *ptrOut=MC_RDT_NET_ADDRESS; - ptrOut++; - memcpy(ptrOut,buf,shift); - ptrOut+=shift; - for(int i=0;i<(int)pRelayManager->m_MyAddress.m_NetAddresses.size();i++) - { - memcpy(ptrOut,&(pRelayManager->m_MyAddress.m_NetAddresses[i]),sizeof(CAddress)); - ptrOut+=sizeof(CAddress); - } - } - } - else + goto exitlbl; + } + for(int c=0;cresize(ptrEnd-ptrStart); - memcpy(&(*payload_relay)[0],ptrStart,ptrEnd-ptrStart); - } + total_size+=((mc_ChunkEntityKey*)ptr)->m_Size; + ptr+=size; } - - ptr+=sizeof(CKeyID); break; default: - strError=strprintf("Unsupported request format (%d, %d)",MC_RMT_MC_ADDRESS_QUERY,*ptr); - return false; + goto exitlbl; } } - return true; + if(response->m_Payload.size() != 1+shift+total_size) + { + goto exitlbl; + } + + ptrOut=&(response->m_Payload[0]); + if(*ptrOut != MC_RDT_CHUNKS) + { + goto exitlbl; + } + ptrOut++; + countOut=(int)mc_GetVarInt(ptrOut,1+shift+total_size,-1,&shiftOut); + if( (countOut != count) || (shift != shiftOut) ) + { + goto exitlbl; + } + ptrOut+=shift; + + ptr=ptrStart+1+shift; + for(int c=0;cm_Size; + chunk=(mc_ChunkEntityKey*)ptr; + sizeOut=chunk->m_Size; + map ::iterator itreq = request_pairs->find(c); + if (itreq != request_pairs->end()) + { + collect_row=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(itreq->second); + uint256 hash; + mc_gState->m_TmpBuffers->m_RpcHasher1->DoubleHash(&ptrOut,sizeOut,&hash); + if(memcmp(&hash,chunk->m_Hash,sizeof(uint256))) + { + goto exitlbl; + } + chunk_err=pwalletTxsMain->m_ChunkDB->AddChunk(chunk->m_Hash,&(chunk->m_Entity),(unsigned char*)collect_row->m_TxID,collect_row->m_Vout,ptrOut,NULL,sizeOut,0,0); + if(chunk_err) + { + goto exitlbl; + } + collect_row->m_State.m_Status |= MC_CCF_DELETED; + } + + ptr+=size; + ptrOut+=sizeOut; + } + + result=true; + +exitlbl: + + pRelayManager->UnLock(); + + return result; } -bool MultichainRelayResponse(uint32_t msg_type_stored, CNode *pto_stored, - uint32_t msg_type_in, uint32_t flags, vector& vPayloadIn,vector& vAddrIn, - uint32_t* msg_type_response,uint32_t *flags_response,vector& vPayloadResponse,vector& vAddrResponse, - uint32_t* msg_type_relay,uint32_t *flags_relay,vector& vPayloadRelay,vector& vAddrRelay,string& strError) +int MultichainResponseScore(mc_RelayResponse *response,mc_ChunkCollectorRow *collect_row) { unsigned char *ptr; unsigned char *ptrEnd; - vector *payload_relay_ptr=NULL; - vector *payload_response_ptr=NULL; - - if(msg_type_response) + unsigned char *ptrStart; + int shift,count,size; + mc_ChunkEntityKey *chunk; + int c; + + if( (response->m_Status & MC_RST_SUCCESS) == 0 ) { - payload_response_ptr=&vPayloadResponse; + return MC_CCW_WORST_RESPONSE_SCORE; } - if(msg_type_relay) + ptrStart=&(response->m_Payload[0]); + + size=sizeof(mc_ChunkEntityKey); + shift=0; + count=0; + + ptr=ptrStart; + ptrEnd=ptr+response->m_Payload.size(); + ptr++; + count=(int)mc_GetVarInt(ptr,ptrEnd-ptr,-1,&shift); + ptr+=shift; + + c=0; + while(cm_Hash,collect_row->m_ChunkDef.m_Hash,MC_CDB_CHUNK_HASH_SIZE) == 0) && + (memcmp(&(chunk->m_Entity),&(collect_row->m_ChunkDef.m_Entity),sizeof(mc_TxEntity)) == 0)) + { + if(chunk->m_Flags & MC_CCF_ERROR_MASK) + { + return MC_CCW_WORST_RESPONSE_SCORE; + } + c=count+1; + } + ptr+=size; + c++; } - - ptr=&vPayloadIn[0]; - ptrEnd=ptr+vPayloadIn.size(); - - strError=""; - switch(msg_type_in) + if(c == count) { - case MC_RMT_MC_ADDRESS_QUERY: - if(msg_type_stored) + return MC_CCW_WORST_RESPONSE_SCORE; + } + return response->m_TryCount+response->m_HopCount; +} + +int MultichainCollectChunks(mc_ChunkCollector* collector) +{ + uint32_t time_now; + vector vChunkDefs; + int row,last_row,last_count; + uint32_t total_size; + mc_ChunkCollectorRow *collect_row; + mc_ChunkCollectorRow *collect_subrow; + time_now=mc_TimeNowAsUInt(); + vector payload; + unsigned char buf[16]; + int shift,count; + unsigned char *ptrOut; + int64_t query_id,request_id; + map query_to_delete; + map requests_to_send; + map responses_to_process; + mc_RelayRequest *request; + mc_RelayRequest *query; + mc_RelayResponse *response; + CRelayResponsePair response_pair; + vector vRows; + CRelayRequestPairs request_pairs; + int best_score,best_response,this_score,not_processed; + + pRelayManager->InvalidateResponsesFromDisconnected(); + + collector->Lock(); + + for(row=0;rowm_MemPool->GetCount();row++) + { + collect_row=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(row); + if( collect_row->m_State.m_Status != MC_CCF_DELETED ) + { + if(collect_row->m_State.m_RequestTimeStamp <= time_now) { - strError=strprintf("Unexpected response message type (%d,%d)",msg_type_stored,msg_type_in);; - goto exitlbl; - } - if(payload_response_ptr == NULL) + if(collect_row->m_State.m_Request) + { + pRelayManager->DeleteRequest(collect_row->m_State.m_Request); + collect_row->m_State.m_Request=0; + } + } + request=NULL; + if(collect_row->m_State.m_Request) { - if(payload_relay_ptr) + request=pRelayManager->FindRequest(collect_row->m_State.m_Request); + if(request == NULL) { - vPayloadRelay=vPayloadIn; - *msg_type_relay=msg_type_in; + collect_row->m_State.m_Request=0; + collect_row->m_State.m_RequestTimeStamp=0; + } + } + if(request) + { + if(request->m_Responses.size()) + { + response_pair.request_id=collect_row->m_State.m_Request; + response_pair.response_id=0; + map::iterator itrsp = responses_to_process.find(response_pair); + if (itrsp == responses_to_process.end()) + { + request_pairs.m_Pairs.clear(); + request_pairs.m_Pairs.insert(make_pair(collect_row->m_State.m_RequestPos,row)); + responses_to_process.insert(make_pair(response_pair,request_pairs)); + } + else + { + itrsp->second.m_Pairs.insert(make_pair(collect_row->m_State.m_RequestPos,row)); + } } + pRelayManager->UnLock(); } else { - if(mc_RelayProcess_Address_Query(ptr,ptrEnd,payload_response_ptr,payload_relay_ptr,strError)) + if(collect_row->m_State.m_Query) { - if(payload_response_ptr && (payload_response_ptr->size() != 0)) + query=pRelayManager->FindRequest(collect_row->m_State.m_Query); + if(query == NULL) { - *msg_type_response=MC_RMT_NODE_DETAILS; + collect_row->m_State.m_Query=0; + collect_row->m_State.m_QueryNextAttempt=time_now+MultichainNextChunkQueryAttempt(collect_row->m_State.m_QueryAttempts); } - if(payload_relay_ptr && (payload_relay_ptr->size() != 0)) + best_response=-1; + best_score=MC_CCW_WORST_RESPONSE_SCORE; + for(int i=0;i<(int)query->m_Responses.size();i++) { - *msg_type_relay=MC_RMT_MC_ADDRESS_QUERY; + this_score=MultichainResponseScore(&(query->m_Responses[i]),collect_row); + if(this_score < best_score) + { + best_score=this_score; + best_response=i; + } + } + if(best_response >= 0) + { + response_pair.request_id=collect_row->m_State.m_Query; + response_pair.response_id=best_response; + map::iterator itrsp = requests_to_send.find(response_pair); + if (itrsp == requests_to_send.end()) + { + request_pairs.m_Pairs.clear(); + request_pairs.m_Pairs.insert(make_pair(row,0)); + requests_to_send.insert(make_pair(response_pair,request_pairs)); + } + else + { + itrsp->second.m_Pairs.insert(make_pair(row,0)); + } } } } - break; - case MC_RMT_NODE_DETAILS: - if(msg_type_stored != MC_RMT_MC_ADDRESS_QUERY) + } + } + + BOOST_FOREACH(PAIRTYPE(const CRelayResponsePair, CRelayRequestPairs)& item, responses_to_process) + { + MultichainProcessChunkResponse(&(item.first),&(item.second.m_Pairs),collector); + } + + BOOST_FOREACH(PAIRTYPE(const CRelayResponsePair, CRelayRequestPairs)& item, requests_to_send) + { + payload.clear(); + shift=mc_PutVarInt(buf,16,requests_to_send.size()); + payload.resize(1+shift+sizeof(mc_ChunkEntityKey)*requests_to_send.size()); + ptrOut=&(payload[0]); + count=0; + BOOST_FOREACH(PAIRTYPE(const int, int)& chunk_row, item.second.m_Pairs) + { + collect_subrow=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(chunk_row.first); + collect_subrow->m_State.m_RequestPos=count; + memcpy(ptrOut,&(collect_subrow->m_ChunkDef),sizeof(mc_ChunkEntityKey)); + ptrOut+=sizeof(mc_ChunkEntityKey); + count++; + } + + request=pRelayManager->FindRequest(item.first.request_id); + if(request == NULL) + { + return false; + } + + response=&(request->m_Responses[item.first.response_id]); + request_id=pRelayManager->SendNextRequest(response,MC_RMT_CHUNK_REQUEST,0,payload); + BOOST_FOREACH(PAIRTYPE(const int, int)& chunk_row, item.second.m_Pairs) + { + collect_subrow=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(chunk_row.first); + collect_subrow->m_State.m_Request=request_id; + } + } + + row=0; + last_row=0; + last_count=0; + total_size=0; + while(row<=collector->m_MemPool->GetCount()) + { + collect_row=NULL; + if(rowm_MemPool->GetCount()) + { + collect_row=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(row); + } + + if( (collect_row == NULL)|| (last_count >= MC_CCW_MAX_CHUNKS_PER_QUERY) || (total_size+collect_row->m_ChunkDef.m_Size > MAX_SIZE-48) ) + { + if(last_count) { - strError=strprintf("Unexpected response message type (%d,%d)",msg_type_stored,msg_type_in);; - goto exitlbl; - } - if(payload_response_ptr) + payload.clear(); + shift=mc_PutVarInt(buf,16,last_count); + payload.resize(1+shift+sizeof(mc_ChunkEntityKey)*last_count); + ptrOut=&(payload[0]); + + *ptrOut=MC_RDT_CHUNK_IDS; + ptrOut++; + memcpy(ptrOut,buf,shift); + ptrOut+=shift; + for(int r=last_row;rm_MemPool->GetRow(row); + if(collect_subrow->m_State.m_Status & MC_CCF_SELECTED) + { + memcpy(ptrOut,&(collect_subrow->m_ChunkDef),sizeof(mc_ChunkEntityKey)); + ptrOut+=sizeof(mc_ChunkEntityKey); + } + } + query_id=pRelayManager->SendRequest(NULL,MC_RMT_CHUNK_QUERY,0,payload); + for(int r=last_row;rm_MemPool->GetRow(row); + if(collect_subrow->m_State.m_Status & MC_CCF_SELECTED) + { + collect_subrow->m_State.m_Status -= MC_CCF_SELECTED; + collect_subrow->m_State.m_Query=query_id; + collect_subrow->m_State.m_QueryAttempts+=1; + collect_subrow->m_State.m_QueryTimeStamp=time_now+MC_CCW_TIMEOUT_QUERY; + } + } + last_row=row; + last_count=0; + total_size=0; + } + } + + if(collect_row) + { + if(collect_row->m_State.m_Status != MC_CCF_DELETED) { - *msg_type_response=MC_RMT_ADD_RESPONSE; + if(collect_row->m_State.m_QueryTimeStamp <= time_now) + { + if(collect_row->m_State.m_Request) + { + pRelayManager->DeleteRequest(collect_row->m_State.m_Request); + collect_row->m_State.m_Request=0; + } + if(collect_row->m_State.m_Query) + { + map::iterator itqry = query_to_delete.find(collect_row->m_State.m_Query); + if (itqry == query_to_delete.end()) + { + query_to_delete.insert(make_pair(collect_row->m_State.m_Query,true)); + } + collect_row->m_State.m_Query=0; + collect_row->m_State.m_QueryNextAttempt=time_now+MultichainNextChunkQueryAttempt(collect_row->m_State.m_QueryAttempts); + } + if(collect_row->m_State.m_QueryNextAttempt <= time_now) + { + if( (collect_row->m_State.m_Status & MC_CCF_ERROR_MASK) == 0) + { + collect_row->m_State.m_Status |= MC_CCF_SELECTED; + last_count++; + } + } + } } - - break; + } + row++; } + + not_processed=0; -exitlbl: - - if(strError.size()) + for(row=0;rowm_MemPool->GetCount();row++) { - return false; + collect_row=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(row); + if( collect_row->m_State.m_Status != MC_CCF_DELETED ) + { + if(collect_row->m_State.m_Query) + { + map::iterator itqry = query_to_delete.find(collect_row->m_State.m_Query); + if (itqry != query_to_delete.end()) + { + itqry->second=false; + } + } + not_processed++; + } } - return true; + BOOST_FOREACH(PAIRTYPE(const int64_t, bool)& item, query_to_delete) + { + if(item.second) + { + pRelayManager->DeleteRequest(item.first); + } + } + + collector->UnLock(); + + return not_processed; } -void mc_Limiter::Zero() -{ - memset(this,0,sizeof(mc_Limiter)); -} -int mc_Limiter::Initialize(int seconds,int measures) +void mc_RelayPayload_ChunkIDs(vector* payload,vector & vChunkDefs,int size) { - Zero(); + unsigned char buf[16]; + int shift; + unsigned char *ptrOut; - if(seconds > MC_LIM_MAX_SECONDS) - { - return MC_ERR_INVALID_PARAMETER_VALUE; - } - if(measures > MC_LIM_MAX_MEASURES) + if(payload) { - return MC_ERR_INVALID_PARAMETER_VALUE; + if(vChunkDefs.size()) + { + shift=mc_PutVarInt(buf,16,vChunkDefs.size()); + payload->resize(1+shift+size*vChunkDefs.size()); + ptrOut=&(*payload)[0]; + + *ptrOut=MC_RDT_CHUNK_IDS; + ptrOut++; + memcpy(ptrOut,buf,shift); + ptrOut+=shift; + + for(int i=0;i<(int)vChunkDefs.size();i++) + { + memcpy(ptrOut,&vChunkDefs[i],size); + ptrOut+=size; + } + } } - m_SecondCount=seconds; - m_MeasureCount=measures; - m_Time=mc_TimeNowAsUInt(); - - return MC_ERR_NOERROR; } -int mc_Limiter::SetLimit(int meausure, int64_t limit) +bool mc_RelayProcess_Chunk_Query(unsigned char *ptrStart,unsigned char *ptrEnd,vector* payload_response,vector* payload_relay,string& strError) { - if( (meausure > MC_LIM_MAX_MEASURES) || (meausure <0) ) - { + unsigned char *ptr; + int shift,count,size; + vector vToRelay; + vector vToRespond; + mc_ChunkEntityKey chunk; + mc_ChunkDBRow chunk_def; + + size=sizeof(mc_ChunkEntityKey); + ptr=ptrStart; + while(ptrm_ChunkDB->GetChunkDef(&chunk_def,chunk.m_Hash,NULL,NULL,-1) == MC_ERR_NOERROR) + { + if(chunk_def.m_Size != chunk.m_Size) + { + chunk.m_Flags |= MC_CCF_WRONG_SIZE; + } + vToRespond.push_back(chunk); + } + else + { + vToRelay.push_back(chunk); + } + ptr+=size; + } + + mc_RelayPayload_ChunkIDs(payload_response,vToRespond,size); + mc_RelayPayload_ChunkIDs(payload_relay,vToRelay,size); + break; + default: + strError=strprintf("Unsupported request format (%d, %d)",MC_RMT_CHUNK_QUERY,*ptr); + return false; + } + } + + return true; +} + +bool mc_RelayProcess_Chunk_Request(unsigned char *ptrStart,unsigned char *ptrEnd,vector* payload_response,vector* payload_relay,string& strError) +{ + unsigned char *ptr; + int shift,count,size; + mc_ChunkEntityKey chunk; + mc_ChunkDBRow chunk_def; + const unsigned char *chunk_found; + unsigned char buf[16]; + size_t chunk_bytes; + unsigned char *ptrOut; + + uint32_t total_size=0; + + mc_gState->m_TmpBuffers->m_RelayTmpBuffer->Clear(); + mc_gState->m_TmpBuffers->m_RelayTmpBuffer->AddElement(); + + size=sizeof(mc_ChunkEntityKey); + ptr=ptrStart; + while(ptrm_ChunkDB->GetChunkDef(&chunk_def,chunk.m_Hash,NULL,NULL,-1) == MC_ERR_NOERROR) + { + if(chunk_def.m_Size != chunk.m_Size) + { + strError="Bad chunk size"; + return false; + } + if(total_size + chunk_def.m_Size > MAX_SIZE-48) + { + strError="Total size of requested chunks is too big"; + return false; + } + chunk_found=pwalletTxsMain->m_ChunkDB->GetChunk(&chunk_def,0,-1,&chunk_bytes); + mc_gState->m_TmpBuffers->m_RelayTmpBuffer->SetData(chunk_found,chunk_bytes); + } + else + { + strError="Chunk not found"; + return false; + } + ptr+=size; + } + + chunk_found=mc_gState->m_TmpBuffers->m_RelayTmpBuffer->GetData(0,&chunk_bytes); + payload_response->resize(1+shift+chunk_bytes); + ptrOut=&(*payload_response)[0]; + + *ptrOut=MC_RDT_CHUNKS; + ptrOut++; + memcpy(ptrOut,buf,shift); + ptrOut+=shift; + memcpy(ptrOut,chunk_found,chunk_bytes); + ptrOut+=chunk_bytes; + + break; + default: + strError=strprintf("Unsupported request format (%d, %d)",MC_RMT_CHUNK_QUERY,*ptr); + return false; + } + } + + return true; +} + +bool mc_RelayProcess_Address_Query(unsigned char *ptrStart,unsigned char *ptrEnd,vector* payload_response,vector* payload_relay,string& strError) +{ + unsigned char *ptr; + unsigned char *ptrOut; + unsigned char buf[16]; + int shift; + CKey key; + + ptr=ptrStart; + while(ptrGetKey(*(CKeyID*)ptr, key)) + { + if(payload_response) + { + shift=mc_PutVarInt(buf,16,pRelayManager->m_MyAddress.m_NetAddresses.size()); + payload_response->resize(1+sizeof(CKeyID)+1+shift+sizeof(CAddress)*pRelayManager->m_MyAddress.m_NetAddresses.size()); + ptrOut=&(*payload_response)[0]; + + *ptrOut=MC_RDT_MC_ADDRESS; + ptrOut++; + *(CKeyID*)ptrOut=pRelayManager->m_MyAddress.m_Address; + ptrOut+=sizeof(CKeyID); + + *ptrOut=MC_RDT_NET_ADDRESS; + ptrOut++; + memcpy(ptrOut,buf,shift); + ptrOut+=shift; + for(int i=0;i<(int)pRelayManager->m_MyAddress.m_NetAddresses.size();i++) + { + memcpy(ptrOut,&(pRelayManager->m_MyAddress.m_NetAddresses[i]),sizeof(CAddress)); + ptrOut+=sizeof(CAddress); + } + } + } + else + { + if(payload_relay) + { + payload_relay->resize(ptrEnd-ptrStart); + memcpy(&(*payload_relay)[0],ptrStart,ptrEnd-ptrStart); + } + } + + ptr+=sizeof(CKeyID); + break; + default: + strError=strprintf("Unsupported request format (%d, %d)",MC_RMT_MC_ADDRESS_QUERY,*ptr); + return false; + } + } + + return true; +} + +bool MultichainRelayResponse(uint32_t msg_type_stored, CNode *pto_stored, + uint32_t msg_type_in, uint32_t flags, vector& vPayloadIn,vector& vAddrIn, + uint32_t* msg_type_response,uint32_t *flags_response,vector& vPayloadResponse,vector& vAddrResponse, + uint32_t* msg_type_relay,uint32_t *flags_relay,vector& vPayloadRelay,vector& vAddrRelay,string& strError) +{ + unsigned char *ptr; + unsigned char *ptrEnd; + vector *payload_relay_ptr=NULL; + vector *payload_response_ptr=NULL; + + if(msg_type_response) + { + payload_response_ptr=&vPayloadResponse; + } + + if(msg_type_relay) + { + payload_relay_ptr=&vPayloadRelay; + } + + ptr=&vPayloadIn[0]; + ptrEnd=ptr+vPayloadIn.size(); + + strError=""; + switch(msg_type_in) + { + case MC_RMT_MC_ADDRESS_QUERY: + if(msg_type_stored) + { + strError=strprintf("Unexpected response message type (%d,%d)",msg_type_stored,msg_type_in);; + goto exitlbl; + } + if(payload_response_ptr == NULL) + { + if(payload_relay_ptr) + { + vPayloadRelay=vPayloadIn; + *msg_type_relay=msg_type_in; + } + } + else + { + if(mc_RelayProcess_Address_Query(ptr,ptrEnd,payload_response_ptr,payload_relay_ptr,strError)) + { + if(payload_response_ptr && (payload_response_ptr->size() != 0)) + { + *msg_type_response=MC_RMT_NODE_DETAILS; + } + if(payload_relay_ptr && (payload_relay_ptr->size() != 0)) + { + *msg_type_relay=MC_RMT_MC_ADDRESS_QUERY; + } + } + } + break; + case MC_RMT_NODE_DETAILS: + if(msg_type_stored != MC_RMT_MC_ADDRESS_QUERY) + { + strError=strprintf("Unexpected response message type (%d,%d)",msg_type_stored,msg_type_in);; + goto exitlbl; + } + if(payload_response_ptr) + { + *msg_type_response=MC_RMT_ADD_RESPONSE; + } + + break; + case MC_RMT_CHUNK_QUERY: + if(msg_type_stored) + { + strError=strprintf("Unexpected response message type (%d,%d)",msg_type_stored,msg_type_in);; + goto exitlbl; + } + if(payload_response_ptr == NULL) + { + if(payload_relay_ptr) + { + vPayloadRelay=vPayloadIn; + *msg_type_relay=msg_type_in; + } + } + else + { + if(mc_RelayProcess_Chunk_Query(ptr,ptrEnd,payload_response_ptr,payload_relay_ptr,strError)) + { + if(payload_response_ptr && (payload_response_ptr->size() != 0)) + { + *msg_type_response=MC_RMT_CHUNK_QUERY_HIT; + } + if(payload_relay_ptr && (payload_relay_ptr->size() != 0)) + { + *msg_type_relay=MC_RMT_CHUNK_QUERY; + } + } + } + break; + case MC_RMT_CHUNK_QUERY_HIT: + if(msg_type_stored != MC_RMT_CHUNK_QUERY) + { + strError=strprintf("Unexpected response message type (%d,%d)",msg_type_stored,msg_type_in);; + goto exitlbl; + } + if(payload_response_ptr) + { + *msg_type_response=MC_RMT_ADD_RESPONSE; + } + break; + case MC_RMT_CHUNK_REQUEST: + if(msg_type_stored != MC_RMT_CHUNK_QUERY_HIT) + { + strError=strprintf("Unexpected response message type (%d,%d)",msg_type_stored,msg_type_in);; + goto exitlbl; + } + if(payload_response_ptr == NULL) + { + if(payload_relay_ptr) + { + vPayloadRelay=vPayloadIn; + *msg_type_relay=msg_type_in; + } + } + else + { + if(mc_RelayProcess_Chunk_Request(ptr,ptrEnd,payload_response_ptr,payload_relay_ptr,strError)) + { + if(payload_response_ptr && (payload_response_ptr->size() != 0)) + { + *msg_type_response=MC_RMT_CHUNK_RESPONSE; + } + if(payload_relay_ptr && (payload_relay_ptr->size() != 0)) + { + *msg_type_relay=MC_RMT_CHUNK_REQUEST; + } + } + } + break; + case MC_RMT_CHUNK_RESPONSE: + if(msg_type_stored != MC_RMT_CHUNK_QUERY_HIT) + { + strError=strprintf("Unexpected response message type (%d,%d)",msg_type_stored,msg_type_in);; + goto exitlbl; + } + if(payload_response_ptr) + { + *msg_type_response=MC_RMT_ADD_RESPONSE; + } + + break; + } + +exitlbl: + + if(strError.size()) + { + return false; + } + + return true; +} + +void mc_Limiter::Zero() +{ + memset(this,0,sizeof(mc_Limiter)); +} + +int mc_Limiter::Initialize(int seconds,int measures) +{ + Zero(); + + if(seconds > MC_LIM_MAX_SECONDS) + { + return MC_ERR_INVALID_PARAMETER_VALUE; + } + if(measures > MC_LIM_MAX_MEASURES) + { + return MC_ERR_INVALID_PARAMETER_VALUE; + } + m_SecondCount=seconds; + m_MeasureCount=measures; + m_Time=mc_TimeNowAsUInt(); + + return MC_ERR_NOERROR; +} + +int mc_Limiter::SetLimit(int meausure, int64_t limit) +{ + if( (meausure > MC_LIM_MAX_MEASURES) || (meausure <0) ) + { return MC_ERR_INVALID_PARAMETER_VALUE; } m_Limits[meausure]=limit*m_SecondCount; @@ -403,6 +1106,17 @@ int64_t mc_RelayManager::AggregateNonce(uint32_t timestamp,uint32_t nonce) return ((int64_t)nonce<<32)+(int64_t)timestamp; } +uint32_t mc_RelayManager::Timestamp(int64_t aggr_nonce) +{ + return aggr_nonce & 0xFFFFFFFF; +} + +uint32_t mc_RelayManager::Nonce(int64_t aggr_nonce) +{ + return (aggr_nonce >> 32) & 0xFFFFFFFF; +} + + void mc_RelayManager::Zero() { m_Semaphore=NULL; @@ -1134,8 +1848,9 @@ void mc_RelayResponse::Zero() m_Nonce=0; m_MsgType=MC_RMT_NONE; m_Flags=0; - m_NodeFrom=NULL; + m_NodeFrom=0; m_HopCount=0; + m_TryCount=0; m_Source=0; m_LastTryTimestamp=0; m_Status=MC_RST_NONE; @@ -1148,7 +1863,7 @@ void mc_RelayRequest::Zero() m_Nonce=0; m_MsgType=MC_RMT_NONE; m_Flags=0; - m_NodeTo=NULL; + m_NodeTo=0; m_ParentNonce=0; m_ParentResponseID=-1; m_LastTryTimestamp=0; @@ -1171,7 +1886,7 @@ int mc_RelayManager::AddRequest(int64_t parent_nonce,int parent_response_id,CNod request.m_Nonce=nonce; request.m_MsgType=msg_type; request.m_Flags=flags; - request.m_NodeTo=pto; + request.m_NodeTo=pto ? pto->GetId() : 0; request.m_ParentNonce=parent_nonce; request.m_ParentResponseID=parent_response_id; request.m_LastTryTimestamp=0; @@ -1192,610 +1907,175 @@ int mc_RelayManager::AddRequest(int64_t parent_nonce,int parent_response_id,CNod } else { - err=MC_ERR_NOT_FOUND; - } - } - - printf("rqst: %lu, mt: %d, node: %d, size: %d, pr: %lu\n",nonce,msg_type,pto ? pto->GetId() : 0,(int)payload.size(),parent_nonce); - m_Requests.insert(make_pair(nonce,request)); - } - else - { - err=MC_ERR_FOUND; - } - - UnLock(); - return err; -} - -int mc_RelayManager::AddResponse(int64_t request,CNode *pfrom,int32_t source,int hop_count,int64_t nonce,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status) -{ - printf("resp: %lu, mt: %d, node: %d, size: %d, rn: %lu\n",request,msg_type,pfrom ? pfrom->GetId() : 0,(int)payload.size(),nonce); - if(request == 0) - { - return MC_ERR_NOERROR; - } - - mc_RelayResponse response; - response.m_Nonce=nonce; - response.m_MsgType=msg_type; - response.m_Flags=flags; - response.m_NodeFrom=pfrom; - response.m_HopCount=hop_count; - response.m_Source=source; - response.m_LastTryTimestamp=0; - response.m_Status=status; - response.m_Payload=payload; - response.m_Requests.clear(); - - if(request) - { - map::iterator itreq = m_Requests.find(request); - if(itreq != m_Requests.end()) - { - itreq->second.m_Responses.push_back(response); - if(status & MC_RST_SUCCESS) - { - itreq->second.m_Status |= MC_RST_SUCCESS; - } - } - else - { - return MC_ERR_NOT_FOUND; - } - } - - return MC_ERR_NOERROR; -} - -int mc_RelayManager::DeleteRequest(int64_t request) -{ - int err=MC_ERR_NOERROR; - - Lock(); - map::iterator itreq = m_Requests.find(request); - if(itreq != m_Requests.end()) - { - m_Requests.erase(itreq); - } - else - { - err= MC_ERR_NOT_FOUND; - } - - UnLock(); - return err; -} - -mc_RelayRequest *mc_RelayManager::FindRequest(int64_t request) -{ - Lock(); - map::iterator itreq = m_Requests.find(request); - if(itreq != m_Requests.end()) - { - return &(itreq->second); - } - - UnLock(); - return NULL; -} - - -int64_t mc_RelayManager::SendRequest(CNode* pto,uint32_t msg_type,uint32_t flags,vector & payload) -{ - uint32_t timestamp=mc_TimeNowAsUInt(); - uint32_t nonce=GenerateNonce(); - int64_t aggr_nonce=AggregateNonce(timestamp,nonce); - vector vEmptyHops; - - vector vSigScriptsEmpty; - - { - LOCK(cs_vNodes); - BOOST_FOREACH(CNode* pnode, vNodes) - { - if( (pto == NULL) || (pnode == pto) ) - { - PushRelay(pnode,0,vEmptyHops,msg_type,timestamp,nonce,0,0,flags,payload,vSigScriptsEmpty,NULL,0); - } - } - } - - if(AddRequest(0,0,pto,aggr_nonce,msg_type,flags,payload,MC_RST_NONE) != MC_ERR_NOERROR) - { - return 0; - } - - return aggr_nonce; -} - -/* -bool ProcessMultichainRelay(CNode* pfrom, CDataStream& vRecv, CValidationState &state, uint32_t verify_flags_in) -{ - uint32_t msg_type_in; - uint32_t verify_flags; - int64_t nonce_received; - int64_t response_to_nonce; - vector vOriginMCAddressIn; - vector vDestinationMCAddressIn; - vector vOriginNetAddressIn; - vector vDestinationNetAddressIn; - vector vPayloadIn; - vector vSigScript; - uint256 payload_hash,payload_hash_in; - uint32_t flags_in; - CKeyID origin_mc_address; - CKeyID destination_mc_address; - bool destination_cannot_be_empty; - vector vchSigOut; - vector vchPubKey; - CPubKey pubkey; - - vRecv >> msg_type_in; - - vRecv >> nonce_received; - vRecv >> vOriginMCAddressIn; - vRecv >> vOriginNetAddressIn; - vRecv >> vDestinationMCAddressIn; - vRecv >> vDestinationNetAddressIn; - vRecv >> flags_in; - vRecv >> response_to_nonce; - vRecv >> payload_hash_in; - vRecv >> vSigScript; - vRecv >> vPayloadIn; - - if(verify_flags & MC_VRA_MESSAGE_TYPE) - { - switch(msg_type_in) - { - case MC_RMT_CHUNK_QUERY: - case MC_RMT_CHUNK_QUERY_HIT: - case MC_RMT_CHUNK_REQUEST: - case MC_RMT_CHUNK_RESPONSE: - break; - default: - LogPrintf("ProcessMultichainRelay() : Unsupported relay message type %d\n",msg_type_in); - return false; - } - } - - if(verify_flags & MC_VRA_ORIGIN_ADDRESS) - { - if(vOriginNetAddressIn.size() == 0) - { - return state.DoS(100, error("ProcessMultichainRelay() : No origin network address"),REJECT_INVALID, "bad-origin-net-address"); - } - if(MCP_ANYONE_CAN_CONNECT == 0) - { - if(vOriginMCAddressIn.size() != sizeof(uint160)) - { - return state.DoS(100, error("ProcessMultichainRelay() : Bad origin address"),REJECT_INVALID, "bad-origin-address"); - } - origin_mc_address=CKeyID(*(uint160*)&vOriginMCAddressIn[0]); - } - } - - if(verify_flags & MC_VRA_DESTINATION_ADDRESS) - { - destination_cannot_be_empty=false; - switch(msg_type_in) - { - case MC_RMT_CHUNK_QUERY_HIT: - case MC_RMT_CHUNK_RESPONSE: - destination_cannot_be_empty=true; - break; - } - if(destination_cannot_be_empty) - { - if( ((vDestinationMCAddressIn.size()) == 0) && (MCP_ANYONE_CAN_CONNECT == 0) ) - { - return state.DoS(100, error("ProcessMultichainRelay() : Empty destination address"),REJECT_INVALID, "bad-destination-address"); - } - if(vOriginNetAddressIn.size() == 0) - { - return state.DoS(100, error("ProcessMultichainRelay() : No destination network address"),REJECT_INVALID, "bad-destination-net-address"); - } - } - if(vDestinationMCAddressIn.size()) - { - if(MCP_ANYONE_CAN_CONNECT == 0) - { - if(vDestinationMCAddressIn.size() != sizeof(uint160)) - { - return state.DoS(100, error("ProcessMultichainRelay() : Bad destination address"),REJECT_INVALID, "bad-destination-address"); - } - destination_mc_address=CKeyID(*(uint160*)&vDestinationMCAddressIn[0]); - } - } - } - - if( verify_flags & MC_VRA_SIGSCRIPT) - { - if(vSigScript.size()) - { - CScript scriptSig((unsigned char*)&vSigScript[0],(unsigned char*)&vSigScript[0]+vSigScript.size()); - - opcodetype opcode; - - CScript::const_iterator pc = scriptSig.begin(); - - if (!scriptSig.GetOp(pc, opcode, vchSigOut)) - { - return state.DoS(100, error("ProcessMultichainRelay() : Cannot extract signature from sigScript"),REJECT_INVALID, "bad-sigscript-signature"); - } - - vchSigOut.resize(vchSigOut.size()-1); - if (!scriptSig.GetOp(pc, opcode, vchPubKey)) - { - return state.DoS(100, error("ProcessMultichainRelay() : Cannot extract pubkey from sigScript"),REJECT_INVALID, "bad-sigscript-pubkey"); - } - - CPubKey pubKeyOut(vchPubKey); - if (!pubKeyOut.IsValid()) - { - return state.DoS(100, error("ProcessMultichainRelay() : Invalid pubkey"),REJECT_INVALID, "bad-sigscript-pubkey"); - } - - pubkey=pubKeyOut; - } - else - { - switch(msg_type_in) - { - case MC_RMT_CHUNK_QUERY: - break; - default: - return state.DoS(100, error("ProcessMultichainRelay() : Missing sigScript"),REJECT_INVALID, "bad-sigscript"); - } + err=MC_ERR_NOT_FOUND; + } } + + printf("rqst: %lu, mt: %d, node: %d, size: %d, pr: %lu\n",nonce,msg_type,pto ? pto->GetId() : 0,(int)payload.size(),parent_nonce); + m_Requests.insert(make_pair(nonce,request)); } + else + { + err=MC_ERR_FOUND; + } - if( verify_flags & MC_VRA_RESPONSE_TO_NONCE ) + UnLock(); + return err; +} + +int mc_RelayManager::AddResponse(int64_t request,CNode *pfrom,int32_t source,int hop_count,int64_t nonce,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status) +{ + printf("resp: %lu, mt: %d, node: %d, size: %d, rn: %lu\n",request,msg_type,pfrom ? pfrom->GetId() : 0,(int)payload.size(),nonce); + if(request == 0) { - if(response_to_nonce == 0) - { - switch(msg_type_in) - { - case MC_RMT_CHUNK_QUERY: - break; - default: - return state.DoS(100, error("ProcessMultichainRelay() : Missing response_to_nonce"),REJECT_INVALID, "bad-nonce"); - } - } + return MC_ERR_NOERROR; } - if( verify_flags & (MC_VRA_PAYLOAD_HASH | MC_VRA_SIGNATURE) ) + mc_RelayResponse response; + response.m_Nonce=nonce; + response.m_MsgType=msg_type; + response.m_Flags=flags; + response.m_NodeFrom=pfrom ? pfrom->GetId() : 0; + response.m_HopCount=hop_count; + response.m_Source=source; + response.m_LastTryTimestamp=0; + response.m_Status=status; + response.m_Payload=payload; + response.m_Requests.clear(); + + if(request) { - CHashWriter ssHash(SER_GETHASH, 0); - ssHash << vPayloadIn; - payload_hash=ssHash.GetHash(); - - if(payload_hash != payload_hash_in) + map::iterator itreq = m_Requests.find(request); + if(itreq != m_Requests.end()) + { + itreq->second.m_Responses.push_back(response); + if(status & MC_RST_SUCCESS) + { + itreq->second.m_Status |= MC_RST_SUCCESS; + } + } + else { - return state.DoS(100, error("ProcessMultichainRelay() : Bad payload hash"),REJECT_INVALID, "bad-payload-hash"); + return MC_ERR_NOT_FOUND; } } - if( verify_flags & MC_VRA_SIGNATURE ) - { - if(vSigScript.size()) - { - CHashWriter ss(SER_GETHASH, 0); - ss << vector((unsigned char*)&payload_hash, (unsigned char*)&payload_hash+32); - ss << vector((unsigned char*)&response_to_nonce, (unsigned char*)&response_to_nonce+sizeof(response_to_nonce)); - uint256 signed_hash=ss.GetHash(); - - if(!pubkey.Verify(signed_hash,vchSigOut)) - { - return state.DoS(100, error("ProcessMultichainRelay() : Wrong signature"),REJECT_INVALID, "bad-signature"); - } - } - } - - return true; -} -*/ -void PushMultiChainRelay(CNode* pto, uint32_t msg_type,vector& path,vector& path_to_follow,vector& payload) -{ - pto->PushMessage("relay", msg_type, path, path_to_follow, payload); + return MC_ERR_NOERROR; } - -void PushMultiChainRelay(CNode* pto, uint32_t msg_type,vector& path,vector& path_to_follow,unsigned char *payload,size_t size) +int mc_RelayManager::DeleteRequest(int64_t request) { - vector vPayload= vector((unsigned char *)payload,(unsigned char *)payload+size); - PushMultiChainRelay(pto,msg_type,path,path_to_follow,vPayload); -} + int err=MC_ERR_NOERROR; -bool MultichainRelayResponse(uint32_t msg_type_in,vector& path,vector& vPayloadIn, - uint32_t* msg_type_response,vector& vPayloadResponse, - uint32_t* msg_type_relay,vector& vPayloadRelay) -{ -/* - switch(msg_type_in) + Lock(); + map::iterator itreq = m_Requests.find(request); + if(itreq != m_Requests.end()) { - case MC_RMT_GLOBAL_PING: - *msg_type_response=MC_RMT_GLOBAL_PONG; - vPayloadResponse=vPayloadIn; - if(path.size() < 4) - { - *msg_type_relay=MC_RMT_GLOBAL_PING; - vPayloadRelay=vPayloadIn; - } - break; - case MC_RMT_GLOBAL_PONG: - string str_path=""; - for(int i=(int)path.size()-1;i>=0;i--) - { - str_path+="->" + path[i].ToString(); - } - LogPrintf("PONG : %s\n",str_path.c_str()); - break; + m_Requests.erase(itreq); + } + else + { + err= MC_ERR_NOT_FOUND; } - */ - return true; + + UnLock(); + return err; } -bool ProcessMultichainRelay(CNode* pfrom, CDataStream& vRecv, CValidationState &state) +mc_RelayRequest *mc_RelayManager::FindRequest(int64_t request) { - uint32_t msg_type_in,msg_type_response,msg_type_relay; - size_t common_size; - CNode* pto; - - vector path; - vector path_to_follow; - vector path_response; - vector path_to_follow_response; - vector vPayloadIn; - vector vPayloadResponse; - vector vPayloadRelay; - - vRecv >> msg_type_in; - vRecv >> path; - vRecv >> path_to_follow; - vRecv >> vPayloadIn; - - msg_type_response=MC_RMT_NONE; - msg_type_relay=MC_RMT_NONE; - pto=NULL; - - common_size=path_to_follow.size(); - if(common_size > path.size()) + Lock(); + map::iterator itreq = m_Requests.find(request); + if(itreq != m_Requests.end()) { - common_size=path.size(); - if(pfrom->addrLocal != (CService)path_to_follow[common_size]) - { - return state.DoS(100, error("ProcessMultichainRelay() : Bad path"),REJECT_INVALID, "bad-relay-path"); - } - } + return &(itreq->second); + } - path.push_back(pfrom->addr); - common_size++; + UnLock(); + return NULL; +} + + +int64_t mc_RelayManager::SendRequest(CNode* pto,uint32_t msg_type,uint32_t flags,vector & payload) +{ + uint32_t timestamp=mc_TimeNowAsUInt(); + uint32_t nonce=GenerateNonce(); + int64_t aggr_nonce=AggregateNonce(timestamp,nonce); + vector vEmptyHops; - if(common_size < path_to_follow.size()) + vector vSigScriptsEmpty; + { LOCK(cs_vNodes); BOOST_FOREACH(CNode* pnode, vNodes) { - if(pnode->addr == path_to_follow[common_size]) + if( (pto == NULL) || (pnode == pto) ) { - pto=pnode; + PushRelay(pnode,0,vEmptyHops,msg_type,timestamp,nonce,0,0,flags,payload,vSigScriptsEmpty,NULL,0); } } - if(pto == NULL) - { - LogPrintf("Request to relay to unconnected peer %s\n",path_to_follow[common_size].ToString().c_str()); - return false; - } } - - if(pto) + + if(AddRequest(0,0,pto,aggr_nonce,msg_type,flags,payload,MC_RST_NONE) != MC_ERR_NOERROR) { - PushMultiChainRelay(pto, msg_type_in,path,path_to_follow,vPayloadIn); - return true; + return 0; } - if(pto == NULL) + return aggr_nonce; +} + +int64_t mc_RelayManager::SendNextRequest(mc_RelayResponse* response,uint32_t msg_type,uint32_t flags,vector & payload) +{ + int64_t aggr_nonce; + vector vEmptyHops; + vector vSigScriptsEmpty; + + aggr_nonce=0; { - if(MultichainRelayResponse(msg_type_in,path,vPayloadIn,&msg_type_response,vPayloadResponse,&msg_type_relay,vPayloadRelay)) + LOCK(cs_vNodes); + BOOST_FOREACH(CNode* pnode, vNodes) { - if(msg_type_relay) - { - BOOST_FOREACH(CNode* pnode, vNodes) - { - int path_index=-1; - for(unsigned int i=0;iaddr == path[i]) - { - path_index=i; - } - } - } - if(path_index < 0) - { - PushMultiChainRelay(pnode, msg_type_relay,path,path_to_follow,vPayloadRelay); - } - } - } - if(msg_type_response) + if( pnode->GetId() == response->m_NodeFrom ) { - for(int i=(int)path.size()-1;i>=0;i--) + aggr_nonce=PushRelay(pnode,0,vEmptyHops,msg_type,0,0,Timestamp(response->m_Nonce),Nonce(response->m_Nonce), + flags,payload,vSigScriptsEmpty,NULL,MC_PRA_GENERATE_TIMESTAMP | MC_PRA_GENERATE_NONCE); + if(AddRequest(0,0,pnode,aggr_nonce,msg_type,flags,payload,MC_RST_NONE) != MC_ERR_NOERROR) { - path_to_follow_response.push_back(path[i]); - PushMultiChainRelay(pfrom, msg_type_response,path_response,path_to_follow_response,vPayloadResponse); + return 0; } } - } - else - { - return false; } } - return true; + return aggr_nonce; } -/* -void mc_RelayManager::PushRelay(CNode* pto, - uint32_t msg_format, - unsigned char hop_count, - uint32_t msg_type, - uint32_t timestamp_to_send, - uint32_t nonce_to_send, - uint32_t timestamp_to_respond, - uint32_t nonce_to_respond, - uint32_t flags, - vector& payload, - vector& sigScripts_to_relay, - CNode* pfrom, - uint32_t action) -{ - vector vOriginAddress; - vector vDestinationAddress; - vector sigScripts; - vectorvSigScript; - CScript sigScript; - uint256 message_hash; - uint32_t timestamp; - uint32_t nonce; - mc_NodeFullAddress origin_to_send; - mc_NodeFullAddress *origin; - mc_NodeFullAddress destination_to_send; - mc_NodeFullAddress *destination; - - nonce=nonce_to_send; - timestamp=timestamp_to_send; - - if(action & MC_PRA_GENERATE_TIMSTAMP) - { - timestamp=mc_TimeNowAsUInt(); - } - - if(action & MC_PRA_GENERATE_NONCE) - { - GetRandBytes((unsigned char*)&nonce, sizeof(nonce)); - } - if(action & (MC_PRA_MY_ORIGIN_MC_ADDRESS | MC_PRA_MY_ORIGIN_NT_ADDRESS) ) - { - InitNodeAddress(&origin_to_send,pto,action); - origin=&origin_to_send; - } - else - { - if(origin_to_relay) - { - origin=origin_to_relay; - } - else - { - origin=&origin_to_send; - } - } - - destination=&destination_to_send; - if(action & MC_PRA_USE_DESTINATION_ADDRESS) - { - if(destination_to_relay) - { - destination=destination_to_relay; - } - } - - if(MCP_ANYONE_CAN_CONNECT == 0) - { - vOriginAddress.resize(sizeof(CKeyID)); - memcpy(&vOriginAddress[0],&(origin->m_Address),sizeof(CKeyID)); - if(destination->m_Address != 0) - { - vDestinationAddress.resize(sizeof(CKeyID)); - memcpy(&vOriginAddress[0],&(origin->m_Address),sizeof(CKeyID)); - } - } - - message_hash=message_hash_to_relay; - - if(action & MC_PRA_CALCULATE_MESSAGE_HASH) - { - CHashWriter ssHash(SER_GETHASH, 0); - ssHash << nonce; - ssHash << msg_type; - ssHash << vOriginAddress; - ssHash << origin->m_NetAddresses; - ssHash << vDestinationAddress; - ssHash << destination->m_NetAddresses; - ssHash << response_to_nonce; - ssHash << flags; - ssHash << payload; - message_hash=ssHash.GetHash(); - } - - if( (action & (MC_PRA_SIGN | MC_PRA_SIGN_WITH_HANDSHAKE_ADDRESS)) && (MCP_ANYONE_CAN_CONNECT == 0) ) +void mc_RelayManager::InvalidateResponsesFromDisconnected() +{ + LOCK(cs_vNodes); + int m=(int)vNodes.size(); + Lock(); + BOOST_FOREACH(PAIRTYPE(const int64_t,mc_RelayRequest)& item, m_Requests) { - CHashWriter ssSig(SER_GETHASH, 0); - ssSig << message_hash; - ssSig << vector((unsigned char*)&response_to_nonce, (unsigned char*)&response_to_nonce+sizeof(response_to_nonce)); - uint256 signed_hash=ssSig.GetHash(); - CKey key; - CKeyID keyID; - CPubKey pkey; - - keyID=origin->m_Address; - if(action & MC_PRA_SIGN_WITH_HANDSHAKE_ADDRESS) - { - if(MCP_ANYONE_CAN_CONNECT == 0) - { - keyID=pto->kAddrLocal; - } - } - - if(pwalletMain->GetKey(keyID, key)) + for(int i=0;i<(int)item.second.m_Responses.size();i++) { - pkey=key.GetPubKey(); - vector vchSig; - sigScript.clear(); - if (key.Sign(signed_hash, vchSig)) + int n=0; + while(nGetId() == item.second.m_Responses[i].m_NodeFrom) + { + n=m+1; + } + n++; } - else + if(n == m) { - LogPrintf("PushRelay(): Internal error: Cannot sign\n"); + item.second.m_Responses[i].m_Status &= ~MC_RST_SUCCESS; + item.second.m_Responses[i].m_Status |= MC_RST_DISCONNECTED; } } - else - { - LogPrintf("PushRelay(): Internal error: Cannot find key for signature\n"); - } - sigScripts.push_back(sigScript); } - - for(unsigned int i=0;iPushMessage("relay", - msg_type, - nonce, - vOriginAddress, - origin->m_NetAddresses, - vDestinationAddress, - destination->m_NetAddresses, - response_to_nonce, - flags, - payload, - message_hash, - path, - sigScripts); - - SetRelayRecord(pto,NULL,msg_type,nonce); - SetRelayRecord(pto,pfrom,msg_type,nonce); + UnLock(); } - -*/ \ No newline at end of file + diff --git a/src/protocol/relay.h b/src/protocol/relay.h index 093cc5de..30b8dd32 100644 --- a/src/protocol/relay.h +++ b/src/protocol/relay.h @@ -56,6 +56,8 @@ #define MC_RDT_UNKNOWN 0 #define MC_RDT_MC_ADDRESS 1 #define MC_RDT_NET_ADDRESS 2 +#define MC_RDT_CHUNK_IDS 11 +#define MC_RDT_CHUNKS 12 #define MC_LIM_MAX_SECONDS 60 #define MC_LIM_MAX_MEASURES 4 @@ -65,16 +67,19 @@ #define MC_RST_SUCCESS 0x00000002 #define MC_RST_PERMANENT_FAILURE 0x00000004 #define MC_RST_TEMPORARY_FAILURE 0x00000008 +#define MC_RST_DISCONNECTED 0x00000010 using namespace std; +struct mc_ChunkCollector; + bool MultichainRelayResponse(uint32_t msg_type_stored, CNode *pto_stored, uint32_t msg_type_in, uint32_t flags, vector& vPayloadIn,vector& vAddrIn, uint32_t* msg_type_response,uint32_t *flags_response,vector& vPayloadResponse,vector& vAddrResponse, uint32_t* msg_type_relay,uint32_t *flags_relay,vector& vPayloadRelay,vector& vAddrRelay, string& strError); - +int MultichainCollectChunks(mc_ChunkCollector* collector); @@ -158,8 +163,9 @@ typedef struct mc_RelayResponse int64_t m_Nonce; uint32_t m_MsgType; uint32_t m_Flags; - CNode *m_NodeFrom; + NodeId m_NodeFrom; int m_HopCount; + int m_TryCount; int32_t m_Source; uint32_t m_LastTryTimestamp; uint32_t m_Status; @@ -179,7 +185,7 @@ typedef struct mc_RelayRequest int64_t m_Nonce; uint32_t m_MsgType; uint32_t m_Flags; - CNode *m_NodeTo; + NodeId m_NodeTo; int64_t m_ParentNonce; int m_ParentResponseID; uint32_t m_LastTryTimestamp; @@ -231,6 +237,8 @@ typedef struct mc_RelayManager int Initialize(); int64_t AggregateNonce(uint32_t timestamp,uint32_t nonce); + uint32_t Timestamp(int64_t aggr_nonce); + uint32_t Nonce(int64_t aggr_nonce); uint32_t GenerateNonce(); void SetDefaults(); void SetMyIPs(uint32_t *ips,int ip_count); @@ -263,10 +271,12 @@ typedef struct mc_RelayManager int AddRequest(int64_t parent_nonce,int parent_response_id,CNode *pto,int64_t nonce,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status); int AddResponse(int64_t request,CNode *pfrom,int32_t source,int hop_count,int64_t nonce,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status); int DeleteRequest(int64_t request); + int ProcessRequest(int64_t request); mc_RelayRequest *FindRequest(int64_t request); - + void InvalidateResponsesFromDisconnected(); int64_t SendRequest(CNode* pto,uint32_t msg_type,uint32_t flags,vector & payload); + int64_t SendNextRequest(mc_RelayResponse* response,uint32_t msg_type,uint32_t flags,vector & payload); } mc_RelayManager; diff --git a/src/rpc/rpccache.cpp b/src/rpc/rpccache.cpp index 23df60bd..d00eabce 100644 --- a/src/rpc/rpccache.cpp +++ b/src/rpc/rpccache.cpp @@ -304,6 +304,7 @@ Value offchain(const Array& params, bool fHelp) addresses.push_back(node_addr.m_NetAddresses[a].ToStringIPPort()); } res.push_back(Pair("addresses",addresses)); + res.push_back(Pair("neighbour",response->m_Source ? false : true)); res_found=true; } } diff --git a/src/wallet/chunkcollector.cpp b/src/wallet/chunkcollector.cpp index 1c32a56d..312e07a9 100644 --- a/src/wallet/chunkcollector.cpp +++ b/src/wallet/chunkcollector.cpp @@ -4,6 +4,17 @@ #include "multichain/multichain.h" #include "wallet/chunkcollector.h" +void mc_ChunkEntityKey::Zero() +{ + memset(this,0, sizeof(mc_ChunkEntityKey)); +} + +void mc_ChunkEntityValue::Zero() +{ + memset(this,0, sizeof(mc_ChunkEntityValue)); +} + + void mc_ChunkCollectorRow::Zero() { memset(this,0, sizeof(mc_ChunkCollectorRow)); @@ -214,10 +225,10 @@ int mc_ChunkCollector::Initialize(mc_ChunkDB *chunk_db,const char *name,uint32_t { return MC_ERR_CORRUPTED; } - collect_row.m_Flags |= MC_CCF_INSERTED; - if(m_ChunkDB->GetChunkDef(&chunk_def,collect_row.m_Hash,NULL,NULL,-1) == MC_ERR_NOERROR) + collect_row.m_State.m_Status |= MC_CCF_INSERTED; + if(m_ChunkDB->GetChunkDef(&chunk_def,collect_row.m_ChunkDef.m_Hash,NULL,NULL,-1) == MC_ERR_NOERROR) { - collect_row.m_Flags |= MC_CCF_DELETED; + collect_row.m_State.m_Status |= MC_CCF_DELETED; } m_MemPool->Add(&collect_row); if(ptr) @@ -272,12 +283,12 @@ int mc_ChunkCollector::InsertChunkInternal( int mprow; collect_row.Zero(); - memcpy(collect_row.m_Hash,hash,MC_CDB_CHUNK_HASH_SIZE); - memcpy(&collect_row.m_Entity,entity,sizeof(mc_TxEntity)); + memcpy(collect_row.m_ChunkDef.m_Hash,hash,MC_CDB_CHUNK_HASH_SIZE); + memcpy(&collect_row.m_ChunkDef.m_Entity,entity,sizeof(mc_TxEntity)); memcpy(collect_row.m_TxID,txid,MC_TDB_TXID_SIZE); collect_row.m_Vout=vout; - collect_row.m_Size=chunk_size; - collect_row.m_Flags=MC_CCF_NEW; + collect_row.m_ChunkDef.m_Size=chunk_size; + collect_row.m_State.m_Status=MC_CCF_NEW; mprow=m_MemPool->Seek(&collect_row); if(mprow<0) @@ -303,11 +314,11 @@ int mc_ChunkCollector::MarkAndClear(uint32_t flag, int unmark) row=(mc_ChunkCollectorRow *)m_MemPool->GetRow(mprow); if(unmark) { - row->m_Flags &= ~flag; + row->m_State.m_Status &= ~flag; } else { - row->m_Flags |= flag; + row->m_State.m_Status |= flag; } } } @@ -333,7 +344,7 @@ int mc_ChunkCollector::CopyFlags() if(mprow >= 0) { row=(mc_ChunkCollectorRow *)m_MemPool->GetRow(mprow); - row->m_Flags=mark_row->m_Flags; + row->m_State.m_Status=mark_row->m_State.m_Status; } } @@ -355,7 +366,7 @@ int mc_ChunkCollector::FillMarkPoolByHash(const unsigned char *hash) for(i=0;iGetCount();i++) { row=(mc_ChunkCollectorRow *)m_MemPool->GetRow(i); - if(memcmp(row->m_Hash,hash,MC_CDB_CHUNK_HASH_SIZE) == 0) + if(memcmp(row->m_ChunkDef.m_Hash,hash,MC_CDB_CHUNK_HASH_SIZE) == 0) { m_MarkPool->Add(row); } @@ -378,9 +389,9 @@ int mc_ChunkCollector::FillMarkPoolByFlag(uint32_t flag, uint32_t not_flag) for(i=0;iGetCount();i++) { row=(mc_ChunkCollectorRow *)m_MemPool->GetRow(i); - if(row->m_Flags & flag) + if(row->m_State.m_Status & flag) { - if( (row->m_Flags & not_flag) == 0) + if( (row->m_State.m_Status & not_flag) == 0) { m_MarkPool->Add(row); } @@ -428,9 +439,9 @@ int mc_ChunkCollector::CommitInternal() { row=(mc_ChunkCollectorRow *)m_MemPool->GetRow(i); - if(row->m_Flags & MC_CCF_DELETED) + if(row->m_State.m_Status & MC_CCF_DELETED) { - if(row->m_Flags & MC_CCF_INSERTED) + if(row->m_State.m_Status & MC_CCF_INSERTED) { commit_required=1; m_DB->Delete((char*)row+m_KeyOffset,m_KeySize,MC_OPT_DB_DATABASE_TRANSACTIONAL); @@ -438,7 +449,7 @@ int mc_ChunkCollector::CommitInternal() } else { - if( (row->m_Flags & MC_CCF_INSERTED) == 0 ) + if( (row->m_State.m_Status & MC_CCF_INSERTED) == 0 ) { commit_required=1; m_DB->Write((char*)row+m_KeyOffset,m_KeySize,(char*)row+m_ValueOffset,m_ValueDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); @@ -464,3 +475,4 @@ int mc_ChunkCollector::CommitInternal() return err; } + diff --git a/src/wallet/chunkcollector.h b/src/wallet/chunkcollector.h index 3d829667..93e1fdbe 100644 --- a/src/wallet/chunkcollector.h +++ b/src/wallet/chunkcollector.h @@ -12,18 +12,48 @@ #define MC_CCF_NEW 0x00000000 #define MC_CCF_INSERTED 0x00000001 #define MC_CCF_DELETED 0x00000002 +#define MC_CCF_SELECTED 0x00000004 +#define MC_CCF_WRONG_SIZE 0x00010000 +#define MC_CCF_ERROR_MASK 0x00FF0000 #define MC_CCF_ALL 0xFFFFFFFF +#define MC_CCW_TIMEOUT_QUERY 60 +#define MC_CCW_TIMEOUT_REQUEST 5 +#define MC_CCW_MAX_CHUNKS_PER_QUERY 64 +#define MC_CCW_WORST_RESPONSE_SCORE 1000 -typedef struct mc_ChunkCollectorRow + +typedef struct mc_ChunkEntityKey { unsigned char m_Hash[MC_CDB_CHUNK_HASH_SIZE]; // Chunk hash mc_TxEntity m_Entity; - unsigned char m_TxID[MC_TDB_TXID_SIZE]; - int m_Vout; uint32_t m_Size; uint32_t m_Flags; + void Zero(); +} mc_ChunkEntityKey; + +typedef struct mc_ChunkEntityValue +{ + int64_t m_Query; + uint32_t m_QueryTimeStamp; + uint32_t m_QueryAttempts; + uint32_t m_QueryNextAttempt; + int64_t m_Request; + uint32_t m_RequestTimeStamp; + uint32_t m_RequestPos; + uint32_t m_Status; + + void Zero(); +} mc_ChunkEntityValue; + +typedef struct mc_ChunkCollectorRow +{ + mc_ChunkEntityKey m_ChunkDef; + unsigned char m_TxID[MC_TDB_TXID_SIZE]; + int m_Vout; + mc_ChunkEntityValue m_State; + void Zero(); } mc_ChunkCollectorRow; @@ -88,7 +118,7 @@ typedef struct mc_ChunkCollector int CopyFlags(); int FillMarkPoolByHash(const unsigned char *hash); int FillMarkPoolByFlag(uint32_t flag, uint32_t not_flag); - + int Commit(); int CommitInternal(); From e4646056b967cf3b15d66a51900309fa38b8fa60 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 23 Apr 2018 16:33:16 +0300 Subject: [PATCH 038/157] Offchain items - manual fetch from the nearest node --- src/protocol/relay.cpp | 30 +++- src/protocol/relay.h | 3 + src/rpc/rpccache.cpp | 266 ++++++++++++++++++++++++++++------ src/rpc/rpcutils.cpp | 18 +++ src/wallet/chunkcollector.cpp | 140 ++++++++++-------- 5 files changed, 347 insertions(+), 110 deletions(-) diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index 0f4679a4..d613f000 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -41,6 +41,7 @@ int MultichainProcessChunkResponse(const CRelayResponsePair *response_pair,map < mc_RelayRequest *request; mc_RelayResponse *response; request=pRelayManager->FindRequest(response_pair->request_id); + if(request == NULL) { return false; @@ -68,7 +69,7 @@ int MultichainProcessChunkResponse(const CRelayResponsePair *response_pair,map < ptr=ptrStart; ptrEnd=ptr+request->m_Payload.size(); - + while(ptrm_Payload.size() != 1+shift+total_size) { goto exitlbl; @@ -102,6 +104,7 @@ int MultichainProcessChunkResponse(const CRelayResponsePair *response_pair,map < { goto exitlbl; } + ptrOut++; countOut=(int)mc_GetVarInt(ptrOut,1+shift+total_size,-1,&shiftOut); if( (countOut != count) || (shift != shiftOut) ) @@ -121,7 +124,7 @@ int MultichainProcessChunkResponse(const CRelayResponsePair *response_pair,map < { collect_row=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(itreq->second); uint256 hash; - mc_gState->m_TmpBuffers->m_RpcHasher1->DoubleHash(&ptrOut,sizeOut,&hash); + mc_gState->m_TmpBuffers->m_RpcHasher1->DoubleHash(ptrOut,sizeOut,&hash); if(memcmp(&hash,chunk->m_Hash,sizeof(uint256))) { goto exitlbl; @@ -254,6 +257,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) { response_pair.request_id=collect_row->m_State.m_Request; response_pair.response_id=0; + printf("coll new rsp: row: %d, id: %lu, %d\n",row,collect_row->m_State.m_Request,collect_row->m_State.m_RequestPos); map::iterator itrsp = responses_to_process.find(response_pair); if (itrsp == responses_to_process.end()) { @@ -265,11 +269,12 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) { itrsp->second.m_Pairs.insert(make_pair(collect_row->m_State.m_RequestPos,row)); } - } + } pRelayManager->UnLock(); } else { + query=NULL; if(collect_row->m_State.m_Query) { query=pRelayManager->FindRequest(collect_row->m_State.m_Query); @@ -278,6 +283,9 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) collect_row->m_State.m_Query=0; collect_row->m_State.m_QueryNextAttempt=time_now+MultichainNextChunkQueryAttempt(collect_row->m_State.m_QueryAttempts); } + } + if(query) + { best_response=-1; best_score=MC_CCW_WORST_RESPONSE_SCORE; for(int i=0;i<(int)query->m_Responses.size();i++) @@ -289,6 +297,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) best_response=i; } } + printf("coll new req: row: %d, id: %lu, rsps: %d, score (%d,%d)\n",row,collect_row->m_State.m_Query,(int)query->m_Responses.size(),best_score,best_response); if(best_response >= 0) { response_pair.request_id=collect_row->m_State.m_Query; @@ -306,6 +315,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) } } } + pRelayManager->UnLock(); } } } @@ -321,6 +331,10 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) shift=mc_PutVarInt(buf,16,requests_to_send.size()); payload.resize(1+shift+sizeof(mc_ChunkEntityKey)*requests_to_send.size()); ptrOut=&(payload[0]); + *ptrOut=MC_RDT_CHUNK_IDS; + ptrOut++; + memcpy(ptrOut,buf,shift); + ptrOut+=shift; count=0; BOOST_FOREACH(PAIRTYPE(const int, int)& chunk_row, item.second.m_Pairs) { @@ -343,6 +357,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) { collect_subrow=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(chunk_row.first); collect_subrow->m_State.m_Request=request_id; + collect_row->m_State.m_RequestTimeStamp=time_now+MC_CCW_TIMEOUT_REQUEST; } } @@ -373,7 +388,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) ptrOut+=shift; for(int r=last_row;rm_MemPool->GetRow(row); + collect_subrow=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(r); if(collect_subrow->m_State.m_Status & MC_CCF_SELECTED) { memcpy(ptrOut,&(collect_subrow->m_ChunkDef),sizeof(mc_ChunkEntityKey)); @@ -383,9 +398,10 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) query_id=pRelayManager->SendRequest(NULL,MC_RMT_CHUNK_QUERY,0,payload); for(int r=last_row;rm_MemPool->GetRow(row); + collect_subrow=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(r); if(collect_subrow->m_State.m_Status & MC_CCF_SELECTED) { + printf("coll new qry: row: %d, id: %lu: att: %d\n",r,query_id,collect_subrow->m_State.m_QueryAttempts); collect_subrow->m_State.m_Status -= MC_CCF_SELECTED; collect_subrow->m_State.m_Query=query_id; collect_subrow->m_State.m_QueryAttempts+=1; @@ -606,6 +622,7 @@ bool mc_RelayProcess_Chunk_Request(unsigned char *ptrStart,unsigned char *ptrEnd } chunk_found=mc_gState->m_TmpBuffers->m_RelayTmpBuffer->GetData(0,&chunk_bytes); + shift=mc_PutVarInt(buf,16,count); payload_response->resize(1+shift+chunk_bytes); ptrOut=&(*payload_response)[0]; @@ -714,6 +731,7 @@ bool MultichainRelayResponse(uint32_t msg_type_stored, CNode *pto_stored, ptr=&vPayloadIn[0]; ptrEnd=ptr+vPayloadIn.size(); +// mc_DumpSize("H",ptr,ptrEnd-ptr,32); strError=""; switch(msg_type_in) { @@ -828,7 +846,7 @@ bool MultichainRelayResponse(uint32_t msg_type_stored, CNode *pto_stored, } break; case MC_RMT_CHUNK_RESPONSE: - if(msg_type_stored != MC_RMT_CHUNK_QUERY_HIT) + if(msg_type_stored != MC_RMT_CHUNK_REQUEST) { strError=strprintf("Unexpected response message type (%d,%d)",msg_type_stored,msg_type_in);; goto exitlbl; diff --git a/src/protocol/relay.h b/src/protocol/relay.h index 30b8dd32..d15da66f 100644 --- a/src/protocol/relay.h +++ b/src/protocol/relay.h @@ -52,6 +52,9 @@ #define MC_RMT_ADD_RESPONSE 0x01000001 #define MC_RMT_ERROR_IN_MESSAGE 0x01000002 #define MC_RMT_NEW_REQUEST 0x01000003 +#define MC_RMT_SPECIAL_MASK 0x10000000 +#define MC_RMT_SPECIAL_COLLECT_CHUNKS 0x10000001 +#define MC_RMT_SPECIAL_VIEW_CHUNKS 0x10000002 #define MC_RDT_UNKNOWN 0 #define MC_RDT_MC_ADDRESS 1 diff --git a/src/rpc/rpccache.cpp b/src/rpc/rpccache.cpp index d00eabce..76cbfd39 100644 --- a/src/rpc/rpccache.cpp +++ b/src/rpc/rpccache.cpp @@ -1,9 +1,12 @@ // Copyright (c) 2014-2017 Coin Sciences Ltd // MultiChain code distributed under the GPLv3 license, see COPYING file. +#include "core/init.h" #include "rpc/rpcutils.h" #include "protocol/relay.h" +#include "wallet/wallettxs.h" #include "net/net.h" +void parseStreamIdentifier(Value stream_identifier,mc_EntityDetails *entity); Value createbinarycache(const Array& params, bool fHelp) @@ -127,7 +130,9 @@ Value offchain(const Array& params, bool fHelp) string request_str=""; mc_RelayRequest *request; mc_RelayResponse *response; + int attempts,delay; Object res; + mc_ChunkCollector collector; bool res_found=false; if(params[0].type() == str_type) @@ -137,6 +142,15 @@ Value offchain(const Array& params, bool fHelp) request_type=MC_RMT_MC_ADDRESS_QUERY; request_str="query for address "; } + if(params[0].get_str() == "getchunks") + { + request_type=MC_RMT_SPECIAL_COLLECT_CHUNKS; + request_str="query for address "; + } + if(params[0].get_str() == "viewchunks") + { + request_type=MC_RMT_SPECIAL_VIEW_CHUNKS; + } } if(request_type == MC_RMT_NONE) @@ -146,34 +160,184 @@ Value offchain(const Array& params, bool fHelp) if(params[1].type() != obj_type) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid request datails, should be object"); + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid request details, should be object"); } - switch(request_type) + if(request_type & MC_RMT_SPECIAL_MASK) { - case MC_RMT_MC_ADDRESS_QUERY: - string addr_to_find=""; - BOOST_FOREACH(const Pair& d, params[1].get_obj()) - { - if(d.name_ == "address") + switch(request_type) + { + case MC_RMT_SPECIAL_COLLECT_CHUNKS: + collector.Initialize(NULL,NULL,0); + attempts=10; + delay=1000; + BOOST_FOREACH(const Pair& d, params[1].get_obj()) { - if(d.value_.type() ==str_type) + if(d.name_ == "attempts") + { + if(d.value_.type() == int_type) + { + attempts=d.value_.get_int(); + } + } + if(d.name_ == "delay") { - addr_to_find=d.value_.get_str(); - request_str+=addr_to_find; + if(d.value_.type() == int_type) + { + delay=d.value_.get_int(); + } } - } - } - - CBitcoinAddress address(addr_to_find); - CKeyID keyID; - if (!address.GetKeyID(keyID)) - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + if(d.name_ == "chunks") + { + if(d.value_.type() == array_type) + { + for(int c=0;c<(int)d.value_.get_array().size();c++) + { + Value cd=d.value_.get_array()[c]; + if(cd.type() != obj_type) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid chunk"); + } + uint256 chunk_hash=0; + int chunk_size=0; + mc_EntityDetails stream_entity; + + mc_TxEntity entity; + entity.Zero(); + BOOST_FOREACH(const Pair& dd, cd.get_obj()) + { + if(dd.name_ == "stream") + { + parseStreamIdentifier(dd.value_.get_str(),&stream_entity); + memcpy(&entity,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,MC_AST_SHORT_TXID_SIZE); + entity.m_EntityType=MC_TET_STREAM; + } + if(dd.name_ == "hash") + { + chunk_hash = ParseHashV(dd.value_.get_str(), "hash"); + } + if(dd.name_ == "size") + { + if(dd.value_.type() != int_type) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid size"); + } + chunk_size=dd.value_.get_int(); + } + } + if(chunk_hash == 0) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Missing hash"); + } + if(chunk_size == 0) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Missing size"); + } + if(entity.m_EntityType == MC_TET_NONE) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Missing stream"); + } + uint256 txid=0; + collector.InsertChunk((unsigned char*)&chunk_hash,&entity,(unsigned char*)&txid,0,chunk_size); + } + } + } + } + + if(collector.m_MemPool->GetCount() == 0) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Missing chunks"); + } - payload.resize(1+sizeof(CKeyID)); - payload[0]=MC_RDT_MC_ADDRESS; - memcpy(&payload[1],&keyID,sizeof(CKeyID)); - break; + break; + case MC_RMT_SPECIAL_VIEW_CHUNKS: + Array arr_res; + BOOST_FOREACH(const Pair& d, params[1].get_obj()) + { + if(d.name_ == "chunks") + { + if(d.value_.type() == array_type) + { + for(int c=0;c<(int)d.value_.get_array().size();c++) + { + Value cd=d.value_.get_array()[c]; + if(cd.type() != obj_type) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid chunk"); + } + uint256 chunk_hash=0; + BOOST_FOREACH(const Pair& dd, cd.get_obj()) + { + if(dd.name_ == "hash") + { + chunk_hash = ParseHashV(dd.value_.get_str(), "hash"); + } + } + if(chunk_hash == 0) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Missing hash"); + } + + unsigned char *chunk_found; + size_t chunk_bytes; + mc_ChunkDBRow chunk_def; + + Object chunk_obj; + chunk_obj.push_back(Pair("hash",chunk_hash.ToString())); + + if(pwalletTxsMain->m_ChunkDB->GetChunkDef(&chunk_def,(unsigned char *)&chunk_hash,NULL,NULL,-1) == MC_ERR_NOERROR) + { + chunk_found=pwalletTxsMain->m_ChunkDB->GetChunk(&chunk_def,0,-1,&chunk_bytes); + if(chunk_found) + { + chunk_obj.push_back(Pair("size",chunk_bytes)); + chunk_obj.push_back(Pair("data",HexStr(chunk_found,chunk_found+chunk_bytes))); + } + else + { + chunk_obj.push_back(Pair("error","Internal error")); + } + } + else + { + chunk_obj.push_back(Pair("error","Chunk not found")); + } + arr_res.push_back(chunk_obj); + } + } + } + } + return arr_res; + } + } + else + { + switch(request_type) + { + case MC_RMT_MC_ADDRESS_QUERY: + string addr_to_find=""; + BOOST_FOREACH(const Pair& d, params[1].get_obj()) + { + if(d.name_ == "address") + { + if(d.value_.type() ==str_type) + { + addr_to_find=d.value_.get_str(); + request_str+=addr_to_find; + } + } + } + + CBitcoinAddress address(addr_to_find); + CKeyID keyID; + if (!address.GetKeyID(keyID)) + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + + payload.resize(1+sizeof(CKeyID)); + payload[0]=MC_RDT_MC_ADDRESS; + memcpy(&payload[1],&keyID,sizeof(CKeyID)); + break; + } } if(params.size() > 2) @@ -192,34 +356,54 @@ Value offchain(const Array& params, bool fHelp) } } + if(request_type & MC_RMT_SPECIAL_MASK) { - LOCK(cs_vNodes); - if(params.size() > 3) + int remaining=0; + for(int a=0;acopyStats(stats); - if( (params[3].get_str() == stats.addrName) || - (params[3].get_str() == CBitcoinAddress(stats.kAddrRemote).ToString()) ) - { - pto=pnode; - } + return 0; } } - - if(pto) - { - request_str="Sending " + request_str + strprintf(" to node %d",pto->GetId()); - } - else + return remaining; + } + else + { { - request_str="Broadcasting " + request_str; + LOCK(cs_vNodes); + if(params.size() > 3) + { + if(params[3].type() != str_type) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid destination"); + } + + BOOST_FOREACH(CNode* pnode, vNodes) + { + CNodeStats stats; + pnode->copyStats(stats); + if( (params[3].get_str() == stats.addrName) || + (params[3].get_str() == CBitcoinAddress(stats.kAddrRemote).ToString()) ) + { + pto=pnode; + } + } + } + + if(pto) + { + request_str="Sending " + request_str + strprintf(" to node %d",pto->GetId()); + } + else + { + request_str="Broadcasting " + request_str; + } } request_id=pRelayManager->SendRequest(pto,request_type,0,payload); diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index 40e4094c..6495725e 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -1325,6 +1325,24 @@ Value DataItemEntry(const CTransaction& tx,int n,set & already_seen,uin } format_item_value=OpReturnFormatEntry(elem,out_size,tx.GetHash(),n,format,&format_text_str,retrieve_status); + if(retrieve_status & MC_OST_CONTROL_NO_DATA) + { + if(format_item_value.type() == obj_type) + { + int chunk_shift; + mc_GetVarInt(chunk_hashes,MC_CDB_CHUNK_HASH_SIZE+16,-1,&chunk_shift); + chunk_hashes+=chunk_shift; + Array chunks; + + for(int chunk=0;chunkToString()); + chunk_hashes+=MC_CDB_CHUNK_HASH_SIZE; + } + format_item_value.get_obj().push_back(Pair("chunks", chunks)); + } + } + already_seen.insert(hash); publishers_set.clear(); diff --git a/src/wallet/chunkcollector.cpp b/src/wallet/chunkcollector.cpp index 312e07a9..4a76b5bc 100644 --- a/src/wallet/chunkcollector.cpp +++ b/src/wallet/chunkcollector.cpp @@ -134,24 +134,27 @@ void mc_ChunkCollector::Dump(const char *message) mc_LogString(fHan,message); - fprintf(fHan,"\nDB\n"); - dbrow.Zero(); - ptr=(unsigned char*)m_DB->Read((char*)&dbrow+m_KeyOffset,m_KeySize,&dbvalue_len,MC_OPT_DB_DATABASE_SEEK_ON_READ,&err); - if(err) + if(m_DB) { - return; - } + fprintf(fHan,"\nDB\n"); + dbrow.Zero(); + ptr=(unsigned char*)m_DB->Read((char*)&dbrow+m_KeyOffset,m_KeySize,&dbvalue_len,MC_OPT_DB_DATABASE_SEEK_ON_READ,&err); + if(err) + { + return; + } - if(ptr) - { - memcpy((char*)&dbrow+m_ValueOffset,ptr,m_ValueDBSize); - while(ptr) + if(ptr) { - mc_MemoryDumpCharSizeToFile(fHan,(char*)&dbrow+m_KeyOffset,0,m_TotalDBSize,56); - ptr=(unsigned char*)m_DB->MoveNext(&err); - if(ptr) + memcpy((char*)&dbrow+m_ValueOffset,ptr,m_ValueDBSize); + while(ptr) { - memcpy((char*)&dbrow+m_KeyOffset,ptr,m_TotalDBSize); + mc_MemoryDumpCharSizeToFile(fHan,(char*)&dbrow+m_KeyOffset,0,m_TotalDBSize,56); + ptr=(unsigned char*)m_DB->MoveNext(&err); + if(ptr) + { + memcpy((char*)&dbrow+m_KeyOffset,ptr,m_TotalDBSize); + } } } } @@ -173,21 +176,24 @@ int mc_ChunkCollector::Initialize(mc_ChunkDB *chunk_db,const char *name,uint32_t mc_ChunkDBRow chunk_def; unsigned char *ptr; - strcpy(m_Name,name); - - m_DB=new mc_Database; - - mc_GetFullFileName(name,"chunks/collect",".db",MC_FOM_RELATIVE_TO_DATADIR | MC_FOM_CREATE_DIR,m_DBName); - - m_DB->SetOption("KeySize",0,m_KeySize); - m_DB->SetOption("ValueSize",0,m_ValueDBSize); - - - err=m_DB->Open(m_DBName,MC_OPT_DB_DATABASE_CREATE_IF_MISSING | MC_OPT_DB_DATABASE_TRANSACTIONAL | MC_OPT_DB_DATABASE_LEVELDB); - - if(err) + if(name) { - return err; + strcpy(m_Name,name); + + m_DB=new mc_Database; + + mc_GetFullFileName(name,"chunks/collect",".db",MC_FOM_RELATIVE_TO_DATADIR | MC_FOM_CREATE_DIR,m_DBName); + + m_DB->SetOption("KeySize",0,m_KeySize); + m_DB->SetOption("ValueSize",0,m_ValueDBSize); + + + err=m_DB->Open(m_DBName,MC_OPT_DB_DATABASE_CREATE_IF_MISSING | MC_OPT_DB_DATABASE_TRANSACTIONAL | MC_OPT_DB_DATABASE_LEVELDB); + + if(err) + { + return err; + } } m_InitMode=mode; @@ -205,50 +211,53 @@ int mc_ChunkCollector::Initialize(mc_ChunkDB *chunk_db,const char *name,uint32_t collect_row.Zero(); - ptr=(unsigned char*)m_DB->Read((char*)&collect_row+m_KeyOffset,m_KeySize,&value_len,MC_OPT_DB_DATABASE_SEEK_ON_READ,&err); - if(err) + if(m_DB) { - return err; - } - - if(ptr) - { - ptr=(unsigned char*)m_DB->MoveNext(&err); - if(ptr) + ptr=(unsigned char*)m_DB->Read((char*)&collect_row+m_KeyOffset,m_KeySize,&value_len,MC_OPT_DB_DATABASE_SEEK_ON_READ,&err); + if(err) { - memcpy((char*)&collect_row,ptr,m_TotalDBSize); + return err; } - while(ptr) - { + + if(ptr) + { ptr=(unsigned char*)m_DB->MoveNext(&err); - if(err) - { - return MC_ERR_CORRUPTED; - } - collect_row.m_State.m_Status |= MC_CCF_INSERTED; - if(m_ChunkDB->GetChunkDef(&chunk_def,collect_row.m_ChunkDef.m_Hash,NULL,NULL,-1) == MC_ERR_NOERROR) - { - collect_row.m_State.m_Status |= MC_CCF_DELETED; - } - m_MemPool->Add(&collect_row); if(ptr) { memcpy((char*)&collect_row,ptr,m_TotalDBSize); } - } - } - else - { - err=m_DB->Write((char*)&collect_row+m_KeyOffset,m_KeySize,(char*)&collect_row+m_ValueOffset,m_ValueSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); - if(err) - { - return err; - } - err=m_DB->Commit(MC_OPT_DB_DATABASE_TRANSACTIONAL); - if(err) + while(ptr) + { + ptr=(unsigned char*)m_DB->MoveNext(&err); + if(err) + { + return MC_ERR_CORRUPTED; + } + collect_row.m_State.m_Status |= MC_CCF_INSERTED; + if(m_ChunkDB->GetChunkDef(&chunk_def,collect_row.m_ChunkDef.m_Hash,NULL,NULL,-1) == MC_ERR_NOERROR) + { + collect_row.m_State.m_Status |= MC_CCF_DELETED; + } + m_MemPool->Add(&collect_row); + if(ptr) + { + memcpy((char*)&collect_row,ptr,m_TotalDBSize); + } + } + } + else { - return err; - } + err=m_DB->Write((char*)&collect_row+m_KeyOffset,m_KeySize,(char*)&collect_row+m_ValueOffset,m_ValueSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + if(err) + { + return err; + } + err=m_DB->Commit(MC_OPT_DB_DATABASE_TRANSACTIONAL); + if(err) + { + return err; + } + } } Dump("Initialize"); @@ -422,6 +431,11 @@ int mc_ChunkCollector::CommitInternal() err=MC_ERR_NOERROR; + if(m_DB == NULL) + { + return MC_ERR_NOT_ALLOWED; + } + if(m_MemPool == m_MemPool1) { m_MemPoolNext=m_MemPool2; From c2e29d7cc981deaf76888a9d7a88f6b893168a27 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 24 Apr 2018 14:46:26 +0300 Subject: [PATCH 039/157] Offchain items, automatic fetching --- src/chainparams/state.h | 1 + src/core/main.cpp | 16 +++++++++++++ src/protocol/handshake.cpp | 10 ++++++++ src/protocol/relay.cpp | 45 +++++++++++++++++++++++------------ src/rpc/rpcserver.cpp | 1 + src/wallet/chunkcollector.cpp | 24 ++++++++++++------- src/wallet/chunkcollector.h | 12 ++++++---- 7 files changed, 81 insertions(+), 28 deletions(-) diff --git a/src/chainparams/state.h b/src/chainparams/state.h index 3bb58d2c..58af4042 100644 --- a/src/chainparams/state.h +++ b/src/chainparams/state.h @@ -34,6 +34,7 @@ #define MC_NPS_INCOMING 0x00000002 #define MC_NPS_MINING 0x00000004 #define MC_NPS_REACCEPT 0x00000008 +#define MC_NPS_OFFCHAIN 0x00000010 #define MC_NPS_ALL 0xFFFFFFFF #define MC_WMD_NONE 0x00000000 diff --git a/src/core/main.cpp b/src/core/main.cpp index 346a3e01..acd3d048 100644 --- a/src/core/main.cpp +++ b/src/core/main.cpp @@ -73,6 +73,7 @@ bool MultichainNode_SendInv(CNode *pnode); bool MultichainNode_AcceptData(CNode *pnode); bool MultichainNode_IgnoreIncoming(CNode *pnode); bool MultichainNode_IsLocal(CNode *pnode); +bool MultichainNode_CollectChunks(); bool IsTxBanned(uint256 txid); int CreateUpgradeLists(int current_height,vector *vParams,vector *vUpgrades); @@ -7295,6 +7296,21 @@ bool SendMessages(CNode* pto, bool fSendTrickle) /* MCHN START */ } pto->fLastIgnoreIncoming=ignore_incoming; + + if(MultichainNode_CollectChunks()) + { + if(pwalletTxsMain->m_ChunkCollector) + { + int64_t time_millis_now=GetTimeMillis(); + + if(pwalletTxsMain->m_ChunkCollector->m_NextTryTimestamp < time_millis_now) + { + MultichainCollectChunks(pwalletTxsMain->m_ChunkCollector); +// if(fDebug)LogPrint("chunks", "Chunks to collect: %d\n", still_to_collect); + pwalletTxsMain->m_ChunkCollector->m_NextTryTimestamp=time_millis_now+100; + } + } + } /* MCHN END */ } return true; diff --git a/src/protocol/handshake.cpp b/src/protocol/handshake.cpp index da62f39d..30f42ab3 100644 --- a/src/protocol/handshake.cpp +++ b/src/protocol/handshake.cpp @@ -111,6 +111,16 @@ bool MultichainNode_IgnoreIncoming(CNode *pnode) return false; } +bool MultichainNode_CollectChunks() +{ + if(mc_gState->m_NodePausedState & MC_NPS_OFFCHAIN) + { + return false; + } + return true; +} + + bool MultichainNode_IsLocal(CNode *pnode) { return (IsLocal(pnode->addr) || pnode->addr.IsRFC1918()) && (pnode->addr.GetPort() == GetListenPort()); diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index d613f000..745a5aa8 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -132,7 +132,10 @@ int MultichainProcessChunkResponse(const CRelayResponsePair *response_pair,map < chunk_err=pwalletTxsMain->m_ChunkDB->AddChunk(chunk->m_Hash,&(chunk->m_Entity),(unsigned char*)collect_row->m_TxID,collect_row->m_Vout,ptrOut,NULL,sizeOut,0,0); if(chunk_err) { - goto exitlbl; + if(chunk_err != MC_ERR_FOUND) + { + goto exitlbl; + } } collect_row->m_State.m_Status |= MC_CCF_DELETED; } @@ -158,7 +161,6 @@ int MultichainResponseScore(mc_RelayResponse *response,mc_ChunkCollectorRow *col int shift,count,size; mc_ChunkEntityKey *chunk; int c; - if( (response->m_Status & MC_RST_SUCCESS) == 0 ) { return MC_CCW_WORST_RESPONSE_SCORE; @@ -231,7 +233,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) for(row=0;rowm_MemPool->GetCount();row++) { collect_row=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(row); - if( collect_row->m_State.m_Status != MC_CCF_DELETED ) + if( (collect_row->m_State.m_Status & MC_CCF_DELETED ) == 0 ) { if(collect_row->m_State.m_RequestTimeStamp <= time_now) { @@ -257,7 +259,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) { response_pair.request_id=collect_row->m_State.m_Request; response_pair.response_id=0; - printf("coll new rsp: row: %d, id: %lu, %d\n",row,collect_row->m_State.m_Request,collect_row->m_State.m_RequestPos); +// printf("coll new rsp: row: %d, id: %lu, %d\n",row,collect_row->m_State.m_Request,collect_row->m_State.m_RequestPos); map::iterator itrsp = responses_to_process.find(response_pair); if (itrsp == responses_to_process.end()) { @@ -282,6 +284,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) { collect_row->m_State.m_Query=0; collect_row->m_State.m_QueryNextAttempt=time_now+MultichainNextChunkQueryAttempt(collect_row->m_State.m_QueryAttempts); + collect_row->m_State.m_Status |= MC_CCF_UPDATED; } } if(query) @@ -297,7 +300,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) best_response=i; } } - printf("coll new req: row: %d, id: %lu, rsps: %d, score (%d,%d)\n",row,collect_row->m_State.m_Query,(int)query->m_Responses.size(),best_score,best_response); +// printf("coll new req: row: %d, id: %lu, rsps: %d, score (%d,%d)\n",row,collect_row->m_State.m_Query,(int)query->m_Responses.size(),best_score,best_response); if(best_response >= 0) { response_pair.request_id=collect_row->m_State.m_Query; @@ -353,6 +356,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) response=&(request->m_Responses[item.first.response_id]); request_id=pRelayManager->SendNextRequest(response,MC_RMT_CHUNK_REQUEST,0,payload); + if(fDebug)LogPrint("chunks","New chunk request %ld, response: %ld, chunks: %d\n",request_id,response->m_Nonce,item.second.m_Pairs.size()); BOOST_FOREACH(PAIRTYPE(const int, int)& chunk_row, item.second.m_Pairs) { collect_subrow=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(chunk_row.first); @@ -396,16 +400,18 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) } } query_id=pRelayManager->SendRequest(NULL,MC_RMT_CHUNK_QUERY,0,payload); + if(fDebug)LogPrint("chunks","New chunk query: %ld, chunks: %d\n",query_id,last_count); for(int r=last_row;rm_MemPool->GetRow(r); if(collect_subrow->m_State.m_Status & MC_CCF_SELECTED) { - printf("coll new qry: row: %d, id: %lu: att: %d\n",r,query_id,collect_subrow->m_State.m_QueryAttempts); +// printf("coll new qry: row: %d, id: %lu: att: %d\n",r,query_id,collect_subrow->m_State.m_QueryAttempts); collect_subrow->m_State.m_Status -= MC_CCF_SELECTED; collect_subrow->m_State.m_Query=query_id; collect_subrow->m_State.m_QueryAttempts+=1; collect_subrow->m_State.m_QueryTimeStamp=time_now+MC_CCW_TIMEOUT_QUERY; + collect_subrow->m_State.m_Status |= MC_CCF_UPDATED; } } last_row=row; @@ -416,7 +422,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) if(collect_row) { - if(collect_row->m_State.m_Status != MC_CCF_DELETED) + if( (collect_row->m_State.m_Status & MC_CCF_DELETED ) == 0 ) { if(collect_row->m_State.m_QueryTimeStamp <= time_now) { @@ -433,7 +439,8 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) query_to_delete.insert(make_pair(collect_row->m_State.m_Query,true)); } collect_row->m_State.m_Query=0; - collect_row->m_State.m_QueryNextAttempt=time_now+MultichainNextChunkQueryAttempt(collect_row->m_State.m_QueryAttempts); + collect_row->m_State.m_QueryNextAttempt=time_now+MultichainNextChunkQueryAttempt(collect_row->m_State.m_QueryAttempts); + collect_row->m_State.m_Status |= MC_CCF_UPDATED; } if(collect_row->m_State.m_QueryNextAttempt <= time_now) { @@ -454,7 +461,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) for(row=0;rowm_MemPool->GetCount();row++) { collect_row=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(row); - if( collect_row->m_State.m_Status != MC_CCF_DELETED ) + if( (collect_row->m_State.m_Status & MC_CCF_DELETED ) == 0 ) { if(collect_row->m_State.m_Query) { @@ -1266,7 +1273,7 @@ void mc_RelayManager::SetRelayRecord(CNode *pto,CNode *pfrom,uint32_t msg_type,u value.m_Count=(it->second).m_Count; it->second=value; } - printf("setrr: %d, ts: %u, nc: %u, mt: %d\n",pto_id,value.m_Timestamp,nonce,msg_type); +// printf("setrr: %d, ts: %u, nc: %u, mt: %d\n",pto_id,value.m_Timestamp,nonce,msg_type); } int mc_RelayManager::GetRelayRecord(CNode *pfrom,uint32_t timestamp,uint32_t nonce,uint32_t* msg_type,CNode **pto) @@ -1278,7 +1285,7 @@ int mc_RelayManager::GetRelayRecord(CNode *pfrom,uint32_t timestamp,uint32_t non { pfrom_id=pfrom->GetId(); } - printf("getrr: %d, ts: %u, nc: %u\n",pfrom_id,timestamp,nonce); +// printf("getrr: %d, ts: %u, nc: %u\n",pfrom_id,timestamp,nonce); const mc_RelayRecordKey key=mc_RelayRecordKey(timestamp,nonce,pfrom_id); map::iterator it = m_RelayRecords.find(key); if (it == m_RelayRecords.end()) @@ -1337,6 +1344,8 @@ uint32_t mc_RelayManager::GenerateNonce() GetRandBytes((unsigned char*)&nonce, sizeof(nonce)); + nonce &= 0x7FFFFFFF; + return nonce; } @@ -1434,7 +1443,9 @@ int64_t mc_RelayManager::PushRelay(CNode* pto, vHops.push_back((int32_t)pfrom->GetId()); } - printf("send: %d, to: %d, from: %d, hc: %d, size: %d, ts: %u, nc: %u\n",msg_type,pto->GetId(),pfrom ? pfrom->GetId() : 0,(int)vHops.size(),(int)payload.size(),timestamp,nonce); +// printf("send: %d, to: %d, from: %d, hc: %d, size: %d, ts: %u, nc: %u\n",msg_type,pto->GetId(),pfrom ? pfrom->GetId() : 0,(int)vHops.size(),(int)payload.size(),timestamp,nonce); + if(fDebug)LogPrint("offchain","Offchain send: %ld, request: %ld, to: %d, from: %d, msg: %d, hops: %d\n", + AggregateNonce(timestamp,nonce),AggregateNonce(timestamp_to_respond,nonce_to_respond),pto->GetId(),pfrom ? pfrom->GetId() : 0,msg_type,(int)vHops.size()); pto->PushMessage("offchain", msg_format, vHops, @@ -1682,7 +1693,9 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, vRecv >> vPayloadIn; vRecv >> vSigScripts; - printf("recv: %d, from: %d, to: %d, hc: %d, size: %d, ts: %u, nc: %u\n",msg_type_in,pfrom->GetId(),pto_stored ? pto_stored->GetId() : 0,hop_count,(int)vPayloadIn.size(),timestamp_received,nonce_received); +// printf("recv: %d, from: %d, to: %d, hc: %d, size: %d, ts: %u, nc: %u\n",msg_type_in,pfrom->GetId(),pto_stored ? pto_stored->GetId() : 0,hop_count,(int)vPayloadIn.size(),timestamp_received,nonce_received); + if(fDebug)LogPrint("offchain","Offchain recv: %ld, request: %ld, from: %d, to: %d, msg: %d, hops: %d\n", + AggregateNonce(timestamp_received,nonce_received),AggregateNonce(timestamp_to_respond,nonce_to_respond),pfrom->GetId(),pto_stored ? pto_stored->GetId() : 0,msg_type_in,hop_count); if( verify_flags & MC_VRA_SIGNATURES ) { @@ -1929,7 +1942,8 @@ int mc_RelayManager::AddRequest(int64_t parent_nonce,int parent_response_id,CNod } } - printf("rqst: %lu, mt: %d, node: %d, size: %d, pr: %lu\n",nonce,msg_type,pto ? pto->GetId() : 0,(int)payload.size(),parent_nonce); +// printf("rqst: %lu, mt: %d, node: %d, size: %d, pr: %lu\n",nonce,msg_type,pto ? pto->GetId() : 0,(int)payload.size(),parent_nonce); + if(fDebug)LogPrint("offchain","Offchain rqst: %ld, to: %d, msg: %d, size: %d\n",nonce,pto ? pto->GetId() : 0,msg_type,(int)payload.size()); m_Requests.insert(make_pair(nonce,request)); } else @@ -1943,7 +1957,8 @@ int mc_RelayManager::AddRequest(int64_t parent_nonce,int parent_response_id,CNod int mc_RelayManager::AddResponse(int64_t request,CNode *pfrom,int32_t source,int hop_count,int64_t nonce,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status) { - printf("resp: %lu, mt: %d, node: %d, size: %d, rn: %lu\n",request,msg_type,pfrom ? pfrom->GetId() : 0,(int)payload.size(),nonce); +// printf("resp: %lu, mt: %d, node: %d, size: %d, rn: %lu\n",request,msg_type,pfrom ? pfrom->GetId() : 0,(int)payload.size(),nonce); + if(fDebug)LogPrint("offchain","Offchain resp: %ld, request: %ld, from: %d, msg: %d, size: %d\n",nonce,request,pfrom ? pfrom->GetId() : 0,msg_type,(int)payload.size()); if(request == 0) { return MC_ERR_NOERROR; diff --git a/src/rpc/rpcserver.cpp b/src/rpc/rpcserver.cpp index 5a18fc28..0a602ff3 100644 --- a/src/rpc/rpcserver.cpp +++ b/src/rpc/rpcserver.cpp @@ -371,6 +371,7 @@ uint32_t GetPausedServices(const char *str) if(memcmp(start,"incoming", ptr-start) == 0)type = MC_NPS_INCOMING; if(memcmp(start,"mining", ptr-start) == 0)type = MC_NPS_MINING; if(memcmp(start,"reaccepting", ptr-start) == 0)type = MC_NPS_REACCEPT; + if(memcmp(start,"offchain", ptr-start) == 0)type = MC_NPS_OFFCHAIN; if(type == 0) { diff --git a/src/wallet/chunkcollector.cpp b/src/wallet/chunkcollector.cpp index 4a76b5bc..3bad978b 100644 --- a/src/wallet/chunkcollector.cpp +++ b/src/wallet/chunkcollector.cpp @@ -25,14 +25,15 @@ void mc_ChunkCollector::Zero() m_DB=NULL; m_ChunkDB=NULL; m_KeyOffset=0; - m_KeySize=MC_CDB_CHUNK_HASH_SIZE+MC_TDB_TXID_SIZE+sizeof(mc_TxEntity)+4; + m_KeySize=MC_TDB_TXID_SIZE+sizeof(int)+sizeof(mc_ChunkEntityKey); m_ValueOffset=m_KeySize; - m_ValueSize=8; - m_ValueDBSize=4; + m_ValueSize=sizeof(mc_ChunkEntityValue); + m_ValueDBSize=28; m_TotalSize=m_KeySize+m_ValueSize; m_TotalDBSize=m_KeySize+m_ValueDBSize; m_Name[0]=0; m_DBName[0]=0; + m_NextTryTimestamp=0; m_MarkPool=NULL; m_MemPool=NULL; @@ -149,7 +150,7 @@ void mc_ChunkCollector::Dump(const char *message) memcpy((char*)&dbrow+m_ValueOffset,ptr,m_ValueDBSize); while(ptr) { - mc_MemoryDumpCharSizeToFile(fHan,(char*)&dbrow+m_KeyOffset,0,m_TotalDBSize,56); + mc_MemoryDumpCharSizeToFile(fHan,(char*)&dbrow+m_KeyOffset,0,m_TotalDBSize,64); ptr=(unsigned char*)m_DB->MoveNext(&err); if(ptr) { @@ -162,7 +163,7 @@ void mc_ChunkCollector::Dump(const char *message) fprintf(fHan,"\nMempool\n"); for(i=0;iGetCount();i++) { - mc_MemoryDumpCharSizeToFile(fHan,m_MemPool->GetRow(i),0,m_TotalSize,56); + mc_MemoryDumpCharSizeToFile(fHan,m_MemPool->GetRow(i),0,m_TotalSize,64); } fprintf(fHan,"\n<<<<<< \tChain height: %6d\t%s\n\n",mc_gState->m_Permissions->m_Block,message); @@ -222,6 +223,7 @@ int mc_ChunkCollector::Initialize(mc_ChunkDB *chunk_db,const char *name,uint32_t if(ptr) { ptr=(unsigned char*)m_DB->MoveNext(&err); + collect_row.Zero(); if(ptr) { memcpy((char*)&collect_row,ptr,m_TotalDBSize); @@ -233,12 +235,13 @@ int mc_ChunkCollector::Initialize(mc_ChunkDB *chunk_db,const char *name,uint32_t { return MC_ERR_CORRUPTED; } - collect_row.m_State.m_Status |= MC_CCF_INSERTED; - if(m_ChunkDB->GetChunkDef(&chunk_def,collect_row.m_ChunkDef.m_Hash,NULL,NULL,-1) == MC_ERR_NOERROR) + collect_row.m_State.m_Status |= MC_CCF_INSERTED; + if(m_ChunkDB->GetChunkDef(&chunk_def,collect_row.m_ChunkDef.m_Hash,&(collect_row.m_ChunkDef.m_Entity),collect_row.m_TxID,collect_row.m_Vout) == MC_ERR_NOERROR) { collect_row.m_State.m_Status |= MC_CCF_DELETED; } m_MemPool->Add(&collect_row); + collect_row.Zero(); if(ptr) { memcpy((char*)&collect_row,ptr,m_TotalDBSize); @@ -436,6 +439,8 @@ int mc_ChunkCollector::CommitInternal() return MC_ERR_NOT_ALLOWED; } + Dump("Before Commit"); + if(m_MemPool == m_MemPool1) { m_MemPoolNext=m_MemPool2; @@ -463,10 +468,12 @@ int mc_ChunkCollector::CommitInternal() } else { - if( (row->m_State.m_Status & MC_CCF_INSERTED) == 0 ) + if( ((row->m_State.m_Status & MC_CCF_INSERTED) == 0 ) || (row->m_State.m_Status & MC_CCF_UPDATED) ) { commit_required=1; + row->m_State.m_Status &= MC_CCF_ERROR_MASK; m_DB->Write((char*)row+m_KeyOffset,m_KeySize,(char*)row+m_ValueOffset,m_ValueDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + row->m_State.m_Status |= MC_CCF_INSERTED; } m_MemPoolNext->Add(row); } @@ -483,7 +490,6 @@ int mc_ChunkCollector::CommitInternal() m_MemPool->Clear(); m_MemPool=m_MemPoolNext; - m_MemPoolNext->Clear(); Dump("Commit"); diff --git a/src/wallet/chunkcollector.h b/src/wallet/chunkcollector.h index 93e1fdbe..b62f621b 100644 --- a/src/wallet/chunkcollector.h +++ b/src/wallet/chunkcollector.h @@ -13,6 +13,7 @@ #define MC_CCF_INSERTED 0x00000001 #define MC_CCF_DELETED 0x00000002 #define MC_CCF_SELECTED 0x00000004 +#define MC_CCF_UPDATED 0x00000008 #define MC_CCF_WRONG_SIZE 0x00010000 #define MC_CCF_ERROR_MASK 0x00FF0000 #define MC_CCF_ALL 0xFFFFFFFF @@ -35,14 +36,17 @@ typedef struct mc_ChunkEntityKey typedef struct mc_ChunkEntityValue { - int64_t m_Query; - uint32_t m_QueryTimeStamp; uint32_t m_QueryAttempts; uint32_t m_QueryNextAttempt; + uint32_t m_Status; + int64_t m_Reserved1; + int64_t m_Reserved2; + int64_t m_Query; + uint32_t m_QueryTimeStamp; + uint32_t m_Reserved3; int64_t m_Request; uint32_t m_RequestTimeStamp; uint32_t m_RequestPos; - uint32_t m_Status; void Zero(); } mc_ChunkEntityValue; @@ -69,7 +73,7 @@ typedef struct mc_ChunkCollector uint32_t m_TotalSize; uint32_t m_ValueDBSize; uint32_t m_TotalDBSize; - + int64_t m_NextTryTimestamp; char m_Name[MC_PRM_NETWORK_NAME_MAX_SIZE+1]; // Chain name char m_DBName[MC_DCT_DB_MAX_PATH]; // Full database name From 02e96c57a984330993c732badbb803765be1dc34 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 30 Apr 2018 18:27:56 +0300 Subject: [PATCH 040/157] Chunk collection and protocol tweaks, not debugged --- src/protocol/relay.cpp | 367 +++++++++++++++------------------- src/protocol/relay.h | 146 +++++++++----- src/rpc/rpccache.cpp | 4 +- src/structs/uint256.cpp | 8 + src/structs/uint256.h | 8 + src/utils/declare.h | 1 + src/utils/utility.cpp | 14 ++ src/wallet/chunkcollector.cpp | 242 ++++++++++++++++++++-- src/wallet/chunkcollector.h | 55 ++++- src/wallet/wallettxs.cpp | 9 +- 10 files changed, 558 insertions(+), 296 deletions(-) diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index 745a5aa8..e5e7aafd 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -20,7 +20,7 @@ uint32_t MultichainNextChunkQueryAttempt(uint32_t attempts) typedef struct CRelayResponsePair { - int64_t request_id; + mc_OffchainMessageID request_id; int response_id; friend bool operator<(const CRelayResponsePair& a, const CRelayResponsePair& b) @@ -56,6 +56,7 @@ int MultichainProcessChunkResponse(const CRelayResponsePair *response_pair,map < int shiftOut,countOut,sizeOut; int chunk_err; mc_ChunkEntityKey *chunk; + mc_ChunkEntityKey *chunkOut; unsigned char *ptrOut; bool result=false; mc_ChunkCollectorRow *collect_row; @@ -84,7 +85,7 @@ int MultichainProcessChunkResponse(const CRelayResponsePair *response_pair,map < } for(int c=0;cm_Size; + total_size+=((mc_ChunkEntityKey*)ptr)->m_Size+size; ptr+=size; } break; @@ -118,6 +119,19 @@ int MultichainProcessChunkResponse(const CRelayResponsePair *response_pair,map < { sizeOut=((mc_ChunkEntityKey*)ptr)->m_Size; chunk=(mc_ChunkEntityKey*)ptr; + chunkOut=(mc_ChunkEntityKey*)ptrOut; + if(chunk->m_Size != chunkOut->m_Size) + { + goto exitlbl; + } + if(memcmp(chunk->m_Hash,chunkOut->m_Hash,sizeof(uint256))) + { + goto exitlbl; + } + if(memcmp(&(chunk->m_Entity),&(chunkOut->m_Entity),sizeof(uint256))) + { + goto exitlbl; + } sizeOut=chunk->m_Size; map ::iterator itreq = request_pairs->find(c); if (itreq != request_pairs->end()) @@ -214,8 +228,8 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) unsigned char buf[16]; int shift,count; unsigned char *ptrOut; - int64_t query_id,request_id; - map query_to_delete; + mc_OffchainMessageID query_id,request_id; + map query_to_delete; map requests_to_send; map responses_to_process; mc_RelayRequest *request; @@ -237,14 +251,14 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) { if(collect_row->m_State.m_RequestTimeStamp <= time_now) { - if(collect_row->m_State.m_Request) + if(!collect_row->m_State.m_Request.IsZero()) { pRelayManager->DeleteRequest(collect_row->m_State.m_Request); collect_row->m_State.m_Request=0; } } request=NULL; - if(collect_row->m_State.m_Request) + if(!collect_row->m_State.m_Request.IsZero()) { request=pRelayManager->FindRequest(collect_row->m_State.m_Request); if(request == NULL) @@ -259,7 +273,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) { response_pair.request_id=collect_row->m_State.m_Request; response_pair.response_id=0; -// printf("coll new rsp: row: %d, id: %lu, %d\n",row,collect_row->m_State.m_Request,collect_row->m_State.m_RequestPos); + printf("coll new rsp: row: %d, id: %s, %d\n",row,collect_row->m_State.m_Request.ToString().c_str(),collect_row->m_State.m_RequestPos); map::iterator itrsp = responses_to_process.find(response_pair); if (itrsp == responses_to_process.end()) { @@ -277,7 +291,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) else { query=NULL; - if(collect_row->m_State.m_Query) + if(!collect_row->m_State.m_Query.IsZero()) { query=pRelayManager->FindRequest(collect_row->m_State.m_Query); if(query == NULL) @@ -300,20 +314,20 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) best_response=i; } } -// printf("coll new req: row: %d, id: %lu, rsps: %d, score (%d,%d)\n",row,collect_row->m_State.m_Query,(int)query->m_Responses.size(),best_score,best_response); if(best_response >= 0) { response_pair.request_id=collect_row->m_State.m_Query; response_pair.response_id=best_response; - map::iterator itrsp = requests_to_send.find(response_pair); + map::iterator itrsp = requests_to_send.find(response_pair); if (itrsp == requests_to_send.end()) - { + { request_pairs.m_Pairs.clear(); request_pairs.m_Pairs.insert(make_pair(row,0)); requests_to_send.insert(make_pair(response_pair,request_pairs)); } else { + printf("coll new req: row: %d, id: %s, rsps: %d, score (%d,%d)\n",row,collect_row->m_State.m_Query.ToString().c_str(),(int)query->m_Responses.size(),best_score,best_response); itrsp->second.m_Pairs.insert(make_pair(row,0)); } } @@ -331,8 +345,8 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) BOOST_FOREACH(PAIRTYPE(const CRelayResponsePair, CRelayRequestPairs)& item, requests_to_send) { payload.clear(); - shift=mc_PutVarInt(buf,16,requests_to_send.size()); - payload.resize(1+shift+sizeof(mc_ChunkEntityKey)*requests_to_send.size()); + shift=mc_PutVarInt(buf,16,item.second.m_Pairs.size()); + payload.resize(1+shift+sizeof(mc_ChunkEntityKey)*item.second.m_Pairs.size()); ptrOut=&(payload[0]); *ptrOut=MC_RDT_CHUNK_IDS; ptrOut++; @@ -340,14 +354,15 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) ptrOut+=shift; count=0; BOOST_FOREACH(PAIRTYPE(const int, int)& chunk_row, item.second.m_Pairs) - { + { collect_subrow=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(chunk_row.first); + printf("S %d\n",chunk_row.first); collect_subrow->m_State.m_RequestPos=count; memcpy(ptrOut,&(collect_subrow->m_ChunkDef),sizeof(mc_ChunkEntityKey)); ptrOut+=sizeof(mc_ChunkEntityKey); count++; } - +// mc_DumpSize("req",&(payload[0]),1+shift+sizeof(mc_ChunkEntityKey)*item.second.m_Pairs.size(),64); request=pRelayManager->FindRequest(item.first.request_id); if(request == NULL) { @@ -356,7 +371,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) response=&(request->m_Responses[item.first.response_id]); request_id=pRelayManager->SendNextRequest(response,MC_RMT_CHUNK_REQUEST,0,payload); - if(fDebug)LogPrint("chunks","New chunk request %ld, response: %ld, chunks: %d\n",request_id,response->m_Nonce,item.second.m_Pairs.size()); + if(fDebug)LogPrint("chunks","New chunk request %s, response: %s, chunks: %d\n",request_id.ToString().c_str(),response->m_MsgID.ToString().c_str(),item.second.m_Pairs.size()); BOOST_FOREACH(PAIRTYPE(const int, int)& chunk_row, item.second.m_Pairs) { collect_subrow=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(chunk_row.first); @@ -400,7 +415,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) } } query_id=pRelayManager->SendRequest(NULL,MC_RMT_CHUNK_QUERY,0,payload); - if(fDebug)LogPrint("chunks","New chunk query: %ld, chunks: %d\n",query_id,last_count); + if(fDebug)LogPrint("chunks","New chunk query: %s, chunks: %d\n",query_id.ToString().c_str(),last_count); for(int r=last_row;rm_MemPool->GetRow(r); @@ -426,14 +441,14 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) { if(collect_row->m_State.m_QueryTimeStamp <= time_now) { - if(collect_row->m_State.m_Request) + if(!collect_row->m_State.m_Request.IsZero()) { pRelayManager->DeleteRequest(collect_row->m_State.m_Request); collect_row->m_State.m_Request=0; } - if(collect_row->m_State.m_Query) + if(!collect_row->m_State.m_Query.IsZero()) { - map::iterator itqry = query_to_delete.find(collect_row->m_State.m_Query); + map::iterator itqry = query_to_delete.find(collect_row->m_State.m_Query); if (itqry == query_to_delete.end()) { query_to_delete.insert(make_pair(collect_row->m_State.m_Query,true)); @@ -463,9 +478,9 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) collect_row=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(row); if( (collect_row->m_State.m_Status & MC_CCF_DELETED ) == 0 ) { - if(collect_row->m_State.m_Query) + if(!collect_row->m_State.m_Query.IsZero()) { - map::iterator itqry = query_to_delete.find(collect_row->m_State.m_Query); + map::iterator itqry = query_to_delete.find(collect_row->m_State.m_Query); if (itqry != query_to_delete.end()) { itqry->second=false; @@ -475,7 +490,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) } } - BOOST_FOREACH(PAIRTYPE(const int64_t, bool)& item, query_to_delete) + BOOST_FOREACH(PAIRTYPE(const mc_OffchainMessageID, bool)& item, query_to_delete) { if(item.second) { @@ -544,11 +559,13 @@ bool mc_RelayProcess_Chunk_Query(unsigned char *ptrStart,unsigned char *ptrEnd,v for(int c=0;cm_ChunkDB->GetChunkDef(&chunk_def,chunk.m_Hash,NULL,NULL,-1) == MC_ERR_NOERROR) { if(chunk_def.m_Size != chunk.m_Size) { chunk.m_Flags |= MC_CCF_WRONG_SIZE; + chunk.m_Size=chunk_def.m_Size; } vToRespond.push_back(chunk); } @@ -605,6 +622,7 @@ bool mc_RelayProcess_Chunk_Request(unsigned char *ptrStart,unsigned char *ptrEnd for(int c=0;cm_ChunkDB->GetChunkDef(&chunk_def,chunk.m_Hash,NULL,NULL,-1) == MC_ERR_NOERROR) { if(chunk_def.m_Size != chunk.m_Size) @@ -618,6 +636,7 @@ bool mc_RelayProcess_Chunk_Request(unsigned char *ptrStart,unsigned char *ptrEnd return false; } chunk_found=pwalletTxsMain->m_ChunkDB->GetChunk(&chunk_def,0,-1,&chunk_bytes); + mc_gState->m_TmpBuffers->m_RelayTmpBuffer->SetData((unsigned char*)&chunk,size); mc_gState->m_TmpBuffers->m_RelayTmpBuffer->SetData(chunk_found,chunk_bytes); } else @@ -1126,20 +1145,6 @@ void mc_RelayManager::MsgTypeSettings(uint32_t msg_type,int latency,int seconds, } } -int64_t mc_RelayManager::AggregateNonce(uint32_t timestamp,uint32_t nonce) -{ - return ((int64_t)nonce<<32)+(int64_t)timestamp; -} - -uint32_t mc_RelayManager::Timestamp(int64_t aggr_nonce) -{ - return aggr_nonce & 0xFFFFFFFF; -} - -uint32_t mc_RelayManager::Nonce(int64_t aggr_nonce) -{ - return (aggr_nonce >> 32) & 0xFFFFFFFF; -} void mc_RelayManager::Zero() @@ -1198,7 +1203,6 @@ void mc_RelayManager::SetDefaults() MsgTypeSettings(MC_RMT_NONE , 0,10,1000,100*1024*1024); MsgTypeSettings(MC_RMT_MC_ADDRESS_QUERY,10,10, 100, 1*1024*1024); MsgTypeSettings(MC_RMT_NODE_DETAILS , 0,10, 100, 1*1024*1024); - MsgTypeSettings(MC_RMT_REJECT , 0,10,1000, 1*1024*1024); MsgTypeSettings(MC_RMT_CHUNK_QUERY ,10,10, 100, 1*1024*1024); MsgTypeSettings(MC_RMT_CHUNK_QUERY_HIT ,30,10, 100, 1*1024*1024); MsgTypeSettings(MC_RMT_CHUNK_REQUEST ,30,10, 100, 1*1024*1024); @@ -1207,8 +1211,8 @@ void mc_RelayManager::SetDefaults() MsgTypeSettings(MC_RMT_NEW_REQUEST ,30,10,1000, 1*1024*1024); - m_MinTimeShift=180; - m_MaxTimeShift=180; + m_MinTimeShift=2 * 6 * Params().TargetSpacing(); + m_MaxTimeShift=2 * 6 * Params().TargetSpacing(); m_MaxResponses=16; } @@ -1234,7 +1238,7 @@ void mc_RelayManager::CheckTime() m_LastTime=time_now; } -void mc_RelayManager::SetRelayRecord(CNode *pto,CNode *pfrom,uint32_t msg_type,uint32_t timestamp,uint32_t nonce) +void mc_RelayManager::SetRelayRecord(CNode *pto,CNode *pfrom,uint32_t msg_type,mc_OffchainMessageID msg_id) { map::iterator itlat = m_Latency.find(msg_type); if (itlat == m_Latency.end()) @@ -1251,7 +1255,7 @@ void mc_RelayManager::SetRelayRecord(CNode *pto,CNode *pfrom,uint32_t msg_type,u { pto_id=pto->GetId(); } - const mc_RelayRecordKey key=mc_RelayRecordKey(timestamp,nonce,pto_id); + const mc_RelayRecordKey key=mc_RelayRecordKey(msg_id,pto_id); mc_RelayRecordValue value; value.m_NodeFrom=0; if(pfrom) @@ -1276,7 +1280,7 @@ void mc_RelayManager::SetRelayRecord(CNode *pto,CNode *pfrom,uint32_t msg_type,u // printf("setrr: %d, ts: %u, nc: %u, mt: %d\n",pto_id,value.m_Timestamp,nonce,msg_type); } -int mc_RelayManager::GetRelayRecord(CNode *pfrom,uint32_t timestamp,uint32_t nonce,uint32_t* msg_type,CNode **pto) +int mc_RelayManager::GetRelayRecord(CNode *pfrom,mc_OffchainMessageID msg_id,uint32_t* msg_type,CNode **pto) { NodeId pfrom_id,pto_id; @@ -1286,7 +1290,7 @@ int mc_RelayManager::GetRelayRecord(CNode *pfrom,uint32_t timestamp,uint32_t non pfrom_id=pfrom->GetId(); } // printf("getrr: %d, ts: %u, nc: %u\n",pfrom_id,timestamp,nonce); - const mc_RelayRecordKey key=mc_RelayRecordKey(timestamp,nonce,pfrom_id); + const mc_RelayRecordKey key=mc_RelayRecordKey(msg_id,pfrom_id); map::iterator it = m_RelayRecords.find(key); if (it == m_RelayRecords.end()) { @@ -1338,6 +1342,19 @@ int mc_RelayManager::GetRelayRecord(CNode *pfrom,uint32_t timestamp,uint32_t non return MC_ERR_NOT_ALLOWED; } +mc_OffchainMessageID mc_RelayManager::GenerateMsgID(uint32_t timestamp) +{ + mc_OffchainMessageID msg_id; + msg_id.m_TimeStamp=timestamp; + GetRandBytes((unsigned char*)&(msg_id.m_Nonce), sizeof(msg_id.m_Nonce)); + return msg_id; +} + +mc_OffchainMessageID mc_RelayManager::GenerateMsgID() +{ + return GenerateMsgID(mc_TimeNowAsUInt()); +} + uint32_t mc_RelayManager::GenerateNonce() { uint32_t nonce; @@ -1349,14 +1366,13 @@ uint32_t mc_RelayManager::GenerateNonce() return nonce; } -int64_t mc_RelayManager::PushRelay(CNode* pto, +mc_OffchainMessageID mc_RelayManager::PushRelay(CNode* pto, uint32_t msg_format, vector &vHops, + vector &vSendPaths, uint32_t msg_type, - uint32_t timestamp_to_send, - uint32_t nonce_to_send, - uint32_t timestamp_to_respond, - uint32_t nonce_to_respond, + mc_OffchainMessageID msg_id, + mc_OffchainMessageID msg_id_to_respond, uint32_t flags, vector& payload, vector& sigScripts_to_relay, @@ -1368,32 +1384,15 @@ int64_t mc_RelayManager::PushRelay(CNode* pto, vector sigScripts; CScript sigScript; uint256 message_hash; - uint32_t timestamp; - uint32_t nonce; - - nonce=nonce_to_send; - timestamp=timestamp_to_send; - - if(action & MC_PRA_GENERATE_TIMESTAMP) - { - timestamp=mc_TimeNowAsUInt(); - } - - if(action & MC_PRA_GENERATE_NONCE) - { - nonce=GenerateNonce(); - } - - int64_t aggr_nonce=AggregateNonce(timestamp,nonce); if( (action & MC_PRA_SIGN_WITH_HANDSHAKE_ADDRESS) && (MCP_ANYONE_CAN_CONNECT == 0) ) { CHashWriter ssHash(SER_GETHASH, 0); ssHash << msg_type; - ssHash << timestamp; - ssHash << nonce; - ssHash << timestamp_to_respond; - ssHash << nonce_to_respond; + ssHash << msg_id.m_TimeStamp; + ssHash << msg_id.m_Nonce; + ssHash << msg_id_to_respond.m_TimeStamp; + ssHash << msg_id_to_respond.m_Nonce; ssHash << flags; ssHash << payload; @@ -1402,7 +1401,7 @@ int64_t mc_RelayManager::PushRelay(CNode* pto, CHashWriter ssSig(SER_GETHASH, 0); ssSig << message_hash; - ssSig << vector((unsigned char*)&aggr_nonce, (unsigned char*)&aggr_nonce+sizeof(aggr_nonce)); + ssSig << vector((unsigned char*)&msg_id_to_respond, (unsigned char*)&msg_id_to_respond+sizeof(msg_id_to_respond)); uint256 signed_hash=ssSig.GetHash(); CKey key; CKeyID keyID; @@ -1444,24 +1443,25 @@ int64_t mc_RelayManager::PushRelay(CNode* pto, } // printf("send: %d, to: %d, from: %d, hc: %d, size: %d, ts: %u, nc: %u\n",msg_type,pto->GetId(),pfrom ? pfrom->GetId() : 0,(int)vHops.size(),(int)payload.size(),timestamp,nonce); - if(fDebug)LogPrint("offchain","Offchain send: %ld, request: %ld, to: %d, from: %d, msg: %d, hops: %d\n", - AggregateNonce(timestamp,nonce),AggregateNonce(timestamp_to_respond,nonce_to_respond),pto->GetId(),pfrom ? pfrom->GetId() : 0,msg_type,(int)vHops.size()); + if(fDebug)LogPrint("offchain","Offchain send: %s, request: %s, to: %d, from: %d, msg: %d, hops: %d, size: %d\n", + msg_id.ToString().c_str(),msg_id_to_respond.ToString().c_str(),pto->GetId(),pfrom ? pfrom->GetId() : 0,msg_type,(int)vHops.size(),(int)payload.size()); pto->PushMessage("offchain", msg_format, vHops, + vSendPaths, msg_type, - timestamp, - nonce, - timestamp_to_respond, - nonce_to_respond, + msg_id.m_TimeStamp, + msg_id.m_Nonce, + msg_id_to_respond.m_TimeStamp, + msg_id_to_respond.m_Nonce, flags, payload, sigScripts); // SetRelayRecord(pto,NULL,msg_type,timestamp,nonce); - SetRelayRecord(pto,pfrom,msg_type,timestamp,nonce); + SetRelayRecord(pto,pfrom,msg_type,msg_id); - return aggr_nonce; + return msg_id; } bool mc_RelayManager::ProcessRelay( CNode* pfrom, @@ -1471,14 +1471,13 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, { uint32_t msg_type_in; uint32_t verify_flags; - uint32_t timestamp_received; - uint32_t nonce_received; - uint32_t timestamp_to_respond; - uint32_t nonce_to_respond; + mc_OffchainMessageID msg_id_received; + mc_OffchainMessageID msg_id_to_respond; vector vPayloadIn; vector vSigScripts; vector vSigScriptsEmpty; vector vHops; + vector vSendPaths; vector vEmptyHops; uint256 message_hash; uint32_t flags_in,flags_response,flags_relay; @@ -1515,6 +1514,12 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, vRecv >> vHops; hop_count=(int)vHops.size(); + vRecv >> vSendPaths; + if(vSendPaths.size()) + { + LogPrintf("ProcessRelay() : Unsupported send path\n"); + return false; + } vRecv >> msg_type_in; switch(msg_type_in) { @@ -1524,9 +1529,6 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, case MC_RMT_NODE_DETAILS: verify_flags |= MC_VRA_IS_RESPONSE | MC_VRA_SIGNATURE_ORIGIN; break; - case MC_RMT_REJECT: - verify_flags |= MC_VRA_IS_RESPONSE | MC_VRA_SIGNATURE_ORIGIN; - break; case MC_RMT_CHUNK_QUERY: verify_flags |= MC_VRA_IS_NOT_RESPONSE; break; @@ -1568,23 +1570,23 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, CheckTime(); - vRecv >> timestamp_received; + vRecv >> msg_id_received.m_TimeStamp; if(verify_flags & MC_VRA_TIMESTAMP) { - if(timestamp_received+m_MinTimeShift < m_LastTime) + if(msg_id_received.m_TimeStamp+m_MinTimeShift < m_LastTime) { - LogPrintf("ProcessRelay() : Timestamp too far in the past: %d\n",timestamp_received); + LogPrintf("ProcessRelay() : Timestamp too far in the past: %d\n",msg_id_received.m_TimeStamp); return false; } - if(timestamp_received > m_LastTime + m_MaxTimeShift) + if(msg_id_received.m_TimeStamp > m_LastTime + m_MaxTimeShift) { - LogPrintf("ProcessRelay() : Timestamp too far in the future: %d\n",timestamp_received); + LogPrintf("ProcessRelay() : Timestamp too far in the future: %d\n",msg_id_received.m_TimeStamp); return false; } } - vRecv >> nonce_received; + vRecv >> msg_id_received.m_Nonce; msg_type_relay_ptr=&msg_type_relay; @@ -1592,7 +1594,7 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, if( verify_flags & (MC_VRA_PROCESS_ONCE | MC_VRA_BROADCAST_ONCE)) { - switch(GetRelayRecord(NULL,timestamp_received,nonce_received,NULL,NULL)) + switch(GetRelayRecord(NULL,msg_id_received,NULL,NULL)) { case MC_ERR_NOERROR: if(verify_flags & MC_VRA_PROCESS_ONCE) @@ -1621,36 +1623,34 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, } - vRecv >> timestamp_to_respond; - vRecv >> nonce_to_respond; + vRecv >> msg_id_to_respond.m_TimeStamp; + vRecv >> msg_id_to_respond.m_Nonce; if( verify_flags & MC_VRA_IS_NOT_RESPONSE ) { - if( (timestamp_to_respond != 0) || (nonce_to_respond != 0) ) + if( (msg_id_to_respond.m_TimeStamp != 0) || (msg_id_to_respond.m_Nonce != 0) ) { return state.DoS(100, error("ProcessRelay() : This message should not be response"),REJECT_INVALID, "bad-nonce"); } } - - int64_t aggr_nonce=AggregateNonce(timestamp_to_respond,nonce_to_respond); - + if( verify_flags & MC_VRA_IS_RESPONSE ) { - if( timestamp_to_respond == 0 ) + if( msg_id_to_respond.m_TimeStamp == 0 ) { return state.DoS(100, error("ProcessRelay() : This message should be response"),REJECT_INVALID, "bad-nonce"); } - if(GetRelayRecord(pfrom,timestamp_to_respond,nonce_to_respond,&msg_type_stored,&pto_stored)) + if(GetRelayRecord(pfrom,msg_id_to_respond,&msg_type_stored,&pto_stored)) { LogPrintf("ProcessRelay() : Response without request from peer %d\n",pfrom->GetId()); return false; } } - SetRelayRecord(NULL,pfrom,MC_RMT_ERROR_IN_MESSAGE,timestamp_received,nonce_received); + SetRelayRecord(NULL,pfrom,MC_RMT_ERROR_IN_MESSAGE,msg_id_received); - if(timestamp_to_respond) + if(msg_id_to_respond.m_TimeStamp) { if(pto_stored) { @@ -1694,17 +1694,17 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, vRecv >> vSigScripts; // printf("recv: %d, from: %d, to: %d, hc: %d, size: %d, ts: %u, nc: %u\n",msg_type_in,pfrom->GetId(),pto_stored ? pto_stored->GetId() : 0,hop_count,(int)vPayloadIn.size(),timestamp_received,nonce_received); - if(fDebug)LogPrint("offchain","Offchain recv: %ld, request: %ld, from: %d, to: %d, msg: %d, hops: %d\n", - AggregateNonce(timestamp_received,nonce_received),AggregateNonce(timestamp_to_respond,nonce_to_respond),pfrom->GetId(),pto_stored ? pto_stored->GetId() : 0,msg_type_in,hop_count); + if(fDebug)LogPrint("offchain","Offchain recv: %s, request: %s, from: %d, to: %d, msg: %d, hops: %d, size: %d\n", + msg_id_received.ToString().c_str(),msg_id_to_respond.ToString().c_str(),pfrom->GetId(),pto_stored ? pto_stored->GetId() : 0,msg_type_in,hop_count,(int)vPayloadIn.size()); if( verify_flags & MC_VRA_SIGNATURES ) { CHashWriter ssHash(SER_GETHASH, 0); ssHash << msg_type_in; - ssHash << timestamp_received; - ssHash << nonce_received; - ssHash << timestamp_to_respond; - ssHash << nonce_to_respond; + ssHash << msg_id_received.m_TimeStamp; + ssHash << msg_id_received.m_Nonce; + ssHash << msg_id_to_respond.m_TimeStamp; + ssHash << msg_id_to_respond.m_Nonce; ssHash << flags_in; ssHash << vPayloadIn; @@ -1746,7 +1746,7 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, CHashWriter ss(SER_GETHASH, 0); ss << vector((unsigned char*)&message_hash, (unsigned char*)&message_hash+32); - ss << vector((unsigned char*)&aggr_nonce, (unsigned char*)&aggr_nonce+sizeof(aggr_nonce)); + ss << vector((unsigned char*)&msg_id_to_respond, (unsigned char*)&msg_id_to_respond+sizeof(msg_id_to_respond)); uint256 signed_hash=ss.GetHash(); if(!pubkey.Verify(signed_hash,vchSigOut)) @@ -1773,7 +1773,7 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, if(pto_stored) { - PushRelay(pto_stored,msg_format,vHops,msg_type_in,timestamp_received,nonce_received,timestamp_to_respond,nonce_to_respond,flags_in, + PushRelay(pto_stored,msg_format,vHops,vSendPaths,msg_type_in,msg_id_received,msg_id_to_respond,flags_in, vPayloadIn,vSigScripts,pfrom,MC_PRA_NONE); } else @@ -1789,16 +1789,16 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, { if(*msg_type_response_ptr != MC_RMT_ADD_RESPONSE) { - PushRelay(pfrom,0,vEmptyHops,*msg_type_response_ptr,m_LastTime,0,timestamp_received,nonce_received,flags_response, - vPayloadResponse,vSigScriptsEmpty,NULL,MC_PRA_GENERATE_NONCE); + PushRelay(pfrom,0,vEmptyHops,vSendPaths,*msg_type_response_ptr,GenerateMsgID(m_LastTime),msg_id_received,flags_response, + vPayloadResponse,vSigScriptsEmpty,NULL,MC_PRA_NONE); } else { - map::iterator itreq = m_Requests.find(AggregateNonce(timestamp_to_respond,nonce_to_respond)); + map::iterator itreq = m_Requests.find(msg_id_to_respond); if(itreq != m_Requests.end()) { Lock(); - AddResponse(itreq->second.m_Nonce,pfrom,vHops.size() ? vHops[0] : 0,hop_count,AggregateNonce(timestamp_received,nonce_received),msg_type_in,flags_in,vPayloadIn,MC_RST_SUCCESS); + AddResponse(itreq->second.m_MsgID,pfrom,vHops.size() ? vHops[0] : 0,hop_count,msg_id_received,msg_type_in,flags_in,vPayloadIn,MC_RST_SUCCESS); UnLock(); } else @@ -1815,7 +1815,7 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, { if(pnode != pfrom) { - PushRelay(pnode,msg_format,vHops,*msg_type_relay_ptr,timestamp_received,nonce_received,timestamp_to_respond,nonce_to_respond,flags_relay, + PushRelay(pnode,msg_format,vHops,vSendPaths,*msg_type_relay_ptr,msg_id_received,msg_id_to_respond,flags_relay, vPayloadRelay,vSigScriptsEmpty,pfrom,MC_PRA_NONE); } } @@ -1829,54 +1829,21 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, } } - if(timestamp_to_respond) + if(msg_id_to_respond.m_TimeStamp) { - SetRelayRecord(NULL,pfrom,msg_type_in,timestamp_received,nonce_received); + SetRelayRecord(NULL,pfrom,msg_type_in,msg_id_received); } else { - SetRelayRecord(NULL,pfrom,MC_RMT_NEW_REQUEST,timestamp_received,nonce_received); + SetRelayRecord(NULL,pfrom,MC_RMT_NEW_REQUEST,msg_id_received); } return true; } -/* -typedef struct mc_RelayResponse -{ - int64_t m_Nonce; - uint32_t m_MsgType; - uint32_t m_Flags; - CNode *m_NodeFrom; - int m_HopCount; - mc_NodeFullAddress m_Source; - uint32_t m_LastTryTimestamp; - vector m_Payload; - vector m_Requests; - - void Zero(); -} mc_RelayResponse; - -typedef struct mc_RelayRequest -{ - int64_t m_Nonce; - uint32_t m_MsgType; - uint32_t m_Flags; - int64_t m_ParentNonce; - int m_ParentResponseID; - uint32_t m_LastTryTimestamp; - int m_TryCount; - uint32_t m_Status; - vector m_Payload; - vector m_Responses; - - void Zero(); -} mc_RelayRequest; -*/ - void mc_RelayResponse::Zero() { - m_Nonce=0; + m_MsgID=0; m_MsgType=MC_RMT_NONE; m_Flags=0; m_NodeFrom=0; @@ -1891,12 +1858,10 @@ void mc_RelayResponse::Zero() void mc_RelayRequest::Zero() { - m_Nonce=0; + m_MsgID=0; m_MsgType=MC_RMT_NONE; m_Flags=0; m_NodeTo=0; - m_ParentNonce=0; - m_ParentResponseID=-1; m_LastTryTimestamp=0; m_TryCount=0; m_Status=MC_RST_NONE; @@ -1904,47 +1869,28 @@ void mc_RelayRequest::Zero() m_Responses.clear(); } -int mc_RelayManager::AddRequest(int64_t parent_nonce,int parent_response_id,CNode *pto,int64_t nonce,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status) +int mc_RelayManager::AddRequest(CNode *pto,mc_OffchainMessageID msg_id,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status) { int err=MC_ERR_NOERROR; Lock(); - map::iterator itreq_this = m_Requests.find(nonce); + map::iterator itreq_this = m_Requests.find(msg_id); if(itreq_this == m_Requests.end()) { mc_RelayRequest request; - request.m_Nonce=nonce; + request.m_MsgID=msg_id; request.m_MsgType=msg_type; request.m_Flags=flags; request.m_NodeTo=pto ? pto->GetId() : 0; - request.m_ParentNonce=parent_nonce; - request.m_ParentResponseID=parent_response_id; request.m_LastTryTimestamp=0; request.m_TryCount=0; request.m_Status=status; request.m_Payload=payload; request.m_Responses.clear(); - if(parent_nonce) - { - map::iterator itreq = m_Requests.find(parent_nonce); - if(itreq != m_Requests.end()) - { - if(parent_response_id < (int)(itreq->second.m_Responses.size())) - { - itreq->second.m_Responses[parent_response_id].m_Requests.push_back(request); - } - } - else - { - err=MC_ERR_NOT_FOUND; - } - } - -// printf("rqst: %lu, mt: %d, node: %d, size: %d, pr: %lu\n",nonce,msg_type,pto ? pto->GetId() : 0,(int)payload.size(),parent_nonce); - if(fDebug)LogPrint("offchain","Offchain rqst: %ld, to: %d, msg: %d, size: %d\n",nonce,pto ? pto->GetId() : 0,msg_type,(int)payload.size()); - m_Requests.insert(make_pair(nonce,request)); + if(fDebug)LogPrint("offchain","Offchain rqst: %s, to: %d, msg: %d, size: %d\n",msg_id.ToString().c_str(),pto ? pto->GetId() : 0,msg_type,(int)payload.size()); + m_Requests.insert(make_pair(msg_id,request)); } else { @@ -1955,17 +1901,18 @@ int mc_RelayManager::AddRequest(int64_t parent_nonce,int parent_response_id,CNod return err; } -int mc_RelayManager::AddResponse(int64_t request,CNode *pfrom,int32_t source,int hop_count,int64_t nonce,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status) +int mc_RelayManager::AddResponse(mc_OffchainMessageID request,CNode *pfrom,int32_t source,int hop_count,mc_OffchainMessageID msg_id,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status) { // printf("resp: %lu, mt: %d, node: %d, size: %d, rn: %lu\n",request,msg_type,pfrom ? pfrom->GetId() : 0,(int)payload.size(),nonce); - if(fDebug)LogPrint("offchain","Offchain resp: %ld, request: %ld, from: %d, msg: %d, size: %d\n",nonce,request,pfrom ? pfrom->GetId() : 0,msg_type,(int)payload.size()); - if(request == 0) + if(fDebug)LogPrint("offchain","Offchain resp: %s, request: %s, from: %d, msg: %d, size: %d\n", + msg_id.ToString().c_str(),request.ToString().c_str(),pfrom ? pfrom->GetId() : 0,msg_type,(int)payload.size()); + if(request.IsZero()) { return MC_ERR_NOERROR; } mc_RelayResponse response; - response.m_Nonce=nonce; + response.m_MsgID=msg_id; response.m_MsgType=msg_type; response.m_Flags=flags; response.m_NodeFrom=pfrom ? pfrom->GetId() : 0; @@ -1976,9 +1923,9 @@ int mc_RelayManager::AddResponse(int64_t request,CNode *pfrom,int32_t source,int response.m_Payload=payload; response.m_Requests.clear(); - if(request) + if(!request.IsZero()) { - map::iterator itreq = m_Requests.find(request); + map::iterator itreq = m_Requests.find(request); if(itreq != m_Requests.end()) { itreq->second.m_Responses.push_back(response); @@ -1996,12 +1943,12 @@ int mc_RelayManager::AddResponse(int64_t request,CNode *pfrom,int32_t source,int return MC_ERR_NOERROR; } -int mc_RelayManager::DeleteRequest(int64_t request) +int mc_RelayManager::DeleteRequest(mc_OffchainMessageID request) { int err=MC_ERR_NOERROR; Lock(); - map::iterator itreq = m_Requests.find(request); + map::iterator itreq = m_Requests.find(request); if(itreq != m_Requests.end()) { m_Requests.erase(itreq); @@ -2015,10 +1962,10 @@ int mc_RelayManager::DeleteRequest(int64_t request) return err; } -mc_RelayRequest *mc_RelayManager::FindRequest(int64_t request) +mc_RelayRequest *mc_RelayManager::FindRequest(mc_OffchainMessageID request) { Lock(); - map::iterator itreq = m_Requests.find(request); + map::iterator itreq = m_Requests.find(request); if(itreq != m_Requests.end()) { return &(itreq->second); @@ -2029,58 +1976,60 @@ mc_RelayRequest *mc_RelayManager::FindRequest(int64_t request) } -int64_t mc_RelayManager::SendRequest(CNode* pto,uint32_t msg_type,uint32_t flags,vector & payload) +mc_OffchainMessageID mc_RelayManager::SendRequest(CNode* pto,uint32_t msg_type,uint32_t flags,vector & payload) { - uint32_t timestamp=mc_TimeNowAsUInt(); - uint32_t nonce=GenerateNonce(); - int64_t aggr_nonce=AggregateNonce(timestamp,nonce); + mc_OffchainMessageID msg_id; vector vEmptyHops; - + vector vEmptySendPaths; vector vSigScriptsEmpty; + msg_id=GenerateMsgID(); + { LOCK(cs_vNodes); BOOST_FOREACH(CNode* pnode, vNodes) { if( (pto == NULL) || (pnode == pto) ) { - PushRelay(pnode,0,vEmptyHops,msg_type,timestamp,nonce,0,0,flags,payload,vSigScriptsEmpty,NULL,0); + PushRelay(pnode,0,vEmptyHops,vEmptySendPaths,msg_type,msg_id,mc_OffchainMessageID(),flags,payload,vSigScriptsEmpty,NULL,0); } } } - if(AddRequest(0,0,pto,aggr_nonce,msg_type,flags,payload,MC_RST_NONE) != MC_ERR_NOERROR) + if(AddRequest(pto,msg_id,msg_type,flags,payload,MC_RST_NONE) != MC_ERR_NOERROR) { - return 0; + return mc_OffchainMessageID(); } - return aggr_nonce; + return msg_id; } -int64_t mc_RelayManager::SendNextRequest(mc_RelayResponse* response,uint32_t msg_type,uint32_t flags,vector & payload) +mc_OffchainMessageID mc_RelayManager::SendNextRequest(mc_RelayResponse* response,uint32_t msg_type,uint32_t flags,vector & payload) { - int64_t aggr_nonce; + mc_OffchainMessageID msg_id; vector vEmptyHops; + vector vEmptySendPaths; vector vSigScriptsEmpty; + + msg_id=GenerateMsgID(); - aggr_nonce=0; { LOCK(cs_vNodes); BOOST_FOREACH(CNode* pnode, vNodes) { if( pnode->GetId() == response->m_NodeFrom ) { - aggr_nonce=PushRelay(pnode,0,vEmptyHops,msg_type,0,0,Timestamp(response->m_Nonce),Nonce(response->m_Nonce), - flags,payload,vSigScriptsEmpty,NULL,MC_PRA_GENERATE_TIMESTAMP | MC_PRA_GENERATE_NONCE); - if(AddRequest(0,0,pnode,aggr_nonce,msg_type,flags,payload,MC_RST_NONE) != MC_ERR_NOERROR) - { - return 0; + msg_id=PushRelay(pnode,0,vEmptyHops,vEmptySendPaths,msg_type,msg_id,response->m_MsgID, + flags,payload,vSigScriptsEmpty,NULL,MC_PRA_NONE); + if(AddRequest(pnode,msg_id,msg_type,flags,payload,MC_RST_NONE) != MC_ERR_NOERROR) + { + return mc_OffchainMessageID(); } } } } - return aggr_nonce; + return msg_id; } @@ -2089,7 +2038,7 @@ void mc_RelayManager::InvalidateResponsesFromDisconnected() LOCK(cs_vNodes); int m=(int)vNodes.size(); Lock(); - BOOST_FOREACH(PAIRTYPE(const int64_t,mc_RelayRequest)& item, m_Requests) + BOOST_FOREACH(PAIRTYPE(const mc_OffchainMessageID,mc_RelayRequest)& item, m_Requests) { for(int i=0;i<(int)item.second.m_Responses.size();i++) { diff --git a/src/protocol/relay.h b/src/protocol/relay.h index d15da66f..53030027 100644 --- a/src/protocol/relay.h +++ b/src/protocol/relay.h @@ -19,8 +19,6 @@ #define MC_PRA_MY_ORIGIN_NT_ADDRESS 0x00000002 #define MC_PRA_USE_DESTINATION_ADDRESS 0x00000004 #define MC_PRA_SIGN_WITH_HANDSHAKE_ADDRESS 0x00000020 -#define MC_PRA_GENERATE_TIMESTAMP 0x00000040 -#define MC_PRA_GENERATE_NONCE 0x00000080 #define MC_VRA_NONE 0x00000000 #define MC_VRA_IS_RESPONSE 0x00000002 @@ -42,25 +40,24 @@ #define MC_VRA_DEFAULT 0x03970000 #define MC_RMT_NONE 0 -#define MC_RMT_REJECT 0x00000001 -#define MC_RMT_MC_ADDRESS_QUERY 0x00000002 -#define MC_RMT_NODE_DETAILS 0x00000003 -#define MC_RMT_CHUNK_QUERY 0x00000101 -#define MC_RMT_CHUNK_QUERY_HIT 0x00000102 -#define MC_RMT_CHUNK_REQUEST 0x00000103 -#define MC_RMT_CHUNK_RESPONSE 0x00000104 -#define MC_RMT_ADD_RESPONSE 0x01000001 -#define MC_RMT_ERROR_IN_MESSAGE 0x01000002 -#define MC_RMT_NEW_REQUEST 0x01000003 -#define MC_RMT_SPECIAL_MASK 0x10000000 -#define MC_RMT_SPECIAL_COLLECT_CHUNKS 0x10000001 -#define MC_RMT_SPECIAL_VIEW_CHUNKS 0x10000002 +#define MC_RMT_MC_ADDRESS_QUERY 0x7971616d //maqy +#define MC_RMT_NODE_DETAILS 0x7464646e //nddt +#define MC_RMT_CHUNK_QUERY 0x79716863 //chqy +#define MC_RMT_CHUNK_QUERY_HIT 0x68716883 //chqh +#define MC_RMT_CHUNK_REQUEST 0x71726863 //chrq +#define MC_RMT_CHUNK_RESPONSE 0x73726863 //chrs +#define MC_RMT_ADD_RESPONSE 0x00800001 +#define MC_RMT_ERROR_IN_MESSAGE 0x00800002 +#define MC_RMT_NEW_REQUEST 0x00800003 +#define MC_RMT_SPECIAL_MASK 0x80000000 +#define MC_RMT_SPECIAL_COLLECT_CHUNKS 0x80000001 +#define MC_RMT_SPECIAL_VIEW_CHUNKS 0x80000002 #define MC_RDT_UNKNOWN 0 -#define MC_RDT_MC_ADDRESS 1 -#define MC_RDT_NET_ADDRESS 2 -#define MC_RDT_CHUNK_IDS 11 -#define MC_RDT_CHUNKS 12 +#define MC_RDT_MC_ADDRESS 0x01 +#define MC_RDT_NET_ADDRESS 0x02 +#define MC_RDT_CHUNK_IDS 0x11 +#define MC_RDT_CHUNKS 0x12 #define MC_LIM_MAX_SECONDS 60 #define MC_LIM_MAX_MEASURES 4 @@ -113,6 +110,60 @@ typedef struct mc_Limiter } mc_Limiter; +typedef struct mc_OffchainMessageID +{ + uint32_t m_TimeStamp; + uint96 m_Nonce; + + mc_OffchainMessageID() + { + m_TimeStamp=0; + m_Nonce=0; + } + + + friend bool operator<(const mc_OffchainMessageID& a, const mc_OffchainMessageID& b) + { + return ((a.m_TimeStamp < b.m_TimeStamp) || + (a.m_TimeStamp == b.m_TimeStamp && a.m_Nonce < b.m_Nonce)); + } + + friend bool operator==(const mc_OffchainMessageID& a, const mc_OffchainMessageID& b) + { + return (a.m_TimeStamp == b.m_TimeStamp && a.m_Nonce == b.m_Nonce); + } + + friend bool operator!=(const mc_OffchainMessageID& a, const mc_OffchainMessageID& b) + { + return (a.m_TimeStamp != b.m_TimeStamp || a.m_Nonce != b.m_Nonce); + } + + + mc_OffchainMessageID& operator=(const mc_OffchainMessageID& b) + { + m_TimeStamp=b.m_TimeStamp; + m_Nonce=b.m_Nonce; + return *this; + } + + mc_OffchainMessageID& operator=(const uint32_t& b) + { + m_TimeStamp=b; + m_Nonce=0; + return *this; + } + + bool IsZero() + { + return (m_TimeStamp == 0 && m_Nonce == 0); + } + + std::string ToString() + { + return strprintf("%s-%d",m_Nonce.ToString().c_str(),m_TimeStamp); + } +} mc_OffchainMessageID; + typedef struct mc_NodeFullAddress { @@ -130,22 +181,19 @@ typedef struct mc_NodeFullAddress typedef struct mc_RelayRecordKey { - uint32_t m_TimeStamp; - uint32_t m_Nonce; + mc_OffchainMessageID m_ID; NodeId m_NodeTo; - mc_RelayRecordKey(uint32_t timestamp,uint32_t nonce,NodeId node) + mc_RelayRecordKey(mc_OffchainMessageID msg_id,NodeId node) { - m_TimeStamp=timestamp; - m_Nonce=nonce; + m_ID=msg_id; m_NodeTo=node; } friend bool operator<(const mc_RelayRecordKey& a, const mc_RelayRecordKey& b) { - return ((a.m_TimeStamp < b.m_TimeStamp) || - (a.m_TimeStamp == b.m_TimeStamp && a.m_Nonce < b.m_Nonce) || - (a.m_TimeStamp == b.m_TimeStamp && a.m_Nonce == b.m_Nonce && a.m_NodeTo < b.m_NodeTo)); + return ((a.m_ID < b.m_ID) || + (a.m_ID == b.m_ID && a.m_NodeTo < b.m_NodeTo)); } } mc_RelayRecordKey; @@ -163,7 +211,7 @@ struct mc_RelayRequest; typedef struct mc_RelayResponse { - int64_t m_Nonce; + mc_OffchainMessageID m_MsgID; uint32_t m_MsgType; uint32_t m_Flags; NodeId m_NodeFrom; @@ -185,12 +233,10 @@ typedef struct mc_RelayResponse typedef struct mc_RelayRequest { - int64_t m_Nonce; + mc_OffchainMessageID m_MsgID; uint32_t m_MsgType; uint32_t m_Flags; NodeId m_NodeTo; - int64_t m_ParentNonce; - int m_ParentResponseID; uint32_t m_LastTryTimestamp; int m_TryCount; uint32_t m_Status; @@ -230,7 +276,7 @@ typedef struct mc_RelayManager map m_Latency; map m_Limiters; map m_RelayRecords; - map m_Requests; + map m_Requests; void Zero(); void Destroy(); @@ -239,27 +285,25 @@ typedef struct mc_RelayManager void UnLock(); int Initialize(); - int64_t AggregateNonce(uint32_t timestamp,uint32_t nonce); - uint32_t Timestamp(int64_t aggr_nonce); - uint32_t Nonce(int64_t aggr_nonce); uint32_t GenerateNonce(); + mc_OffchainMessageID GenerateMsgID(uint32_t timestamp); + mc_OffchainMessageID GenerateMsgID(); void SetDefaults(); void SetMyIPs(uint32_t *ips,int ip_count); void MsgTypeSettings(uint32_t msg_type,int latency,int seconds,int64_t serves_per_second,int64_t bytes_per_second); void InitNodeAddress(mc_NodeFullAddress* node_address,CNode* pto,uint32_t action); void InitNodeAddress(mc_NodeFullAddress* node_address,CKeyID& mc_address, vector& net_addresses); void CheckTime(); - void SetRelayRecord(CNode *pto,CNode *pfrom,uint32_t msg_type,uint32_t timestamp,uint32_t nonce); - int GetRelayRecord(CNode *pfrom,uint32_t timestamp,uint32_t nonce,uint32_t *msg_type,CNode **pto); + void SetRelayRecord(CNode *pto,CNode *pfrom,uint32_t msg_type,mc_OffchainMessageID msg_id); + int GetRelayRecord(CNode *pfrom,mc_OffchainMessageID msg_id,uint32_t *msg_type,CNode **pto); - int64_t PushRelay ( CNode* pto, + mc_OffchainMessageID PushRelay(CNode* pto, uint32_t msg_format, vector &vHops, + vector &vSendPaths, uint32_t msg_type, - uint32_t timestamp_to_send, - uint32_t nonce_to_send, - uint32_t timestamp_to_respond, - uint32_t nonce_to_respond, + mc_OffchainMessageID msg_id, + mc_OffchainMessageID msg_id_to_respond, uint32_t flags, vector& payload, vector& sigScripts_to_relay, @@ -271,15 +315,19 @@ typedef struct mc_RelayManager CValidationState &state, uint32_t verify_flags_in); - int AddRequest(int64_t parent_nonce,int parent_response_id,CNode *pto,int64_t nonce,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status); - int AddResponse(int64_t request,CNode *pfrom,int32_t source,int hop_count,int64_t nonce,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status); - int DeleteRequest(int64_t request); - int ProcessRequest(int64_t request); - mc_RelayRequest *FindRequest(int64_t request); + int AddRequest(CNode *pto,mc_OffchainMessageID msg_id,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status); +// int AddRequest(int64_t parent_nonce,int parent_response_id,CNode *pto,int64_t nonce,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status); + int AddResponse(mc_OffchainMessageID request,CNode *pfrom,int32_t source,int hop_count,mc_OffchainMessageID msg_id,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status); + //int AddResponse(int64_t request,CNode *pfrom,int32_t source,int hop_count,int64_t nonce,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status); + int DeleteRequest(mc_OffchainMessageID request); + int ProcessRequest(mc_OffchainMessageID request); + mc_RelayRequest *FindRequest(mc_OffchainMessageID request); void InvalidateResponsesFromDisconnected(); - int64_t SendRequest(CNode* pto,uint32_t msg_type,uint32_t flags,vector & payload); - int64_t SendNextRequest(mc_RelayResponse* response,uint32_t msg_type,uint32_t flags,vector & payload); + mc_OffchainMessageID SendRequest(CNode* pto,uint32_t msg_type,uint32_t flags,vector & payload); +// int64_t SendRequest(CNode* pto,uint32_t msg_type,uint32_t flags,vector & payload); + mc_OffchainMessageID SendNextRequest(mc_RelayResponse* response,uint32_t msg_type,uint32_t flags,vector & payload); +// int64_t SendNextRequest(mc_RelayResponse* response,uint32_t msg_type,uint32_t flags,vector & payload); } mc_RelayManager; diff --git a/src/rpc/rpccache.cpp b/src/rpc/rpccache.cpp index 76cbfd39..c8a5923a 100644 --- a/src/rpc/rpccache.cpp +++ b/src/rpc/rpccache.cpp @@ -125,7 +125,7 @@ Value offchain(const Array& params, bool fHelp) uint32_t request_type=MC_RMT_NONE; int timeout=5; vector payload; - int64_t request_id; + mc_OffchainMessageID request_id; CNode *pto=NULL; string request_str=""; mc_RelayRequest *request; @@ -408,7 +408,7 @@ Value offchain(const Array& params, bool fHelp) request_id=pRelayManager->SendRequest(pto,request_type,0,payload); printf("%s",request_str.c_str()); - printf(". Request ID: %lu\n",request_id); + printf(". Request ID: %s\n",request_id.ToString().c_str()); } uint32_t time_now; diff --git a/src/structs/uint256.cpp b/src/structs/uint256.cpp index cc5d9154..fa4f9efb 100644 --- a/src/structs/uint256.cpp +++ b/src/structs/uint256.cpp @@ -216,6 +216,14 @@ unsigned int base_uint::bits() const return 0; } +// Explicit instantiations for base_uint<96> +template int base_uint<96>::CompareTo(const base_uint<96>&) const; +template bool base_uint<96>::EqualTo(uint64_t) const; +template std::string base_uint<96>::GetHex() const; +template std::string base_uint<96>::ToString() const; + + + // Explicit instantiations for base_uint<160> template base_uint<160>::base_uint(const std::string&); template base_uint<160>::base_uint(const std::vector&); diff --git a/src/structs/uint256.h b/src/structs/uint256.h index 7f39dd75..5647410e 100644 --- a/src/structs/uint256.h +++ b/src/structs/uint256.h @@ -286,6 +286,14 @@ class base_uint } }; +/** 96-bit unsigned big integer. */ +class uint96 : public base_uint<96> { +public: + uint96() {} + uint96(const base_uint<96>& b) : base_uint<96>(b) {} + uint96(uint64_t b) : base_uint<96>(b) {} +}; + /** 160-bit unsigned big integer. */ class uint160 : public base_uint<160> { public: diff --git a/src/utils/declare.h b/src/utils/declare.h index 9dcae8dd..a42c99fa 100644 --- a/src/utils/declare.h +++ b/src/utils/declare.h @@ -206,6 +206,7 @@ void *mc_New(int Size); void mc_Delete(void *ptr); void mc_PutLE(void *dest,void *src,int dest_size); int64_t mc_GetLE(void *src,int size); +uint32_t mc_SwapBytes32(uint32_t src); int mc_BackupFile(const char *network_name,const char *filename, const char *extension,int options); int mc_RecoverFile(const char *network_name,const char *filename, const char *extension,int options); FILE *mc_OpenFile(const char *network_name,const char *filename, const char *extension,const char *mode, int options); diff --git a/src/utils/utility.cpp b/src/utils/utility.cpp index 31c5334b..270ebbce 100644 --- a/src/utils/utility.cpp +++ b/src/utils/utility.cpp @@ -89,6 +89,20 @@ int64_t mc_GetLE(void *src,int size) return result; // Assuming all systems are little endian } +uint32_t mc_SwapBytes32(uint32_t src) +{ + uint32_t res=0; + unsigned char *pr; + unsigned char *ps; + pr=(unsigned char*)&res; + ps=(unsigned char*)&src; + for(int i=0;i<4;i++) + { + pr[3-i]=ps[i]; + } + return res; +} + void mc_print(const char *message) { printf("%s\n",message); diff --git a/src/wallet/chunkcollector.cpp b/src/wallet/chunkcollector.cpp index 3bad978b..2e9d3003 100644 --- a/src/wallet/chunkcollector.cpp +++ b/src/wallet/chunkcollector.cpp @@ -20,27 +20,48 @@ void mc_ChunkCollectorRow::Zero() memset(this,0, sizeof(mc_ChunkCollectorRow)); } +void mc_ChunkCollectorDBRow::Zero() +{ + memset(this,0, sizeof(mc_ChunkCollectorDBRow)); +} + uint32_t m_QueryNextAttempt; + int m_Vout; + unsigned char m_TxID[MC_TDB_TXID_SIZE]; + mc_TxEntity m_Entity; + unsigned char m_Hash[MC_CDB_CHUNK_HASH_SIZE]; // Chunk hash + + uint32_t m_Size; + uint32_t m_Flags; + uint32_t m_QueryAttempts; + uint32_t m_Status; + int64_t m_Reserved1; + int64_t m_Reserved2; + void mc_ChunkCollector::Zero() { m_DB=NULL; m_ChunkDB=NULL; m_KeyOffset=0; - m_KeySize=MC_TDB_TXID_SIZE+sizeof(int)+sizeof(mc_ChunkEntityKey); + m_KeyDBOffset=0; + m_KeySize=MC_TDB_TXID_SIZE+sizeof(int)+sizeof(mc_ChunkEntityKey)+3*sizeof(uint32_t); + m_KeyDBSize=MC_TDB_TXID_SIZE+sizeof(int)+sizeof(uint32_t)+MC_CDB_CHUNK_HASH_SIZE+sizeof(mc_TxEntity); // 96 m_ValueOffset=m_KeySize; + m_ValueDBOffset=m_KeyDBSize; m_ValueSize=sizeof(mc_ChunkEntityValue); - m_ValueDBSize=28; + m_ValueDBSize=4*sizeof(uint32_t)+2*sizeof(int64_t); m_TotalSize=m_KeySize+m_ValueSize; - m_TotalDBSize=m_KeySize+m_ValueDBSize; + m_TotalDBSize=m_KeyDBSize+m_ValueDBSize; m_Name[0]=0; m_DBName[0]=0; m_NextTryTimestamp=0; - m_MarkPool=NULL; m_MemPool=NULL; m_MemPoolNext=NULL; m_MemPool1=NULL; m_MemPool2=NULL; + m_MaxMemPoolSize=MC_CCW_DEFAULT_MEMPOOL_SIZE; + m_Semaphore=NULL; m_LockedBy=0; @@ -70,7 +91,7 @@ int mc_ChunkCollector::Destroy() { delete m_MemPool2; } - + if(m_Semaphore) { __US_SemDestroy(m_Semaphore); @@ -108,6 +129,132 @@ int mc_ChunkCollector::Lock() return Lock(1,0); } +void mc_ChunkCollector::SetDBRow(mc_ChunkCollectorRow* collect_row) +{ + m_DBRow.Zero(); + m_DBRow.m_QueryNextAttempt=mc_SwapBytes32(collect_row->m_DBNextAttempt); + m_DBRow.m_Vout=collect_row->m_Vout; + memcpy(m_DBRow.m_TxID,collect_row->m_TxID,MC_TDB_TXID_SIZE); + memcpy(&(m_DBRow.m_Entity),&(collect_row->m_ChunkDef.m_Entity),sizeof(mc_TxEntity)); + memcpy(m_DBRow.m_Hash,collect_row->m_ChunkDef.m_Hash,MC_CDB_CHUNK_HASH_SIZE); + m_DBRow.m_Size=collect_row->m_ChunkDef.m_Size; + m_DBRow.m_Flags=collect_row->m_ChunkDef.m_Flags; + m_DBRow.m_QueryAttempts=collect_row->m_State.m_QueryAttempts; + m_DBRow.m_Status=collect_row->m_State.m_Status; +} + +void mc_ChunkCollector::GetDBRow(mc_ChunkCollectorRow* collect_row) +{ + collect_row->Zero(); + collect_row->m_DBNextAttempt=mc_SwapBytes32(m_DBRow.m_QueryNextAttempt); + collect_row->m_State.m_QueryAttempts=collect_row->m_DBNextAttempt; + collect_row->m_Vout=m_DBRow.m_Vout; + memcpy(collect_row->m_TxID,m_DBRow.m_TxID,MC_TDB_TXID_SIZE); + memcpy(&(collect_row->m_ChunkDef.m_Entity),&(m_DBRow.m_Entity),sizeof(mc_TxEntity)); + memcpy(collect_row->m_ChunkDef.m_Hash,m_DBRow.m_Hash,MC_CDB_CHUNK_HASH_SIZE); + collect_row->m_ChunkDef.m_Size=m_DBRow.m_Size; + collect_row->m_ChunkDef.m_Flags=m_DBRow.m_Flags; + collect_row->m_State.m_QueryAttempts=m_DBRow.m_QueryAttempts; + collect_row->m_State.m_Status=m_DBRow.m_Status; + collect_row->m_State.m_Status |= MC_CCF_INSERTED; +} + +int mc_ChunkCollector::DeleteDBRow(mc_ChunkCollectorRow *collect_row) +{ + SetDBRow(collect_row); + return m_DB->Delete((char*)&m_DBRow+m_KeyDBOffset,m_KeyDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); +} + +int mc_ChunkCollector::InsertDBRow(mc_ChunkCollectorRow *collect_row) +{ + collect_row->m_State.m_Status &= MC_CCF_ERROR_MASK; + SetDBRow(collect_row); + collect_row->m_State.m_Status |= MC_CCF_INSERTED; + return m_DB->Write((char*)&m_DBRow+m_KeyDBOffset,m_KeyDBSize,(char*)&m_DBRow+m_ValueDBOffset,m_ValueDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); +} + +int mc_ChunkCollector::UpdateDBRow(mc_ChunkCollectorRow *collect_row) +{ + int err; + collect_row->m_State.m_Status &= MC_CCF_ERROR_MASK; + SetDBRow(collect_row); + err=m_DB->Delete((char*)&m_DBRow+m_KeyOffset,m_KeySize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + if(err) + { + return err; + } + collect_row->m_DBNextAttempt=collect_row->m_State.m_QueryNextAttempt; + err=m_DBRow.m_QueryNextAttempt=mc_SwapBytes32(collect_row->m_DBNextAttempt); + collect_row->m_State.m_Status |= MC_CCF_INSERTED; + return m_DB->Write((char*)&m_DBRow+m_KeyDBOffset,m_KeyDBSize,(char*)&m_DBRow+m_ValueDBOffset,m_ValueDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); +} + +int mc_ChunkCollector::SeekDB(void *dbrow) +{ + int err,value_len; + unsigned char *ptr; + + err=MC_ERR_NOERROR; + ptr=(unsigned char*)m_DB->Read((char*)dbrow+m_KeyDBOffset,m_KeyDBSize,&value_len,MC_OPT_DB_DATABASE_SEEK_ON_READ,&err); + if(ptr==NULL) + { + return MC_ERR_NOT_FOUND; + } + + memcpy((unsigned char*)&m_DBRow+m_KeyDBOffset,(unsigned char*)dbrow,m_KeyDBSize); + memcpy((unsigned char*)&m_DBRow+m_ValueDBOffset,ptr,m_ValueDBSize); + memcpy(&m_LastDBRow,&m_DBRow,m_TotalDBSize); + + return err; +} + +int mc_ChunkCollector::ReadFromDB(mc_Buffer *mempool,int rows) +{ + int err; + mc_ChunkCollectorRow collect_row; + mc_ChunkDBRow chunk_def; + unsigned char *ptr; + int row; + if(rows <= 0) + { + return MC_ERR_NOERROR; + } + + err=MC_ERR_NOERROR; + + ptr=NULL; + row=0; + while(rowMoveNext(&err); + if(err) + { + return MC_ERR_CORRUPTED; + } + if(ptr) + { + memcpy((char*)&m_DBRow,ptr,m_TotalDBSize); + GetDBRow(&collect_row); + collect_row.m_State.m_Status |= MC_CCF_INSERTED; + if(m_ChunkDB->GetChunkDef(&chunk_def,collect_row.m_ChunkDef.m_Hash,&(collect_row.m_ChunkDef.m_Entity),collect_row.m_TxID,collect_row.m_Vout) == MC_ERR_NOERROR) + { + collect_row.m_State.m_Status |= MC_CCF_DELETED; + } + mempool->Add(&collect_row); + row++; + } + else + { + row=rows; + } + } + + memcpy(&m_LastDBRow,&m_DBRow,m_TotalDBSize); + + return MC_ERR_NOERROR; +} + + void mc_ChunkCollector::Dump(const char *message) { int i; @@ -116,8 +263,6 @@ void mc_ChunkCollector::Dump(const char *message) { return; } - mc_ChunkCollectorRow dbrow; - unsigned char *ptr; int dbvalue_len,err; char ShortName[65]; @@ -138,8 +283,8 @@ void mc_ChunkCollector::Dump(const char *message) if(m_DB) { fprintf(fHan,"\nDB\n"); - dbrow.Zero(); - ptr=(unsigned char*)m_DB->Read((char*)&dbrow+m_KeyOffset,m_KeySize,&dbvalue_len,MC_OPT_DB_DATABASE_SEEK_ON_READ,&err); + m_DBRow.Zero(); + ptr=(unsigned char*)m_DB->Read((char*)&m_DBRow+m_KeyDBOffset,m_KeyDBSize,&dbvalue_len,MC_OPT_DB_DATABASE_SEEK_ON_READ,&err); if(err) { return; @@ -147,14 +292,14 @@ void mc_ChunkCollector::Dump(const char *message) if(ptr) { - memcpy((char*)&dbrow+m_ValueOffset,ptr,m_ValueDBSize); + memcpy((char*)&m_DBRow+m_ValueDBOffset,ptr,m_ValueDBSize); while(ptr) { - mc_MemoryDumpCharSizeToFile(fHan,(char*)&dbrow+m_KeyOffset,0,m_TotalDBSize,64); + mc_MemoryDumpCharSizeToFile(fHan,(char*)&m_DBRow+m_KeyDBOffset,0,m_TotalDBSize,64); ptr=(unsigned char*)m_DB->MoveNext(&err); if(ptr) { - memcpy((char*)&dbrow+m_KeyOffset,ptr,m_TotalDBSize); + memcpy((char*)&m_DBRow+m_KeyDBOffset,ptr,m_TotalDBSize); } } } @@ -172,11 +317,7 @@ void mc_ChunkCollector::Dump(const char *message) int mc_ChunkCollector::Initialize(mc_ChunkDB *chunk_db,const char *name,uint32_t mode) { - int err,value_len; - mc_ChunkCollectorRow collect_row; - mc_ChunkDBRow chunk_def; - unsigned char *ptr; - + int err=MC_ERR_NOERROR; if(name) { strcpy(m_Name,name); @@ -209,11 +350,42 @@ int mc_ChunkCollector::Initialize(mc_ChunkDB *chunk_db,const char *name,uint32_t err=m_MemPool2->Initialize(m_KeySize,m_TotalSize,MC_BUF_MODE_MAP); m_MemPool=m_MemPool1; - - collect_row.Zero(); + + m_DBRow.Zero(); if(m_DB) { + err=SeekDB(&m_DBRow); + if(err) + { + if(err != MC_ERR_NOT_FOUND) + { + return err; + } + } + if(err != MC_ERR_NOT_FOUND) + { + err=ReadFromDB(m_MemPool,m_MaxMemPoolSize); + if(err) + { + return err; + } + } + else + { + err=m_DB->Write((char*)&m_DBRow+m_KeyDBOffset,m_KeyDBSize,(char*)&m_DBRow+m_ValueDBOffset,m_ValueDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + if(err) + { + return err; + } + err=m_DB->Commit(MC_OPT_DB_DATABASE_TRANSACTIONAL); + if(err) + { + return err; + } + } + } +/* ptr=(unsigned char*)m_DB->Read((char*)&collect_row+m_KeyOffset,m_KeySize,&value_len,MC_OPT_DB_DATABASE_SEEK_ON_READ,&err); if(err) { @@ -250,7 +422,7 @@ int mc_ChunkCollector::Initialize(mc_ChunkDB *chunk_db,const char *name,uint32_t } else { - err=m_DB->Write((char*)&collect_row+m_KeyOffset,m_KeySize,(char*)&collect_row+m_ValueOffset,m_ValueSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + err=m_DB->Write((char*)&collect_row+m_KeyOffset,m_KeySize,(char*)&collect_row+m_ValueOffset,m_ValueDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); if(err) { return err; @@ -262,6 +434,7 @@ int mc_ChunkCollector::Initialize(mc_ChunkDB *chunk_db,const char *name,uint32_t } } } + */ Dump("Initialize"); @@ -463,18 +636,34 @@ int mc_ChunkCollector::CommitInternal() if(row->m_State.m_Status & MC_CCF_INSERTED) { commit_required=1; - m_DB->Delete((char*)row+m_KeyOffset,m_KeySize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + DeleteDBRow(row); +// m_DB->Delete((char*)row+m_KeyOffset,m_KeySize,MC_OPT_DB_DATABASE_TRANSACTIONAL); } } else { - if( ((row->m_State.m_Status & MC_CCF_INSERTED) == 0 ) || (row->m_State.m_Status & MC_CCF_UPDATED) ) + if( (row->m_State.m_Status & MC_CCF_INSERTED) == 0 ) { commit_required=1; + InsertDBRow(row); + } + else + { + if(row->m_State.m_Status & MC_CCF_UPDATED) + { + commit_required=1; + UpdateDBRow(row); + } + } +/* + if( ((row->m_State.m_Status & MC_CCF_INSERTED) == 0 ) || (row->m_State.m_Status & MC_CCF_UPDATED) ) + { + row->m_State.m_Status &= MC_CCF_ERROR_MASK; m_DB->Write((char*)row+m_KeyOffset,m_KeySize,(char*)row+m_ValueOffset,m_ValueDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); row->m_State.m_Status |= MC_CCF_INSERTED; } + */ m_MemPoolNext->Add(row); } } @@ -487,6 +676,15 @@ int mc_ChunkCollector::CommitInternal() return err; } } + + if(m_MemPoolNext->GetCount() < m_MaxMemPoolSize) + { + err=SeekDB(&m_LastDBRow); + if(err != MC_ERR_NOERROR) + { + ReadFromDB(m_MemPoolNext,m_MaxMemPoolSize); + } + } m_MemPool->Clear(); m_MemPool=m_MemPoolNext; diff --git a/src/wallet/chunkcollector.h b/src/wallet/chunkcollector.h index b62f621b..3e7b3a33 100644 --- a/src/wallet/chunkcollector.h +++ b/src/wallet/chunkcollector.h @@ -5,6 +5,7 @@ #define MULTICHAIN_CHUNKCOLLECTOR_H #include "utils/declare.h" +#include "protocol/relay.h" #include "wallet/chunkdb.h" #include "wallet/wallettxdb.h" @@ -18,10 +19,11 @@ #define MC_CCF_ERROR_MASK 0x00FF0000 #define MC_CCF_ALL 0xFFFFFFFF -#define MC_CCW_TIMEOUT_QUERY 60 -#define MC_CCW_TIMEOUT_REQUEST 5 +#define MC_CCW_TIMEOUT_QUERY 14 +#define MC_CCW_TIMEOUT_REQUEST 3 #define MC_CCW_MAX_CHUNKS_PER_QUERY 64 #define MC_CCW_WORST_RESPONSE_SCORE 1000 +#define MC_CCW_DEFAULT_MEMPOOL_SIZE 1000 typedef struct mc_ChunkEntityKey @@ -39,23 +41,43 @@ typedef struct mc_ChunkEntityValue uint32_t m_QueryAttempts; uint32_t m_QueryNextAttempt; uint32_t m_Status; - int64_t m_Reserved1; - int64_t m_Reserved2; - int64_t m_Query; + uint32_t m_Reserved1; + mc_OffchainMessageID m_Query; + mc_OffchainMessageID m_Request; uint32_t m_QueryTimeStamp; - uint32_t m_Reserved3; - int64_t m_Request; + uint32_t m_Reserved2; uint32_t m_RequestTimeStamp; uint32_t m_RequestPos; void Zero(); } mc_ChunkEntityValue; +typedef struct mc_ChunkCollectorDBRow +{ + uint32_t m_QueryNextAttempt; + int m_Vout; + unsigned char m_TxID[MC_TDB_TXID_SIZE]; + mc_TxEntity m_Entity; + unsigned char m_Hash[MC_CDB_CHUNK_HASH_SIZE]; // Chunk hash + + uint32_t m_Size; + uint32_t m_Flags; + uint32_t m_QueryAttempts; + uint32_t m_Status; + int64_t m_Reserved1; + int64_t m_Reserved2; + + void Zero(); +} mc_ChunkCollectorDBRow; + typedef struct mc_ChunkCollectorRow { mc_ChunkEntityKey m_ChunkDef; - unsigned char m_TxID[MC_TDB_TXID_SIZE]; + uint32_t m_DBNextAttempt; int m_Vout; + unsigned char m_TxID[MC_TDB_TXID_SIZE]; + uint32_t m_Reserved1; + uint32_t m_Reserved2; mc_ChunkEntityValue m_State; void Zero(); @@ -67,13 +89,17 @@ typedef struct mc_ChunkCollector mc_Database *m_DB; // Database object mc_ChunkDB *m_ChunkDB; uint32_t m_KeyOffset; + uint32_t m_KeyDBOffset; uint32_t m_KeySize; + uint32_t m_KeyDBSize; uint32_t m_ValueOffset; + uint32_t m_ValueDBOffset; uint32_t m_ValueSize; - uint32_t m_TotalSize; uint32_t m_ValueDBSize; + uint32_t m_TotalSize; uint32_t m_TotalDBSize; int64_t m_NextTryTimestamp; + int m_MaxMemPoolSize; char m_Name[MC_PRM_NETWORK_NAME_MAX_SIZE+1]; // Chain name char m_DBName[MC_DCT_DB_MAX_PATH]; // Full database name @@ -84,6 +110,9 @@ typedef struct mc_ChunkCollector mc_Buffer *m_MemPool1; mc_Buffer *m_MemPool2; + mc_ChunkCollectorDBRow m_DBRow; + mc_ChunkCollectorDBRow m_LastDBRow; + void *m_Semaphore; // mc_TxDB object semaphore uint64_t m_LockedBy; // ID of the thread locking it @@ -99,6 +128,14 @@ typedef struct mc_ChunkCollector Destroy(); } + void SetDBRow(mc_ChunkCollectorRow *collect_row); + void GetDBRow(mc_ChunkCollectorRow *collect_row); + int DeleteDBRow(mc_ChunkCollectorRow *collect_row); + int UpdateDBRow(mc_ChunkCollectorRow *collect_row); + int InsertDBRow(mc_ChunkCollectorRow *collect_row); + int SeekDB(void *dbrow); + int ReadFromDB(mc_Buffer *mempool,int rows); + int Initialize( // Initialization mc_ChunkDB *chunk_db, const char *name, // Chain name diff --git a/src/wallet/wallettxs.cpp b/src/wallet/wallettxs.cpp index e6f1bb12..497d205c 100644 --- a/src/wallet/wallettxs.cpp +++ b/src/wallet/wallettxs.cpp @@ -506,15 +506,14 @@ int mc_WalletTxs::Commit(mc_TxImport *import) { if(imp->m_ImportID == 0) { - if(m_ChunkCollector) - { - err=m_ChunkCollector->Commit(); - } - if(err == MC_ERR_NOERROR) { err=m_ChunkDB->Commit(imp->m_Block+1); } + if(m_ChunkCollector) + { + err=m_ChunkCollector->Commit(); + } } } m_Database->Lock(1,0); From 5ef2b719c2107328d7be792e6f2ee6672690a982 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 1 May 2018 12:40:47 +0300 Subject: [PATCH 041/157] Offchain items, automatic fetching --- src/protocol/relay.cpp | 78 ++++++++++++++++++++++++----------- src/protocol/relay.h | 2 +- src/rpc/rpccache.cpp | 17 +++++++- src/wallet/chunkcollector.cpp | 31 ++++++++++---- src/wallet/chunkcollector.h | 5 ++- 5 files changed, 97 insertions(+), 36 deletions(-) diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index e5e7aafd..48894f23 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -18,6 +18,13 @@ uint32_t MultichainNextChunkQueryAttempt(uint32_t attempts) return 86400*365*10; } +string mc_MsgTypeStr(uint32_t msg_type) +{ + char *ptr; + ptr=(char*)&msg_type; + return strprintf("%c%c%c%c",ptr[0],ptr[1],ptr[2],ptr[3]); +} + typedef struct CRelayResponsePair { mc_OffchainMessageID request_id; @@ -41,7 +48,6 @@ int MultichainProcessChunkResponse(const CRelayResponsePair *response_pair,map < mc_RelayRequest *request; mc_RelayResponse *response; request=pRelayManager->FindRequest(response_pair->request_id); - if(request == NULL) { return false; @@ -59,6 +65,7 @@ int MultichainProcessChunkResponse(const CRelayResponsePair *response_pair,map < mc_ChunkEntityKey *chunkOut; unsigned char *ptrOut; bool result=false; + string strError=""; mc_ChunkCollectorRow *collect_row; uint32_t total_size=0; @@ -97,12 +104,14 @@ int MultichainProcessChunkResponse(const CRelayResponsePair *response_pair,map < if(response->m_Payload.size() != 1+shift+total_size) { + strError="Total size mismatch"; goto exitlbl; } ptrOut=&(response->m_Payload[0]); if(*ptrOut != MC_RDT_CHUNKS) { + strError="Unsupported payload format"; goto exitlbl; } @@ -110,6 +119,7 @@ int MultichainProcessChunkResponse(const CRelayResponsePair *response_pair,map < countOut=(int)mc_GetVarInt(ptrOut,1+shift+total_size,-1,&shiftOut); if( (countOut != count) || (shift != shiftOut) ) { + strError="Chunk count mismatch"; goto exitlbl; } ptrOut+=shift; @@ -120,16 +130,20 @@ int MultichainProcessChunkResponse(const CRelayResponsePair *response_pair,map < sizeOut=((mc_ChunkEntityKey*)ptr)->m_Size; chunk=(mc_ChunkEntityKey*)ptr; chunkOut=(mc_ChunkEntityKey*)ptrOut; + ptrOut+=size; if(chunk->m_Size != chunkOut->m_Size) { + strError="Chunk info size mismatch"; goto exitlbl; } if(memcmp(chunk->m_Hash,chunkOut->m_Hash,sizeof(uint256))) { + strError="Chunk info hash mismatch"; goto exitlbl; } - if(memcmp(&(chunk->m_Entity),&(chunkOut->m_Entity),sizeof(uint256))) + if(memcmp(&(chunk->m_Entity),&(chunkOut->m_Entity),sizeof(mc_TxEntity))) { + strError="Chunk info entity mismatch"; goto exitlbl; } sizeOut=chunk->m_Size; @@ -141,6 +155,7 @@ int MultichainProcessChunkResponse(const CRelayResponsePair *response_pair,map < mc_gState->m_TmpBuffers->m_RpcHasher1->DoubleHash(ptrOut,sizeOut,&hash); if(memcmp(&hash,chunk->m_Hash,sizeof(uint256))) { + strError="Chunk data hash mismatch"; goto exitlbl; } chunk_err=pwalletTxsMain->m_ChunkDB->AddChunk(chunk->m_Hash,&(chunk->m_Entity),(unsigned char*)collect_row->m_TxID,collect_row->m_Vout,ptrOut,NULL,sizeOut,0,0); @@ -148,6 +163,7 @@ int MultichainProcessChunkResponse(const CRelayResponsePair *response_pair,map < { if(chunk_err != MC_ERR_FOUND) { + strError=strprintf("Internal chunk DB error: %d",chunk_err); goto exitlbl; } } @@ -164,6 +180,10 @@ int MultichainProcessChunkResponse(const CRelayResponsePair *response_pair,map < pRelayManager->UnLock(); + if(strError.size()) + { + if(fDebug)LogPrint("chunks","Bad response from peer %d: %s\n",response->m_NodeFrom,strError.c_str()); + } return result; } @@ -273,7 +293,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) { response_pair.request_id=collect_row->m_State.m_Request; response_pair.response_id=0; - printf("coll new rsp: row: %d, id: %s, %d\n",row,collect_row->m_State.m_Request.ToString().c_str(),collect_row->m_State.m_RequestPos); +// printf("coll new rsp: row: %d, id: %s, %d\n",row,collect_row->m_State.m_Request.ToString().c_str(),collect_row->m_State.m_RequestPos); map::iterator itrsp = responses_to_process.find(response_pair); if (itrsp == responses_to_process.end()) { @@ -321,13 +341,14 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) map::iterator itrsp = requests_to_send.find(response_pair); if (itrsp == requests_to_send.end()) { +// printf("coll new req: row: %d, id: %s, rsps: %d, score (%d,%d)\n",row,collect_row->m_State.m_Query.ToString().c_str(),(int)query->m_Responses.size(),best_score,best_response); request_pairs.m_Pairs.clear(); request_pairs.m_Pairs.insert(make_pair(row,0)); requests_to_send.insert(make_pair(response_pair,request_pairs)); } else { - printf("coll new req: row: %d, id: %s, rsps: %d, score (%d,%d)\n",row,collect_row->m_State.m_Query.ToString().c_str(),(int)query->m_Responses.size(),best_score,best_response); +// printf("coll old req: row: %d, id: %s, rsps: %d, score (%d,%d)\n",row,collect_row->m_State.m_Query.ToString().c_str(),(int)query->m_Responses.size(),best_score,best_response); itrsp->second.m_Pairs.insert(make_pair(row,0)); } } @@ -356,7 +377,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) BOOST_FOREACH(PAIRTYPE(const int, int)& chunk_row, item.second.m_Pairs) { collect_subrow=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(chunk_row.first); - printf("S %d\n",chunk_row.first); +// printf("S %d\n",chunk_row.first); collect_subrow->m_State.m_RequestPos=count; memcpy(ptrOut,&(collect_subrow->m_ChunkDef),sizeof(mc_ChunkEntityKey)); ptrOut+=sizeof(mc_ChunkEntityKey); @@ -371,12 +392,13 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) response=&(request->m_Responses[item.first.response_id]); request_id=pRelayManager->SendNextRequest(response,MC_RMT_CHUNK_REQUEST,0,payload); - if(fDebug)LogPrint("chunks","New chunk request %s, response: %s, chunks: %d\n",request_id.ToString().c_str(),response->m_MsgID.ToString().c_str(),item.second.m_Pairs.size()); + if(fDebug)LogPrint("chunks","New chunk request: %s, response: %s, chunks: %d\n",request_id.ToString().c_str(),response->m_MsgID.ToString().c_str(),item.second.m_Pairs.size()); BOOST_FOREACH(PAIRTYPE(const int, int)& chunk_row, item.second.m_Pairs) { collect_subrow=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(chunk_row.first); collect_subrow->m_State.m_Request=request_id; - collect_row->m_State.m_RequestTimeStamp=time_now+MC_CCW_TIMEOUT_REQUEST; + collect_subrow->m_State.m_RequestTimeStamp=time_now+MC_CCW_TIMEOUT_REQUEST; +// printf("T %d %d %s\n",chunk_row.first,collect_subrow->m_State.m_RequestPos,collect_subrow->m_State.m_Request.ToString().c_str()); } } @@ -392,7 +414,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) collect_row=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(row); } - if( (collect_row == NULL)|| (last_count >= MC_CCW_MAX_CHUNKS_PER_QUERY) || (total_size+collect_row->m_ChunkDef.m_Size > MAX_SIZE-48) ) + if( (collect_row == NULL)|| (last_count >= MC_CCW_MAX_CHUNKS_PER_QUERY) || (total_size+collect_row->m_ChunkDef.m_Size + sizeof(mc_ChunkEntityKey)> MAX_SIZE-48) ) { if(last_count) { @@ -415,7 +437,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) } } query_id=pRelayManager->SendRequest(NULL,MC_RMT_CHUNK_QUERY,0,payload); - if(fDebug)LogPrint("chunks","New chunk query: %s, chunks: %d\n",query_id.ToString().c_str(),last_count); + if(fDebug)LogPrint("chunks","New chunk query: %s, chunks: %d, rows [%d-%d)\n",query_id.ToString().c_str(),last_count,last_row,row); for(int r=last_row;rm_MemPool->GetRow(r); @@ -499,6 +521,15 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) } collector->UnLock(); + if(not_processed < collector->m_MaxMemPoolSize/2) + { + if(collector->m_NextAutoCommitTimestamp < GetTimeMillis()) + { + collector->Commit(); + collector->m_NextAutoCommitTimestamp=GetTimeMillis()+collector->m_AutoCommitDelay; + } + } + return not_processed; } @@ -559,7 +590,7 @@ bool mc_RelayProcess_Chunk_Query(unsigned char *ptrStart,unsigned char *ptrEnd,v for(int c=0;cm_ChunkDB->GetChunkDef(&chunk_def,chunk.m_Hash,NULL,NULL,-1) == MC_ERR_NOERROR) { if(chunk_def.m_Size != chunk.m_Size) @@ -622,7 +653,7 @@ bool mc_RelayProcess_Chunk_Request(unsigned char *ptrStart,unsigned char *ptrEnd for(int c=0;cm_ChunkDB->GetChunkDef(&chunk_def,chunk.m_Hash,NULL,NULL,-1) == MC_ERR_NOERROR) { if(chunk_def.m_Size != chunk.m_Size) @@ -630,7 +661,8 @@ bool mc_RelayProcess_Chunk_Request(unsigned char *ptrStart,unsigned char *ptrEnd strError="Bad chunk size"; return false; } - if(total_size + chunk_def.m_Size > MAX_SIZE-48) + total_size+=chunk_def.m_Size+size; + if(total_size > MAX_SIZE-48) { strError="Total size of requested chunks is too big"; return false; @@ -1443,8 +1475,8 @@ mc_OffchainMessageID mc_RelayManager::PushRelay(CNode* pto, } // printf("send: %d, to: %d, from: %d, hc: %d, size: %d, ts: %u, nc: %u\n",msg_type,pto->GetId(),pfrom ? pfrom->GetId() : 0,(int)vHops.size(),(int)payload.size(),timestamp,nonce); - if(fDebug)LogPrint("offchain","Offchain send: %s, request: %s, to: %d, from: %d, msg: %d, hops: %d, size: %d\n", - msg_id.ToString().c_str(),msg_id_to_respond.ToString().c_str(),pto->GetId(),pfrom ? pfrom->GetId() : 0,msg_type,(int)vHops.size(),(int)payload.size()); + if(fDebug)LogPrint("offchain","Offchain send: %s, request: %s, to: %d, from: %d, msg: %s, hops: %d, size: %d\n", + msg_id.ToString().c_str(),msg_id_to_respond.ToString().c_str(),pto->GetId(),pfrom ? pfrom->GetId() : 0,mc_MsgTypeStr(msg_type).c_str(),(int)vHops.size(),(int)payload.size()); pto->PushMessage("offchain", msg_format, vHops, @@ -1544,7 +1576,7 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, default: if(verify_flags & MC_VRA_MESSAGE_TYPE) { - LogPrintf("ProcessRelay() : Unsupported relay message type %d\n",msg_type_in); + LogPrintf("ProcessRelay() : Unsupported relay message type %s\n",mc_MsgTypeStr(msg_type_in).c_str()); return false; } break; @@ -1554,7 +1586,7 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, { if(hop_count) { - LogPrintf("ProcessRelay() : Unsupported hop count %d for msg type %d\n",hop_count,msg_type_in); + LogPrintf("ProcessRelay() : Unsupported hop count %d for msg type %s\n",hop_count,mc_MsgTypeStr(msg_type_in).c_str()); return false; } } @@ -1563,7 +1595,7 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, { if(hop_count > 1) { - LogPrintf("ProcessRelay() : Unsupported hop count %d for msg type %d\n",hop_count,msg_type_in); + LogPrintf("ProcessRelay() : Unsupported hop count %d for msg type %s\n",hop_count,mc_MsgTypeStr(msg_type_in).c_str()); return false; } } @@ -1694,8 +1726,8 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, vRecv >> vSigScripts; // printf("recv: %d, from: %d, to: %d, hc: %d, size: %d, ts: %u, nc: %u\n",msg_type_in,pfrom->GetId(),pto_stored ? pto_stored->GetId() : 0,hop_count,(int)vPayloadIn.size(),timestamp_received,nonce_received); - if(fDebug)LogPrint("offchain","Offchain recv: %s, request: %s, from: %d, to: %d, msg: %d, hops: %d, size: %d\n", - msg_id_received.ToString().c_str(),msg_id_to_respond.ToString().c_str(),pfrom->GetId(),pto_stored ? pto_stored->GetId() : 0,msg_type_in,hop_count,(int)vPayloadIn.size()); + if(fDebug)LogPrint("offchain","Offchain recv: %s, request: %s, from: %d, to: %d, msg: %s, hops: %d, size: %d\n", + msg_id_received.ToString().c_str(),msg_id_to_respond.ToString().c_str(),pfrom->GetId(),pto_stored ? pto_stored->GetId() : 0,mc_MsgTypeStr(msg_type_in).c_str(),hop_count,(int)vPayloadIn.size()); if( verify_flags & MC_VRA_SIGNATURES ) { @@ -1823,7 +1855,7 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, } else { - LogPrintf("ProcessRelay() : Error processing request %08X from peer %d: %s\n",msg_type_in,pfrom->GetId(),strError.c_str()); + LogPrintf("ProcessRelay() : Error processing %s (request %s) from peer %d: %s\n",mc_MsgTypeStr(msg_type_in).c_str(),pfrom->GetId(),strError.c_str()); return false; } } @@ -1889,7 +1921,7 @@ int mc_RelayManager::AddRequest(CNode *pto,mc_OffchainMessageID msg_id,uint32_t request.m_Payload=payload; request.m_Responses.clear(); - if(fDebug)LogPrint("offchain","Offchain rqst: %s, to: %d, msg: %d, size: %d\n",msg_id.ToString().c_str(),pto ? pto->GetId() : 0,msg_type,(int)payload.size()); + if(fDebug)LogPrint("offchain","Offchain rqst: %s, to: %d, msg: %s, size: %d\n",msg_id.ToString().c_str(),pto ? pto->GetId() : 0,mc_MsgTypeStr(msg_type).c_str(),(int)payload.size()); m_Requests.insert(make_pair(msg_id,request)); } else @@ -1904,8 +1936,8 @@ int mc_RelayManager::AddRequest(CNode *pto,mc_OffchainMessageID msg_id,uint32_t int mc_RelayManager::AddResponse(mc_OffchainMessageID request,CNode *pfrom,int32_t source,int hop_count,mc_OffchainMessageID msg_id,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status) { // printf("resp: %lu, mt: %d, node: %d, size: %d, rn: %lu\n",request,msg_type,pfrom ? pfrom->GetId() : 0,(int)payload.size(),nonce); - if(fDebug)LogPrint("offchain","Offchain resp: %s, request: %s, from: %d, msg: %d, size: %d\n", - msg_id.ToString().c_str(),request.ToString().c_str(),pfrom ? pfrom->GetId() : 0,msg_type,(int)payload.size()); + if(fDebug)LogPrint("offchain","Offchain resp: %s, request: %s, from: %d, msg: %s, size: %d\n", + msg_id.ToString().c_str(),request.ToString().c_str(),pfrom ? pfrom->GetId() : 0,mc_MsgTypeStr(msg_type).c_str(),(int)payload.size()); if(request.IsZero()) { return MC_ERR_NOERROR; diff --git a/src/protocol/relay.h b/src/protocol/relay.h index 53030027..6811ef9c 100644 --- a/src/protocol/relay.h +++ b/src/protocol/relay.h @@ -43,7 +43,7 @@ #define MC_RMT_MC_ADDRESS_QUERY 0x7971616d //maqy #define MC_RMT_NODE_DETAILS 0x7464646e //nddt #define MC_RMT_CHUNK_QUERY 0x79716863 //chqy -#define MC_RMT_CHUNK_QUERY_HIT 0x68716883 //chqh +#define MC_RMT_CHUNK_QUERY_HIT 0x68716863 //chqh #define MC_RMT_CHUNK_REQUEST 0x71726863 //chrq #define MC_RMT_CHUNK_RESPONSE 0x73726863 //chrs #define MC_RMT_ADD_RESPONSE 0x00800001 diff --git a/src/rpc/rpccache.cpp b/src/rpc/rpccache.cpp index c8a5923a..dd6de530 100644 --- a/src/rpc/rpccache.cpp +++ b/src/rpc/rpccache.cpp @@ -202,6 +202,8 @@ Value offchain(const Array& params, bool fHelp) int chunk_size=0; mc_EntityDetails stream_entity; + uint256 txid=0; + int vout=0; mc_TxEntity entity; entity.Zero(); BOOST_FOREACH(const Pair& dd, cd.get_obj()) @@ -216,6 +218,18 @@ Value offchain(const Array& params, bool fHelp) { chunk_hash = ParseHashV(dd.value_.get_str(), "hash"); } + if(dd.name_ == "txid") + { + txid = ParseHashV(dd.value_.get_str(), "hash"); + } + if(dd.name_ == "vout") + { + if(dd.value_.type() != int_type) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid vout"); + } + vout=dd.value_.get_int(); + } if(dd.name_ == "size") { if(dd.value_.type() != int_type) @@ -237,8 +251,7 @@ Value offchain(const Array& params, bool fHelp) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Missing stream"); } - uint256 txid=0; - collector.InsertChunk((unsigned char*)&chunk_hash,&entity,(unsigned char*)&txid,0,chunk_size); + collector.InsertChunk((unsigned char*)&chunk_hash,&entity,(unsigned char*)&txid,vout,chunk_size); } } } diff --git a/src/wallet/chunkcollector.cpp b/src/wallet/chunkcollector.cpp index 2e9d3003..35711344 100644 --- a/src/wallet/chunkcollector.cpp +++ b/src/wallet/chunkcollector.cpp @@ -53,6 +53,7 @@ void mc_ChunkCollector::Zero() m_TotalDBSize=m_KeyDBSize+m_ValueDBSize; m_Name[0]=0; m_DBName[0]=0; + m_NextAutoCommitTimestamp=0; m_NextTryTimestamp=0; m_MarkPool=NULL; m_MemPool=NULL; @@ -61,6 +62,7 @@ void mc_ChunkCollector::Zero() m_MemPool2=NULL; m_MaxMemPoolSize=MC_CCW_DEFAULT_MEMPOOL_SIZE; + m_AutoCommitDelay=MC_CCW_DEFAULT_AUTOCOMMIT_DELAY; m_Semaphore=NULL; m_LockedBy=0; @@ -147,7 +149,7 @@ void mc_ChunkCollector::GetDBRow(mc_ChunkCollectorRow* collect_row) { collect_row->Zero(); collect_row->m_DBNextAttempt=mc_SwapBytes32(m_DBRow.m_QueryNextAttempt); - collect_row->m_State.m_QueryAttempts=collect_row->m_DBNextAttempt; + collect_row->m_State.m_QueryNextAttempt=collect_row->m_DBNextAttempt; collect_row->m_Vout=m_DBRow.m_Vout; memcpy(collect_row->m_TxID,m_DBRow.m_TxID,MC_TDB_TXID_SIZE); memcpy(&(collect_row->m_ChunkDef.m_Entity),&(m_DBRow.m_Entity),sizeof(mc_TxEntity)); @@ -178,13 +180,13 @@ int mc_ChunkCollector::UpdateDBRow(mc_ChunkCollectorRow *collect_row) int err; collect_row->m_State.m_Status &= MC_CCF_ERROR_MASK; SetDBRow(collect_row); - err=m_DB->Delete((char*)&m_DBRow+m_KeyOffset,m_KeySize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + err=m_DB->Delete((char*)&m_DBRow+m_KeyDBOffset,m_KeyDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); if(err) { return err; } collect_row->m_DBNextAttempt=collect_row->m_State.m_QueryNextAttempt; - err=m_DBRow.m_QueryNextAttempt=mc_SwapBytes32(collect_row->m_DBNextAttempt); + m_DBRow.m_QueryNextAttempt=mc_SwapBytes32(collect_row->m_DBNextAttempt); collect_row->m_State.m_Status |= MC_CCF_INSERTED; return m_DB->Write((char*)&m_DBRow+m_KeyDBOffset,m_KeyDBSize,(char*)&m_DBRow+m_ValueDBOffset,m_ValueDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); } @@ -210,7 +212,7 @@ int mc_ChunkCollector::SeekDB(void *dbrow) int mc_ChunkCollector::ReadFromDB(mc_Buffer *mempool,int rows) { - int err; + int err,mprow; mc_ChunkCollectorRow collect_row; mc_ChunkDBRow chunk_def; unsigned char *ptr; @@ -240,8 +242,12 @@ int mc_ChunkCollector::ReadFromDB(mc_Buffer *mempool,int rows) { collect_row.m_State.m_Status |= MC_CCF_DELETED; } - mempool->Add(&collect_row); - row++; + mprow=mempool->Seek(&collect_row); + if(mprow < 0) + { + mempool->Add(&collect_row); + row++; + } } else { @@ -326,7 +332,7 @@ int mc_ChunkCollector::Initialize(mc_ChunkDB *chunk_db,const char *name,uint32_t mc_GetFullFileName(name,"chunks/collect",".db",MC_FOM_RELATIVE_TO_DATADIR | MC_FOM_CREATE_DIR,m_DBName); - m_DB->SetOption("KeySize",0,m_KeySize); + m_DB->SetOption("KeySize",0,m_KeyDBSize); m_DB->SetOption("ValueSize",0,m_ValueDBSize); @@ -604,6 +610,7 @@ int mc_ChunkCollector::CommitInternal() int i; mc_ChunkCollectorRow *row; int err,commit_required; + uint32_t time_now; err=MC_ERR_NOERROR; @@ -614,6 +621,8 @@ int mc_ChunkCollector::CommitInternal() Dump("Before Commit"); + time_now=mc_TimeNowAsUInt(); + if(m_MemPool == m_MemPool1) { m_MemPoolNext=m_MemPool2; @@ -664,7 +673,10 @@ int mc_ChunkCollector::CommitInternal() row->m_State.m_Status |= MC_CCF_INSERTED; } */ - m_MemPoolNext->Add(row); + if(!row->m_State.m_Query.IsZero() || (row->m_State.m_QueryNextAttempt <= time_now)) + { + m_MemPoolNext->Add(row); + } } } @@ -679,8 +691,9 @@ int mc_ChunkCollector::CommitInternal() if(m_MemPoolNext->GetCount() < m_MaxMemPoolSize) { + m_LastDBRow.Zero(); err=SeekDB(&m_LastDBRow); - if(err != MC_ERR_NOERROR) + if(err == MC_ERR_NOERROR) { ReadFromDB(m_MemPoolNext,m_MaxMemPoolSize); } diff --git a/src/wallet/chunkcollector.h b/src/wallet/chunkcollector.h index 3e7b3a33..208a717a 100644 --- a/src/wallet/chunkcollector.h +++ b/src/wallet/chunkcollector.h @@ -21,7 +21,8 @@ #define MC_CCW_TIMEOUT_QUERY 14 #define MC_CCW_TIMEOUT_REQUEST 3 -#define MC_CCW_MAX_CHUNKS_PER_QUERY 64 +#define MC_CCW_MAX_CHUNKS_PER_QUERY 4 +#define MC_CCW_DEFAULT_AUTOCOMMIT_DELAY 200 #define MC_CCW_WORST_RESPONSE_SCORE 1000 #define MC_CCW_DEFAULT_MEMPOOL_SIZE 1000 @@ -98,6 +99,8 @@ typedef struct mc_ChunkCollector uint32_t m_ValueDBSize; uint32_t m_TotalSize; uint32_t m_TotalDBSize; + int64_t m_AutoCommitDelay; + int64_t m_NextAutoCommitTimestamp; int64_t m_NextTryTimestamp; int m_MaxMemPoolSize; From e4101148ad97e0280e94ff7da6eb79f8c6106bdb Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 1 May 2018 17:15:45 +0300 Subject: [PATCH 042/157] Offchain items, API tweaks --- src/Makefile.am | 1 + src/chainparams/state.h | 1 + src/core/main.cpp | 11 +- src/protocol/handshake.cpp | 4 + src/rpc/rpccache.cpp | 411 ------------------------------------ src/rpc/rpcclient.cpp | 4 +- src/rpc/rpcdebug.cpp | 418 +++++++++++++++++++++++++++++++++++++ src/rpc/rpclist.cpp | 2 +- src/rpc/rpcrawdata.cpp | 2 +- src/rpc/rpcserver.cpp | 3 +- src/rpc/rpcserver.h | 2 +- src/rpc/rpcstreams.cpp | 2 +- src/rpc/rpcutils.cpp | 24 ++- 13 files changed, 456 insertions(+), 429 deletions(-) create mode 100644 src/rpc/rpcdebug.cpp diff --git a/src/Makefile.am b/src/Makefile.am index 102b99aa..9258b421 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -193,6 +193,7 @@ libbitcoin_server_a_SOURCES = \ rpc/rpcrawdata.cpp \ rpc/rpcutils.cpp \ rpc/rpccache.cpp \ + rpc/rpcdebug.cpp \ rpc/rpchelp.cpp \ rpc/rpcblockchain.cpp \ rpc/rpcmining.cpp \ diff --git a/src/chainparams/state.h b/src/chainparams/state.h index 58af4042..d3f6436c 100644 --- a/src/chainparams/state.h +++ b/src/chainparams/state.h @@ -35,6 +35,7 @@ #define MC_NPS_MINING 0x00000004 #define MC_NPS_REACCEPT 0x00000008 #define MC_NPS_OFFCHAIN 0x00000010 +#define MC_NPS_CHUNKS 0x00000020 #define MC_NPS_ALL 0xFFFFFFFF #define MC_WMD_NONE 0x00000000 diff --git a/src/core/main.cpp b/src/core/main.cpp index acd3d048..bf5a9696 100644 --- a/src/core/main.cpp +++ b/src/core/main.cpp @@ -5785,12 +5785,15 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, CValidationState state; if(pRelayManager) { - if(!pRelayManager->ProcessRelay(pfrom,vRecv,state,MC_VRA_DEFAULT)) + if( (mc_gState->m_NodePausedState & MC_NPS_OFFCHAIN) == 0 ) { - int nDos = 0; - if (state.IsInvalid(nDos) && nDos > 0) + if(!pRelayManager->ProcessRelay(pfrom,vRecv,state,MC_VRA_DEFAULT)) { - Misbehaving(pfrom->GetId(), nDos); + int nDos = 0; + if (state.IsInvalid(nDos) && nDos > 0) + { + Misbehaving(pfrom->GetId(), nDos); + } } } } diff --git a/src/protocol/handshake.cpp b/src/protocol/handshake.cpp index 30f42ab3..3b25285f 100644 --- a/src/protocol/handshake.cpp +++ b/src/protocol/handshake.cpp @@ -117,6 +117,10 @@ bool MultichainNode_CollectChunks() { return false; } + if(mc_gState->m_NodePausedState & MC_NPS_CHUNKS) + { + return false; + } return true; } diff --git a/src/rpc/rpccache.cpp b/src/rpc/rpccache.cpp index dd6de530..f98aacc6 100644 --- a/src/rpc/rpccache.cpp +++ b/src/rpc/rpccache.cpp @@ -1,12 +1,7 @@ // Copyright (c) 2014-2017 Coin Sciences Ltd // MultiChain code distributed under the GPLv3 license, see COPYING file. -#include "core/init.h" #include "rpc/rpcutils.h" -#include "protocol/relay.h" -#include "wallet/wallettxs.h" -#include "net/net.h" -void parseStreamIdentifier(Value stream_identifier,mc_EntityDetails *entity); Value createbinarycache(const Array& params, bool fHelp) @@ -117,409 +112,3 @@ Value deletebinarycache(const Array& params, bool fHelp) } -Value offchain(const Array& params, bool fHelp) -{ - if (fHelp || params.size() < 2 || params.size() > 4) - throw runtime_error("Help message not found\n"); - - uint32_t request_type=MC_RMT_NONE; - int timeout=5; - vector payload; - mc_OffchainMessageID request_id; - CNode *pto=NULL; - string request_str=""; - mc_RelayRequest *request; - mc_RelayResponse *response; - int attempts,delay; - Object res; - mc_ChunkCollector collector; - bool res_found=false; - - if(params[0].type() == str_type) - { - if(params[0].get_str() == "findaddress") - { - request_type=MC_RMT_MC_ADDRESS_QUERY; - request_str="query for address "; - } - if(params[0].get_str() == "getchunks") - { - request_type=MC_RMT_SPECIAL_COLLECT_CHUNKS; - request_str="query for address "; - } - if(params[0].get_str() == "viewchunks") - { - request_type=MC_RMT_SPECIAL_VIEW_CHUNKS; - } - } - - if(request_type == MC_RMT_NONE) - { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid request type"); - } - - if(params[1].type() != obj_type) - { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid request details, should be object"); - } - - if(request_type & MC_RMT_SPECIAL_MASK) - { - switch(request_type) - { - case MC_RMT_SPECIAL_COLLECT_CHUNKS: - collector.Initialize(NULL,NULL,0); - attempts=10; - delay=1000; - BOOST_FOREACH(const Pair& d, params[1].get_obj()) - { - if(d.name_ == "attempts") - { - if(d.value_.type() == int_type) - { - attempts=d.value_.get_int(); - } - } - if(d.name_ == "delay") - { - if(d.value_.type() == int_type) - { - delay=d.value_.get_int(); - } - } - if(d.name_ == "chunks") - { - if(d.value_.type() == array_type) - { - for(int c=0;c<(int)d.value_.get_array().size();c++) - { - Value cd=d.value_.get_array()[c]; - if(cd.type() != obj_type) - { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid chunk"); - } - uint256 chunk_hash=0; - int chunk_size=0; - mc_EntityDetails stream_entity; - - uint256 txid=0; - int vout=0; - mc_TxEntity entity; - entity.Zero(); - BOOST_FOREACH(const Pair& dd, cd.get_obj()) - { - if(dd.name_ == "stream") - { - parseStreamIdentifier(dd.value_.get_str(),&stream_entity); - memcpy(&entity,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,MC_AST_SHORT_TXID_SIZE); - entity.m_EntityType=MC_TET_STREAM; - } - if(dd.name_ == "hash") - { - chunk_hash = ParseHashV(dd.value_.get_str(), "hash"); - } - if(dd.name_ == "txid") - { - txid = ParseHashV(dd.value_.get_str(), "hash"); - } - if(dd.name_ == "vout") - { - if(dd.value_.type() != int_type) - { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid vout"); - } - vout=dd.value_.get_int(); - } - if(dd.name_ == "size") - { - if(dd.value_.type() != int_type) - { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid size"); - } - chunk_size=dd.value_.get_int(); - } - } - if(chunk_hash == 0) - { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Missing hash"); - } - if(chunk_size == 0) - { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Missing size"); - } - if(entity.m_EntityType == MC_TET_NONE) - { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Missing stream"); - } - collector.InsertChunk((unsigned char*)&chunk_hash,&entity,(unsigned char*)&txid,vout,chunk_size); - } - } - } - } - - if(collector.m_MemPool->GetCount() == 0) - { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Missing chunks"); - } - - break; - case MC_RMT_SPECIAL_VIEW_CHUNKS: - Array arr_res; - BOOST_FOREACH(const Pair& d, params[1].get_obj()) - { - if(d.name_ == "chunks") - { - if(d.value_.type() == array_type) - { - for(int c=0;c<(int)d.value_.get_array().size();c++) - { - Value cd=d.value_.get_array()[c]; - if(cd.type() != obj_type) - { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid chunk"); - } - uint256 chunk_hash=0; - BOOST_FOREACH(const Pair& dd, cd.get_obj()) - { - if(dd.name_ == "hash") - { - chunk_hash = ParseHashV(dd.value_.get_str(), "hash"); - } - } - if(chunk_hash == 0) - { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Missing hash"); - } - - unsigned char *chunk_found; - size_t chunk_bytes; - mc_ChunkDBRow chunk_def; - - Object chunk_obj; - chunk_obj.push_back(Pair("hash",chunk_hash.ToString())); - - if(pwalletTxsMain->m_ChunkDB->GetChunkDef(&chunk_def,(unsigned char *)&chunk_hash,NULL,NULL,-1) == MC_ERR_NOERROR) - { - chunk_found=pwalletTxsMain->m_ChunkDB->GetChunk(&chunk_def,0,-1,&chunk_bytes); - if(chunk_found) - { - chunk_obj.push_back(Pair("size",chunk_bytes)); - chunk_obj.push_back(Pair("data",HexStr(chunk_found,chunk_found+chunk_bytes))); - } - else - { - chunk_obj.push_back(Pair("error","Internal error")); - } - } - else - { - chunk_obj.push_back(Pair("error","Chunk not found")); - } - arr_res.push_back(chunk_obj); - } - } - } - } - return arr_res; - } - } - else - { - switch(request_type) - { - case MC_RMT_MC_ADDRESS_QUERY: - string addr_to_find=""; - BOOST_FOREACH(const Pair& d, params[1].get_obj()) - { - if(d.name_ == "address") - { - if(d.value_.type() ==str_type) - { - addr_to_find=d.value_.get_str(); - request_str+=addr_to_find; - } - } - } - - CBitcoinAddress address(addr_to_find); - CKeyID keyID; - if (!address.GetKeyID(keyID)) - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); - - payload.resize(1+sizeof(CKeyID)); - payload[0]=MC_RDT_MC_ADDRESS; - memcpy(&payload[1],&keyID,sizeof(CKeyID)); - break; - } - } - - if(params.size() > 2) - { - if(params[2].type() != int_type) - { - timeout=params[2].get_int(); - if( (timeout <= 0) || (timeout > 15 ) ) - { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid timeout"); - } - } - else - { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid timeout"); - } - } - - if(request_type & MC_RMT_SPECIAL_MASK) - { - int remaining=0; - for(int a=0;a 3) - { - if(params[3].type() != str_type) - { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid destination"); - } - - BOOST_FOREACH(CNode* pnode, vNodes) - { - CNodeStats stats; - pnode->copyStats(stats); - if( (params[3].get_str() == stats.addrName) || - (params[3].get_str() == CBitcoinAddress(stats.kAddrRemote).ToString()) ) - { - pto=pnode; - } - } - } - - if(pto) - { - request_str="Sending " + request_str + strprintf(" to node %d",pto->GetId()); - } - else - { - request_str="Broadcasting " + request_str; - } - } - - request_id=pRelayManager->SendRequest(pto,request_type,0,payload); - printf("%s",request_str.c_str()); - printf(". Request ID: %s\n",request_id.ToString().c_str()); - } - - uint32_t time_now; - uint32_t time_stop; - - time_now=mc_TimeNowAsUInt(); - time_stop=time_now+timeout; - res_found=false; - - while(time_nowFindRequest(request_id); - if(request) - { - if(request->m_Responses.size()) - { - switch(request_type) - { - case MC_RMT_MC_ADDRESS_QUERY: - mc_NodeFullAddress node_addr; - int count,shift; - bool take_it=true; - unsigned char *ptr; - unsigned char *ptrEnd; - for(int r=0;r<(int)request->m_Responses.size();r++) - { - if(!res_found) - { - response=&(request->m_Responses[r]); - ptr=&(response->m_Payload[0]); - ptrEnd=ptr+response->m_Payload.size(); - while( (ptr (ptrEnd-ptr)) - { - take_it=false; - } - else - { - node_addr.m_Address=*(CKeyID*)ptr; - ptr+=sizeof(CKeyID); - } - break; - case MC_RDT_NET_ADDRESS: - ptr++; - count=(int)mc_GetVarInt(ptr,ptrEnd-ptr,-1,&shift); - ptr+=shift; - if(count*(int)sizeof(CAddress) > (ptrEnd-ptr)) - { - take_it=false; - } - else - { - for(int a=0;am_Source ? false : true)); - res_found=true; - } - } - } - - break; - } - } - pRelayManager->UnLock(); - if(res_found) - { - pRelayManager->DeleteRequest(request_id); - return res; - } - } - __US_Sleep(100); - time_now=mc_TimeNowAsUInt(); - } - - return Value::null; -} diff --git a/src/rpc/rpcclient.cpp b/src/rpc/rpcclient.cpp index ecb76abd..ea08effc 100644 --- a/src/rpc/rpcclient.cpp +++ b/src/rpc/rpcclient.cpp @@ -36,8 +36,8 @@ static const CRPCConvertParam vRPCConvertParams[] = { "send", 1 }, /* MCHN START */ // { "setruntimeparam", 1 }, - { "offchain", 1 }, - { "offchain", 2 }, + { "debug", 1 }, + { "debug", 2 }, { "createkeypairs", 0 }, { "combineunspent", 1 }, { "combineunspent", 2 }, diff --git a/src/rpc/rpcdebug.cpp b/src/rpc/rpcdebug.cpp new file mode 100644 index 00000000..19550018 --- /dev/null +++ b/src/rpc/rpcdebug.cpp @@ -0,0 +1,418 @@ +// Copyright (c) 2014-2017 Coin Sciences Ltd +// MultiChain code distributed under the GPLv3 license, see COPYING file. + +#include "core/init.h" +#include "rpc/rpcutils.h" +#include "protocol/relay.h" +#include "wallet/wallettxs.h" +#include "net/net.h" +void parseStreamIdentifier(Value stream_identifier,mc_EntityDetails *entity); + + + +Value debug(const Array& params, bool fHelp) +{ + if (fHelp || params.size() < 2 || params.size() > 4) + throw runtime_error("Help message not found\n"); + + uint32_t request_type=MC_RMT_NONE; + int timeout=5; + vector payload; + mc_OffchainMessageID request_id; + CNode *pto=NULL; + string request_str=""; + mc_RelayRequest *request; + mc_RelayResponse *response; + int attempts,delay; + Object res; + mc_ChunkCollector collector; + bool res_found=false; + + if(params[0].type() == str_type) + { + if(params[0].get_str() == "findaddress") + { + request_type=MC_RMT_MC_ADDRESS_QUERY; + request_str="query for address "; + } + if(params[0].get_str() == "getchunks") + { + request_type=MC_RMT_SPECIAL_COLLECT_CHUNKS; + request_str="query for address "; + } + if(params[0].get_str() == "viewchunks") + { + request_type=MC_RMT_SPECIAL_VIEW_CHUNKS; + } + } + + if(request_type == MC_RMT_NONE) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid request type"); + } + + if(params[1].type() != obj_type) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid request details, should be object"); + } + + if(request_type & MC_RMT_SPECIAL_MASK) + { + switch(request_type) + { + case MC_RMT_SPECIAL_COLLECT_CHUNKS: + collector.Initialize(NULL,NULL,0); + attempts=10; + delay=1000; + BOOST_FOREACH(const Pair& d, params[1].get_obj()) + { + if(d.name_ == "attempts") + { + if(d.value_.type() == int_type) + { + attempts=d.value_.get_int(); + } + } + if(d.name_ == "delay") + { + if(d.value_.type() == int_type) + { + delay=d.value_.get_int(); + } + } + if(d.name_ == "chunks") + { + if(d.value_.type() == array_type) + { + for(int c=0;c<(int)d.value_.get_array().size();c++) + { + Value cd=d.value_.get_array()[c]; + if(cd.type() != obj_type) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid chunk"); + } + uint256 chunk_hash=0; + int chunk_size=0; + mc_EntityDetails stream_entity; + + uint256 txid=0; + int vout=0; + mc_TxEntity entity; + entity.Zero(); + BOOST_FOREACH(const Pair& dd, cd.get_obj()) + { + if(dd.name_ == "stream") + { + parseStreamIdentifier(dd.value_.get_str(),&stream_entity); + memcpy(&entity,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,MC_AST_SHORT_TXID_SIZE); + entity.m_EntityType=MC_TET_STREAM; + } + if(dd.name_ == "hash") + { + chunk_hash = ParseHashV(dd.value_.get_str(), "hash"); + } + if(dd.name_ == "txid") + { + txid = ParseHashV(dd.value_.get_str(), "hash"); + } + if(dd.name_ == "vout") + { + if(dd.value_.type() != int_type) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid vout"); + } + vout=dd.value_.get_int(); + } + if(dd.name_ == "size") + { + if(dd.value_.type() != int_type) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid size"); + } + chunk_size=dd.value_.get_int(); + } + } + if(chunk_hash == 0) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Missing hash"); + } + if(chunk_size == 0) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Missing size"); + } + if(entity.m_EntityType == MC_TET_NONE) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Missing stream"); + } + collector.InsertChunk((unsigned char*)&chunk_hash,&entity,(unsigned char*)&txid,vout,chunk_size); + } + } + } + } + + if(collector.m_MemPool->GetCount() == 0) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Missing chunks"); + } + + break; + case MC_RMT_SPECIAL_VIEW_CHUNKS: + Array arr_res; + BOOST_FOREACH(const Pair& d, params[1].get_obj()) + { + if(d.name_ == "chunks") + { + if(d.value_.type() == array_type) + { + for(int c=0;c<(int)d.value_.get_array().size();c++) + { + Value cd=d.value_.get_array()[c]; + if(cd.type() != obj_type) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid chunk"); + } + uint256 chunk_hash=0; + BOOST_FOREACH(const Pair& dd, cd.get_obj()) + { + if(dd.name_ == "hash") + { + chunk_hash = ParseHashV(dd.value_.get_str(), "hash"); + } + } + if(chunk_hash == 0) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Missing hash"); + } + + unsigned char *chunk_found; + size_t chunk_bytes; + mc_ChunkDBRow chunk_def; + + Object chunk_obj; + chunk_obj.push_back(Pair("hash",chunk_hash.ToString())); + + if(pwalletTxsMain->m_ChunkDB->GetChunkDef(&chunk_def,(unsigned char *)&chunk_hash,NULL,NULL,-1) == MC_ERR_NOERROR) + { + chunk_found=pwalletTxsMain->m_ChunkDB->GetChunk(&chunk_def,0,-1,&chunk_bytes); + if(chunk_found) + { + chunk_obj.push_back(Pair("size",chunk_bytes)); + chunk_obj.push_back(Pair("data",HexStr(chunk_found,chunk_found+chunk_bytes))); + } + else + { + chunk_obj.push_back(Pair("error","Internal error")); + } + } + else + { + chunk_obj.push_back(Pair("error","Chunk not found")); + } + arr_res.push_back(chunk_obj); + } + } + } + } + return arr_res; + } + } + else + { + switch(request_type) + { + case MC_RMT_MC_ADDRESS_QUERY: + string addr_to_find=""; + BOOST_FOREACH(const Pair& d, params[1].get_obj()) + { + if(d.name_ == "address") + { + if(d.value_.type() ==str_type) + { + addr_to_find=d.value_.get_str(); + request_str+=addr_to_find; + } + } + } + + CBitcoinAddress address(addr_to_find); + CKeyID keyID; + if (!address.GetKeyID(keyID)) + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + + payload.resize(1+sizeof(CKeyID)); + payload[0]=MC_RDT_MC_ADDRESS; + memcpy(&payload[1],&keyID,sizeof(CKeyID)); + break; + } + } + + if(params.size() > 2) + { + if(params[2].type() != int_type) + { + timeout=params[2].get_int(); + if( (timeout <= 0) || (timeout > 15 ) ) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid timeout"); + } + } + else + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid timeout"); + } + } + + if(request_type & MC_RMT_SPECIAL_MASK) + { + int remaining=0; + for(int a=0;a 3) + { + if(params[3].type() != str_type) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid destination"); + } + + BOOST_FOREACH(CNode* pnode, vNodes) + { + CNodeStats stats; + pnode->copyStats(stats); + if( (params[3].get_str() == stats.addrName) || + (params[3].get_str() == CBitcoinAddress(stats.kAddrRemote).ToString()) ) + { + pto=pnode; + } + } + } + + if(pto) + { + request_str="Sending " + request_str + strprintf(" to node %d",pto->GetId()); + } + else + { + request_str="Broadcasting " + request_str; + } + } + + request_id=pRelayManager->SendRequest(pto,request_type,0,payload); + printf("%s",request_str.c_str()); + printf(". Request ID: %s\n",request_id.ToString().c_str()); + } + + uint32_t time_now; + uint32_t time_stop; + + time_now=mc_TimeNowAsUInt(); + time_stop=time_now+timeout; + res_found=false; + + while(time_nowFindRequest(request_id); + if(request) + { + if(request->m_Responses.size()) + { + switch(request_type) + { + case MC_RMT_MC_ADDRESS_QUERY: + mc_NodeFullAddress node_addr; + int count,shift; + bool take_it=true; + unsigned char *ptr; + unsigned char *ptrEnd; + for(int r=0;r<(int)request->m_Responses.size();r++) + { + if(!res_found) + { + response=&(request->m_Responses[r]); + ptr=&(response->m_Payload[0]); + ptrEnd=ptr+response->m_Payload.size(); + while( (ptr (ptrEnd-ptr)) + { + take_it=false; + } + else + { + node_addr.m_Address=*(CKeyID*)ptr; + ptr+=sizeof(CKeyID); + } + break; + case MC_RDT_NET_ADDRESS: + ptr++; + count=(int)mc_GetVarInt(ptr,ptrEnd-ptr,-1,&shift); + ptr+=shift; + if(count*(int)sizeof(CAddress) > (ptrEnd-ptr)) + { + take_it=false; + } + else + { + for(int a=0;am_Source ? false : true)); + res_found=true; + } + } + } + + break; + } + } + pRelayManager->UnLock(); + if(res_found) + { + pRelayManager->DeleteRequest(request_id); + return res; + } + } + __US_Sleep(100); + time_now=mc_TimeNowAsUInt(); + } + + return Value::null; +} diff --git a/src/rpc/rpclist.cpp b/src/rpc/rpclist.cpp index 6ab5e9b2..da6dd990 100644 --- a/src/rpc/rpclist.cpp +++ b/src/rpc/rpclist.cpp @@ -121,7 +121,7 @@ static const CRPCCommand vRPCCommands[] = { "rawtransactions", "appendrawchange", &appendrawchange, false, false, true }, { "hidden", "appendrawmetadata", &appendrawmetadata, false, false, true }, { "rawtransactions", "appendrawdata", &appendrawmetadata, false, false, true }, - { "hidden", "offchain", &offchain, false, true, true }, + { "hidden", "debug", &debug, false, true, true }, /* MCHN END */ /* Utility functions */ diff --git a/src/rpc/rpcrawdata.cpp b/src/rpc/rpcrawdata.cpp index 5a00ff21..b3ada1ac 100644 --- a/src/rpc/rpcrawdata.cpp +++ b/src/rpc/rpcrawdata.cpp @@ -290,7 +290,7 @@ vector ParseRawFormattedData(const Value *value,uint32_t *data_fo { if(total_size > MAX_OP_RETURN_RELAY) { - *strError="Cannot read binary cache item"; + *strError="Binary cache item too big"; *errorCode=RPC_NOT_SUPPORTED; close(fHan); } diff --git a/src/rpc/rpcserver.cpp b/src/rpc/rpcserver.cpp index 0a602ff3..c8fa5c60 100644 --- a/src/rpc/rpcserver.cpp +++ b/src/rpc/rpcserver.cpp @@ -341,7 +341,7 @@ Value stop(const Array& params, bool fHelp) string AllowedPausedServices() { - string ret="incoming,mining"; + string ret="incoming,mining,offchain"; return ret; } @@ -372,6 +372,7 @@ uint32_t GetPausedServices(const char *str) if(memcmp(start,"mining", ptr-start) == 0)type = MC_NPS_MINING; if(memcmp(start,"reaccepting", ptr-start) == 0)type = MC_NPS_REACCEPT; if(memcmp(start,"offchain", ptr-start) == 0)type = MC_NPS_OFFCHAIN; + if(memcmp(start,"chunks", ptr-start) == 0)type = MC_NPS_CHUNKS; if(type == 0) { diff --git a/src/rpc/rpcserver.h b/src/rpc/rpcserver.h index c830f181..00c1ba59 100644 --- a/src/rpc/rpcserver.h +++ b/src/rpc/rpcserver.h @@ -192,7 +192,7 @@ extern json_spirit::Value getaccount(const json_spirit::Array& params, bool fHel extern json_spirit::Value getaddressesbyaccount(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value sendtoaddress(const json_spirit::Array& params, bool fHelp); /* MCHN START */ -extern json_spirit::Value offchain(const json_spirit::Array& params, bool fHelp); +extern json_spirit::Value debug(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value createkeypairs(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value getaddresses(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value createbinarycache(const json_spirit::Array& params, bool fHelp); diff --git a/src/rpc/rpcstreams.cpp b/src/rpc/rpcstreams.cpp index 0a46a8a9..9431c86f 100644 --- a/src/rpc/rpcstreams.cpp +++ b/src/rpc/rpcstreams.cpp @@ -608,7 +608,7 @@ Value publishfrom(const Array& params, bool fHelp) int errorCode=RPC_INVALID_PARAMETER; vector vChunkHashes; - dataData=ParseRawFormattedData(&(params[3]),&data_format,lpDetailsScript,MC_RFD_OPTION_NONE,&out_options,&errorCode,&strError); + dataData=ParseRawFormattedData(&(params[3]),&data_format,lpDetailsScript,in_options,&out_options,&errorCode,&strError); if(strError.size()) { diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index 6495725e..3e17316c 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -1325,21 +1325,24 @@ Value DataItemEntry(const CTransaction& tx,int n,set & already_seen,uin } format_item_value=OpReturnFormatEntry(elem,out_size,tx.GetHash(),n,format,&format_text_str,retrieve_status); + Array chunks; if(retrieve_status & MC_OST_CONTROL_NO_DATA) { if(format_item_value.type() == obj_type) { - int chunk_shift; - mc_GetVarInt(chunk_hashes,MC_CDB_CHUNK_HASH_SIZE+16,-1,&chunk_shift); - chunk_hashes+=chunk_shift; - Array chunks; - for(int chunk=0;chunkToString()); + int chunk_shift,chunk_size; + Object chunk_obj; + + chunk_size=mc_GetVarInt(chunk_hashes,MC_CDB_CHUNK_HASH_SIZE+16,-1,&chunk_shift); + chunk_hashes+=chunk_shift; + chunk_obj.push_back(Pair("hash", ((uint256*)chunk_hashes)->ToString())); + chunk_obj.push_back(Pair("size", chunk_size)); + chunks.push_back(chunk_obj); chunk_hashes+=MC_CDB_CHUNK_HASH_SIZE; } - format_item_value.get_obj().push_back(Pair("chunks", chunks)); +// format_item_value.get_obj().push_back(Pair("chunks", chunks)); } } @@ -1385,6 +1388,13 @@ Value DataItemEntry(const CTransaction& tx,int n,set & already_seen,uin } entry.push_back(Pair("data", format_item_value)); entry.push_back(Pair("offchain", (retrieve_status & MC_OST_STORAGE_MASK) == MC_OST_OFF_CHAIN)); + if(retrieve_status & MC_OST_CONTROL_NO_DATA) + { + if((retrieve_status & MC_OST_STORAGE_MASK) == MC_OST_OFF_CHAIN) + { + entry.push_back(Pair("chunks", chunks)); + } + } return entry; } From 3e7fc1b94242c2e58d313bb75e28672fc0ae5935 Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 2 May 2018 00:46:49 +0300 Subject: [PATCH 043/157] Avoiding connecting to seed if -addnodeonly --- src/utils/utilwrapper.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/utils/utilwrapper.cpp b/src/utils/utilwrapper.cpp index 75b790bb..e26501a1 100644 --- a/src/utils/utilwrapper.cpp +++ b/src/utils/utilwrapper.cpp @@ -201,6 +201,10 @@ const char *mc_Params::SeedNode() const char* mc_State::GetSeedNode() { const char *seed_node; + if(GetBoolArg("-addnodeonly",false)) + { + return NULL; + } seed_node=mc_gState->m_Params->SeedNode(); if(seed_node == NULL) { From 77c8b1508887ae60bbdf25f57561f4e335464de7 Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 2 May 2018 08:47:04 +0300 Subject: [PATCH 044/157] Avoiding reading seed.dat if -addnodeonly --- src/utils/utilwrapper.cpp | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/src/utils/utilwrapper.cpp b/src/utils/utilwrapper.cpp index e26501a1..8de0cea6 100644 --- a/src/utils/utilwrapper.cpp +++ b/src/utils/utilwrapper.cpp @@ -144,25 +144,28 @@ void mc_Params::Parse(int argc, const char* const argv[],int exe_type) int err; const char *seed_node; - mapConfig=new mc_MapStringString; - - err=mc_ReadGeneralConfigFile(mapConfig,mc_gState->m_Params->NetworkName(),"seed",".dat"); - - if(err == MC_ERR_NOERROR) + if(!GetBoolArg("-addnodeonly",false)) { - seed_node=mapConfig->Get("seed"); + mapConfig=new mc_MapStringString; - if(seed_node) + err=mc_ReadGeneralConfigFile(mapConfig,mc_gState->m_Params->NetworkName(),"seed",".dat"); + + if(err == MC_ERR_NOERROR) { - if(strlen(seed_node) <= MC_DCT_SEED_NODE_MAX_SIZE) + seed_node=mapConfig->Get("seed"); + + if(seed_node) { - strcpy(m_Arguments[m_NumArguments],seed_node); - length+=strlen(seed_node); + if(strlen(seed_node) <= MC_DCT_SEED_NODE_MAX_SIZE) + { + strcpy(m_Arguments[m_NumArguments],seed_node); + length+=strlen(seed_node); + } } } - } - delete mapConfig; + delete mapConfig; + } m_NumArguments++; } } @@ -201,10 +204,6 @@ const char *mc_Params::SeedNode() const char* mc_State::GetSeedNode() { const char *seed_node; - if(GetBoolArg("-addnodeonly",false)) - { - return NULL; - } seed_node=mc_gState->m_Params->SeedNode(); if(seed_node == NULL) { From 4ec87ddb117e79043cf544275a15b0fc1fefc9d6 Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 2 May 2018 12:24:15 +0300 Subject: [PATCH 045/157] Offchain items logging tweaks --- src/protocol/relay.cpp | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index 48894f23..f222f5b9 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -167,6 +167,10 @@ int MultichainProcessChunkResponse(const CRelayResponsePair *response_pair,map < goto exitlbl; } } + else + { + LogPrint("chunks","Retrieved chunk %s\n",(*(uint256*)(chunk->m_Hash)).ToString().c_str()); + } collect_row->m_State.m_Status |= MC_CCF_DELETED; } @@ -590,7 +594,6 @@ bool mc_RelayProcess_Chunk_Query(unsigned char *ptrStart,unsigned char *ptrEnd,v for(int c=0;cm_ChunkDB->GetChunkDef(&chunk_def,chunk.m_Hash,NULL,NULL,-1) == MC_ERR_NOERROR) { if(chunk_def.m_Size != chunk.m_Size) @@ -653,7 +656,7 @@ bool mc_RelayProcess_Chunk_Request(unsigned char *ptrStart,unsigned char *ptrEnd for(int c=0;cm_ChunkDB->GetChunkDef(&chunk_def,chunk.m_Hash,NULL,NULL,-1) == MC_ERR_NOERROR) { if(chunk_def.m_Size != chunk.m_Size) @@ -796,7 +799,7 @@ bool MultichainRelayResponse(uint32_t msg_type_stored, CNode *pto_stored, case MC_RMT_MC_ADDRESS_QUERY: if(msg_type_stored) { - strError=strprintf("Unexpected response message type (%d,%d)",msg_type_stored,msg_type_in);; + strError=strprintf("Unexpected response message type (%s,%s)",mc_MsgTypeStr(msg_type_stored).c_str(),mc_MsgTypeStr(msg_type_in).c_str()); goto exitlbl; } if(payload_response_ptr == NULL) @@ -825,7 +828,7 @@ bool MultichainRelayResponse(uint32_t msg_type_stored, CNode *pto_stored, case MC_RMT_NODE_DETAILS: if(msg_type_stored != MC_RMT_MC_ADDRESS_QUERY) { - strError=strprintf("Unexpected response message type (%d,%d)",msg_type_stored,msg_type_in);; + strError=strprintf("Unexpected response message type (%s,%s)",mc_MsgTypeStr(msg_type_stored).c_str(),mc_MsgTypeStr(msg_type_in).c_str()); goto exitlbl; } if(payload_response_ptr) @@ -837,7 +840,7 @@ bool MultichainRelayResponse(uint32_t msg_type_stored, CNode *pto_stored, case MC_RMT_CHUNK_QUERY: if(msg_type_stored) { - strError=strprintf("Unexpected response message type (%d,%d)",msg_type_stored,msg_type_in);; + strError=strprintf("Unexpected response message type (%s,%s)",mc_MsgTypeStr(msg_type_stored).c_str(),mc_MsgTypeStr(msg_type_in).c_str()); goto exitlbl; } if(payload_response_ptr == NULL) @@ -866,7 +869,7 @@ bool MultichainRelayResponse(uint32_t msg_type_stored, CNode *pto_stored, case MC_RMT_CHUNK_QUERY_HIT: if(msg_type_stored != MC_RMT_CHUNK_QUERY) { - strError=strprintf("Unexpected response message type (%d,%d)",msg_type_stored,msg_type_in);; + strError=strprintf("Unexpected response message type (%s,%s)",mc_MsgTypeStr(msg_type_stored).c_str(),mc_MsgTypeStr(msg_type_in).c_str()); goto exitlbl; } if(payload_response_ptr) @@ -877,7 +880,7 @@ bool MultichainRelayResponse(uint32_t msg_type_stored, CNode *pto_stored, case MC_RMT_CHUNK_REQUEST: if(msg_type_stored != MC_RMT_CHUNK_QUERY_HIT) { - strError=strprintf("Unexpected response message type (%d,%d)",msg_type_stored,msg_type_in);; + strError=strprintf("Unexpected response message type (%s,%s)",mc_MsgTypeStr(msg_type_stored).c_str(),mc_MsgTypeStr(msg_type_in).c_str()); goto exitlbl; } if(payload_response_ptr == NULL) @@ -906,7 +909,7 @@ bool MultichainRelayResponse(uint32_t msg_type_stored, CNode *pto_stored, case MC_RMT_CHUNK_RESPONSE: if(msg_type_stored != MC_RMT_CHUNK_REQUEST) { - strError=strprintf("Unexpected response message type (%d,%d)",msg_type_stored,msg_type_in);; + strError=strprintf("Unexpected response message type (%s,%s)",mc_MsgTypeStr(msg_type_stored).c_str(),mc_MsgTypeStr(msg_type_in).c_str()); goto exitlbl; } if(payload_response_ptr) @@ -1855,7 +1858,8 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, } else { - LogPrintf("ProcessRelay() : Error processing %s (request %s) from peer %d: %s\n",mc_MsgTypeStr(msg_type_in).c_str(),pfrom->GetId(),strError.c_str()); + LogPrintf("ProcessRelay() : Error processing %s (request %s) from peer %d: %s\n",mc_MsgTypeStr(msg_type_in).c_str(), + msg_id_received.ToString().c_str(),pfrom->GetId(),strError.c_str()); return false; } } From 59b197aa0ca502bd7aaab1d0ef1cc2c6b35cf992 Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 2 May 2018 12:26:53 +0300 Subject: [PATCH 046/157] Offchain items, logging tweaks --- src/protocol/relay.cpp | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index f222f5b9..cd35d890 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -1543,7 +1543,7 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, vRecv >> msg_format; if(msg_format != 0) { - LogPrintf("ProcessRelay() : Unsupported message format %08X\n",msg_format); + LogPrintf("ProcessOffchain() : Unsupported message format %08X\n",msg_format); return false; } @@ -1552,7 +1552,7 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, vRecv >> vSendPaths; if(vSendPaths.size()) { - LogPrintf("ProcessRelay() : Unsupported send path\n"); + LogPrintf("ProcessOffchain() : Unsupported send path\n"); return false; } vRecv >> msg_type_in; @@ -1579,7 +1579,7 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, default: if(verify_flags & MC_VRA_MESSAGE_TYPE) { - LogPrintf("ProcessRelay() : Unsupported relay message type %s\n",mc_MsgTypeStr(msg_type_in).c_str()); + LogPrintf("ProcessOffchain() : Unsupported relay message type %s\n",mc_MsgTypeStr(msg_type_in).c_str()); return false; } break; @@ -1589,7 +1589,7 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, { if(hop_count) { - LogPrintf("ProcessRelay() : Unsupported hop count %d for msg type %s\n",hop_count,mc_MsgTypeStr(msg_type_in).c_str()); + LogPrintf("ProcessOffchain() : Unsupported hop count %d for msg type %s\n",hop_count,mc_MsgTypeStr(msg_type_in).c_str()); return false; } } @@ -1598,7 +1598,7 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, { if(hop_count > 1) { - LogPrintf("ProcessRelay() : Unsupported hop count %d for msg type %s\n",hop_count,mc_MsgTypeStr(msg_type_in).c_str()); + LogPrintf("ProcessOffchain() : Unsupported hop count %d for msg type %s\n",hop_count,mc_MsgTypeStr(msg_type_in).c_str()); return false; } } @@ -1611,12 +1611,12 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, { if(msg_id_received.m_TimeStamp+m_MinTimeShift < m_LastTime) { - LogPrintf("ProcessRelay() : Timestamp too far in the past: %d\n",msg_id_received.m_TimeStamp); + LogPrintf("ProcessOffchain() : Timestamp too far in the past: %d\n",msg_id_received.m_TimeStamp); return false; } if(msg_id_received.m_TimeStamp > m_LastTime + m_MaxTimeShift) { - LogPrintf("ProcessRelay() : Timestamp too far in the future: %d\n",msg_id_received.m_TimeStamp); + LogPrintf("ProcessOffchain() : Timestamp too far in the future: %d\n",msg_id_received.m_TimeStamp); return false; } } @@ -1644,7 +1644,7 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, case MC_ERR_ERROR_IN_SCRIPT: // We already processed this message, it has errors return false; case MC_ERR_NOT_ALLOWED: - LogPrintf("ProcessRelay() : Processing this message is not allowed by current limits or requesting peer was disconnected\n"); + LogPrintf("ProcessOffchain() : Processing this message is not allowed by current limits or requesting peer was disconnected\n"); return false; } } @@ -1665,7 +1665,7 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, { if( (msg_id_to_respond.m_TimeStamp != 0) || (msg_id_to_respond.m_Nonce != 0) ) { - return state.DoS(100, error("ProcessRelay() : This message should not be response"),REJECT_INVALID, "bad-nonce"); + return state.DoS(100, error("ProcessOffchain() : This message should not be response"),REJECT_INVALID, "bad-nonce"); } } @@ -1673,12 +1673,12 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, { if( msg_id_to_respond.m_TimeStamp == 0 ) { - return state.DoS(100, error("ProcessRelay() : This message should be response"),REJECT_INVALID, "bad-nonce"); + return state.DoS(100, error("ProcessOffchain() : This message should be response"),REJECT_INVALID, "bad-nonce"); } if(GetRelayRecord(pfrom,msg_id_to_respond,&msg_type_stored,&pto_stored)) { - LogPrintf("ProcessRelay() : Response without request from peer %d\n",pfrom->GetId()); + LogPrintf("ProcessOffchain() : Response without request from peer %d\n",pfrom->GetId()); return false; } } @@ -1749,7 +1749,7 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, { if(vSigScripts.size() < 1) { - return state.DoS(100, error("ProcessRelay() : Missing sigScript"),REJECT_INVALID, "bad-sigscript"); + return state.DoS(100, error("ProcessOffchain() : Missing sigScript"),REJECT_INVALID, "bad-sigscript"); } if(vSigScripts[0].size()) { @@ -1761,19 +1761,19 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, if (!scriptSig.GetOp(pc, opcode, vchSigOut)) { - return state.DoS(100, error("ProcessRelay() : Cannot extract signature from sigScript"),REJECT_INVALID, "bad-sigscript-signature"); + return state.DoS(100, error("ProcessOffchain() : Cannot extract signature from sigScript"),REJECT_INVALID, "bad-sigscript-signature"); } vchSigOut.resize(vchSigOut.size()-1); if (!scriptSig.GetOp(pc, opcode, vchPubKey)) { - return state.DoS(100, error("ProcessRelay() : Cannot extract pubkey from sigScript"),REJECT_INVALID, "bad-sigscript-pubkey"); + return state.DoS(100, error("ProcessOffchain() : Cannot extract pubkey from sigScript"),REJECT_INVALID, "bad-sigscript-pubkey"); } CPubKey pubKeyOut(vchPubKey); if (!pubKeyOut.IsValid()) { - return state.DoS(100, error("ProcessRelay() : Invalid pubkey"),REJECT_INVALID, "bad-sigscript-pubkey"); + return state.DoS(100, error("ProcessOffchain() : Invalid pubkey"),REJECT_INVALID, "bad-sigscript-pubkey"); } pubkey=pubKeyOut; @@ -1786,12 +1786,12 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, if(!pubkey.Verify(signed_hash,vchSigOut)) { - return state.DoS(100, error("ProcessRelay() : Wrong signature"),REJECT_INVALID, "bad-signature"); + return state.DoS(100, error("ProcessOffchain() : Wrong signature"),REJECT_INVALID, "bad-signature"); } } else { - return state.DoS(100, error("ProcessRelay() : Empty sigScript"),REJECT_INVALID, "bad-sigscript"); + return state.DoS(100, error("ProcessOffchain() : Empty sigScript"),REJECT_INVALID, "bad-sigscript"); } } } @@ -1838,7 +1838,7 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, } else { - LogPrintf("ProcessRelay() : Response without stored request from peer %d\n",pfrom->GetId()); + LogPrintf("ProcessOffchain() : Response without stored request from peer %d\n",pfrom->GetId()); return false; } } @@ -1858,7 +1858,7 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, } else { - LogPrintf("ProcessRelay() : Error processing %s (request %s) from peer %d: %s\n",mc_MsgTypeStr(msg_type_in).c_str(), + LogPrintf("ProcessOffchain() : Error processing %s (request %s) from peer %d: %s\n",mc_MsgTypeStr(msg_type_in).c_str(), msg_id_received.ToString().c_str(),pfrom->GetId(),strError.c_str()); return false; } From d8c102ba0659f02f31c65a533b79848dc0f171f6 Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 2 May 2018 17:21:25 +0300 Subject: [PATCH 047/157] Fixed fetching multiple chunks in one message --- src/chainparams/globals.h | 1 + src/core/main.h | 1 + src/protocol/relay.cpp | 5 +++-- src/utils/utilwrapper.cpp | 4 ++++ src/wallet/chunkcollector.h | 4 ++-- 5 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/chainparams/globals.h b/src/chainparams/globals.h index fd1baabe..03aa51c0 100644 --- a/src/chainparams/globals.h +++ b/src/chainparams/globals.h @@ -15,6 +15,7 @@ unsigned int MAX_BLOCK_SIGOPS = 20000; unsigned int MAX_TX_SIGOPS = 4000; // main.h int COINBASE_MATURITY = 100; // main.h unsigned int MAX_SIZE = 0x02000000; // serialize,h +unsigned int OFFCHAIN_MSG_PADDING = 2048; // main.h int64_t COIN = 100000000; // amount.h int64_t CENT = 1000000; // amount.h int64_t MAX_MONEY = 21000000 * COIN; // amount.h diff --git a/src/core/main.h b/src/core/main.h index 9eca8043..4095809c 100644 --- a/src/core/main.h +++ b/src/core/main.h @@ -74,6 +74,7 @@ static const unsigned int DEFAULT_MAX_SUCCESSORS_FROM_ONE_NODE = 10; extern int MAX_OP_RETURN_SHOWN; extern int MAX_FORMATTED_DATA_DEPTH; extern int MIN_BLOCKS_BETWEEN_UPGRADES; +extern unsigned int OFFCHAIN_MSG_PADDING; /* MCHN END */ /** The maximum size of a blk?????.dat file (since 0.8) */ extern unsigned int MAX_BLOCKFILE_SIZE; // MCHN global diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index cd35d890..f69af137 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -418,7 +418,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) collect_row=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(row); } - if( (collect_row == NULL)|| (last_count >= MC_CCW_MAX_CHUNKS_PER_QUERY) || (total_size+collect_row->m_ChunkDef.m_Size + sizeof(mc_ChunkEntityKey)> MAX_SIZE-48) ) + if( (collect_row == NULL)|| (last_count >= MC_CCW_MAX_CHUNKS_PER_QUERY) || (total_size+collect_row->m_ChunkDef.m_Size + sizeof(mc_ChunkEntityKey)> MAX_SIZE-OFFCHAIN_MSG_PADDING) ) { if(last_count) { @@ -489,6 +489,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) { collect_row->m_State.m_Status |= MC_CCF_SELECTED; last_count++; + total_size+=collect_row->m_ChunkDef.m_Size + sizeof(mc_ChunkEntityKey); } } } @@ -665,7 +666,7 @@ bool mc_RelayProcess_Chunk_Request(unsigned char *ptrStart,unsigned char *ptrEnd return false; } total_size+=chunk_def.m_Size+size; - if(total_size > MAX_SIZE-48) + if(total_size > MAX_SIZE-OFFCHAIN_MSG_PADDING) { strError="Total size of requested chunks is too big"; return false; diff --git a/src/utils/utilwrapper.cpp b/src/utils/utilwrapper.cpp index 8de0cea6..fa2099c6 100644 --- a/src/utils/utilwrapper.cpp +++ b/src/utils/utilwrapper.cpp @@ -791,6 +791,10 @@ int mc_MultichainParams::SetGlobals() { MAX_SIZE *= 2; } + while(MAX_CHUNK_SIZE+OFFCHAIN_MSG_PADDING>MAX_SIZE) + { + MAX_SIZE *= 2; + } MIN_BLOCKS_BETWEEN_UPGRADES=(unsigned int)GetInt64Param("timingupgrademingap"); MAX_STANDARD_TX_SIZE=(unsigned int)GetInt64Param("maxstdtxsize"); MAX_SCRIPT_ELEMENT_SIZE=(unsigned int)GetInt64Param("maxstdelementsize"); diff --git a/src/wallet/chunkcollector.h b/src/wallet/chunkcollector.h index 208a717a..9f249b99 100644 --- a/src/wallet/chunkcollector.h +++ b/src/wallet/chunkcollector.h @@ -19,8 +19,8 @@ #define MC_CCF_ERROR_MASK 0x00FF0000 #define MC_CCF_ALL 0xFFFFFFFF -#define MC_CCW_TIMEOUT_QUERY 14 -#define MC_CCW_TIMEOUT_REQUEST 3 +#define MC_CCW_TIMEOUT_QUERY 30 +#define MC_CCW_TIMEOUT_REQUEST 5 #define MC_CCW_MAX_CHUNKS_PER_QUERY 4 #define MC_CCW_DEFAULT_AUTOCOMMIT_DELAY 200 #define MC_CCW_WORST_RESPONSE_SCORE 1000 From fff83fad63b7be5047a9cdac0a3e0d1ebb315384 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 8 May 2018 13:31:51 +0300 Subject: [PATCH 048/157] Fixed unsubscribe when rows in mempool in wallettxdb --- src/wallet/wallettxdb.cpp | 25 ++++++++++++++++++------- src/wallet/wallettxdb.h | 1 + 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/wallet/wallettxdb.cpp b/src/wallet/wallettxdb.cpp index 16fb6087..3bc5554c 100644 --- a/src/wallet/wallettxdb.cpp +++ b/src/wallet/wallettxdb.cpp @@ -222,6 +222,8 @@ void mc_TxDB::Zero() m_LobFileNamePrefix[0]=0x00; m_LogFileName[0]=0x00; + m_UnsubscribeMemPoolSize=0; + m_Mode=MC_WMD_NONE; m_Semaphore=NULL; m_LockedBy=0; @@ -1609,15 +1611,21 @@ int mc_TxDB::Commit(mc_TxImport *import) goto exitlbl; } if(ptr == NULL) + { + if((imp->m_ImportID != 0) || (i>=m_UnsubscribeMemPoolSize)) // first rows may be deleted by Unsubscribe + { + err=MC_ERR_CORRUPTED; + goto exitlbl; + } + } + else { - err=MC_ERR_CORRUPTED; - goto exitlbl; + memcpy((char*)&erow+m_Database->m_ValueOffset,ptr,m_Database->m_ValueSize); + erow.m_LastSubKeyPos=lperow->m_LastSubKeyPos; + erow.SwapPosBytes(); + err=m_Database->m_DB->Write((char*)&erow+m_Database->m_KeyOffset,m_Database->m_KeySize,(char*)&erow+m_Database->m_ValueOffset,m_Database->m_ValueSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + erow.SwapPosBytes(); } - memcpy((char*)&erow+m_Database->m_ValueOffset,ptr,m_Database->m_ValueSize); - erow.m_LastSubKeyPos=lperow->m_LastSubKeyPos; - erow.SwapPosBytes(); - err=m_Database->m_DB->Write((char*)&erow+m_Database->m_KeyOffset,m_Database->m_KeySize,(char*)&erow+m_Database->m_ValueOffset,m_Database->m_ValueSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); - erow.SwapPosBytes(); } } lperow->m_Pos=lperow->m_TempPos; // m_Pos in mempool was 0 - to support search by TxID in mempool @@ -1680,6 +1688,7 @@ int mc_TxDB::Commit(mc_TxImport *import) } } + m_UnsubscribeMemPoolSize=0; FlushDataFile(m_DBStat.m_LastFileID); err=m_Database->m_DB->Commit(MC_OPT_DB_DATABASE_TRANSACTIONAL); @@ -2699,6 +2708,8 @@ int mc_TxDB::Unsubscribe(mc_Buffer* lpEntities) deleted_items=0; } + m_UnsubscribeMemPoolSize=m_MemPools[0]->GetCount(); + exitlbl: return err; diff --git a/src/wallet/wallettxdb.h b/src/wallet/wallettxdb.h index f101c642..cff9f761 100644 --- a/src/wallet/wallettxdb.h +++ b/src/wallet/wallettxdb.h @@ -238,6 +238,7 @@ typedef struct mc_TxDB char m_LobFileNamePrefix[MC_DCT_DB_MAX_PATH]; // Full data file name char m_LogFileName[MC_DCT_DB_MAX_PATH]; // Full log file name + int m_UnsubscribeMemPoolSize; // Size of the mempool when unsubscribed uint32_t m_Mode; void *m_Semaphore; // mc_TxDB object semaphore uint64_t m_LockedBy; // ID of the thread locking it From e65527fbc1071ceb798b7bb5af2511a7a69457c9 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 8 May 2018 15:59:40 +0300 Subject: [PATCH 049/157] Deleting chunks on unsubscribe --- src/entities/asset.h | 1 + src/protocol/multichainscript.cpp | 22 +- src/rpc/rpcclient.cpp | 1 + src/rpc/rpcdebug.cpp | 330 +++++++++++++++++++- src/rpc/rpcstreams.cpp | 16 +- src/utils/dbwrapper.cpp | 29 +- src/utils/dbwrapper.h | 1 + src/utils/declare.h | 3 + src/utils/tools.cpp | 11 + src/utils/utility.cpp | 24 +- src/wallet/chunkdb.cpp | 500 ++++++++++++++++++++++++------ src/wallet/chunkdb.h | 10 +- src/wallet/wallettxdb.cpp | 12 +- src/wallet/wallettxdb.h | 4 +- src/wallet/wallettxs.cpp | 20 +- src/wallet/wallettxs.h | 2 +- 16 files changed, 865 insertions(+), 121 deletions(-) diff --git a/src/entities/asset.h b/src/entities/asset.h index 79518edb..4387288a 100644 --- a/src/entities/asset.h +++ b/src/entities/asset.h @@ -69,6 +69,7 @@ #define MC_ENT_SPRM_CHUNK_SIZE 0x85 #define MC_ENT_SPRM_CHUNK_DETAILS 0x86 #define MC_ENT_SPRM_CHUNK_DATA 0x87 +#define MC_ENT_SPRM_ITEM_COUNT 0x88 #define MC_ENT_FLAG_OFFSET_IS_SET 0x00000001 diff --git a/src/protocol/multichainscript.cpp b/src/protocol/multichainscript.cpp index 9e3500d8..c81c3029 100644 --- a/src/protocol/multichainscript.cpp +++ b/src/protocol/multichainscript.cpp @@ -301,11 +301,12 @@ int mc_Script::SetParamValue(const char *param_name,const size_t param_name_size return MC_ERR_NOERROR; } -uint32_t mc_GetParamFromDetailsScript(const unsigned char *ptr,uint32_t total,uint32_t offset,uint32_t* param_value_start,size_t *bytes,int *err) +uint32_t mc_GetParamFromDetailsScriptErr(const unsigned char *ptr,uint32_t total,uint32_t offset,uint32_t* param_value_start,size_t *bytes,int *err) { int shift,name_size,value_size,size; - *param_value_start=0; + *param_value_start=total; + *bytes=0; *err=MC_ERR_NOERROR; if(offset>=total) @@ -340,6 +341,9 @@ uint32_t mc_GetParamFromDetailsScript(const unsigned char *ptr,uint32_t total,ui *err=MC_ERR_ERROR_IN_SCRIPT; return total; } + + *bytes=value_size; + *param_value_start=offset+name_size+shift; size=name_size+shift+value_size; if(offset+size>total) @@ -348,8 +352,6 @@ uint32_t mc_GetParamFromDetailsScript(const unsigned char *ptr,uint32_t total,ui return total; } - *bytes=value_size; - *param_value_start=offset+name_size+shift; return offset+size; } @@ -357,7 +359,15 @@ uint32_t mc_GetParamFromDetailsScript(const unsigned char *ptr,uint32_t total,ui uint32_t mc_GetParamFromDetailsScript(const unsigned char *ptr,uint32_t total,uint32_t offset,uint32_t* param_value_start,size_t *bytes) { int err; - return mc_GetParamFromDetailsScript(ptr,total,offset,param_value_start,bytes,&err); + uint32_t new_offset; + new_offset=mc_GetParamFromDetailsScriptErr(ptr,total,offset,param_value_start,bytes,&err); + + if(err) + { + *param_value_start=0; + *bytes=0; + } + return new_offset; } uint32_t mc_FindSpecialParamInDetailsScript(const unsigned char *ptr,uint32_t total,uint32_t param,size_t *bytes) @@ -426,7 +436,7 @@ int mc_VerifyDetailsScript(const unsigned char *script,uint32_t script_size) while(offsetSetOption("KeySize",0,key_size); + m_DB->SetOption("ValueSize",0,value_size); + + + err=m_DB->Open(m_DBName,MC_OPT_DB_DATABASE_CREATE_IF_MISSING | MC_OPT_DB_DATABASE_TRANSACTIONAL | MC_OPT_DB_DATABASE_LEVELDB); + if(err) + { + return -1; + } + + kbuf=new char[key_size]; + vbuf=new char[value_size]; + + memset(kbuf,0,key_size); + memset(vbuf,0,value_size); + + ptr=(unsigned char*)m_DB->Read(kbuf,key_size,&value_len,0,&err); + if(err) + { + return -2; + } + + size=0; + if(ptr) + { + size=mc_GetLE(ptr,8); + } + else + { + err=m_DB->Write(kbuf,key_size,vbuf,value_size,MC_OPT_DB_DATABASE_TRANSACTIONAL); + if(err) + { + return -3; + } + + err=m_DB->Commit(MC_OPT_DB_DATABASE_TRANSACTIONAL); + if(err) + { + return -4; + } + } + + delete [] kbuf; + delete [] vbuf; + + *lpDB=m_DB; + + return size; +} + +void mcd_CloseDatabase(mc_Database *m_DB) +{ + if(m_DB) + { + m_DB->Close(); + delete m_DB; + } +} + +int64_t mcd_AddRows(mc_Database *m_DB,int key_size,int value_size,int row_count,int64_t prev_rows) +{ + int err; + int per_commit_count=1000; + int commit_count=row_count/per_commit_count; + int64_t total_rows=prev_rows; + char *kbuf; + char *vbuf; + kbuf=new char[key_size]; + vbuf=new char[value_size]; + + double tb,ta; + tb=mc_TimeNowAsDouble(); + for(int c=0;cWrite(kbuf,key_size,vbuf,value_size,MC_OPT_DB_DATABASE_TRANSACTIONAL); + if(err) + { + return total_rows; + } + } + memset(kbuf,0,key_size); + memset(vbuf,0,value_size); + total_rows+=per_commit_count; + mc_PutLE(vbuf,&total_rows,8); + err=m_DB->Write(kbuf,key_size,vbuf,value_size,MC_OPT_DB_DATABASE_TRANSACTIONAL); + if(err) + { + return total_rows; + } + err=m_DB->Commit(MC_OPT_DB_DATABASE_TRANSACTIONAL); + if(err) + { + return total_rows; + } + } + ta=mc_TimeNowAsDouble(); + printf("%8.3f\n",ta-tb); + + delete [] kbuf; + delete [] vbuf; + + return total_rows; +} + +double mcd_ReadRows(mc_Database *m_DB,int key_size,int row_count,int read_type) +{ + int err,value_len; + int options; + char *kbuf; + char *vbuf; + kbuf=new char[key_size]; + int64_t sum=0; + + options=0; + if(read_type == 1 ) + { + options=MC_OPT_DB_DATABASE_SEEK_ON_READ; + } + if(read_type == 2 ) + { + options=MC_OPT_DB_DATABASE_NEXT_ON_READ; + } + double tb,ta; + tb=mc_TimeNowAsDouble(); + for(int r=0;rRead(kbuf,key_size,&value_len,options,&err); + if(err) + { + return 0; + } + if(vbuf) + { + if(read_type == 2) + { + sum+=mc_GetLE(vbuf,4); + } + else + { + return 0; + } + } + else + { + if(read_type == 2) + { + return 0; + } + } + } + ta=mc_TimeNowAsDouble(); + + delete [] kbuf; + printf("%8.3f %ld\n",ta-tb,sum); + return ta-tb; +} + + +Value mcd_DebugRequest(string method,const Object& params) +{ + if(method == "dbopen") + { + mc_Database *m_DB; + string dbname=mcd_ParamStringValue(params,"dbname",""); + int key_size=mcd_ParamIntValue(params,"keysize",-1); + int value_size=mcd_ParamIntValue(params,"valuesize",-1); + int sleep=mcd_ParamIntValue(params,"sleep",1000); + int64_t res; + if( (dbname.size() == 0) || (key_size < 0) || (value_size < 0) ) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameters"); + } + res=mcd_OpenDatabase(mc_gState->m_NetworkParams->Name(),dbname.c_str(),key_size,value_size,&m_DB); + __US_Sleep(sleep); + mcd_CloseDatabase(m_DB); + return res; + } + if(method == "dbwrite") + { + mc_Database *m_DB; + string dbname=mcd_ParamStringValue(params,"dbname",""); + int key_size=mcd_ParamIntValue(params,"keysize",-1); + int value_size=mcd_ParamIntValue(params,"valuesize",-1); + int row_count=mcd_ParamIntValue(params,"rows",-1); + int sleep=mcd_ParamIntValue(params,"sleep",1000); + int64_t res; + if( (dbname.size() == 0) || (key_size < 0) || (value_size < 0) || (row_count < 0) ) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameters"); + } + res=mcd_OpenDatabase(mc_gState->m_NetworkParams->Name(),dbname.c_str(),key_size,value_size,&m_DB); + res=mcd_AddRows(m_DB,key_size,value_size,row_count,res); + __US_Sleep(sleep); + mcd_CloseDatabase(m_DB); + return res; + } + if(method == "dbread") + { + mc_Database *m_DB; + string dbname=mcd_ParamStringValue(params,"dbname",""); + int key_size=mcd_ParamIntValue(params,"keysize",-1); + int value_size=mcd_ParamIntValue(params,"valuesize",-1); + int read_type=mcd_ParamIntValue(params,"type",-1); + int row_count=mcd_ParamIntValue(params,"rows",-1); + int sleep=mcd_ParamIntValue(params,"sleep",1000); + double dres; + if( (dbname.size() == 0) || (key_size < 0) || (read_type < 0) || (value_size < 0) || (row_count < 0) ) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameters"); + } + mcd_OpenDatabase(mc_gState->m_NetworkParams->Name(),dbname.c_str(),key_size,value_size,&m_DB); + dres=mcd_ReadRows(m_DB,key_size,row_count,read_type); + __US_Sleep(sleep); + mcd_CloseDatabase(m_DB); + return dres; + } + if(method == "chunksdump") + { + int force=mcd_ParamIntValue(params,"force",0); + pwalletTxsMain->m_ChunkDB->Dump("Debug",force); + return Value::null; + } + if(method == "chunkscommit") + { + pwalletTxsMain->m_ChunkDB->Commit(-3); + return Value::null; + } + if(method == "walletdump") + { + int force=mcd_ParamIntValue(params,"force",0); + pwalletTxsMain->m_Database->Dump("Debug",force); + return Value::null; + } + + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid request type"); + + return Value::null; +} Value debug(const Array& params, bool fHelp) { @@ -45,17 +337,19 @@ Value debug(const Array& params, bool fHelp) request_type=MC_RMT_SPECIAL_VIEW_CHUNKS; } } - - if(request_type == MC_RMT_NONE) + + if(params[1].type() != obj_type) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid request type"); + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid request details, should be object"); } - if(params[1].type() != obj_type) + if(request_type == MC_RMT_NONE) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid request details, should be object"); + return mcd_DebugRequest(params[0].get_str(),params[1].get_obj()); +// throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid request type"); } + if(request_type & MC_RMT_SPECIAL_MASK) { switch(request_type) @@ -172,12 +466,35 @@ Value debug(const Array& params, bool fHelp) throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid chunk"); } uint256 chunk_hash=0; + uint256 txid=0; + int vout=-1; + mc_TxEntity entity; + mc_EntityDetails stream_entity; + entity.Zero(); BOOST_FOREACH(const Pair& dd, cd.get_obj()) { if(dd.name_ == "hash") { chunk_hash = ParseHashV(dd.value_.get_str(), "hash"); } + if(dd.name_ == "stream") + { + parseStreamIdentifier(dd.value_.get_str(),&stream_entity); + memcpy(&entity,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,MC_AST_SHORT_TXID_SIZE); + entity.m_EntityType=MC_TET_STREAM; + } + if(dd.name_ == "txid") + { + txid = ParseHashV(dd.value_.get_str(), "hash"); + } + if(dd.name_ == "vout") + { + if(dd.value_.type() != int_type) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid vout"); + } + vout=dd.value_.get_int(); + } } if(chunk_hash == 0) { @@ -191,7 +508,8 @@ Value debug(const Array& params, bool fHelp) Object chunk_obj; chunk_obj.push_back(Pair("hash",chunk_hash.ToString())); - if(pwalletTxsMain->m_ChunkDB->GetChunkDef(&chunk_def,(unsigned char *)&chunk_hash,NULL,NULL,-1) == MC_ERR_NOERROR) + if(pwalletTxsMain->m_ChunkDB->GetChunkDef(&chunk_def,(unsigned char *)&chunk_hash, + (entity.m_EntityType == MC_TET_NONE) ? NULL: &entity,(txid == 0) ? NULL : (unsigned char*)&txid,vout) == MC_ERR_NOERROR) { chunk_found=pwalletTxsMain->m_ChunkDB->GetChunk(&chunk_def,0,-1,&chunk_bytes); if(chunk_found) diff --git a/src/rpc/rpcstreams.cpp b/src/rpc/rpcstreams.cpp index 9431c86f..9545a19f 100644 --- a/src/rpc/rpcstreams.cpp +++ b/src/rpc/rpcstreams.cpp @@ -783,7 +783,7 @@ Value unsubscribe(const Array& params, bool fHelp) throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported with this wallet version. To get this functionality, run \"multichaind -walletdbversion=2 -rescan\" "); } - + bool purge=false; vector inputEntities; vector inputStrings; if(params[0].type() == str_type) @@ -795,6 +795,18 @@ Value unsubscribe(const Array& params, bool fHelp) inputStrings=ParseStringList(params[0]); } + if(params.size() > 1) + { + if(params[1].type() == bool_type) + { + purge=params[1].get_bool(); + } + else + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid value for 'purge' field, should be boolean"); + } + } + for(int is=0;is<(int)inputStrings.size();is++) { mc_EntityDetails entity_to_subscribe; @@ -847,7 +859,7 @@ Value unsubscribe(const Array& params, bool fHelp) if(fNewFound) { - if(pwalletTxsMain->Unsubscribe(streams)) + if(pwalletTxsMain->Unsubscribe(streams,purge)) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Couldn't unsubscribe from stream"); } diff --git a/src/utils/dbwrapper.cpp b/src/utils/dbwrapper.cpp index d76f05c4..106cff3c 100644 --- a/src/utils/dbwrapper.cpp +++ b/src/utils/dbwrapper.cpp @@ -410,10 +410,12 @@ char *cs_Database::Read(char *key,int key_len,int *value_len,int Options,int *er { char *err = NULL; const char *lpIterRead; + const char *lpIterReadKey; char *lpRead; char *lpNewBuffer; int NewSize; size_t vallen; + size_t kallen; int klen=key_len; @@ -435,6 +437,7 @@ char *cs_Database::Read(char *key,int key_len,int *value_len,int Options,int *er lpRead=NULL; lpIterRead=NULL; + lpIterReadKey=NULL; switch(m_Options & MC_OPT_DB_DATABASE_TYPE_MASK) { @@ -443,7 +446,7 @@ char *cs_Database::Read(char *key,int key_len,int *value_len,int Options,int *er // *error=MC_ERR_NOT_SUPPORTED; // return NULL; - if(Options & MC_OPT_DB_DATABASE_SEEK_ON_READ) + if(Options & (MC_OPT_DB_DATABASE_SEEK_ON_READ | MC_OPT_DB_DATABASE_NEXT_ON_READ) ) { // *error=MC_ERR_OPERATION_NOT_SUPPORTED; // return NULL; @@ -459,10 +462,10 @@ char *cs_Database::Read(char *key,int key_len,int *value_len,int Options,int *er leveldb_iter_seek((leveldb_iterator_t*)m_Iterator, key, klen); if(leveldb_iter_valid((leveldb_iterator_t*)m_Iterator)) { - lpIterRead=leveldb_iter_key((leveldb_iterator_t*)m_Iterator, &vallen); - if(lpIterRead) + lpIterReadKey=leveldb_iter_key((leveldb_iterator_t*)m_Iterator, &kallen); + if(lpIterReadKey) { - if( ((int)vallen == klen) && (memcmp(lpIterRead,key,klen) == 0) ) + if( ((int)kallen == klen) && ( ((Options & MC_OPT_DB_DATABASE_NEXT_ON_READ) != 0) || (memcmp(lpIterReadKey,key,klen) == 0) ) ) { lpIterRead=leveldb_iter_value((leveldb_iterator_t*)m_Iterator, &vallen); } @@ -499,9 +502,9 @@ char *cs_Database::Read(char *key,int key_len,int *value_len,int Options,int *er break; } - if(*value_len+1>m_ReadBufferSize) + if(*value_len+klen+1>m_ReadBufferSize) { - NewSize=((*value_len)/MC_DCT_DB_READ_BUFFER_SIZE + 1) * MC_DCT_DB_READ_BUFFER_SIZE; + NewSize=((*value_len+klen)/MC_DCT_DB_READ_BUFFER_SIZE + 1) * MC_DCT_DB_READ_BUFFER_SIZE; lpNewBuffer=(char*)mc_New(NewSize); if(lpNewBuffer == NULL) { @@ -518,12 +521,22 @@ char *cs_Database::Read(char *key,int key_len,int *value_len,int Options,int *er if(lpRead) { memcpy(m_ReadBuffer,lpRead,*value_len); + m_ReadBuffer[*value_len]=0; } if(lpIterRead) { - memcpy(m_ReadBuffer,lpIterRead,*value_len); + if(Options & MC_OPT_DB_DATABASE_NEXT_ON_READ) + { + memcpy(m_ReadBuffer,lpIterReadKey,klen); + memcpy(m_ReadBuffer+klen,lpIterRead,*value_len); + m_ReadBuffer[klen+*value_len]=0; + } + else + { + memcpy(m_ReadBuffer,lpIterRead,*value_len); + m_ReadBuffer[*value_len]=0; + } } - m_ReadBuffer[*value_len]=0; switch(m_Options & MC_OPT_DB_DATABASE_TYPE_MASK) { diff --git a/src/utils/dbwrapper.h b/src/utils/dbwrapper.h index 2339365d..790f1799 100644 --- a/src/utils/dbwrapper.h +++ b/src/utils/dbwrapper.h @@ -25,6 +25,7 @@ #define MC_OPT_DB_DATABASE_READONLY 0x00000004 #define MC_OPT_DB_DATABASE_SEEK_ON_READ 0x00000010 #define MC_OPT_DB_DATABASE_DELAYED_OPEN 0x00000020 +#define MC_OPT_DB_DATABASE_NEXT_ON_READ 0x00000040 #define MC_OPT_DB_DATABASE_OPTION_MASK 0x000FFFFF #define MC_OPT_DB_DATABASE_LEVELDB 0x00100000 #define MC_OPT_DB_DATABASE_FSR_UTXOC_BLOCKS 0x00200000 diff --git a/src/utils/declare.h b/src/utils/declare.h index a42c99fa..8cb3f42b 100644 --- a/src/utils/declare.h +++ b/src/utils/declare.h @@ -35,6 +35,7 @@ typedef struct mc_MapStringIndex void Remove(const char* key,int size); int Get(const char* key); int Get(const unsigned char* key,int size); + void Set(const unsigned char* key,int size,int value); void Destroy(); void Clear(); } mc_MapStringIndex; @@ -90,6 +91,7 @@ typedef struct mc_Buffer int Seek(void *lpKey); unsigned char *GetRow(int RowID); int PutRow(int RowID,const void *lpKey,const void *lpValue); + int UpdateRow(int RowID,const void *lpKey,const void *lpValue); int GetCount(); int SetCount(int count); @@ -259,6 +261,7 @@ void mc_StringLowerCase(char *buf,uint32_t len); int mc_StringCompareCaseInsensitive(const char *str1,const char *str2,int len); uint32_t mc_GetParamFromDetailsScript(const unsigned char *ptr,uint32_t total,uint32_t offset,uint32_t* param_value_start,size_t *bytes); +uint32_t mc_GetParamFromDetailsScriptErr(const unsigned char *ptr,uint32_t total,uint32_t offset,uint32_t* param_value_start,size_t *bytes, int *err); uint32_t mc_FindSpecialParamInDetailsScript(const unsigned char *ptr,uint32_t total,uint32_t param,size_t *bytes); uint32_t mc_FindNamedParamInDetailsScript(const unsigned char *ptr,uint32_t total,const char *param,size_t *bytes); diff --git a/src/utils/tools.cpp b/src/utils/tools.cpp index 770194d8..ede25ea9 100644 --- a/src/utils/tools.cpp +++ b/src/utils/tools.cpp @@ -68,6 +68,17 @@ int mc_MapStringIndex::Get(const unsigned char* key,int size) return value; } +void mc_MapStringIndex::Set(const unsigned char* key,int size, int value) +{ + std::map::iterator it; + it=((std::map*)mapObject)->find(string(key,key+size)); + if (it != ((std::map*)mapObject)->end()) + { + it->second=value; + } +} + + void mc_MapStringString::Init() { mapObject=new std::map; diff --git a/src/utils/utility.cpp b/src/utils/utility.cpp index 270ebbce..3ebf0d70 100644 --- a/src/utils/utility.cpp +++ b/src/utils/utility.cpp @@ -478,6 +478,21 @@ int mc_Buffer::Add(const void *lpKeyValue) return Add(lpKeyValue,(unsigned char*)lpKeyValue+m_KeySize); } +int mc_Buffer::UpdateRow(int RowID,const void *lpKey,const void *lpValue) +{ + if(RowID>=m_Count) + { + return MC_ERR_INTERNAL_ERROR; + } + + if(m_lpIndex) + { + m_lpIndex->Remove((char*)GetRow(RowID),m_RowSize-m_KeySize); + } + + return PutRow(RowID,lpKey,lpValue); +} + int mc_Buffer::PutRow(int RowID,const void *lpKey,const void *lpValue) { unsigned char *ptr; @@ -501,7 +516,14 @@ int mc_Buffer::PutRow(int RowID,const void *lpKey,const void *lpValue) if(m_lpIndex) { - m_lpIndex->Add((unsigned char*)lpKey,m_KeySize,RowID); + if(m_lpIndex->Get((unsigned char*)lpKey,m_KeySize) >= 0) + { + m_lpIndex->Set((unsigned char*)lpKey,m_KeySize,RowID); + } + else + { + m_lpIndex->Add((unsigned char*)lpKey,m_KeySize,RowID); + } } return MC_ERR_NOERROR; diff --git a/src/wallet/chunkdb.cpp b/src/wallet/chunkdb.cpp index d156fd4a..616dc183 100644 --- a/src/wallet/chunkdb.cpp +++ b/src/wallet/chunkdb.cpp @@ -291,6 +291,269 @@ int mc_ChunkDB::AddEntity(mc_TxEntity* entity, uint32_t flags) return err; } +int mc_ChunkDB::RemoveEntityInternal(mc_TxEntity *entity) +{ + int err; + mc_SubscriptionDBRow subscription; + mc_SubscriptionDBRow *old_subscription; + mc_ChunkDBRow chunk_def; + char msg[256]; + char enthex[65]; + char enthex_dir[65]; + unsigned char* buf; + int FileHan; + char FileName[MC_DCT_DB_MAX_PATH]; + int chunk_found=0; + int count=0; + + err=MC_ERR_NOERROR; + sprintf_hex(enthex,entity->m_EntityID,MC_TDB_ENTITY_ID_SIZE); + + subscription.Zero(); + subscription.m_RecordType=MC_CDB_TYPE_SUBSCRIPTION; + memcpy(&subscription.m_Entity,entity,sizeof(mc_TxEntity)); + + subscription.m_Entity.m_EntityType &= MC_TET_TYPE_MASK; + sprintf_hex(enthex_dir,subscription.m_Entity.m_EntityID,MC_AST_SHORT_TXID_SIZE); + + old_subscription=FindSubscription(&subscription.m_Entity); + + if(old_subscription == NULL) + { + return MC_ERR_NOERROR; + } + + CommitInternal(-4); + + memcpy(&subscription,old_subscription,sizeof(mc_SubscriptionDBRow)); + subscription.m_Entity.m_EntityType |= MC_TET_DELETED; + + + err=m_DB->Write((char*)&subscription+m_KeyOffset,m_KeySize,(char*)&subscription+m_ValueOffset,m_ValueSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + if(err == MC_ERR_NOERROR) + { + err=m_DB->Delete((char*)old_subscription+m_KeyOffset,m_KeySize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + } + if(err == MC_ERR_NOERROR) + { + err=m_DB->Commit(MC_OPT_DB_DATABASE_TRANSACTIONAL); + } + + if(err) + { + goto exitlbl; + } + + m_Subscriptions->PutRow(old_subscription->m_SubscriptionID,&subscription,(char*)&subscription+m_ValueOffset); + + sprintf(msg,"Entity (%08X, %s) unlinked successfully",entity->m_EntityType,enthex); + LogString(msg); + + m_TmpScript->Resize(MC_CDB_MAX_FILE_READ_BUFFER_SIZE,1); + buf=m_TmpScript->m_lpData; + + chunk_def.Zero(); + chunk_def.m_SubscriptionID=old_subscription->m_SubscriptionID; + + for(int file_id=0;file_id<=old_subscription->m_LastFileID;file_id++) + { + err=MC_ERR_NOERROR; + + SetFileName(FileName,old_subscription,file_id); + FileHan=open(FileName,_O_BINARY | O_RDONLY, S_IRUSR | S_IWUSR); + if(FileHan<=0) + { + err=MC_ERR_INTERNAL_ERROR; + } + + if(err==MC_ERR_NOERROR) + { + int64_t file_size=0; + int64_t file_offset=0; + int64_t buf_offset,buf_tail,buf_size,offset; + uint32_t param_value_start; + size_t bytes; + buf_offset=0; + buf_tail=0; + file_size=lseek64(FileHan,0,SEEK_END); + lseek64(FileHan,0,SEEK_SET); + while(file_offsetfile_size-file_offset) + { + buf_size=file_size-file_offset; + } + if(read(FileHan,buf+buf_tail,buf_size) != buf_size) + { + err=MC_ERR_INTERNAL_ERROR; + } + if(err==MC_ERR_NOERROR) + { + file_offset+=buf_size; + buf_size+=buf_tail; + buf_tail=0; + buf_offset=0; + + while(buf_offset < buf_size) + { + offset=mc_GetParamFromDetailsScriptErr(buf,buf_size,buf_offset,¶m_value_start,&bytes,&err); + if(err) + { + buf_tail=buf_size-buf_offset; + if(param_value_startMC_CDB_MAX_FILE_READ_BUFFER_SIZE) + { + buf_tail=0; + file_offset+=param_value_start+bytes-buf_size; + if(file_offsetDelete((char*)&chunk_def+m_KeyOffset,m_KeySize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + chunk_def.SwapPosBytes(); + count++; + chunk_def.Zero(); + chunk_def.m_SubscriptionID=old_subscription->m_SubscriptionID; + } + if(err==MC_ERR_NOERROR) + { + if(bytes != MC_CDB_CHUNK_HASH_SIZE) + { + err=MC_ERR_CORRUPTED; + } + else + { + chunk_found=1; + memcpy(chunk_def.m_Hash,buf+param_value_start,bytes); + } + } + break; + case MC_ENT_SPRM_ITEM_COUNT: + if(bytes != sizeof(uint32_t)) + { + err=MC_ERR_CORRUPTED; + } + else + { + chunk_def.m_Pos=(uint32_t)mc_GetLE(buf+param_value_start,bytes); + } + break; + } + buf_offset=offset; + } + } + if(err==MC_ERR_NOERROR) + { + if(count >= 1000) + { + if(err == MC_ERR_NOERROR) + { + err=m_DB->Commit(MC_OPT_DB_DATABASE_TRANSACTIONAL); + count=0; + } + } + } + if(err) + { + buf_offset=buf_size; + file_offset=file_size; + } + } + } + else + { + file_offset=file_size; + } + } + } + + + if(FileHan>0) + { + close(FileHan); + } + } + + if(err == MC_ERR_NOERROR) + { + if(chunk_found) + { + chunk_def.SwapPosBytes(); + err=m_DB->Delete((char*)&chunk_def+m_KeyOffset,m_KeySize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + chunk_def.SwapPosBytes(); + count++; + } + } + if(err == MC_ERR_NOERROR) + { + if(count) + { + err=m_DB->Commit(MC_OPT_DB_DATABASE_TRANSACTIONAL); + count=0; + } + } + + if(err == MC_ERR_NOERROR) + { + char dir_name[64]; + sprintf(dir_name,"chunks/data/stream-%s",enthex_dir); + + mc_RemoveDir(mc_gState->m_Params->NetworkName(),dir_name); + } +exitlbl: + + + if(err) + { + sprintf(msg,"Could not remove entity (%08X, %s), error: %d",entity->m_EntityType,enthex,err); + LogString(msg); + } + else + { + sprintf(msg,"Entity (%08X, %s) removed successfully",entity->m_EntityType,enthex); + LogString(msg); + } + + return err; +} + +int mc_ChunkDB::RemoveEntity(mc_TxEntity *entity) +{ + int err; + + Lock(); + err=RemoveEntityInternal(entity); + UnLock(); + + return err; +} + + int mc_ChunkDB::FindSubscription(const mc_TxEntity *entity,mc_SubscriptionDBRow *subscription) { mc_SubscriptionDBRow *found; @@ -465,10 +728,18 @@ int mc_ChunkDB::Initialize(const char *name,uint32_t mode) } void mc_ChunkDB::Dump(const char *message) +{ + Dump(message,0); +} + +void mc_ChunkDB::Dump(const char *message, int force) { - if((m_DBStat.m_InitMode & MC_WMD_DEBUG) == 0) + if(force == 0) { - return; + if((m_DBStat.m_InitMode & MC_WMD_DEBUG) == 0) + { + return; + } } mc_ChunkDBRow dbrow; @@ -540,6 +811,11 @@ int mc_ScriptMatchesTxIDAndVOut(unsigned char *ptr,size_t bytes,const unsigned c size_t value_size; uint32_t offset; + if(ptr == NULL) + { + return MC_ERR_NOT_FOUND; + } + offset=mc_FindSpecialParamInDetailsScript(ptr,bytes,MC_ENT_SPRM_SOURCE_TXID,&value_size); if(offset != bytes) { @@ -613,13 +889,16 @@ int mc_ChunkDB::GetChunkDefInternal( { on_disk_items=chunk_def->m_TmpOnDiskItems; total_items=chunk_def->m_ItemCount; - while(chunk_def->m_Pos < total_items) + while(chunk_def->m_Pos+on_disk_items < total_items) { // ptr=(unsigned char *)m_ChunkData->GetData(chunk_def->m_InternalFileOffset,&bytes); - ptr=GetChunkInternal(chunk_def,-1,-1,&bytes); - if(mc_ScriptMatchesTxIDAndVOut(ptr,bytes,txid,vout) == MC_ERR_NOERROR) + if( (chunk_def->m_TxIDStart == 0) || (chunk_def->m_TxIDStart == (uint32_t)mc_GetLE((void*)txid,4))) { - return MC_ERR_NOERROR; + ptr=GetChunkInternal(chunk_def,-1,-1,&bytes); + if(mc_ScriptMatchesTxIDAndVOut(ptr,bytes,txid,vout) == MC_ERR_NOERROR) + { + return MC_ERR_NOERROR; + } } chunk_def->m_Pos+=1; if(chunk_def->m_Pos < total_items) @@ -631,35 +910,16 @@ int mc_ChunkDB::GetChunkDefInternal( } else { - chunk_def->m_Pos=total_items; + chunk_def->m_Pos=total_items-on_disk_items; } } } } else { - if(entity == NULL) + if(mempool_row) { - if(mempool_row) - { - while(chunk_def->m_NextSubscriptionID) - { - chunk_def->m_SubscriptionID=chunk_def->m_NextSubscriptionID; - mprow=m_MemPool->Seek((unsigned char*)chunk_def); - if(mprow >= 0) - { - if(mempool_row) - { - *mempool_row=mprow; - } - memcpy(chunk_def,(mc_ChunkDBRow *)m_MemPool->GetRow(mprow),sizeof(mc_ChunkDBRow)); - } - else - { - return MC_ERR_INTERNAL_ERROR; - } - } - } + *mempool_row=mprow; } return MC_ERR_NOERROR; } @@ -667,29 +927,69 @@ int mc_ChunkDB::GetChunkDefInternal( { err=MC_ERR_NOT_FOUND; } + else + { + mprow=-1; + chunk_def->m_Pos=0; + } } - else + + if(mprow < 0) { chunk_def->SwapPosBytes(); - ptr=(unsigned char*)m_DB->Read((char*)chunk_def+m_KeyOffset,m_KeySize,&value_len,0,&err); + ptr=(unsigned char*)m_DB->Read((char*)chunk_def+m_KeyOffset,m_KeySize,&value_len,MC_OPT_DB_DATABASE_NEXT_ON_READ,&err); chunk_def->SwapPosBytes(); if(err) { return err; } + if(ptr) + { + if(entity) + { + if(memcmp((char*)chunk_def+m_KeyOffset,ptr,m_KeySize)) + { + ptr=NULL; + } + } + else + { + if(memcmp(chunk_def->m_Hash,ptr,MC_CDB_CHUNK_HASH_SIZE)) + { + ptr=NULL; + } + } + } + if(ptr) { - memcpy((char*)chunk_def+m_ValueOffset,ptr,m_ValueSize); +// mc_DumpSize("E",chunk_def,m_TotalSize,40); +// mc_DumpSize("F",ptr,m_TotalSize,40); + memcpy((char*)chunk_def+m_ValueOffset,ptr+m_ValueOffset,m_ValueSize); + on_disk_items=chunk_def->m_ItemCount; +// mc_DumpSize("G",chunk_def,m_TotalSize,40); + if(entity == NULL) + { + if(chunk_def->m_NextSubscriptionID == 0) + { + chunk_def->m_NextSubscriptionID=((mc_ChunkDBRow*)ptr)->m_SubscriptionID; + } + } +// mc_DumpSize("H",chunk_def,m_TotalSize,40); if(txid) { - ptr=GetChunkInternal(chunk_def,-1,-1,&bytes); +// ptr=GetChunkInternal(chunk_def,-1,-1,&bytes); total_items=chunk_def->m_ItemCount; - while(ptr) + while(chunk_def->m_Pos < on_disk_items) { - if(mc_ScriptMatchesTxIDAndVOut(ptr,bytes,txid,vout) == MC_ERR_NOERROR) + if( (chunk_def->m_TxIDStart == 0) || (chunk_def->m_TxIDStart == (uint32_t)mc_GetLE((void*)txid,4))) { - return MC_ERR_NOERROR; + ptr=GetChunkInternal(chunk_def,-1,-1,&bytes); + if(mc_ScriptMatchesTxIDAndVOut(ptr,bytes,txid,vout) == MC_ERR_NOERROR) + { + return MC_ERR_NOERROR; + } } chunk_def->m_Pos+=1; @@ -706,22 +1006,19 @@ int mc_ChunkDB::GetChunkDefInternal( if(ptr) { memcpy((char*)chunk_def+m_ValueOffset,ptr,m_ValueSize); - ptr=GetChunkInternal(chunk_def,-1,-1,&bytes); +// ptr=GetChunkInternal(chunk_def,-1,-1,&bytes); } else { - chunk_def->m_Pos=total_items; + chunk_def->m_Pos=on_disk_items; } } - else - { - ptr=NULL; - } } err=MC_ERR_NOT_FOUND; } else { +/* if(entity == NULL) { if(mempool_row) @@ -748,6 +1045,7 @@ int mc_ChunkDB::GetChunkDefInternal( } } } + */ return MC_ERR_NOERROR; } } @@ -789,7 +1087,7 @@ int mc_ChunkDB::AddChunkInternal( { int err; int add_null_row,add_entity_row; - int total_items,on_disk_items; + int total_items,on_disk_items,pos; int mempool_entity_row; int mempool_last_null_row; uint32_t timestamp; @@ -893,6 +1191,8 @@ int mc_ChunkDB::AddChunkInternal( m_TmpScript->AddElement(); timestamp=mc_TimeNowAsUInt(); + pos=total_items-on_disk_items; + m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_TIMESTAMP,(unsigned char*)×tamp,sizeof(timestamp)); m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_CHUNK_HASH,hash,MC_CDB_CHUNK_HASH_SIZE); if(txid) @@ -903,19 +1203,28 @@ int mc_ChunkDB::AddChunkInternal( m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_SOURCE_VOUT,(unsigned char*)&vout,sizeof(vout)); } } +/* else { m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_SOURCE_TXID,null_txid,MC_TDB_TXID_SIZE); m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_SOURCE_VOUT,(unsigned char*)&vout,sizeof(vout)); } - m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_CHUNK_SIZE,(unsigned char*)&chunk_size,sizeof(chunk_size)); + */ if(details_size) { m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_CHUNK_DETAILS,details,details_size); } - if(chunk_size) + if(total_items == 0) + { + m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_CHUNK_SIZE,(unsigned char*)&chunk_size,sizeof(chunk_size)); + if(chunk_size) + { + m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_CHUNK_DATA,chunk,chunk_size); + } + } + else { - m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_CHUNK_DATA,chunk,chunk_size); + m_TmpScript->SetSpecialParamValue(MC_ENT_SPRM_ITEM_COUNT,(unsigned char*)&total_items,sizeof(total_items)); } chunk_def.m_Size=chunk_size; @@ -938,15 +1247,24 @@ int mc_ChunkDB::AddChunkInternal( chunk_def.m_Pos=0; chunk_def.m_ItemCount=1; chunk_def.m_TmpOnDiskItems=0; +/* chunk_def.m_PrevSubscriptionID=0; + */ chunk_def.m_NextSubscriptionID=subscription->m_SubscriptionID; m_MemPool->Add(&chunk_def,(char*)&chunk_def+m_ValueOffset); } chunk_def.m_SubscriptionID=subscription->m_SubscriptionID; - chunk_def.m_Pos=total_items-on_disk_items; + chunk_def.m_Pos=pos; chunk_def.m_ItemCount=total_items+1; chunk_def.m_TmpOnDiskItems=on_disk_items; + chunk_def.m_NextSubscriptionID=0; + if(txid) + { + chunk_def.m_TxIDStart=(uint32_t)mc_GetLE((void*)txid,4); + } + +/* if(add_null_row) { chunk_def.m_PrevSubscriptionID=0; @@ -965,6 +1283,7 @@ int mc_ChunkDB::AddChunkInternal( chunk_def.m_NextSubscriptionID=entity_chunk_def.m_NextSubscriptionID; } } +*/ m_MemPool->Add(&chunk_def,(char*)&chunk_def+m_ValueOffset); @@ -973,6 +1292,7 @@ int mc_ChunkDB::AddChunkInternal( ((mc_ChunkDBRow *)m_MemPool->GetRow(mempool_entity_row))->m_ItemCount += 1; } + if(mempool_last_null_row >= 0) { if(add_entity_row) @@ -980,7 +1300,8 @@ int mc_ChunkDB::AddChunkInternal( ((mc_ChunkDBRow *)m_MemPool->GetRow(mempool_last_null_row))->m_NextSubscriptionID = subscription->m_SubscriptionID; } } - + + sprintf_hex(enthex,entity->m_EntityID,MC_TDB_ENTITY_ID_SIZE); sprintf_hex(chunk_hex,hash,MC_CDB_CHUNK_HASH_SIZE); sprintf(msg,"New Chunk %s, size %d, flags %08X, entity (%08X, %s)",chunk_hex,chunk_size,flags,entity->m_EntityType,enthex); @@ -1036,8 +1357,25 @@ unsigned char *mc_ChunkDB::GetChunkInternal(mc_ChunkDBRow *chunk_def, char FileName[MC_DCT_DB_MAX_PATH]; int FileHan; uint32_t read_from; + mc_ChunkDBRow chunk_def_zero; ptr=NULL; + if((offset >= 0) && (chunk_def->m_Pos > 0) ) + { + subscription_id=chunk_def->m_SubscriptionID; + if(subscription_id == 0) + { + subscription_id=chunk_def->m_NextSubscriptionID; + } + subscription=(mc_SubscriptionDBRow *)m_Subscriptions->GetRow(subscription_id); + + if(GetChunkDefInternal(&chunk_def_zero,chunk_def->m_Hash,&(subscription->m_Entity),NULL,0,NULL) == MC_ERR_NOERROR) + { + return GetChunkInternal(&chunk_def_zero,offset,len,bytes); + } + return NULL; + } + FileHan=0; if(chunk_def->m_InternalFileID < 0) { @@ -1191,7 +1529,7 @@ int mc_ChunkDB::CommitInternal(int block) { int r,s; int err; - int last_file_id, last_file_offset; +// int last_file_id, last_file_offset; uint32_t size; int value_len; unsigned char *ptr; @@ -1209,24 +1547,21 @@ int mc_ChunkDB::CommitInternal(int block) } Dump("Before Commit"); - last_file_id=-1; - last_file_offset=0; +// last_file_id=-1; +// last_file_offset=0; for(r=0;rGetCount();r++) { chunk_def=(mc_ChunkDBRow *)m_MemPool->GetRow(r); - s=chunk_def->m_SubscriptionID; - if(s == 0) - { - s=chunk_def->m_NextSubscriptionID; - } - - subscription=(mc_SubscriptionDBRow *)m_Subscriptions->GetRow(s); - subscription->m_TmpFlags |= MC_CDB_TMP_FLAG_SHOULD_COMMIT; - - if(last_file_id < 0) + if(chunk_def->m_SubscriptionID) { + s=chunk_def->m_SubscriptionID; + + subscription=(mc_SubscriptionDBRow *)m_Subscriptions->GetRow(s); + subscription->m_TmpFlags |= MC_CDB_TMP_FLAG_SHOULD_COMMIT; + + size=chunk_def->m_Size+chunk_def->m_HeaderSize; if(subscription->m_LastFileSize+size > MC_CDB_MAX_FILE_SIZE) // New file is needed { @@ -1234,7 +1569,7 @@ int mc_ChunkDB::CommitInternal(int block) subscription->m_LastFileID+=1; subscription->m_LastFileSize=0; } - + err=AddToFile(GetChunkInternal(chunk_def,-1,-1,NULL),size, subscription,subscription->m_LastFileID,subscription->m_LastFileSize); if(err) @@ -1243,36 +1578,22 @@ int mc_ChunkDB::CommitInternal(int block) LogString(msg); return err; } - + chunk_def->m_InternalFileID=subscription->m_LastFileID; chunk_def->m_InternalFileOffset=subscription->m_LastFileSize; chunk_def->m_Pos+=chunk_def->m_TmpOnDiskItems; - - if(chunk_def->m_SubscriptionID == 0) - { - last_file_id=subscription->m_LastFileID; - last_file_offset=subscription->m_LastFileSize; - } subscription->m_LastFileSize+=size; subscription->m_Count+=1; subscription->m_FullSize+=chunk_def->m_Size; - + m_DBStat.m_Count+=1; m_DBStat.m_FullSize+=chunk_def->m_Size; - } - else - { - chunk_def->m_InternalFileID=last_file_id; - chunk_def->m_InternalFileOffset=last_file_offset; - chunk_def->m_Pos+=chunk_def->m_TmpOnDiskItems; - last_file_id=-1; - } - if(chunk_def->m_TmpOnDiskItems) - { - if(chunk_def->m_SubscriptionID) + if(chunk_def->m_TmpOnDiskItems) { + if(chunk_def->m_SubscriptionID) + { entity_chunk_def.Zero(); memcpy(&entity_chunk_def,chunk_def,sizeof(mc_ChunkDBRow)); entity_chunk_def.m_Pos=0; @@ -1296,17 +1617,18 @@ int mc_ChunkDB::CommitInternal(int block) goto exitlbl; } } + } } + + chunk_def->m_TmpOnDiskItems=0; + chunk_def->SwapPosBytes(); + err=m_DB->Write((char*)chunk_def+m_KeyOffset,m_KeySize,(char*)chunk_def+m_ValueOffset,m_ValueSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + chunk_def->SwapPosBytes(); + if(err) + { + goto exitlbl; + } } - - chunk_def->m_TmpOnDiskItems=0; - chunk_def->SwapPosBytes(); - err=m_DB->Write((char*)chunk_def+m_KeyOffset,m_KeySize,(char*)chunk_def+m_ValueOffset,m_ValueSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); - chunk_def->SwapPosBytes(); - if(err) - { - goto exitlbl; - } } for(s=0;sGetCount();s++) diff --git a/src/wallet/chunkdb.h b/src/wallet/chunkdb.h index 0a7897dc..4293d4d7 100644 --- a/src/wallet/chunkdb.h +++ b/src/wallet/chunkdb.h @@ -18,6 +18,7 @@ #define MC_CDB_MAX_FILE_SIZE 0x40000000 // Maximal data file size, 1GB #define MC_CDB_MAX_CHUNK_DATA_POOL_SIZE 0x8000000 // Maximal size of chunk pool before commit, 128MB +#define MC_CDB_MAX_FILE_READ_BUFFER_SIZE 0x0100000 // Maximal size of chunk pool before commit, 4MB #define MC_CDB_MAX_CHUNK_EXTRA_SIZE 1024 #define MC_CDB_MAX_MEMPOOL_SIZE 1024 @@ -96,11 +97,11 @@ typedef struct mc_ChunkDBRow uint32_t m_Flags; // Flags passed from higher level uint32_t m_HeaderSize; // Header size uint32_t m_StorageFlags; // Internal flags - int32_t m_ItemCount; // Number of times this chunk appears in subscription (if m_Pos=0) + int32_t m_ItemCount; // Number of times this chunk appears in subscription (if m_Pos=0) int32_t m_TmpOnDiskItems; - int32_t m_InternalFileID; // Data file ID + int32_t m_InternalFileID; // Data file ID uint32_t m_InternalFileOffset; // Offset in the data file - int32_t m_PrevSubscriptionID; // Prev Subscription ID for this hash + uint32_t m_TxIDStart; // First bytes of TxID int32_t m_NextSubscriptionID; // Next Subscription ID for this hash void Zero(); @@ -151,6 +152,8 @@ typedef struct mc_ChunkDB int AddSubscription(mc_SubscriptionDBRow *subscription); int AddEntity(mc_TxEntity *entity,uint32_t flags); // Adds entity int AddEntityInternal(mc_TxEntity *entity,uint32_t flags); + int RemoveEntity(mc_TxEntity *entity); + int RemoveEntityInternal(mc_TxEntity *entity); mc_SubscriptionDBRow *FindSubscription(const mc_TxEntity *entity); // Finds subscription int FindSubscription(const mc_TxEntity *entity,mc_SubscriptionDBRow *subscription); // Finds subscription @@ -223,6 +226,7 @@ typedef struct mc_ChunkDB void Zero(); int Destroy(); void Dump(const char *message); + void Dump(const char *message, int force); void LogString(const char *message); diff --git a/src/wallet/wallettxdb.cpp b/src/wallet/wallettxdb.cpp index 3bc5554c..e624cb3e 100644 --- a/src/wallet/wallettxdb.cpp +++ b/src/wallet/wallettxdb.cpp @@ -510,10 +510,18 @@ int mc_TxDB::Destroy() } void mc_TxDB::Dump(const char *message) +{ + Dump(message,0); +} + +void mc_TxDB::Dump(const char *message, int force) { - if((m_Mode & MC_WMD_DEBUG) == 0) + if(force == 0) { - return; + if((m_Mode & MC_WMD_DEBUG) == 0) + { + return; + } } mc_TxEntityRow edbRow; diff --git a/src/wallet/wallettxdb.h b/src/wallet/wallettxdb.h index cff9f761..e852f41e 100644 --- a/src/wallet/wallettxdb.h +++ b/src/wallet/wallettxdb.h @@ -33,12 +33,13 @@ #define MC_TET_SUBKEY_STREAM_KEY 0x00000046 #define MC_TET_SUBKEY_STREAM_PUBLISHER 0x00000047 #define MC_TET_SUBKEY 0x00000040 -#define MC_TET_TYPE_MASK 0x000000FF +#define MC_TET_TYPE_MASK 0x040000FF #define MC_TET_CHAINPOS 0x00000100 #define MC_TET_TIMERECEIVED 0x00000200 #define MC_TET_ORDERMASK 0x0000FF00 #define MC_TET_DB_STAT 0x01000000 #define MC_TET_IMPORT 0x02000000 +#define MC_TET_DELETED 0x04000000 #define MC_TET_SPECIALMASK 0xFF000000 #define MC_EFL_NOT_IN_SYNC 0x01000000 @@ -385,6 +386,7 @@ typedef struct mc_TxDB void Zero(); int Destroy(); void Dump(const char *message); + void Dump(const char *message, int force); int Lock(int write_mode, int allow_secondary); void UnLock(); diff --git a/src/wallet/wallettxs.cpp b/src/wallet/wallettxs.cpp index 497d205c..75f6142c 100644 --- a/src/wallet/wallettxs.cpp +++ b/src/wallet/wallettxs.cpp @@ -1303,9 +1303,9 @@ int mc_WalletTxs::GetListSize(mc_TxEntity *entity,int generation,int *confirmed) return res; } -int mc_WalletTxs::Unsubscribe(mc_Buffer* lpEntities) +int mc_WalletTxs::Unsubscribe(mc_Buffer* lpEntities,bool purge) { - int err; + int err,j; if((m_Mode & MC_WMD_TXS) == 0) { return MC_ERR_NOT_SUPPORTED; @@ -1316,6 +1316,22 @@ int mc_WalletTxs::Unsubscribe(mc_Buffer* lpEntities) } m_Database->Lock(1,0); err=m_Database->Unsubscribe(lpEntities); + if(err==MC_ERR_NOERROR) + { + if(mc_gState->m_Features->Chunks()) + { + if(purge) + { + if(m_ChunkDB) + { + for(j=0;jGetCount();j++) + { + m_ChunkDB->RemoveEntity((mc_TxEntity*)lpEntities->GetRow(j)); + } + } + } + } + } if(fDebug)LogPrint("wallet","wtxs: Unsubscribed from %d entities\n",lpEntities->GetCount()); m_Database->UnLock(); return err; diff --git a/src/wallet/wallettxs.h b/src/wallet/wallettxs.h index fb662754..5b40c90b 100644 --- a/src/wallet/wallettxs.h +++ b/src/wallet/wallettxs.h @@ -136,7 +136,7 @@ typedef struct mc_WalletTxs int GetRow( mc_TxEntityRow *erow); - int Unsubscribe(mc_Buffer *lpEntities); // List of the entities to unsubscribe from + int Unsubscribe(mc_Buffer *lpEntities,bool purge); // List of the entities to unsubscribe from mc_TxImport *StartImport( // Starts new import mc_Buffer *lpEntities, // List of entities to import From 5b492c2d87897f0a593c4681c4a1ea3de4b335a2 Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 10 May 2018 08:52:23 +0300 Subject: [PATCH 050/157] Offchain request expiration --- src/chainparams/paramlist.h | 4 +- src/protocol/relay.cpp | 78 +++++++++++++++++++++++++++++-------- src/protocol/relay.h | 2 + src/rpc/rpcdebug.cpp | 53 +++++++++++++++++++++++++ src/wallet/chunkcollector.h | 1 + 5 files changed, 120 insertions(+), 18 deletions(-) diff --git a/src/chainparams/paramlist.h b/src/chainparams/paramlist.h index 8ff21da9..8415db50 100644 --- a/src/chainparams/paramlist.h +++ b/src/chainparams/paramlist.h @@ -35,11 +35,11 @@ static const mc_OneMultichainParam MultichainParamArray[] = "maximumchunksize","", "Maximum block size in bytes."}, { "maximumchunksize" , "maximum-chunk-size" , - MC_PRM_UINT32 | MC_PRM_USER | MC_PRM_CLONE , -1, 16777216, 256, 67108864, 0.0, 20003, 0, "-mc-maximumchunksize", + MC_PRM_UINT32 | MC_PRM_USER | MC_PRM_CLONE , -1, 1048576, 256, 16777216, 0.0, 20003, 0, "-mc-maximumchunksize", "maximumchunkcount","", "Maximum chunk size for off-chain items in bytes."}, { "maximumchunkcount" , "maximum-chunk-count" , - MC_PRM_UINT32 | MC_PRM_USER | MC_PRM_CLONE , -1, 128, 16, 1024, 0.0, 20003, 0, "-mc-maximumchunkcount", + MC_PRM_UINT32 | MC_PRM_USER | MC_PRM_CLONE , -1, 1024, 16, 2048, 0.0, 20003, 0, "-mc-maximumchunkcount", "timingupgrademingap","", "Maximum number of chunks in one off-chain item."}, { "timingupgrademingap" , "timing-upgrade-min-gap" , diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index f69af137..e76680a9 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -52,7 +52,6 @@ int MultichainProcessChunkResponse(const CRelayResponsePair *response_pair,map < { return false; } - response=&(request->m_Responses[response_pair->response_id]); unsigned char *ptr; @@ -82,6 +81,9 @@ int MultichainProcessChunkResponse(const CRelayResponsePair *response_pair,map < { switch(*ptr) { + case MC_RDT_EXPIRATION: + ptr+=5; + break; case MC_RDT_CHUNK_IDS: ptr++; count=(int)mc_GetVarInt(ptr,ptrEnd-ptr,-1,&shift); @@ -124,7 +126,7 @@ int MultichainProcessChunkResponse(const CRelayResponsePair *response_pair,map < } ptrOut+=shift; - ptr=ptrStart+1+shift; + ptr=ptrStart+1+shift+5; for(int c=0;cm_Size; @@ -241,10 +243,10 @@ int MultichainResponseScore(mc_RelayResponse *response,mc_ChunkCollectorRow *col int MultichainCollectChunks(mc_ChunkCollector* collector) { - uint32_t time_now; + uint32_t time_now,expiration,dest_expiration; vector vChunkDefs; int row,last_row,last_count; - uint32_t total_size; + uint32_t total_size,max_total_size; mc_ChunkCollectorRow *collect_row; mc_ChunkCollectorRow *collect_subrow; time_now=mc_TimeNowAsUInt(); @@ -371,8 +373,22 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) { payload.clear(); shift=mc_PutVarInt(buf,16,item.second.m_Pairs.size()); - payload.resize(1+shift+sizeof(mc_ChunkEntityKey)*item.second.m_Pairs.size()); + payload.resize(5+1+shift+sizeof(mc_ChunkEntityKey)*item.second.m_Pairs.size()); + request=pRelayManager->FindRequest(item.first.request_id); + if(request == NULL) + { + return false; + } + + response=&(request->m_Responses[item.first.response_id]); + + expiration=time_now+MC_CCW_TIMEOUT_REQUEST; + dest_expiration=expiration+response->m_TimeDiff; ptrOut=&(payload[0]); + *ptrOut=MC_RDT_EXPIRATION; + ptrOut++; + mc_PutLE(ptrOut,&dest_expiration,sizeof(dest_expiration)); + ptrOut+=sizeof(dest_expiration); *ptrOut=MC_RDT_CHUNK_IDS; ptrOut++; memcpy(ptrOut,buf,shift); @@ -388,20 +404,13 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) count++; } // mc_DumpSize("req",&(payload[0]),1+shift+sizeof(mc_ChunkEntityKey)*item.second.m_Pairs.size(),64); - request=pRelayManager->FindRequest(item.first.request_id); - if(request == NULL) - { - return false; - } - - response=&(request->m_Responses[item.first.response_id]); request_id=pRelayManager->SendNextRequest(response,MC_RMT_CHUNK_REQUEST,0,payload); if(fDebug)LogPrint("chunks","New chunk request: %s, response: %s, chunks: %d\n",request_id.ToString().c_str(),response->m_MsgID.ToString().c_str(),item.second.m_Pairs.size()); BOOST_FOREACH(PAIRTYPE(const int, int)& chunk_row, item.second.m_Pairs) { collect_subrow=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(chunk_row.first); collect_subrow->m_State.m_Request=request_id; - collect_subrow->m_State.m_RequestTimeStamp=time_now+MC_CCW_TIMEOUT_REQUEST; + collect_subrow->m_State.m_RequestTimeStamp=expiration; // printf("T %d %d %s\n",chunk_row.first,collect_subrow->m_State.m_RequestPos,collect_subrow->m_State.m_Request.ToString().c_str()); } } @@ -410,6 +419,12 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) last_row=0; last_count=0; total_size=0; + + max_total_size=(MC_CCW_TIMEOUT_REQUEST-1)*MC_CCW_MAX_MBS_PER_SECOND*1024*1024; + if(max_total_size > MAX_SIZE-OFFCHAIN_MSG_PADDING) + { + max_total_size=MAX_SIZE-OFFCHAIN_MSG_PADDING; + } while(row<=collector->m_MemPool->GetCount()) { collect_row=NULL; @@ -418,7 +433,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) collect_row=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(row); } - if( (collect_row == NULL)|| (last_count >= MC_CCW_MAX_CHUNKS_PER_QUERY) || (total_size+collect_row->m_ChunkDef.m_Size + sizeof(mc_ChunkEntityKey)> MAX_SIZE-OFFCHAIN_MSG_PADDING) ) + if( (collect_row == NULL)|| (last_count >= MC_CCW_MAX_CHUNKS_PER_QUERY) || (total_size+collect_row->m_ChunkDef.m_Size + sizeof(mc_ChunkEntityKey)> max_total_size) ) { if(last_count) { @@ -635,6 +650,8 @@ bool mc_RelayProcess_Chunk_Request(unsigned char *ptrStart,unsigned char *ptrEnd unsigned char *ptrOut; uint32_t total_size=0; + uint32_t max_total_size=MAX_SIZE-OFFCHAIN_MSG_PADDING; + uint32_t expiration=0; mc_gState->m_TmpBuffers->m_RelayTmpBuffer->Clear(); mc_gState->m_TmpBuffers->m_RelayTmpBuffer->AddElement(); @@ -645,6 +662,22 @@ bool mc_RelayProcess_Chunk_Request(unsigned char *ptrStart,unsigned char *ptrEnd { switch(*ptr) { + case MC_RDT_EXPIRATION: + ptr++; + expiration=(uint32_t)mc_GetLE(ptr,sizeof(expiration)); + ptr+=sizeof(expiration); + if(expiration+1 < pRelayManager->m_LastTime) + { + strError="Less than 1s for request expiration"; + return false; + } + if(expiration-35 > pRelayManager->m_LastTime) // We are supposed to store query_hit record only for 30s, something is wrong + { + strError="Expiration is too far in the future"; + return false; + } + max_total_size=MC_CCW_MAX_MBS_PER_SECOND*(expiration-pRelayManager->m_LastTime)*1024*1024; + break; case MC_RDT_CHUNK_IDS: ptr++; count=(int)mc_GetVarInt(ptr,ptrEnd-ptr,-1,&shift); @@ -668,7 +701,12 @@ bool mc_RelayProcess_Chunk_Request(unsigned char *ptrStart,unsigned char *ptrEnd total_size+=chunk_def.m_Size+size; if(total_size > MAX_SIZE-OFFCHAIN_MSG_PADDING) { - strError="Total size of requested chunks is too big"; + strError="Total size of requested chunks is too big for message"; + return false; + } + if(total_size > max_total_size) + { + strError="Total size of requested chunks is too big for response expiration"; return false; } chunk_found=pwalletTxsMain->m_ChunkDB->GetChunk(&chunk_def,0,-1,&chunk_bytes); @@ -701,6 +739,12 @@ bool mc_RelayProcess_Chunk_Request(unsigned char *ptrStart,unsigned char *ptrEnd return false; } } + + if(total_size > max_total_size) + { + strError="Total size of requested chunks is too big for response expiration"; + return false; + } return true; } @@ -1891,6 +1935,7 @@ void mc_RelayResponse::Zero() m_Status=MC_RST_NONE; m_Payload.clear(); m_Requests.clear(); + m_TimeDiff=0; } void mc_RelayRequest::Zero() @@ -1949,8 +1994,9 @@ int mc_RelayManager::AddResponse(mc_OffchainMessageID request,CNode *pfrom,int32 } mc_RelayResponse response; - response.m_MsgID=msg_id; + response.m_MsgID=msg_id; response.m_MsgType=msg_type; + response.m_TimeDiff=msg_id.m_TimeStamp-m_LastTime; response.m_Flags=flags; response.m_NodeFrom=pfrom ? pfrom->GetId() : 0; response.m_HopCount=hop_count; diff --git a/src/protocol/relay.h b/src/protocol/relay.h index 6811ef9c..69c016bf 100644 --- a/src/protocol/relay.h +++ b/src/protocol/relay.h @@ -56,6 +56,7 @@ #define MC_RDT_UNKNOWN 0 #define MC_RDT_MC_ADDRESS 0x01 #define MC_RDT_NET_ADDRESS 0x02 +#define MC_RDT_EXPIRATION 0x03 #define MC_RDT_CHUNK_IDS 0x11 #define MC_RDT_CHUNKS 0x12 @@ -212,6 +213,7 @@ struct mc_RelayRequest; typedef struct mc_RelayResponse { mc_OffchainMessageID m_MsgID; + int m_TimeDiff; uint32_t m_MsgType; uint32_t m_Flags; NodeId m_NodeFrom; diff --git a/src/rpc/rpcdebug.cpp b/src/rpc/rpcdebug.cpp index ce49f3d7..e40f49fa 100644 --- a/src/rpc/rpcdebug.cpp +++ b/src/rpc/rpcdebug.cpp @@ -282,6 +282,7 @@ Value mcd_DebugRequest(string method,const Object& params) if(method == "chunksdump") { int force=mcd_ParamIntValue(params,"force",0); + string message=mcd_ParamStringValue(params,"message","Debug"); pwalletTxsMain->m_ChunkDB->Dump("Debug",force); return Value::null; } @@ -293,9 +294,61 @@ Value mcd_DebugRequest(string method,const Object& params) if(method == "walletdump") { int force=mcd_ParamIntValue(params,"force",0); + string message=mcd_ParamStringValue(params,"message","Debug"); pwalletTxsMain->m_Database->Dump("Debug",force); return Value::null; } + if(method == "publishrandom") + { + int size=mcd_ParamIntValue(params,"size",-1); + string stream=mcd_ParamStringValue(params,"stream",""); + string options=mcd_ParamStringValue(params,"options",""); + string key=mcd_ParamStringValue(params,"key",""); + if( (stream.size() == 0) || (size < 0)) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameters"); + } + Array cbc_params; + string bcname=createbinarycache(cbc_params,false).get_str(); + int part_size=100000; + int total_size=0; + int num_parts=(size-1)/part_size+1; + char *vbuf; + vbuf=new char[part_size]; + for(int p=0;p Date: Thu, 10 May 2018 12:10:21 +0300 Subject: [PATCH 051/157] Throttling offchain chunk requests --- src/protocol/relay.cpp | 97 ++++++++++++++++++++++++++++++------- src/protocol/relay.h | 4 +- src/wallet/chunkcollector.h | 2 +- 3 files changed, 84 insertions(+), 19 deletions(-) diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index e76680a9..e852f191 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -193,18 +193,31 @@ int MultichainProcessChunkResponse(const CRelayResponsePair *response_pair,map < return result; } -int MultichainResponseScore(mc_RelayResponse *response,mc_ChunkCollectorRow *collect_row) +int MultichainResponseScore(mc_RelayResponse *response,mc_ChunkCollectorRow *collect_row,map& destination_loads,uint32_t max_total_size) { unsigned char *ptr; unsigned char *ptrEnd; unsigned char *ptrStart; int shift,count,size; + int64_t total_size; mc_ChunkEntityKey *chunk; int c; if( (response->m_Status & MC_RST_SUCCESS) == 0 ) { return MC_CCW_WORST_RESPONSE_SCORE; } + + total_size=0; + map::iterator itdld = destination_loads.find(response->SourceID()); + if (itdld != destination_loads.end()) + { + total_size=itdld->second; + } + + if(total_size + collect_row->m_ChunkDef.m_Size + sizeof(mc_ChunkEntityKey) >= max_total_size) + { + return MC_CCW_WORST_RESPONSE_SCORE; + } ptrStart=&(response->m_Payload[0]); @@ -238,7 +251,8 @@ int MultichainResponseScore(mc_RelayResponse *response,mc_ChunkCollectorRow *col { return MC_CCW_WORST_RESPONSE_SCORE; } - return response->m_TryCount+response->m_HopCount; + + return (response->m_TryCount+response->m_HopCount)*1024*1024+total_size/1024; } int MultichainCollectChunks(mc_ChunkCollector* collector) @@ -258,6 +272,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) map query_to_delete; map requests_to_send; map responses_to_process; + map destination_loads; mc_RelayRequest *request; mc_RelayRequest *query; mc_RelayResponse *response; @@ -314,7 +329,56 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) } pRelayManager->UnLock(); } - else + } + } + + BOOST_FOREACH(PAIRTYPE(const CRelayResponsePair, CRelayRequestPairs)& item, responses_to_process) + { + MultichainProcessChunkResponse(&(item.first),&(item.second.m_Pairs),collector); + } + + max_total_size=(MC_CCW_TIMEOUT_REQUEST-2)*MC_CCW_MAX_MBS_PER_SECOND*1024*1024; + if(max_total_size > MAX_SIZE-OFFCHAIN_MSG_PADDING) + { + max_total_size=MAX_SIZE-OFFCHAIN_MSG_PADDING; + } + + for(row=0;rowm_MemPool->GetCount();row++) + { + collect_row=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(row); + if( (collect_row->m_State.m_Status & MC_CCF_DELETED ) == 0 ) + { + request=NULL; + if(!collect_row->m_State.m_Request.IsZero()) + { + request=pRelayManager->FindRequest(collect_row->m_State.m_Request); + if(request == NULL) + { + collect_row->m_State.m_Request=0; + collect_row->m_State.m_RequestTimeStamp=0; + } + } + if(request) + { + map::iterator itdld = destination_loads.find(request->m_DestinationID); + if (itdld == destination_loads.end()) + { + destination_loads.insert(make_pair(request->m_DestinationID,collect_row->m_ChunkDef.m_Size + sizeof(mc_ChunkEntityKey))); + } + else + { + itdld->second+=collect_row->m_ChunkDef.m_Size + sizeof(mc_ChunkEntityKey); + } + } + } + } + + for(row=0;rowm_MemPool->GetCount();row++) + { + collect_row=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(row); + if( (collect_row->m_State.m_Status & MC_CCF_DELETED ) == 0 ) + { + if(collect_row->m_State.m_Request.IsZero()) { query=NULL; if(!collect_row->m_State.m_Query.IsZero()) @@ -333,7 +397,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) best_score=MC_CCW_WORST_RESPONSE_SCORE; for(int i=0;i<(int)query->m_Responses.size();i++) { - this_score=MultichainResponseScore(&(query->m_Responses[i]),collect_row); + this_score=MultichainResponseScore(&(query->m_Responses[i]),collect_row,destination_loads,max_total_size); if(this_score < best_score) { best_score=this_score; @@ -363,11 +427,6 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) } } } - - BOOST_FOREACH(PAIRTYPE(const CRelayResponsePair, CRelayRequestPairs)& item, responses_to_process) - { - MultichainProcessChunkResponse(&(item.first),&(item.second.m_Pairs),collector); - } BOOST_FOREACH(PAIRTYPE(const CRelayResponsePair, CRelayRequestPairs)& item, requests_to_send) { @@ -420,11 +479,6 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) last_count=0; total_size=0; - max_total_size=(MC_CCW_TIMEOUT_REQUEST-1)*MC_CCW_MAX_MBS_PER_SECOND*1024*1024; - if(max_total_size > MAX_SIZE-OFFCHAIN_MSG_PADDING) - { - max_total_size=MAX_SIZE-OFFCHAIN_MSG_PADDING; - } while(row<=collector->m_MemPool->GetCount()) { collect_row=NULL; @@ -1947,11 +2001,12 @@ void mc_RelayRequest::Zero() m_LastTryTimestamp=0; m_TryCount=0; m_Status=MC_RST_NONE; + m_DestinationID=0; m_Payload.clear(); m_Responses.clear(); } -int mc_RelayManager::AddRequest(CNode *pto,mc_OffchainMessageID msg_id,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status) +int mc_RelayManager::AddRequest(CNode *pto,int64_t destination,mc_OffchainMessageID msg_id,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status) { int err=MC_ERR_NOERROR; @@ -1968,6 +2023,7 @@ int mc_RelayManager::AddRequest(CNode *pto,mc_OffchainMessageID msg_id,uint32_t request.m_LastTryTimestamp=0; request.m_TryCount=0; request.m_Status=status; + request.m_DestinationID=destination; request.m_Payload=payload; request.m_Responses.clear(); @@ -1983,6 +2039,13 @@ int mc_RelayManager::AddRequest(CNode *pto,mc_OffchainMessageID msg_id,uint32_t return err; } +int64_t mc_RelayResponse::SourceID() +{ + int64_t result=(int64_t)m_NodeFrom; + result = (result << 32) + (int64_t)m_Source; + return result; +} + int mc_RelayManager::AddResponse(mc_OffchainMessageID request,CNode *pfrom,int32_t source,int hop_count,mc_OffchainMessageID msg_id,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status) { // printf("resp: %lu, mt: %d, node: %d, size: %d, rn: %lu\n",request,msg_type,pfrom ? pfrom->GetId() : 0,(int)payload.size(),nonce); @@ -2079,7 +2142,7 @@ mc_OffchainMessageID mc_RelayManager::SendRequest(CNode* pto,uint32_t msg_type,u } } - if(AddRequest(pto,msg_id,msg_type,flags,payload,MC_RST_NONE) != MC_ERR_NOERROR) + if(AddRequest(pto,0,msg_id,msg_type,flags,payload,MC_RST_NONE) != MC_ERR_NOERROR) { return mc_OffchainMessageID(); } @@ -2104,7 +2167,7 @@ mc_OffchainMessageID mc_RelayManager::SendNextRequest(mc_RelayResponse* response { msg_id=PushRelay(pnode,0,vEmptyHops,vEmptySendPaths,msg_type,msg_id,response->m_MsgID, flags,payload,vSigScriptsEmpty,NULL,MC_PRA_NONE); - if(AddRequest(pnode,msg_id,msg_type,flags,payload,MC_RST_NONE) != MC_ERR_NOERROR) + if(AddRequest(pnode,response->SourceID(),msg_id,msg_type,flags,payload,MC_RST_NONE) != MC_ERR_NOERROR) { return mc_OffchainMessageID(); } diff --git a/src/protocol/relay.h b/src/protocol/relay.h index 69c016bf..4df4daab 100644 --- a/src/protocol/relay.h +++ b/src/protocol/relay.h @@ -231,6 +231,7 @@ typedef struct mc_RelayResponse } void Zero(); + int64_t SourceID(); } mc_RelayResponse; typedef struct mc_RelayRequest @@ -242,6 +243,7 @@ typedef struct mc_RelayRequest uint32_t m_LastTryTimestamp; int m_TryCount; uint32_t m_Status; + int64_t m_DestinationID; vector m_Payload; vector m_Responses; @@ -317,7 +319,7 @@ typedef struct mc_RelayManager CValidationState &state, uint32_t verify_flags_in); - int AddRequest(CNode *pto,mc_OffchainMessageID msg_id,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status); + int AddRequest(CNode *pto,int64_t destination,mc_OffchainMessageID msg_id,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status); // int AddRequest(int64_t parent_nonce,int parent_response_id,CNode *pto,int64_t nonce,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status); int AddResponse(mc_OffchainMessageID request,CNode *pfrom,int32_t source,int hop_count,mc_OffchainMessageID msg_id,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status); //int AddResponse(int64_t request,CNode *pfrom,int32_t source,int hop_count,int64_t nonce,uint32_t msg_type,uint32_t flags,vector & payload,uint32_t status); diff --git a/src/wallet/chunkcollector.h b/src/wallet/chunkcollector.h index 4f593cc8..5a3291ea 100644 --- a/src/wallet/chunkcollector.h +++ b/src/wallet/chunkcollector.h @@ -23,7 +23,7 @@ #define MC_CCW_TIMEOUT_REQUEST 5 #define MC_CCW_MAX_CHUNKS_PER_QUERY 4 #define MC_CCW_DEFAULT_AUTOCOMMIT_DELAY 200 -#define MC_CCW_WORST_RESPONSE_SCORE 1000 +#define MC_CCW_WORST_RESPONSE_SCORE 1048576000 #define MC_CCW_DEFAULT_MEMPOOL_SIZE 1000 #define MC_CCW_MAX_MBS_PER_SECOND 8 From c7d54bd5536917f4f54dab29695c68fa77224e7c Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 10 May 2018 19:08:02 +0300 Subject: [PATCH 052/157] Fixed offchain request deletion and expiration timestamp --- src/protocol/relay.cpp | 32 +++++++++++++++++++++++++++++--- src/wallet/chunkcollector.h | 6 +++--- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index e852f191..adb34842 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -335,6 +335,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) BOOST_FOREACH(PAIRTYPE(const CRelayResponsePair, CRelayRequestPairs)& item, responses_to_process) { MultichainProcessChunkResponse(&(item.first),&(item.second.m_Pairs),collector); + pRelayManager->DeleteRequest(item.first.request_id); } max_total_size=(MC_CCW_TIMEOUT_REQUEST-2)*MC_CCW_MAX_MBS_PER_SECOND*1024*1024; @@ -371,6 +372,17 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) } } } + else + { + if(!collect_row->m_State.m_Query.IsZero()) + { + map::iterator itqry = query_to_delete.find(collect_row->m_State.m_Query); + if (itqry == query_to_delete.end()) + { + query_to_delete.insert(make_pair(collect_row->m_State.m_Query,true)); + } + } + } } for(row=0;rowm_MemPool->GetCount();row++) @@ -421,6 +433,15 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) // printf("coll old req: row: %d, id: %s, rsps: %d, score (%d,%d)\n",row,collect_row->m_State.m_Query.ToString().c_str(),(int)query->m_Responses.size(),best_score,best_response); itrsp->second.m_Pairs.insert(make_pair(row,0)); } + map::iterator itdld = destination_loads.find(query->m_Responses[best_response].SourceID()); + if (itdld == destination_loads.end()) + { + destination_loads.insert(make_pair(query->m_Responses[best_response].SourceID(),collect_row->m_ChunkDef.m_Size + sizeof(mc_ChunkEntityKey))); + } + else + { + itdld->second+=collect_row->m_ChunkDef.m_Size + sizeof(mc_ChunkEntityKey); + } } } pRelayManager->UnLock(); @@ -442,7 +463,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) response=&(request->m_Responses[item.first.response_id]); expiration=time_now+MC_CCW_TIMEOUT_REQUEST; - dest_expiration=expiration+response->m_TimeDiff; + dest_expiration=expiration+response->m_MsgID.m_TimeStamp-request->m_MsgID.m_TimeStamp;// response->m_TimeDiff; ptrOut=&(payload[0]); *ptrOut=MC_RDT_EXPIRATION; ptrOut++; @@ -2010,7 +2031,6 @@ int mc_RelayManager::AddRequest(CNode *pto,int64_t destination,mc_OffchainMessag { int err=MC_ERR_NOERROR; - Lock(); map::iterator itreq_this = m_Requests.find(msg_id); if(itreq_this == m_Requests.end()) { @@ -2035,7 +2055,6 @@ int mc_RelayManager::AddRequest(CNode *pto,int64_t destination,mc_OffchainMessag err=MC_ERR_FOUND; } - UnLock(); return err; } @@ -2098,6 +2117,8 @@ int mc_RelayManager::DeleteRequest(mc_OffchainMessageID request) if(itreq != m_Requests.end()) { m_Requests.erase(itreq); + if(fDebug)LogPrint("offchain","Offchain delete: %s, msg: %s, size: %d. Open requests: %d\n",itreq->second.m_MsgID.ToString().c_str(), + mc_MsgTypeStr(itreq->second.m_MsgType).c_str(),(int)itreq->second.m_Payload.size(),(int)m_Requests.size()); } else { @@ -2131,6 +2152,7 @@ mc_OffchainMessageID mc_RelayManager::SendRequest(CNode* pto,uint32_t msg_type,u msg_id=GenerateMsgID(); + Lock(); { LOCK(cs_vNodes); BOOST_FOREACH(CNode* pnode, vNodes) @@ -2144,6 +2166,7 @@ mc_OffchainMessageID mc_RelayManager::SendRequest(CNode* pto,uint32_t msg_type,u if(AddRequest(pto,0,msg_id,msg_type,flags,payload,MC_RST_NONE) != MC_ERR_NOERROR) { + UnLock(); return mc_OffchainMessageID(); } @@ -2159,6 +2182,7 @@ mc_OffchainMessageID mc_RelayManager::SendNextRequest(mc_RelayResponse* response msg_id=GenerateMsgID(); + Lock(); { LOCK(cs_vNodes); BOOST_FOREACH(CNode* pnode, vNodes) @@ -2169,12 +2193,14 @@ mc_OffchainMessageID mc_RelayManager::SendNextRequest(mc_RelayResponse* response flags,payload,vSigScriptsEmpty,NULL,MC_PRA_NONE); if(AddRequest(pnode,response->SourceID(),msg_id,msg_type,flags,payload,MC_RST_NONE) != MC_ERR_NOERROR) { + UnLock(); return mc_OffchainMessageID(); } } } } + UnLock(); return msg_id; } diff --git a/src/wallet/chunkcollector.h b/src/wallet/chunkcollector.h index 5a3291ea..f06b03f0 100644 --- a/src/wallet/chunkcollector.h +++ b/src/wallet/chunkcollector.h @@ -19,9 +19,9 @@ #define MC_CCF_ERROR_MASK 0x00FF0000 #define MC_CCF_ALL 0xFFFFFFFF -#define MC_CCW_TIMEOUT_QUERY 30 -#define MC_CCW_TIMEOUT_REQUEST 5 -#define MC_CCW_MAX_CHUNKS_PER_QUERY 4 +#define MC_CCW_TIMEOUT_QUERY 35 +#define MC_CCW_TIMEOUT_REQUEST 10 +#define MC_CCW_MAX_CHUNKS_PER_QUERY 16 #define MC_CCW_DEFAULT_AUTOCOMMIT_DELAY 200 #define MC_CCW_WORST_RESPONSE_SCORE 1048576000 #define MC_CCW_DEFAULT_MEMPOOL_SIZE 1000 From 77217f0823425c254a7242cbf4ab1302a14786db Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 13 May 2018 15:01:44 +0300 Subject: [PATCH 053/157] Stream restrictions --- src/entities/asset.cpp | 33 +++++++++++ src/entities/asset.h | 8 +++ src/permissions/permission.h | 2 + src/protocol/multichainscript.cpp | 46 +++++---------- src/protocol/multichainscript.h | 1 + src/protocol/multichaintx.cpp | 8 +++ src/rpc/rpcrawdata.cpp | 97 ++++++++++++++++++++++++++++++- src/rpc/rpcstreams.cpp | 86 +++++++++++++++++++++++++-- src/rpc/rpcutils.cpp | 10 +++- src/rpc/rpcutils.h | 1 + 10 files changed, 250 insertions(+), 42 deletions(-) diff --git a/src/entities/asset.cpp b/src/entities/asset.cpp index 3ac3c255..c7952e2d 100644 --- a/src/entities/asset.cpp +++ b/src/entities/asset.cpp @@ -538,6 +538,7 @@ void mc_EntityDetails::Set(mc_EntityLedgerRow* row) } m_Permissions=0; + m_Restrictions=0; switch(m_LedgerRow.m_EntityType) { case MC_ENT_TYPE_ASSET: @@ -588,12 +589,26 @@ void mc_EntityDetails::Set(mc_EntityLedgerRow* row) value_offset=mc_FindSpecialParamInDetailsScript(m_LedgerRow.m_Script,m_LedgerRow.m_ScriptSize,MC_ENT_SPRM_PERMISSIONS,&value_size); if(value_offset <= m_LedgerRow.m_ScriptSize) { + if(m_Permissions & MC_PTP_WRITE) + { + m_Permissions -= MC_PTP_WRITE; + } + m_Permissions |= MC_PTP_SPECIFIED; if((value_size>0) && (value_size<=4)) { m_Permissions |= (uint32_t)mc_GetLE(m_LedgerRow.m_Script+value_offset,value_size); } } + value_offset=mc_FindSpecialParamInDetailsScript(m_LedgerRow.m_Script,m_LedgerRow.m_ScriptSize,MC_ENT_SPRM_RESTRICTIONS,&value_size); + if(value_offset <= m_LedgerRow.m_ScriptSize) + { + if((value_size>0) && (value_size<=4)) + { + m_Restrictions |= (uint32_t)mc_GetLE(m_LedgerRow.m_Script+value_offset,value_size); + } + } + } mc_ZeroABRaw(m_FullRef); @@ -1731,11 +1746,29 @@ uint32_t mc_EntityDetails::Permissions() return m_Permissions; } +uint32_t mc_EntityDetails::Restrictions() +{ + return m_Restrictions; +} + int mc_EntityDetails::AnyoneCanWrite() { unsigned char *ptr; size_t bytes; + + if(m_Permissions & MC_PTP_SPECIFIED) + { + if(mc_gState->m_Features->OffChainData()) + { + if(m_Permissions & MC_PTP_WRITE) + { + return 0; + } + return 1; + } + } + ptr=(unsigned char *)GetSpecialParam(MC_ENT_SPRM_ANYONE_CAN_WRITE,&bytes); if(ptr) { diff --git a/src/entities/asset.h b/src/entities/asset.h index 4387288a..ecb6ae29 100644 --- a/src/entities/asset.h +++ b/src/entities/asset.h @@ -25,6 +25,11 @@ #define MC_AST_ASSET_REF_TYPE_GENESIS 256 #define MC_AST_ASSET_REF_TYPE_SPECIAL 512 +#define MC_ENT_ENTITY_RESTRICTION_NONE 0x00000000 +#define MC_ENT_ENTITY_RESTRICTION_ONCHAIN 0x00000001 +#define MC_ENT_ENTITY_RESTRICTION_OFFCHAIN 0x00000002 + + #define MC_ENT_REF_SIZE 10 #define MC_ENT_REF_PREFIX_SIZE 2 @@ -57,6 +62,7 @@ #define MC_ENT_SPRM_ANYONE_CAN_WRITE 0x04 #define MC_ENT_SPRM_JSON_DETAILS 0x05 #define MC_ENT_SPRM_PERMISSIONS 0x06 +#define MC_ENT_SPRM_RESTRICTIONS 0x07 #define MC_ENT_SPRM_ASSET_MULTIPLE 0x41 #define MC_ENT_SPRM_UPGRADE_PROTOCOL_VERSION 0x42 #define MC_ENT_SPRM_UPGRADE_START_BLOCK 0x43 @@ -148,6 +154,7 @@ typedef struct mc_EntityDetails char m_Name[MC_ENT_MAX_NAME_SIZE+6]; // Entity name uint32_t m_Flags; uint32_t m_Permissions; + uint32_t m_Restrictions; unsigned char m_Reserved[36]; mc_EntityLedgerRow m_LedgerRow; void Zero(); @@ -166,6 +173,7 @@ typedef struct mc_EntityDetails // int HasFollowOns(); int AllowedFollowOns(); uint32_t Permissions(); + uint32_t Restrictions(); int AnyoneCanWrite(); int UpgradeProtocolVersion(); uint32_t UpgradeStartBlock(); diff --git a/src/permissions/permission.h b/src/permissions/permission.h index 61d39fb8..54114203 100644 --- a/src/permissions/permission.h +++ b/src/permissions/permission.h @@ -7,6 +7,7 @@ #include "utils/declare.h" #include "utils/dbwrapper.h" +#define MC_PTP_NONE 0x00000000 #define MC_PTP_CONNECT 0x00000001 #define MC_PTP_SEND 0x00000002 #define MC_PTP_RECEIVE 0x00000004 @@ -19,6 +20,7 @@ #define MC_PTP_UPGRADE 0x00010000 #define MC_PTP_BLOCK_MINER 0x01000000 #define MC_PTP_BLOCK_INDEX 0x02000000 +#define MC_PTP_SPECIFIED 0x80000000 #define MC_PTP_ALL 0x00FFFFFF #define MC_PTP_GLOBAL_ALL 0x00003137 diff --git a/src/protocol/multichainscript.cpp b/src/protocol/multichainscript.cpp index c81c3029..a9dd21a5 100644 --- a/src/protocol/multichainscript.cpp +++ b/src/protocol/multichainscript.cpp @@ -53,7 +53,7 @@ int mc_Script::Zero() m_AllocElements=0; m_AllocSize=0; m_ScriptType=MC_DCT_SCRIPT_TYPE_REGULAR; - + m_Restrictions=MC_ENT_ENTITY_RESTRICTION_NONE; return MC_ERR_NOERROR; } @@ -2535,43 +2535,13 @@ int mc_Script::SetChunkDefHash(unsigned char *hash,int size) int mc_Script::ExtractAndDeleteDataFormat(uint32_t *format) { return ExtractAndDeleteDataFormat(format,NULL,NULL,NULL); -/* - int elem,err; - - if(format) - { - *format=MC_SCR_DATA_FORMAT_UNKNOWN; - } - - if(mc_gState->m_Features->FormattedData() == 0) - { - return MC_ERR_NOERROR; - } - - if(m_NumElements < 2) - { - return MC_ERR_NOERROR; - } - - elem=m_NumElements-2; - - SetElement(elem); - if( (err=GetDataFormat(format)) != MC_ERR_WRONG_SCRIPT ) - { - if( (mc_gState->m_Features->OffChainData() == 0) || (GetChunkDef(NULL,NULL,NULL,NULL) == MC_ERR_WRONG_SCRIPT) ) - { - DeleteElement(elem); - } - return err; - } - - return MC_ERR_NOERROR; - */ } int mc_Script::ExtractAndDeleteDataFormat(uint32_t *format,unsigned char** hashes,int *chunk_count,int64_t *total_size) { int elem,err; + + m_Restrictions=MC_ENT_ENTITY_RESTRICTION_NONE; if(format) { @@ -2588,12 +2558,21 @@ int mc_Script::ExtractAndDeleteDataFormat(uint32_t *format,unsigned char** hashe return MC_ERR_NOERROR; } + if(m_NumElements >= 1) + { + if(m_lpCoord[(m_NumElements-1)*2+1] > 0) + { + m_Restrictions |= MC_ENT_ENTITY_RESTRICTION_ONCHAIN; + } + } + if(m_NumElements < 2) { return MC_ERR_NOERROR; } + elem=m_NumElements-2; SetElement(elem); @@ -2624,6 +2603,7 @@ int mc_Script::ExtractAndDeleteDataFormat(uint32_t *format,unsigned char** hashe SetElement(elem); while( (elem >= 0 ) && ((err=GetChunkDef(format,hashes,chunk_count,total_size)) == MC_ERR_NOERROR) ) { + m_Restrictions |= MC_ENT_ENTITY_RESTRICTION_OFFCHAIN; DeleteElement(elem); elem--; if(elem < 0) diff --git a/src/protocol/multichainscript.h b/src/protocol/multichainscript.h index 9f5156a8..b7b918fb 100644 --- a/src/protocol/multichainscript.h +++ b/src/protocol/multichainscript.h @@ -29,6 +29,7 @@ typedef struct mc_Script int m_AllocElements; int m_AllocSize; int m_ScriptType; + uint32_t m_Restrictions; mc_Script() { diff --git a/src/protocol/multichaintx.cpp b/src/protocol/multichaintx.cpp index 40ecad71..977ee6bf 100644 --- a/src/protocol/multichaintx.cpp +++ b/src/protocol/multichaintx.cpp @@ -792,6 +792,14 @@ bool MultiChainTransaction_CheckStreamItem(mc_EntityDetails *entity, unsigned char item_key[MC_ENT_MAX_ITEM_KEY_SIZE]; int item_key_size; + if(mc_gState->m_Features->OffChainData()) + { + if(mc_gState->m_TmpScript->m_Restrictions & entity->m_Restrictions) + { + reason="Metadata script rejected - stream restrictions violation"; + return false; + } + } // Multiple keys, if not allowed, check for count is made in different place for (int e = 1; e < mc_gState->m_TmpScript->GetNumElements()-1; e++) { diff --git a/src/rpc/rpcrawdata.cpp b/src/rpc/rpcrawdata.cpp index b3ada1ac..01e8a81d 100644 --- a/src/rpc/rpcrawdata.cpp +++ b/src/rpc/rpcrawdata.cpp @@ -701,6 +701,53 @@ CScript RawDataScriptFollowOn(Value *param,mc_EntityDetails *entity,mc_Script *l return scriptOpReturn; } +bool RawDataParseRestrictParameter(const Value& param,uint32_t *restrict,uint32_t *permissions,string *strError) +{ + *restrict=0; + *permissions=0; + + uint32_t match; + char* ptr; + char* start; + char* ptrEnd; + char c; + + if(param.type() != str_type) + { + *strError="Invalid restrict field, should be string"; + return false; + } + + ptr=(char*)param.get_str().c_str(); + ptrEnd=ptr+strlen(ptr); + start=ptr; + + while(ptr<=ptrEnd) + { + c=*ptr; + if( (c == ',') || (c ==0x00)) + { + if(ptr > start) + { + match=0; + if(memcmp(start,"write", ptr-start) == 0){match = 1; *permissions |= MC_PTP_WRITE ;} + if(memcmp(start,"onchain", ptr-start) == 0){match = 1; *restrict |= MC_ENT_ENTITY_RESTRICTION_ONCHAIN;} + if(memcmp(start,"offchain", ptr-start) == 0){match = 1; *restrict |= MC_ENT_ENTITY_RESTRICTION_OFFCHAIN;} + + if(match == 0) + { + *strError="Unsupported restriction"; + return false; + } + start=ptr+1; + } + } + ptr++; + } + + return true; +} + CScript RawDataScriptCreateStream(Value *param,mc_Script *lpDetails,mc_Script *lpDetailsScript,int *errorCode,string *strError) { CScript scriptOpReturn=CScript(); @@ -710,6 +757,8 @@ CScript RawDataScriptCreateStream(Value *param,mc_Script *lpDetails,mc_Script *l const unsigned char *script; string entity_name; int is_open=0; + uint32_t restrict; + uint32_t permissions=MC_PTP_WRITE; bool missing_name=true; bool missing_open=true; @@ -746,7 +795,7 @@ CScript RawDataScriptCreateStream(Value *param,mc_Script *lpDetails,mc_Script *l { if(!missing_open) { - *strError=string("open field can appear only once in the object"); + *strError=string("open/restrict field can appear only once in the object"); } if(d.value_.type() == bool_type) { @@ -756,10 +805,49 @@ CScript RawDataScriptCreateStream(Value *param,mc_Script *lpDetails,mc_Script *l { *strError=string("Invalid open"); } - lpDetails->SetSpecialParamValue(MC_ENT_SPRM_ANYONE_CAN_WRITE,(unsigned char*)&is_open,1); + if(mc_gState->m_Features->OffChainData() == 0) + { + lpDetails->SetSpecialParamValue(MC_ENT_SPRM_ANYONE_CAN_WRITE,(unsigned char*)&is_open,1); + } + else + { + permissions=is_open ? MC_PTP_NONE : MC_PTP_WRITE; + } missing_open=false; field_parsed=true; } + if(d.name_ == "restrict") + { + if(mc_gState->m_Features->OffChainData() == 0) + { + *strError=string("Per-stream restrictions not supported for this protocol version"); + *errorCode=RPC_NOT_SUPPORTED; + } + else + { + if(!missing_open) + { + *strError=string("open/restrict field can appear only once in the object"); + } + if(RawDataParseRestrictParameter(d.value_,&restrict,&permissions,strError)) + { + if(restrict & MC_ENT_ENTITY_RESTRICTION_OFFCHAIN) + { + if(restrict & MC_ENT_ENTITY_RESTRICTION_ONCHAIN) + { + *strError=string("Stream cannot be restricted from both onchain and offchain items"); + *errorCode=RPC_NOT_SUPPORTED; + } + } + if(restrict) + { + lpDetails->SetSpecialParamValue(MC_ENT_SPRM_RESTRICTIONS,(unsigned char*)&restrict,1); + } + } + missing_open=false; + field_parsed=true; + } + } if(d.name_ == "details") { if(!missing_details) @@ -777,6 +865,11 @@ CScript RawDataScriptCreateStream(Value *param,mc_Script *lpDetails,mc_Script *l } } + if(mc_gState->m_Features->OffChainData()) + { + lpDetails->SetSpecialParamValue(MC_ENT_SPRM_PERMISSIONS,(unsigned char*)&permissions,1); + } + if(strError->size() == 0) { lpDetailsScript->Clear(); diff --git a/src/rpc/rpcstreams.cpp b/src/rpc/rpcstreams.cpp index 9545a19f..4a3e744d 100644 --- a/src/rpc/rpcstreams.cpp +++ b/src/rpc/rpcstreams.cpp @@ -271,9 +271,6 @@ Value createstreamfromcmd(const Array& params, bool fHelp) stream_name=params[2].get_str(); } - if(params[3].type() != bool_type) - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid open flag, should be boolean"); - if(stream_name == "*") { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid stream name: *"); @@ -306,10 +303,69 @@ Value createstreamfromcmd(const Array& params, bool fHelp) lpDetails->Clear(); lpDetails->AddElement(); - if(params[3].get_bool()) + + if(mc_gState->m_Features->OffChainData()) + { + string strError; + uint32_t permissions=0; + uint32_t restrict; + if(params[3].type() != bool_type) + { + if(params[3].type() == obj_type) + { + BOOST_FOREACH(const Pair& d, params[3].get_obj()) + { + if(d.name_ == "restrict") + { + if(RawDataParseRestrictParameter(d.value_,&restrict,&permissions,&strError)) + { + if(restrict & MC_ENT_ENTITY_RESTRICTION_OFFCHAIN) + { + if(restrict & MC_ENT_ENTITY_RESTRICTION_ONCHAIN) + { + throw JSONRPCError(RPC_NOT_SUPPORTED, "Stream cannot be restricted from both onchain and offchain items"); + } + } + } + else + { + throw JSONRPCError(RPC_INVALID_PARAMETER, strError); + } + } + else + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid field, should be restrict"); + } + } + } + else + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid open flag, should be boolean or object"); + } + } + else + { + permissions = params[3].get_bool() ? MC_PTP_NONE : MC_PTP_WRITE; + } + lpDetails->SetSpecialParamValue(MC_ENT_SPRM_PERMISSIONS,(unsigned char*)&permissions,1); + if(restrict) + { + lpDetails->SetSpecialParamValue(MC_ENT_SPRM_RESTRICTIONS,(unsigned char*)&restrict,1); + } + + } + else { - unsigned char b=1; - lpDetails->SetSpecialParamValue(MC_ENT_SPRM_ANYONE_CAN_WRITE,&b,1); + if(params[3].type() != bool_type) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid open flag, should be boolean or object"); + } + + if(params[3].get_bool()) + { + unsigned char b=1; + lpDetails->SetSpecialParamValue(MC_ENT_SPRM_ANYONE_CAN_WRITE,&b,1); + } } if(stream_name.size()) { @@ -588,6 +644,24 @@ Value publishfrom(const Array& params, bool fHelp) } } + if( mc_gState->m_Features->OffChainData() ) + { + if(in_options & MC_RFD_OPTION_OFFCHAIN) + { + if(stream_entity.Restrictions() & MC_ENT_ENTITY_RESTRICTION_OFFCHAIN) + { + throw JSONRPCError(RPC_NOT_ALLOWED, "Publishing offchain items is not allowed to this stream"); + } + } + else + { + if(stream_entity.Restrictions() & MC_ENT_ENTITY_RESTRICTION_ONCHAIN) + { + throw JSONRPCError(RPC_NOT_ALLOWED, "Publishing onchain items is not allowed to this stream"); + } + } + } + if(keys.size() > 1) { if( mc_gState->m_Features->MultipleStreamKeys() == 0 ) diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index 3e17316c..4de12d57 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -625,8 +625,16 @@ Object StreamEntry(const unsigned char *txid,uint32_t output_level) { entry.push_back(Pair("open",false)); } + if(mc_gState->m_Features->OffChainData()) + { + Object pObject; + pObject.push_back(Pair("write",(entity.Permissions() & MC_PTP_WRITE) ? true : false)); + pObject.push_back(Pair("onchain",(entity.Restrictions() & MC_ENT_ENTITY_RESTRICTION_ONCHAIN) ? true : false)); + pObject.push_back(Pair("offchain",(entity.Restrictions() & MC_ENT_ENTITY_RESTRICTION_OFFCHAIN) ? true : false)); + entry.push_back(Pair("restrict",pObject)); + } } - + size_t value_size; int64_t offset,new_offset; diff --git a/src/rpc/rpcutils.h b/src/rpc/rpcutils.h index 02cc7178..672d72b3 100644 --- a/src/rpc/rpcutils.h +++ b/src/rpc/rpcutils.h @@ -140,6 +140,7 @@ void AppendOffChainFormatData(uint32_t data_format,uint32_t out_options,mc_Scrip int mc_BinaryCacheFile(string id,int mode); void mc_RemoveBinaryCacheFile(string id); string OffChainError(uint32_t status,int *errorCode); +bool RawDataParseRestrictParameter(const Value& param,uint32_t *restrict,uint32_t *permissions,string *strError); #endif /* RPCMULTICHAINUTILS_H */ From 40a08a0f83ac6f886acf6f6981289e7d62591acf Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 13 May 2018 15:38:26 +0300 Subject: [PATCH 054/157] Stream restrictions, help messages --- src/rpc/rpchelp.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/rpc/rpchelp.cpp b/src/rpc/rpchelp.cpp index 2e4c0d0e..550a3f27 100644 --- a/src/rpc/rpchelp.cpp +++ b/src/rpc/rpchelp.cpp @@ -1402,6 +1402,11 @@ void mc_InitRPCHelpMap06() "1. \"entity-type\" (string, required) upgrade\n" "2. \"upgrade-name\" (string, required) Upgrade name, if not \"\" should be unique.\n" "3. open (boolean, required ) Should be false\n" + " or \n" + "3. restrictions (object, optional) Stream restrictions\n" + " {\n" + " \"restrict\" : \"restrictions\" (string, optional) Stream restrictions, comma delimited. Possible values: write,offchain,onchain\n" + " }\n" "4 custom-fields (object, required) a json object with custom fields\n" " {\n" " \"protocol-version\": version (numeric, optional) Protocol version to upgrade to\n" @@ -1435,6 +1440,11 @@ void mc_InitRPCHelpMap06() "2. entity-type (string, required) stream\n" "3. \"stream-name\" (string, required) Stream name, if not \"\" should be unique.\n" "4. open (boolean, required) Allow anyone to publish in this stream\n" + " or \n" + "4. restrictions (object, optional) Stream restrictions\n" + " {\n" + " \"restrict\" : \"restrictions\" (string, optional) Stream restrictions, comma delimited. Possible values: write,offchain,onchain\n" + " }\n" "5 custom-fields (object, optional) a json object with custom fields\n" " {\n" " \"param-name\": \"param-value\" (strings, required) The key is the parameter name, the value is parameter value\n" @@ -3756,6 +3766,7 @@ void mc_InitRPCHelpMap16() " \"create\" : \"stream\" (string, required) stream\n" " \"name\" : \"stream-name\" (string, optional) Stream name\n" " \"open\" : true|false (string, optional, default: false) If true, anyone can publish\n" + " \"restrict\" : \"restrictions\" (string, optional) Stream restrictions, comma delimited. Possible values: write,offchain,onchain\n" " \"details\" : (object, optional) A json object with custom fields\n" " {\n" " \"param-name\": \"param-value\" (strings, required) The key is the parameter name, the value is parameter value\n" From 8b53dfb37647c35c72e775a636e83ad029e65daa Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 13 May 2018 20:30:31 +0300 Subject: [PATCH 055/157] Fixed realyed hop count --- src/protocol/relay.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index adb34842..3f87924d 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -1632,6 +1632,7 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, vector vSigScripts; vector vSigScriptsEmpty; vector vHops; + vector vHopsToRelay; vector vSendPaths; vector vEmptyHops; uint256 message_hash; @@ -1970,7 +1971,8 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, { if(pnode != pfrom) { - PushRelay(pnode,msg_format,vHops,vSendPaths,*msg_type_relay_ptr,msg_id_received,msg_id_to_respond,flags_relay, + vHopsToRelay=vHops; + PushRelay(pnode,msg_format,vHopsToRelay,vSendPaths,*msg_type_relay_ptr,msg_id_received,msg_id_to_respond,flags_relay, vPayloadRelay,vSigScriptsEmpty,pfrom,MC_PRA_NONE); } } From 0e5cfebaf2209991bc50ec904ad66b426eac538f Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 13 May 2018 20:30:57 +0300 Subject: [PATCH 056/157] Adding 127.0.0.1 to list of local ips --- src/net/net.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/net/net.cpp b/src/net/net.cpp index e1b5b288..0984c328 100644 --- a/src/net/net.cpp +++ b/src/net/net.cpp @@ -329,6 +329,11 @@ bool SeenLocal(const CService& addr) /** check whether a given address is potentially local */ bool IsLocal(const CService& addr) { + if(addr.IsLocal()) + { + return 1; + } + LOCK(cs_mapLocalHost); return mapLocalHost.count(addr) > 0; } From 1d1b048def50bafd7b68b127dd480163250cac33 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 14 May 2018 15:30:35 +0300 Subject: [PATCH 057/157] minimum-offset-fee --- src/chainparams/globals.h | 1 + src/core/main.cpp | 20 ++++++++--- src/multichain/multichain.h | 1 + src/protocol/multichainblock.cpp | 3 +- src/protocol/multichaintx.cpp | 62 ++++++++++++++++++++++++++++---- src/utils/utilwrapper.cpp | 1 + src/wallet/walletcoins.cpp | 22 +++++++++++- 7 files changed, 97 insertions(+), 13 deletions(-) diff --git a/src/chainparams/globals.h b/src/chainparams/globals.h index 03aa51c0..8253f6fa 100644 --- a/src/chainparams/globals.h +++ b/src/chainparams/globals.h @@ -5,6 +5,7 @@ #define GLOBALS_H mc_State* mc_gState; +unsigned int MIN_OFFCHAIN_FEE = 0; // new unsigned int MIN_RELAY_TX_FEE = 1000; // new unsigned int MAX_OP_RETURN_RELAY = 40; // standard.h unsigned int MAX_BLOCK_SIZE = 1000000; // block.h diff --git a/src/core/main.cpp b/src/core/main.cpp index bf5a9696..1d027c3b 100644 --- a/src/core/main.cpp +++ b/src/core/main.cpp @@ -53,6 +53,7 @@ bool AcceptMultiChainTransaction(const CTransaction& tx, int offset, bool accept, string& reason, + int64_t *mandatory_fee_out, uint32_t *replay); bool ExtractDestinationScriptValid(const CScript& scriptPubKey, CTxDestination& addressRet); bool AcceptAssetTransfers(const CTransaction& tx, const CCoinsViewCache &inputs, string& reason); @@ -1625,16 +1626,27 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa /* MCHN START */ uint32_t replay=0; + int64_t mandatory_fee; int permissions_from,permissions_to; permissions_from=mc_gState->m_Permissions->m_MempoolPermissions->GetCount(); - if(!AcceptMultiChainTransaction(tx,view,-1,true,reason, &replay)) + if(!AcceptMultiChainTransaction(tx,view,-1,true,reason, &mandatory_fee, &replay)) { return state.DoS(0, error("AcceptToMemoryPool: : AcceptMultiChainTransaction failed %s : %s", hash.ToString(),reason), REJECT_NONSTANDARD, reason); } + if(mandatory_fee) + { + txMinFee += mandatory_fee; + if (fLimitFree && nFees < txMinFee) + return state.DoS(0, error("AcceptToMemoryPool : not enough fees (including mandatory) %s, %d < %d", + hash.ToString(), nFees, txMinFee), + REJECT_INSUFFICIENTFEE, "insufficient fee"); + } + + if(fAddToWallet) { int err=pwalletTxsMain->AddTx(NULL,tx,-1,NULL,-1,0); @@ -2370,7 +2382,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin { const CTransaction &tx = block.vtx[i]; string reason; - if(!AcceptMultiChainTransaction(tx,view,offset,true,reason,NULL)) + if(!AcceptMultiChainTransaction(tx,view,offset,true,reason,NULL,NULL)) { return state.DoS(100, error(reason.c_str()), REJECT_INVALID, "bad-transaction"); @@ -2529,7 +2541,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin string reason; if(!fJustCheck) { - if(!AcceptMultiChainTransaction(tx,view,offset,true,reason,NULL)) + if(!AcceptMultiChainTransaction(tx,view,offset,true,reason,NULL,NULL)) { return state.DoS(0, error("ConnectBlock: : AcceptMultiChainTransaction failed %s : %s", tx.GetHash().ToString(),reason), @@ -2560,7 +2572,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin string reason; if(!fJustCheck) { - if(!AcceptMultiChainTransaction(tx,view,coinbase_offset,true,reason,NULL)) + if(!AcceptMultiChainTransaction(tx,view,coinbase_offset,true,reason,NULL,NULL)) { return false; } diff --git a/src/multichain/multichain.h b/src/multichain/multichain.h index 79380bcc..72be1973 100644 --- a/src/multichain/multichain.h +++ b/src/multichain/multichain.h @@ -16,6 +16,7 @@ extern unsigned int MIN_RELAY_TX_FEE; +extern unsigned int MIN_OFFCHAIN_FEE; #endif /* MULTICHAIN_H */ diff --git a/src/protocol/multichainblock.cpp b/src/protocol/multichainblock.cpp index a72754b6..6cdb6a0f 100644 --- a/src/protocol/multichainblock.cpp +++ b/src/protocol/multichainblock.cpp @@ -21,6 +21,7 @@ bool AcceptMultiChainTransaction(const CTransaction& tx, int offset, bool accept, string& reason, + int64_t *mandatory_fee_out, uint32_t *replay); bool AcceptAdminMinerPermissions(const CTransaction& tx, int offset, @@ -369,7 +370,7 @@ bool ReplayMemPool(CTxMemPool& pool, int from,bool accept) CCoinsViewCache view(&dummy); CCoinsViewMemPool viewMemPool(pcoinsTip, pool); view.SetBackend(viewMemPool); - if(!AcceptMultiChainTransaction(tx,view,-1,accept,reason,NULL)) + if(!AcceptMultiChainTransaction(tx,view,-1,accept,reason,NULL,NULL)) { removed_type="rejected"; } diff --git a/src/protocol/multichaintx.cpp b/src/protocol/multichaintx.cpp index 977ee6bf..b490bba5 100644 --- a/src/protocol/multichaintx.cpp +++ b/src/protocol/multichaintx.cpp @@ -54,6 +54,9 @@ typedef struct CMultiChainTxDetails int details_script_type; // Entity details script type - new/update uint32_t new_entity_type; // New entity type int new_entity_output; // Output where new entity is defined + int64_t total_offchain_size; // Total size of offchain items + int64_t total_value_in; // Total amount in inputs + int64_t total_value_out; // Total amount in outputs CMultiChainTxDetails() { @@ -97,7 +100,9 @@ void CMultiChainTxDetails::Zero() details_script_type=-1; new_entity_type=MC_ENT_TYPE_NONE; new_entity_output=-1; - + total_offchain_size=0; + total_value_in=0; + total_value_out=0; } bool CMultiChainTxDetails::IsRelevantInput(int vin, int vout) @@ -338,6 +343,8 @@ bool MultiChainTransaction_CheckInputs(const CTransaction& tx, const CScript& script1 = coins->vout[prevout.n].scriptPubKey; CScript::const_iterator pc1 = script1.begin(); + details->total_value_in+=coins->vout[prevout.n].nValue; + txnouttype typeRet; int nRequiredRet; vector addressRets; @@ -559,9 +566,9 @@ void MultiChainTransaction_FillAdminPermissionsBeforeTx(const CTransaction& tx, } } -bool MultiChainTransaction_VerifyAndDeleteDataFormatElements(string& reason) +bool MultiChainTransaction_VerifyAndDeleteDataFormatElements(string& reason,int64_t *total_size) { - if(mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(NULL)) + if(mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(NULL,NULL,NULL,total_size)) { reason="Error in data format script"; return false; @@ -580,12 +587,15 @@ bool MultiChainTransaction_CheckOpReturnScript(const CTransaction& tx, uint32_t timestamp,approval; vector addressRets; CTxDestination single_address; + int64_t total_offchain_size; + total_offchain_size=0; // mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(NULL); // Format scripts are eliminated for protocol checks - if(!MultiChainTransaction_VerifyAndDeleteDataFormatElements(reason)) + if(!MultiChainTransaction_VerifyAndDeleteDataFormatElements(reason,&total_offchain_size)) { return false; } + details->total_offchain_size+=total_offchain_size; if(!details->fScriptHashAllFound) { @@ -849,7 +859,7 @@ bool MultiChainTransaction_CheckEntityItem(const CTransaction& tx, mc_EntityDetails entity; // mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(NULL); // Format scripts are eliminated for protocol checks - if(!MultiChainTransaction_VerifyAndDeleteDataFormatElements(reason)) + if(!MultiChainTransaction_VerifyAndDeleteDataFormatElements(reason,NULL)) { return false; } @@ -1277,6 +1287,8 @@ bool MultiChainTransaction_CheckOutputs(const CTransaction& tx, for (unsigned int vout = 0; vout < tx.vout.size(); vout++) // Basic checks, destinations and simple grants { + details->total_value_out+=tx.vout[vout].nValue; + MultiChainTransaction_SetTmpOutputScript(tx.vout[vout].scriptPubKey); if(mc_gState->m_TmpScript->IsOpReturnScript()) @@ -1311,8 +1323,7 @@ bool MultiChainTransaction_CheckOutputs(const CTransaction& tx, if(!MultiChainTransaction_ProcessPermissions(tx,offset,vout,permission_type,false,details,reason)) { return false; - } - + } } } @@ -1364,6 +1375,31 @@ bool MultiChainTransaction_CheckOutputs(const CTransaction& tx, return true; } +int64_t MultiChainTransaction_OffchainFee(int64_t total_offchain_size) // Total size of offchain items +{ + return (MIN_OFFCHAIN_FEE*total_offchain_size + 999)/ 1000; +} + +bool MultiChainTransaction_CheckMandatoryFee(CMultiChainTxDetails *details, // Tx details object + int64_t *mandatory_fee, // Mandatory Fee + string& reason) // Error message +{ + *mandatory_fee = MultiChainTransaction_OffchainFee(details->total_offchain_size); + + if(*mandatory_fee) + { + if(details->total_value_in-details->total_value_out < *mandatory_fee) + { + reason="Insufficient mandatory fee"; + return false; + } + } + + return true; +} + + + bool MultiChainTransaction_ProcessAssetIssuance(const CTransaction& tx, // Tx to check int offset, // Tx offset in block, -1 if in memppol bool accept, // Accept to mempools if successful @@ -1968,10 +2004,12 @@ bool AcceptMultiChainTransaction (const CTransaction& tx, int offset, // Tx offset in block, -1 if in memppol bool accept, // Accept to mempools if successful string& reason, // Error message + int64_t *mandatory_fee_out, // Mandatory fee uint32_t *replay) // Replay flag - if tx should be rechecked or only permissions { CMultiChainTxDetails details; bool fReject=false; + int64_t mandatory_fee=0; if(mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) { @@ -1998,6 +2036,11 @@ bool AcceptMultiChainTransaction (const CTransaction& tx, goto exitlbl; } + if(!MultiChainTransaction_CheckMandatoryFee(&details,&mandatory_fee,reason)) + { + fReject=true; + goto exitlbl; + } // Asset genesis/followon if(!MultiChainTransaction_ProcessAssetIssuance(tx,offset,accept,&details,reason)) { @@ -2037,6 +2080,11 @@ bool AcceptMultiChainTransaction (const CTransaction& tx, mc_gState->m_Permissions->RollBackToCheckPoint(); } + if(mandatory_fee_out) + { + *mandatory_fee_out=mandatory_fee; + } + if(replay) { *replay=0; diff --git a/src/utils/utilwrapper.cpp b/src/utils/utilwrapper.cpp index fa2099c6..c3843c79 100644 --- a/src/utils/utilwrapper.cpp +++ b/src/utils/utilwrapper.cpp @@ -777,6 +777,7 @@ int mc_MultichainParams::SetGlobals() m_ProtocolVersion=ProtocolVersion(); MIN_RELAY_TX_FEE=(unsigned int)GetInt64Param("minimumrelayfee"); + MIN_OFFCHAIN_FEE=(unsigned int)GetInt64Param("minimumoffchainfee"); MAX_OP_RETURN_RELAY=(unsigned int)GetInt64Param("maxstdopreturnsize"); MAX_OP_RETURN_RELAY=GetArg("-datacarriersize", MAX_OP_RETURN_RELAY); MAX_BLOCK_SIZE=(unsigned int)GetInt64Param("maximumblocksize"); diff --git a/src/wallet/walletcoins.cpp b/src/wallet/walletcoins.cpp index ac718714..b5b1559e 100644 --- a/src/wallet/walletcoins.cpp +++ b/src/wallet/walletcoins.cpp @@ -14,6 +14,8 @@ #include "custom/custom.h" extern mc_WalletTxs* pwalletTxsMain; +void MultiChainTransaction_SetTmpOutputScript(const CScript& script1); +int64_t MultiChainTransaction_OffchainFee(int64_t total_offchain_size); using namespace std; @@ -1497,6 +1499,8 @@ CAmount BuildAssetTransaction(CWallet *lpWallet, CAmount nTotalChangeValue=0; CAmount default_change_output; CAmount change_amount; + CAmount mandatory_fee=0; + int64_t total_offchain_size=0; for(int i=0;iGetCount();i++) // Finding relevant asset groups and calculating native currency total { @@ -1594,6 +1598,22 @@ CAmount BuildAssetTransaction(CWallet *lpWallet, } } + if(MIN_OFFCHAIN_FEE) + { + total_offchain_size=0; + BOOST_FOREACH (const PAIRTYPE(CScript, CAmount)& s, vecSend) // Original outputs + { + MultiChainTransaction_SetTmpOutputScript(s.first); + mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(NULL,NULL,NULL,&total_offchain_size); + } + mandatory_fee=MultiChainTransaction_OffchainFee(total_offchain_size); + } + + if(nFeeRet == 0) + { + nFeeRet=mandatory_fee; + } + missing_amount=nFeeRet+(change_count+extra_change_count)*default_change_output-nTotalInValue; if(missing_amount > 0) // Inputs don't carry enough native currency, go out and select additional coins { @@ -1860,7 +1880,7 @@ CAmount BuildAssetTransaction(CWallet *lpWallet, break; } - CAmount nFeeNeeded = lpWallet->GetMinimumFee(nBytes, nTxConfirmTarget, mempool); + CAmount nFeeNeeded = lpWallet->GetMinimumFee(nBytes, nTxConfirmTarget, mempool)+mandatory_fee; // If we made it here and we aren't even able to meet the relay fee on the next pass, give up // because we must be at the maximum allowed fee. From f13a1ecca554c74340184e6491d6c3207cf25ed2 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 14 May 2018 17:05:03 +0300 Subject: [PATCH 058/157] Fixed create stream restrict --- src/rpc/rpcstreams.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/rpcstreams.cpp b/src/rpc/rpcstreams.cpp index 4a3e744d..d8b6069f 100644 --- a/src/rpc/rpcstreams.cpp +++ b/src/rpc/rpcstreams.cpp @@ -308,7 +308,7 @@ Value createstreamfromcmd(const Array& params, bool fHelp) { string strError; uint32_t permissions=0; - uint32_t restrict; + uint32_t restrict=0; if(params[3].type() != bool_type) { if(params[3].type() == obj_type) From 483d8feda7c43143d94fe592682ec65fa5fe05cb Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 14 May 2018 18:04:39 +0300 Subject: [PATCH 059/157] Fixed no coins with send permission error message --- src/wallet/walletcoins.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/wallet/walletcoins.cpp b/src/wallet/walletcoins.cpp index b5b1559e..bdfb0ec3 100644 --- a/src/wallet/walletcoins.cpp +++ b/src/wallet/walletcoins.cpp @@ -2407,7 +2407,7 @@ bool CreateAssetGroupingTransaction(CWallet *lpWallet, const vector Date: Mon, 14 May 2018 18:14:03 +0300 Subject: [PATCH 060/157] Fixed setting of autosubscribe value --- src/rpc/rpcmisc.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/rpc/rpcmisc.cpp b/src/rpc/rpcmisc.cpp index a7f258ed..724a5d1d 100644 --- a/src/rpc/rpcmisc.cpp +++ b/src/rpc/rpcmisc.cpp @@ -474,19 +474,27 @@ Value setruntimeparam(const json_spirit::Array& params, bool fHelp) { string autosubscribe=params[1].get_str(); uint32_t mode=MC_WMD_NONE; + bool found=false; if(autosubscribe=="streams") { mode |= MC_WMD_AUTOSUBSCRIBE_STREAMS; + found=true; } if(autosubscribe=="assets") { mode |= MC_WMD_AUTOSUBSCRIBE_ASSETS; + found=true; } if( (autosubscribe=="assets,streams") || (autosubscribe=="streams,assets")) { mode |= MC_WMD_AUTOSUBSCRIBE_STREAMS; mode |= MC_WMD_AUTOSUBSCRIBE_ASSETS; + found=true; } + if(!found) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter value"); + } if(pwalletTxsMain) { From cc9023f6c9886fe160cfb80865dd0fd2a3364843 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 14 May 2018 18:29:06 +0300 Subject: [PATCH 061/157] Fixed non-integer param in getblock --- src/rpc/rpcblockchain.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/rpc/rpcblockchain.cpp b/src/rpc/rpcblockchain.cpp index f80fb3a1..3eb59925 100644 --- a/src/rpc/rpcblockchain.cpp +++ b/src/rpc/rpcblockchain.cpp @@ -34,6 +34,7 @@ Array PerOutputDataEntries(const CTxOut& txout,mc_Script *lpScript,uint256 txid, string EncodeHexTx(const CTransaction& tx); int OrphanPoolSize(); bool paramtobool(Value param); +bool StringToInt(string str,int *value); /* MCHN END */ void ScriptPubKeyToJSON(const CScript& scriptPubKey, Object& out, bool fIncludeHex); @@ -397,7 +398,13 @@ Value getblock(const Array& params, bool fHelp) std::string strHash = params[0].get_str(); if(strHash.size() < 64) { - int nHeight = atoi(params[0].get_str().c_str()); + int nHeight; + if(!StringToInt(params[0].get_str(),&nHeight)) + { + throw JSONRPCError(RPC_BLOCK_NOT_FOUND, "Block height should be integer"); + } + +// int nHeight = atoi(params[0].get_str().c_str()); if (nHeight < 0 || nHeight > chainActive.Height()) throw JSONRPCError(RPC_BLOCK_NOT_FOUND, "Block height out of range"); From 40e61149927b6fe543623cfd927526d8b2b9c8b7 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 15 May 2018 08:37:32 +0300 Subject: [PATCH 062/157] Fixed missing help commands --- src/rpc/rpchelp.cpp | 52 +++++++++++++++++++++++++++++++++++++++++-- src/rpc/rpcwallet.cpp | 46 +++----------------------------------- 2 files changed, 53 insertions(+), 45 deletions(-) diff --git a/src/rpc/rpchelp.cpp b/src/rpc/rpchelp.cpp index 550a3f27..e00e382d 100644 --- a/src/rpc/rpchelp.cpp +++ b/src/rpc/rpchelp.cpp @@ -3765,7 +3765,7 @@ void mc_InitRPCHelpMap16() " {\n" " \"create\" : \"stream\" (string, required) stream\n" " \"name\" : \"stream-name\" (string, optional) Stream name\n" - " \"open\" : true|false (string, optional, default: false) If true, anyone can publish\n" + " \"open\" : true|false (boolean, optional, default: false) If true, anyone can publish\n" " \"restrict\" : \"restrictions\" (string, optional) Stream restrictions, comma delimited. Possible values: write,offchain,onchain\n" " \"details\" : (object, optional) A json object with custom fields\n" " {\n" @@ -4042,7 +4042,55 @@ void mc_InitRPCHelpMap17() + HelpExampleRpc("deletebinarycache", "\"TjnVWwHYEg4\"") )); - + mapHelpStrings.insert(std::make_pair("walletpassphrase", + "walletpassphrase \"passphrase\" timeout\n" + "\nStores the wallet decryption key in memory for 'timeout' seconds.\n" + "This is needed prior to performing transactions related to private keys such as sending assets\n" + "\nArguments:\n" + "1. \"passphrase\" (string, required) The wallet passphrase\n" + "2. timeout (numeric, required) The time to keep the decryption key in seconds.\n" + "\nNote:\n" + "Issuing the walletpassphrase command while the wallet is already unlocked will set a new unlock\n" + "time that overrides the old one.\n" + "\nExamples:\n" + "\nunlock the wallet for 60 seconds\n" + + HelpExampleCli("walletpassphrase", "\"my pass phrase\" 60") + + "\nLock the wallet again (before 60 seconds)\n" + + HelpExampleCli("walletlock", "") + + "\nAs json rpc call\n" + + HelpExampleRpc("walletpassphrase", "\"my pass phrase\", 60") + )); + + + mapHelpStrings.insert(std::make_pair("walletpassphrasechange", + "walletpassphrasechange \"oldpassphrase\" \"newpassphrase\"\n" + "\nChanges the wallet passphrase from 'oldpassphrase' to 'newpassphrase'.\n" + "\nArguments:\n" + "1. \"oldpassphrase\" (string, required) The current passphrase\n" + "2. \"newpassphrase\" (string, required) The new passphrase\n" + "\nExamples:\n" + + HelpExampleCli("walletpassphrasechange", "\"old one\" \"new one\"") + + HelpExampleRpc("walletpassphrasechange", "\"old one\", \"new one\"") + )); + + mapHelpStrings.insert(std::make_pair("walletlock", + "walletlock\n" + "\nRemoves the wallet encryption key from memory, locking the wallet.\n" + "After calling this method, you will need to call walletpassphrase again\n" + "before being able to call any methods which require the wallet to be unlocked.\n" + "\nExamples:\n" + "\nSet the passphrase for 2 minutes to perform a transaction\n" + + HelpExampleCli("walletpassphrase", "\"my pass phrase\" 120") + + "\nPerform a send (requires passphrase set)\n" + + HelpExampleCli("sendtoaddress", "\"1M72Sfpbz1BPpXFHz9m3CdqATR44Jvaydd\" 1.0") + + "\nClear the passphrase since we are done before 2 minutes is up\n" + + HelpExampleCli("walletlock", "") + + "\nAs json rpc call\n" + + HelpExampleRpc("walletlock", "") + )); + + + mapHelpStrings.insert(std::make_pair("AAAAAAA", "" )); diff --git a/src/rpc/rpcwallet.cpp b/src/rpc/rpcwallet.cpp index b42a6546..54f4acc8 100644 --- a/src/rpc/rpcwallet.cpp +++ b/src/rpc/rpcwallet.cpp @@ -1953,24 +1953,7 @@ static void LockWallet(CWallet* pWallet) Value walletpassphrase(const Array& params, bool fHelp) { if (pwalletMain->IsCrypted() && (fHelp || params.size() != 2)) // MCHN - throw runtime_error( - "walletpassphrase \"passphrase\" timeout\n" - "\nStores the wallet decryption key in memory for 'timeout' seconds.\n" - "This is needed prior to performing transactions related to private keys such as sending assets\n" - "\nArguments:\n" - "1. \"passphrase\" (string, required) The wallet passphrase\n" - "2. timeout (numeric, required) The time to keep the decryption key in seconds.\n" - "\nNote:\n" - "Issuing the walletpassphrase command while the wallet is already unlocked will set a new unlock\n" - "time that overrides the old one.\n" - "\nExamples:\n" - "\nunlock the wallet for 60 seconds\n" - + HelpExampleCli("walletpassphrase", "\"my pass phrase\" 60") + - "\nLock the wallet again (before 60 seconds)\n" - + HelpExampleCli("walletlock", "") + - "\nAs json rpc call\n" - + HelpExampleRpc("walletpassphrase", "\"my pass phrase\", 60") - ); + throw runtime_error("Help message not found\n"); if (fHelp) return true; @@ -2008,16 +1991,7 @@ Value walletpassphrase(const Array& params, bool fHelp) Value walletpassphrasechange(const Array& params, bool fHelp) { if (pwalletMain->IsCrypted() && (fHelp || params.size() != 2)) - throw runtime_error( - "walletpassphrasechange \"oldpassphrase\" \"newpassphrase\"\n" - "\nChanges the wallet passphrase from 'oldpassphrase' to 'newpassphrase'.\n" - "\nArguments:\n" - "1. \"oldpassphrase\" (string) The current passphrase\n" - "2. \"newpassphrase\" (string) The new passphrase\n" - "\nExamples:\n" - + HelpExampleCli("walletpassphrasechange", "\"old one\" \"new one\"") - + HelpExampleRpc("walletpassphrasechange", "\"old one\", \"new one\"") - ); + throw runtime_error("Help message not found\n"); if (fHelp) return true; @@ -2049,21 +2023,7 @@ Value walletpassphrasechange(const Array& params, bool fHelp) Value walletlock(const Array& params, bool fHelp) { if (pwalletMain->IsCrypted() && (fHelp || params.size() != 0)) - throw runtime_error( - "walletlock\n" - "\nRemoves the wallet encryption key from memory, locking the wallet.\n" - "After calling this method, you will need to call walletpassphrase again\n" - "before being able to call any methods which require the wallet to be unlocked.\n" - "\nExamples:\n" - "\nSet the passphrase for 2 minutes to perform a transaction\n" - + HelpExampleCli("walletpassphrase", "\"my pass phrase\" 120") + - "\nPerform a send (requires passphrase set)\n" - + HelpExampleCli("sendtoaddress", "\"1M72Sfpbz1BPpXFHz9m3CdqATR44Jvaydd\" 1.0") + - "\nClear the passphrase since we are done before 2 minutes is up\n" - + HelpExampleCli("walletlock", "") + - "\nAs json rpc call\n" - + HelpExampleRpc("walletlock", "") - ); + throw runtime_error("Help message not found\n"); if (fHelp) return true; From 569159059ac93a13957fba4995bbabf4b37fd2a9 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 15 May 2018 10:18:31 +0300 Subject: [PATCH 063/157] Fixed wallet encryption for compilation on Ubuntu 18 --- src/wallet/crypter.cpp | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/src/wallet/crypter.cpp b/src/wallet/crypter.cpp index 83804f68..9c7102ca 100644 --- a/src/wallet/crypter.cpp +++ b/src/wallet/crypter.cpp @@ -59,16 +59,19 @@ bool CCrypter::Encrypt(const CKeyingMaterial& vchPlaintext, std::vector (nCLen); - EVP_CIPHER_CTX ctx; +// EVP_CIPHER_CTX ctx; + EVP_CIPHER_CTX* ctx = EVP_CIPHER_CTX_new(); bool fOk = true; - EVP_CIPHER_CTX_init(&ctx); - if (fOk) fOk = EVP_EncryptInit_ex(&ctx, EVP_aes_256_cbc(), NULL, chKey, chIV) != 0; - if (fOk) fOk = EVP_EncryptUpdate(&ctx, &vchCiphertext[0], &nCLen, &vchPlaintext[0], nLen) != 0; - if (fOk) fOk = EVP_EncryptFinal_ex(&ctx, (&vchCiphertext[0]) + nCLen, &nFLen) != 0; - EVP_CIPHER_CTX_cleanup(&ctx); + EVP_CIPHER_CTX_init(ctx); + if (fOk) fOk = EVP_EncryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, chKey, chIV) != 0; + if (fOk) fOk = EVP_EncryptUpdate(ctx, &vchCiphertext[0], &nCLen, &vchPlaintext[0], nLen) != 0; + if (fOk) fOk = EVP_EncryptFinal_ex(ctx, (&vchCiphertext[0]) + nCLen, &nFLen) != 0; + EVP_CIPHER_CTX_cleanup(ctx); + EVP_CIPHER_CTX_free(ctx); + if (!fOk) return false; vchCiphertext.resize(nCLen + nFLen); @@ -86,16 +89,19 @@ bool CCrypter::Decrypt(const std::vector& vchCiphertext, CKeyingM vchPlaintext = CKeyingMaterial(nPLen); - EVP_CIPHER_CTX ctx; +// EVP_CIPHER_CTX ctx; + EVP_CIPHER_CTX* ctx = EVP_CIPHER_CTX_new(); bool fOk = true; - EVP_CIPHER_CTX_init(&ctx); - if (fOk) fOk = EVP_DecryptInit_ex(&ctx, EVP_aes_256_cbc(), NULL, chKey, chIV) != 0; - if (fOk) fOk = EVP_DecryptUpdate(&ctx, &vchPlaintext[0], &nPLen, &vchCiphertext[0], nLen) != 0; - if (fOk) fOk = EVP_DecryptFinal_ex(&ctx, (&vchPlaintext[0]) + nPLen, &nFLen) != 0; - EVP_CIPHER_CTX_cleanup(&ctx); + EVP_CIPHER_CTX_init(ctx); + if (fOk) fOk = EVP_DecryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, chKey, chIV) != 0; + if (fOk) fOk = EVP_DecryptUpdate(ctx, &vchPlaintext[0], &nPLen, &vchCiphertext[0], nLen) != 0; + if (fOk) fOk = EVP_DecryptFinal_ex(ctx, (&vchPlaintext[0]) + nPLen, &nFLen) != 0; + EVP_CIPHER_CTX_cleanup(ctx); + EVP_CIPHER_CTX_free(ctx); + if (!fOk) return false; vchPlaintext.resize(nPLen + nFLen); From 1b1e46f6dc5480dc6d703cf4dd5fa19cfddce3bb Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 15 May 2018 18:29:31 +0300 Subject: [PATCH 064/157] Offchain items help messages tweaks --- src/rpc/rpchelp.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/rpc/rpchelp.cpp b/src/rpc/rpchelp.cpp index e00e382d..5c4e4617 100644 --- a/src/rpc/rpchelp.cpp +++ b/src/rpc/rpchelp.cpp @@ -1393,6 +1393,11 @@ void mc_InitRPCHelpMap06() "1. \"entity-type\" (string, required) stream\n" "2. \"stream-name\" (string, required) Stream name, if not \"\" should be unique.\n" "3. open (boolean, required ) Allow anyone to publish in this stream\n" + " or \n" + "3. restrictions (object, optional) Stream restrictions\n" + " {\n" + " \"restrict\" : \"restrictions\" (string, optional) Stream restrictions, comma delimited. Possible values: write,offchain,onchain\n" + " }\n" "4 custom-fields (object, optional) a json object with custom fields\n" " {\n" " \"param-name\": \"param-value\" (strings, required) The key is the parameter name, the value is parameter value\n" @@ -1402,11 +1407,6 @@ void mc_InitRPCHelpMap06() "1. \"entity-type\" (string, required) upgrade\n" "2. \"upgrade-name\" (string, required) Upgrade name, if not \"\" should be unique.\n" "3. open (boolean, required ) Should be false\n" - " or \n" - "3. restrictions (object, optional) Stream restrictions\n" - " {\n" - " \"restrict\" : \"restrictions\" (string, optional) Stream restrictions, comma delimited. Possible values: write,offchain,onchain\n" - " }\n" "4 custom-fields (object, required) a json object with custom fields\n" " {\n" " \"protocol-version\": version (numeric, optional) Protocol version to upgrade to\n" @@ -3488,6 +3488,7 @@ void mc_InitRPCHelpMap15() "1. \"asset-identifier\" (string, required) Asset identifier - one of the following: asset txid, asset reference, asset name.\n" " or\n" "1. entity-identifier(s) (array, optional) A json array of stream or asset identifiers \n" + "2. purge (boolean, optional, default=false) Purge all offchain data for the stream\n" "\nResult:\n" "\nExamples:\n" + HelpExampleCli("unsubscribe", "\"test-stream\"") From 0631ff3e77a7497ed0c9f7060d272620a2440ce7 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 15 May 2018 20:33:36 +0300 Subject: [PATCH 065/157] Import from block --- src/rpc/rpcclient.cpp | 1 + src/rpc/rpcdump.cpp | 66 +++++++++++++++++------- src/rpc/rpcmisc.cpp | 2 + src/rpc/rpcstreams.cpp | 2 +- src/rpc/rpcutils.cpp | 90 ++++++++++++++++++++++++++++++++- src/rpc/rpcutils.h | 2 + src/wallet/wallet.cpp | 104 ++++++++++++++++++++------------------ src/wallet/wallet.h | 2 +- src/wallet/wallettxdb.cpp | 38 +++++++++++--- src/wallet/wallettxdb.h | 4 +- src/wallet/wallettxs.cpp | 4 +- src/wallet/wallettxs.h | 2 +- 12 files changed, 236 insertions(+), 81 deletions(-) diff --git a/src/rpc/rpcclient.cpp b/src/rpc/rpcclient.cpp index ba9f5747..5d49d62a 100644 --- a/src/rpc/rpcclient.cpp +++ b/src/rpc/rpcclient.cpp @@ -269,6 +269,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "importprivkey", 2 }, { "importaddress", 0 }, { "importaddress", 2 }, + { "importwallet", 1 }, { "verifychain", 0 }, { "verifychain", 1 }, { "keypoolrefill", 0 }, diff --git a/src/rpc/rpcdump.cpp b/src/rpc/rpcdump.cpp index 4ba5aff8..dedc3881 100644 --- a/src/rpc/rpcdump.cpp +++ b/src/rpc/rpcdump.cpp @@ -15,6 +15,7 @@ #include "wallet/wallet.h" /* MCHN START */ #include "wallet/wallettxs.h" +#include "rpc/rpcutils.h" /* MCHN END */ #include @@ -90,8 +91,11 @@ Value importprivkey(const Array& params, bool fHelp) // Whether to perform rescan after import bool fRescan = true; + int start_block=0; if (params.size() > 2) - fRescan = params[2].get_bool(); + { + start_block=ParseRescanParameter(params[2],&fRescan); + } bool fNewFound=false; @@ -167,7 +171,7 @@ Value importprivkey(const Array& params, bool fHelp) } if (fRescan) { - pwalletMain->ScanForWalletTransactions(chainActive.Genesis(), true, true); + pwalletMain->ScanForWalletTransactions(chainActive[start_block], true, true); } return Value::null; @@ -186,8 +190,11 @@ Value importaddress(const Array& params, bool fHelp) // Whether to perform rescan after import bool fRescan = true; + int start_block=0; if (params.size() > 2) - fRescan = params[2].get_bool(); + { + start_block=ParseRescanParameter(params[2],&fRescan); + } bool fNewFound=false; vector inputStrings=ParseStringList(params[0]); @@ -211,10 +218,13 @@ Value importaddress(const Array& params, bool fHelp) inputAddresses.push_back(address); inputScripts.push_back(script); - if (::IsMine(*pwalletMain, script) == ISMINE_SPENDABLE) + if( (mc_gState->m_WalletMode & MC_WMD_ADDRESS_TXS) == 0 ) { -// throw JSONRPCError(RPC_WALLET_ERROR, "The wallet already contains the private key for this address or script"); - return Value::null; + if (::IsMine(*pwalletMain, script) == ISMINE_SPENDABLE) + { + // throw JSONRPCError(RPC_WALLET_ERROR, "The wallet already contains the private key for this address or script"); + return Value::null; + } } if (!pwalletMain->HaveWatchOnly(script)) @@ -223,9 +233,12 @@ Value importaddress(const Array& params, bool fHelp) } } - if(!fNewFound) + if( (mc_gState->m_WalletMode & MC_WMD_ADDRESS_TXS) == 0 ) { - return Value::null; + if(!fNewFound) + { + return Value::null; + } } pwalletMain->MarkDirty(); @@ -282,7 +295,7 @@ Value importaddress(const Array& params, bool fHelp) { if (fRescan) { - pwalletMain->ScanForWalletTransactions(chainActive.Genesis(), true, true); + pwalletMain->ScanForWalletTransactions(chainActive[start_block], true, true); pwalletMain->ReacceptWalletTransactions(); } } @@ -292,11 +305,18 @@ Value importaddress(const Array& params, bool fHelp) Value importwallet(const Array& params, bool fHelp) { - if (fHelp || params.size() != 1) + if (fHelp || params.size() < 1 || params.size() > 2) throw runtime_error("Help message not found\n"); EnsureWalletIsUnlocked(); + bool fRescan = true; + int start_block=0; + if (params.size() > 1) + { + start_block=ParseRescanParameter(params[1],&fRescan); + } + ifstream file; file.open(params[0].get_str().c_str(), std::ios::in | std::ios::ate); if (!file.is_open()) @@ -328,9 +348,11 @@ Value importwallet(const Array& params, bool fHelp) CPubKey pubkey = key.GetPubKey(); assert(key.VerifyPubKey(pubkey)); CKeyID keyid = pubkey.GetID(); + bool fAlreadyHave=false; if (pwalletMain->HaveKey(keyid)) { LogPrintf("Skipping import of %s (key already present)\n", CBitcoinAddress(keyid).ToString()); - continue; +// continue; + fAlreadyHave=true; } int64_t nTime = DecodeDumpTime(vstr[1]); std::string strLabel; @@ -348,11 +370,14 @@ Value importwallet(const Array& params, bool fHelp) } } LogPrintf("Importing %s...\n", CBitcoinAddress(keyid).ToString()); - if (!pwalletMain->AddKeyPubKey(key, pubkey)) { - fGood = false; - continue; + if(!fAlreadyHave) + { + if (!pwalletMain->AddKeyPubKey(key, pubkey)) { + fGood = false; + continue; + } + pwalletMain->mapKeyMetadata[keyid].nCreateTime = nTime; } - pwalletMain->mapKeyMetadata[keyid].nCreateTime = nTime; if (fLabel) pwalletMain->SetAddressBook(keyid, strLabel, "receive"); @@ -385,8 +410,15 @@ Value importwallet(const Array& params, bool fHelp) /* MCHN START */ if(mc_gState->m_WalletMode & MC_WMD_ADDRESS_TXS) { - LogPrintf("Rescanning all %i blocks\n", chainActive.Height()); - pwalletMain->ScanForWalletTransactions(chainActive.Genesis(),false,true); + if(start_block) + { + LogPrintf("Rescanning last %i blocks\n", chainActive.Height()-start_block+1); + } + else + { + LogPrintf("Rescanning all %i blocks\n", chainActive.Height()); + } + pwalletMain->ScanForWalletTransactions(chainActive[start_block],false,true); pwalletMain->MarkDirty(); } else diff --git a/src/rpc/rpcmisc.cpp b/src/rpc/rpcmisc.cpp index 724a5d1d..73971832 100644 --- a/src/rpc/rpcmisc.cpp +++ b/src/rpc/rpcmisc.cpp @@ -82,6 +82,7 @@ Value getinfo(const Array& params, bool fHelp) obj.push_back(Pair("burnaddress", BurnAddress(Params().Base58Prefix(CChainParams::PUBKEY_ADDRESS)))); obj.push_back(Pair("incomingpaused", (mc_gState->m_NodePausedState & MC_NPS_INCOMING) ? true : false)); obj.push_back(Pair("miningpaused", (mc_gState->m_NodePausedState & MC_NPS_MINING) ? true : false)); + obj.push_back(Pair("offchainpaused", (mc_gState->m_NodePausedState & MC_NPS_OFFCHAIN) ? true : false)); /* MCHN END */ #ifdef ENABLE_WALLET @@ -746,6 +747,7 @@ void SetSynchronizedFlag(CTxDestination &dest,Object &ret) if(entStat.m_Flags & MC_EFL_NOT_IN_SYNC) { ret.push_back(Pair("synchronized",false)); + ret.push_back(Pair("startblock",entStat.m_LastImportedBlock+1)); } else { diff --git a/src/rpc/rpcstreams.cpp b/src/rpc/rpcstreams.cpp index d8b6069f..d6177565 100644 --- a/src/rpc/rpcstreams.cpp +++ b/src/rpc/rpcstreams.cpp @@ -840,7 +840,7 @@ Value subscribe(const Array& params, bool fHelp) if (fRescan && fNewFound) { - pwalletMain->ScanForWalletTransactions(chainActive.Genesis(), true, true); + pwalletMain->ScanForWalletTransactions(chainActive.Genesis(), true, true, true); } return Value::null; diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index 4de12d57..e3555ca5 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -3586,6 +3586,92 @@ bool ParseIntRange(string str,int *from,int *to) return true; } + + +int ParseBlockIdentifier(Value blockset_identifier) +{ + if(blockset_identifier.type() == obj_type) + { + int64_t starttime=-1; + BOOST_FOREACH(const Pair& d, blockset_identifier.get_obj()) + { + if(d.name_ == "starttime") + { + if(starttime >= 0) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Multiple starttime"); + } + + starttime=d.value_.get_int64(); + if( (starttime<0) || (starttime > 0xffffffff)) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid starttime"); + } + } + else + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid block identifier"); + } + } + if(starttime<0) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid block identifier"); + } + for(int block=0; blocknTime >= starttime) + { + return block; + } + } + return chainActive.Height()+1; + } + else + { + if(blockset_identifier.type() == int_type) + { + int block=blockset_identifier.get_int(); + if(block < 0) + { + block=chainActive.Height()+block+1; + } + if(block<0) + { + block=0; + } + return block; + } + } + + return -1; +} + +int ParseRescanParameter(Value rescan_identifier, bool *fRescan) +{ + int start_block=ParseBlockIdentifier(rescan_identifier); + *fRescan=false; + if(start_block >= 0) + { + if(start_block <= chainActive.Height()) + { + *fRescan=true; + } + } + else + { + if(rescan_identifier.type() == bool_type) + { + start_block=0; + *fRescan=rescan_identifier.get_bool(); + } + else + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid rescan"); + } + } + return start_block; +} + vector ParseBlockSetIdentifier(Value blockset_identifier) { vector block_set; @@ -3633,11 +3719,11 @@ vector ParseBlockSetIdentifier(Value blockset_identifier) if(starttime < 0) throw JSONRPCError(RPC_INVALID_PARAMETER, "Missing starttime"); if(endtime < 0) - throw JSONRPCError(RPC_INVALID_PARAMETER, "Missing starttime"); + throw JSONRPCError(RPC_INVALID_PARAMETER, "Missing endtime"); if(starttime <= endtime) { - for(int block=0; blocknTime >= starttime) && (chainActive[block]->nTime <= endtime) ) { diff --git a/src/rpc/rpcutils.h b/src/rpc/rpcutils.h index 672d72b3..1fe1a683 100644 --- a/src/rpc/rpcutils.h +++ b/src/rpc/rpcutils.h @@ -130,6 +130,8 @@ void ParseRawAction(string action,bool& lock_it, bool& sign_it,bool& send_it); bool paramtobool(Value param); int paramtoint(Value param,bool check_for_min,int min_value,string error_message); int64_t paramtoint64(Value param,bool check_for_min,int64_t min_value,string error_message); +int ParseBlockIdentifier(Value blockset_identifier); +int ParseRescanParameter(Value rescan_identifier, bool *fRescan); vector ParseBlockSetIdentifier(Value blockset_identifier); vector ParseRawFormattedData(const Value *value,uint32_t *data_format,mc_Script *lpDetailsScript,uint32_t in_options,uint32_t *out_options,int *errorCode,string *strError); void ParseRawDetails(const Value *value,mc_Script *lpDetails,mc_Script *lpDetailsScript,int *errorCode,string *strError); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index cd25d3b5..68d1da0b 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1280,7 +1280,7 @@ bool CWalletTx::WriteToDisk() /* MCHN START */ -mc_TxImport *StartImport(CWallet *lpWallet,bool fOnlyUnsynced, int block, int *err) +mc_TxImport *StartImport(CWallet *lpWallet,bool fOnlyUnsynced, bool fOnlySubscriptions, int block, int *err) { vector vAddressesToImport; vector vStreamsToImport; @@ -1304,41 +1304,45 @@ mc_TxImport *StartImport(CWallet *lpWallet,bool fOnlyUnsynced, int block, int *e if(mc_gState->m_WalletMode & MC_WMD_ADDRESS_TXS) { - BOOST_FOREACH(const PAIRTYPE(CBitcoinAddress, CAddressBookData)& item, lpWallet->mapAddressBook) + if(!fOnlySubscriptions) { - const CBitcoinAddress& address = item.first; - mc_TxEntityStat entStat; - CTxDestination addressRet=address.Get(); - const CKeyID *lpKeyID=boost::get (&addressRet); - const CScriptID *lpScriptID=boost::get (&addressRet); - - entity.Zero(); - if(lpKeyID) + BOOST_FOREACH(const PAIRTYPE(CBitcoinAddress, CAddressBookData)& item, lpWallet->mapAddressBook) { - memcpy(entity.m_EntityID,lpKeyID,MC_TDB_ENTITY_ID_SIZE); - entity.m_EntityType=MC_TET_PUBKEY_ADDRESS | MC_TET_CHAINPOS; - } - if(lpScriptID) - { - memcpy(entity.m_EntityID,lpScriptID,MC_TDB_ENTITY_ID_SIZE); - entity.m_EntityType=MC_TET_SCRIPT_ADDRESS | MC_TET_CHAINPOS; - } + const CBitcoinAddress& address = item.first; + mc_TxEntityStat entStat; + CTxDestination addressRet=address.Get(); + const CKeyID *lpKeyID=boost::get (&addressRet); + const CScriptID *lpScriptID=boost::get (&addressRet); + + entity.Zero(); + if(lpKeyID) + { + memcpy(entity.m_EntityID,lpKeyID,MC_TDB_ENTITY_ID_SIZE); + entity.m_EntityType=MC_TET_PUBKEY_ADDRESS | MC_TET_CHAINPOS; + } + if(lpScriptID) + { + memcpy(entity.m_EntityID,lpScriptID,MC_TDB_ENTITY_ID_SIZE); + entity.m_EntityType=MC_TET_SCRIPT_ADDRESS | MC_TET_CHAINPOS; + } - if(entity.m_EntityType) - { - entStat.Zero(); - memcpy(&entStat,&entity,sizeof(mc_TxEntity)); - if(pwalletTxsMain->FindEntity(&entStat)) + if(entity.m_EntityType) { - if(!fOnlyUnsynced || ((entStat.m_Flags & MC_EFL_NOT_IN_SYNC) != 0) ) + entStat.Zero(); + memcpy(&entStat,&entity,sizeof(mc_TxEntity)); + if(pwalletTxsMain->FindEntity(&entStat)) + { + if(!fOnlyUnsynced || + (((entStat.m_Flags & MC_EFL_NOT_IN_SYNC) != 0) && (entStat.m_LastImportedBlock > block) ) ) + { + vAddressesToImport.push_back(address); + } + } + else { - vAddressesToImport.push_back(address); + vAddressesToImport.push_back(address); } } - else - { - vAddressesToImport.push_back(address); - } } } } @@ -1350,20 +1354,15 @@ mc_TxImport *StartImport(CWallet *lpWallet,bool fOnlyUnsynced, int block, int *e for(i=0;iGetCount();i++) { lpent=(mc_TxEntityStat*)m_ChainEntities->GetRow(i); - switch(lpent->m_Entity.m_EntityType & MC_TET_TYPE_MASK) + if(lpent->m_Entity.IsSubscription()) { - case MC_TET_STREAM: - case MC_TET_STREAM_KEY: - case MC_TET_STREAM_PUBLISHER: - case MC_TET_ASSET: - if(lpent->m_Entity.m_EntityType & MC_TET_CHAINPOS) + if(lpent->m_Entity.m_EntityType & MC_TET_CHAINPOS) + { + if(lpent->m_Flags & MC_EFL_NOT_IN_SYNC) { - if(lpent->m_Flags & MC_EFL_NOT_IN_SYNC) - { - vStreamsToImport.push_back(lpent->m_Entity); - } + vStreamsToImport.push_back(lpent->m_Entity); } - break; + } } } } @@ -1426,21 +1425,26 @@ mc_TxImport *StartImport(CWallet *lpWallet,bool fOnlyUnsynced, int block, int *e } if(take_it) { - entity.Zero(); - memcpy(entity.m_EntityID,vStreamsToImport[i].m_EntityID,MC_TDB_ENTITY_ID_SIZE); - entity.m_EntityType=vStreamsToImport[i].m_EntityType; - lpEntities->Add(&entity,NULL); - entity.m_EntityType=(vStreamsToImport[i].m_EntityType - MC_TET_CHAINPOS) | MC_TET_TIMERECEIVED; - lpEntities->Add(&entity,NULL); ptr=(unsigned char *)stream_entity.GetRef(); + b=chainActive.Height(); if(stream_entity.IsUnconfirmedGenesis() == 0) { b=(int)mc_GetLE(ptr,4)-1; + } + + if(b > block) + { if(b < block_to_start_from) { block_to_start_from=b; } - } + entity.Zero(); + memcpy(entity.m_EntityID,vStreamsToImport[i].m_EntityID,MC_TDB_ENTITY_ID_SIZE); + entity.m_EntityType=vStreamsToImport[i].m_EntityType; + lpEntities->Add(&entity,NULL); + entity.m_EntityType=(vStreamsToImport[i].m_EntityType - MC_TET_CHAINPOS) | MC_TET_TIMERECEIVED; + lpEntities->Add(&entity,NULL); + } } } } @@ -1469,7 +1473,7 @@ mc_TxImport *StartImport(CWallet *lpWallet,bool fOnlyUnsynced, int block, int *e * from or to us. If fUpdate is true, found transactions that already * exist in the wallet will be updated. */ -int CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate,bool fOnlyUnsynced) +int CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate,bool fOnlyUnsynced,bool fOnlySubscriptions) { int ret = 0; int64_t nNow = GetTime(); @@ -1481,7 +1485,7 @@ int CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate,bo /* MCHN START */ mc_TxImport *imp; int err; - imp=StartImport(this,fOnlyUnsynced,pindex->nHeight-1,&err); + imp=StartImport(this,fOnlyUnsynced,fOnlySubscriptions,pindex->nHeight-1,&err); /* MCHN END */ /* MCHN START */ @@ -1590,7 +1594,7 @@ int CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate,bo } } - err=pwalletTxsMain->CompleteImport(imp); + err=pwalletTxsMain->CompleteImport(imp,((pindexStart->nHeight > 0) & !fOnlySubscriptions) ? MC_EFL_NOT_IN_SYNC_AFTER_IMPORT : 0); } else { diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 7c0dddb0..e5162209 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -461,7 +461,7 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface void SyncTransaction(const CTransaction& tx, const CBlock* pblock); bool AddToWalletIfInvolvingMe(const CTransaction& tx, const CBlock* pblock, bool fUpdate); void EraseFromWallet(const uint256 &hash); - int ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate = false, bool fOnlyUnsynced = false); + int ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate = false, bool fOnlyUnsynced = false,bool fOnlySubscriptions = false); void ReacceptWalletTransactions(); void ResendWalletTransactions(bool fForce = false); CAmount GetBalance() const; diff --git a/src/wallet/wallettxdb.cpp b/src/wallet/wallettxdb.cpp index e624cb3e..6c2521a9 100644 --- a/src/wallet/wallettxdb.cpp +++ b/src/wallet/wallettxdb.cpp @@ -22,6 +22,19 @@ void mc_TxEntity::Init(unsigned char *entity_id,uint32_t entity_type) m_EntityType=entity_type; } +int mc_TxEntity::IsSubscription() +{ + switch(m_EntityType & MC_TET_TYPE_MASK) + { + case MC_TET_STREAM: + case MC_TET_STREAM_KEY: + case MC_TET_STREAM_PUBLISHER: + case MC_TET_ASSET: + return 1; + } + return 0; +} + void mc_TxEntityRow::Zero() { memset(this,0,sizeof(mc_TxEntityRow)); @@ -684,7 +697,14 @@ int mc_TxDB::AddEntity(mc_TxImport *import,mc_TxEntity *entity,uint32_t flags) { stat.m_PosInImport=((mc_TxEntityStat*)import->m_Entities->GetRow(import->m_Entities->GetCount()-1))->m_PosInImport+1; } - stat.m_LastImportedBlock=m_DBStat.m_Block; // Block the entity was added on, relevant if out-of-sync + if(flags & MC_EFL_NOT_IN_SYNC) + { + stat.m_LastImportedBlock=m_DBStat.m_Block; // Block the entity was added on, relevant if out-of-sync + } + else + { + stat.m_LastImportedBlock=-1; + } stat.m_TimeAdded=mc_TimeNowAsUInt(); stat.m_Flags=flags; // Mainly for out-of-sync flag @@ -2409,6 +2429,7 @@ mc_TxImport *mc_TxDB::StartImport(mc_Buffer *lpEntities,int block,int *err) count=0; lpEntStat=(m_Imports+slot)->GetEntity(j); lpEntStat->m_Flags |= MC_EFL_NOT_IN_SYNC; + lpEntStat->m_LastImportedBlock=block; row=m_Imports->FindEntity(lpEntStat); if(row >= 0) // Old entity { @@ -2723,7 +2744,7 @@ int mc_TxDB::Unsubscribe(mc_Buffer* lpEntities) return err; } -int mc_TxDB::CompleteImport(mc_TxImport *import) +int mc_TxDB::CompleteImport(mc_TxImport *import,uint32_t flags) { char msg[256]; char enthex[65]; @@ -2800,12 +2821,17 @@ int mc_TxDB::CompleteImport(mc_TxImport *import) m_Imports->AddEntity(lpent); lpdel=m_Imports->GetEntity(row); } - - lpdel->m_LastImportedBlock=import->m_Block; + +// lpdel->m_LastImportedBlock=import->m_Block; + lpdel->m_LastImportedBlock=lpent->m_LastImportedBlock; edbImport.m_Flags=lpdel->m_Flags; + if(lpdel->m_Flags & MC_EFL_NOT_IN_SYNC) { - lpdel->m_Flags-=MC_EFL_NOT_IN_SYNC; + if( (lpdel->m_Entity.IsSubscription() != 0) || ( (flags & MC_EFL_NOT_IN_SYNC_AFTER_IMPORT) == 0) ) + { + lpdel->m_Flags-=MC_EFL_NOT_IN_SYNC; + } } @@ -2816,7 +2842,7 @@ int mc_TxDB::CompleteImport(mc_TxImport *import) edbImport.m_Block=import->m_Block; edbImport.m_Pos=lpdel->m_PosInImport; edbImport.m_LastPos=lpdel->m_LastPos; - edbImport.m_LastImportedBlock=import->m_Block; + edbImport.m_LastImportedBlock=lpent->m_LastImportedBlock;//import->m_Block; edbImport.m_TimeAdded=lpdel->m_TimeAdded; edbImport.m_Flags=lpdel->m_Flags; sprintf_hex(enthex,edbImport.m_Entity.m_EntityID,MC_TDB_ENTITY_ID_SIZE); diff --git a/src/wallet/wallettxdb.h b/src/wallet/wallettxdb.h index e852f41e..de57b43d 100644 --- a/src/wallet/wallettxdb.h +++ b/src/wallet/wallettxdb.h @@ -43,6 +43,7 @@ #define MC_TET_SPECIALMASK 0xFF000000 #define MC_EFL_NOT_IN_SYNC 0x01000000 +#define MC_EFL_NOT_IN_SYNC_AFTER_IMPORT 0x02000000 #define MC_EFL_UNSUBSCRIBED 0x10000000 #define MC_SFL_NONE 0x00000000 @@ -64,6 +65,7 @@ typedef struct mc_TxEntity uint32_t m_EntityType; // Entity type, MC_TET_ constants void Zero(); void Init(unsigned char *entity_id,uint32_t entity_type); + int IsSubscription(); } mc_TxEntity; typedef struct mc_TxEntityRowExtension @@ -373,7 +375,7 @@ typedef struct mc_TxDB int ImportGetBlock( // Returns last processed block in the import mc_TxImport *import); - int CompleteImport(mc_TxImport *import); // Completes import - merges with chain + int CompleteImport(mc_TxImport *import,uint32_t flags); // Completes import - merges with chain int DropImport(mc_TxImport *import); // Drops uncompleted import diff --git a/src/wallet/wallettxs.cpp b/src/wallet/wallettxs.cpp index 75f6142c..af99bcbb 100644 --- a/src/wallet/wallettxs.cpp +++ b/src/wallet/wallettxs.cpp @@ -1396,7 +1396,7 @@ int mc_WalletTxs::ImportGetBlock(mc_TxImport *import) } -int mc_WalletTxs::CompleteImport(mc_TxImport *import) +int mc_WalletTxs::CompleteImport(mc_TxImport *import,uint32_t flags) { int err,import_pos,gen,count; @@ -1414,7 +1414,7 @@ int mc_WalletTxs::CompleteImport(mc_TxImport *import) gen=import->m_ImportID; - err=m_Database->CompleteImport(import); + err=m_Database->CompleteImport(import,flags); if(m_Database->m_DBStat.m_Block != m_Database->m_Imports->m_Block) diff --git a/src/wallet/wallettxs.h b/src/wallet/wallettxs.h index 5b40c90b..53c0821c 100644 --- a/src/wallet/wallettxs.h +++ b/src/wallet/wallettxs.h @@ -151,7 +151,7 @@ typedef struct mc_WalletTxs int ImportGetBlock( // Returns last processed block in the import mc_TxImport *import); - int CompleteImport(mc_TxImport *import); // Completes import - merges with chain + int CompleteImport(mc_TxImport *import,uint32_t flags); // Completes import - merges with chain int DropImport(mc_TxImport *import); // Drops uncompleted import From 70783641b20b99d1b22b60eb68e41dca72c58e66 Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 16 May 2018 07:40:42 +0300 Subject: [PATCH 066/157] Fixed restrict/open fields in liststreams output --- src/rpc/rpcutils.cpp | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index e3555ca5..a36674eb 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -617,22 +617,25 @@ Object StreamEntry(const unsigned char *txid,uint32_t output_level) if(output_level & 0x0004) { - if(entity.AnyoneCanWrite()) - { - entry.push_back(Pair("open",true)); - } - else - { - entry.push_back(Pair("open",false)); + if(mc_gState->m_Compatibility & MC_VCM_1_0) + { + if(entity.AnyoneCanWrite()) + { + entry.push_back(Pair("open",true)); + } + else + { + entry.push_back(Pair("open",false)); + } } + Object pObject; + pObject.push_back(Pair("write",entity.AnyoneCanWrite() ? false : true)); if(mc_gState->m_Features->OffChainData()) { - Object pObject; - pObject.push_back(Pair("write",(entity.Permissions() & MC_PTP_WRITE) ? true : false)); pObject.push_back(Pair("onchain",(entity.Restrictions() & MC_ENT_ENTITY_RESTRICTION_ONCHAIN) ? true : false)); pObject.push_back(Pair("offchain",(entity.Restrictions() & MC_ENT_ENTITY_RESTRICTION_OFFCHAIN) ? true : false)); - entry.push_back(Pair("restrict",pObject)); } + entry.push_back(Pair("restrict",pObject)); } From faf77a3adadc10cb6d6a819b6424d19d8c4200d6 Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 16 May 2018 09:11:10 +0300 Subject: [PATCH 067/157] chainbalance in getblockchaininfo --- src/rpc/rpcblockchain.cpp | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/rpc/rpcblockchain.cpp b/src/rpc/rpcblockchain.cpp index 3eb59925..e24928d6 100644 --- a/src/rpc/rpcblockchain.cpp +++ b/src/rpc/rpcblockchain.cpp @@ -618,6 +618,38 @@ Value getblockchaininfo(const Array& params, bool fHelp) obj.push_back(Pair("difficulty", (double)GetDifficulty())); obj.push_back(Pair("verificationprogress", Checkpoints::GuessVerificationProgress(chainActive.Tip()))); obj.push_back(Pair("chainwork", chainActive.Tip()->nChainWork.GetHex())); + + + double chain_balance=0.; + if(COIN) + { + int chain_height=(int)chainActive.Height(); + int64_t epoch_size=Params().SubsidyHalvingInterval(); + int complete_epochs=chain_height / epoch_size; + int blocks_in_this_epoch=chain_height%epoch_size+1; + int64_t total_value=0; + int64_t epoch_value=MCP_INITIAL_BLOCK_REWARD; + for(int epoch=0;epoch>= 1; + } + total_value+=epoch_value*(int64_t)blocks_in_this_epoch; + if(chain_height >= 0) + { + total_value-=MCP_INITIAL_BLOCK_REWARD; // Genesis block reward is unspendable + } + if(MCP_FIRST_BLOCK_REWARD >= 0) + { + if(chain_height >= 1) + { + total_value+=MCP_FIRST_BLOCK_REWARD-MCP_INITIAL_BLOCK_REWARD; + } + } + chain_balance=(double)total_value/(double)COIN; + } + obj.push_back(Pair("chainbalance", chain_balance)); + return obj; } From ea9d824ca294c5b5f01bf88ea4c82fdff72ffb44 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 21 May 2018 17:29:47 +0300 Subject: [PATCH 068/157] Flushing wallet on startup --- src/core/init-cold.cpp | 3 +++ src/core/init.cpp | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/core/init-cold.cpp b/src/core/init-cold.cpp index 96438fb9..13bdd723 100644 --- a/src/core/init-cold.cpp +++ b/src/core/init-cold.cpp @@ -1232,6 +1232,9 @@ bool AppInit2_Cold(boost::thread_group& threadGroup,int OutputPipe) LogPrintf("mapAddressBook.size() = %u\n", pwalletMain ? pwalletMain->mapAddressBook.size() : 0); #endif + if (pwalletMain) + bitdb.Flush(false); + #ifdef ENABLE_WALLET if (pwalletMain) { // Run a thread to flush wallet periodically diff --git a/src/core/init.cpp b/src/core/init.cpp index a122efef..7167e0f8 100644 --- a/src/core/init.cpp +++ b/src/core/init.cpp @@ -2403,6 +2403,9 @@ bool AppInit2(boost::thread_group& threadGroup,int OutputPipe) LogPrintf("mapAddressBook.size() = %u\n", pwalletMain ? pwalletMain->mapAddressBook.size() : 0); #endif + if (pwalletMain) + bitdb.Flush(false); + if(!GetBoolArg("-offline",false)) { StartNode(threadGroup); From c6b7751b72a2db2e36373db8c0e0094860aca3a4 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 21 May 2018 17:46:38 +0300 Subject: [PATCH 069/157] Fixed long asset names in raw transactions --- src/rpc/rpcrawdata.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/rpc/rpcrawdata.cpp b/src/rpc/rpcrawdata.cpp index 01e8a81d..0ba46299 100644 --- a/src/rpc/rpcrawdata.cpp +++ b/src/rpc/rpcrawdata.cpp @@ -525,8 +525,18 @@ CScript RawDataScriptIssue(Value *param,mc_Script *lpDetails,mc_Script *lpDetail if(d.value_.type() != null_type && !d.value_.get_str().empty()) { entity_name=d.value_.get_str(); + + if(entity_name == "*") + { + *strError=string("Invalid asset name"); + } + if(entity_name.size()) { + if(entity_name.size() > MC_ENT_MAX_NAME_SIZE) + { + *strError=string("Invalid asset name - too long"); + } lpDetails->SetSpecialParamValue(MC_ENT_SPRM_NAME,(const unsigned char*)(entity_name.c_str()),entity_name.size()); } } From aca8974ff59b36f4c8785faa7a65c0fe6439841d Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 22 May 2018 13:35:54 +0300 Subject: [PATCH 070/157] Keypool tweaks --- src/core/init-cold.cpp | 2 +- src/core/init.cpp | 2 +- src/wallet/wallet.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/init-cold.cpp b/src/core/init-cold.cpp index 13bdd723..14ef8046 100644 --- a/src/core/init-cold.cpp +++ b/src/core/init-cold.cpp @@ -189,7 +189,7 @@ std::string HelpMessage_Cold() #ifdef ENABLE_WALLET strUsage += "\n" + _("Wallet options:") + "\n"; strUsage += " -disablewallet " + _("Do not load the wallet and disable wallet RPC calls") + "\n"; - strUsage += " -keypool= " + strprintf(_("Set key pool size to (default: %u)"), 100) + "\n"; + strUsage += " -keypool= " + strprintf(_("Set key pool size to (default: %u)"), 1) + "\n"; if (GetBoolArg("-help-debug", false)) strUsage += " -mintxfee= " + strprintf(_("Fees (in BTC/Kb) smaller than this are considered zero fee for transaction creation (default: %s)"), FormatMoney(CWallet::minTxFee.GetFeePerK())) + "\n"; strUsage += " -salvagewallet " + _("Attempt to recover private keys from a corrupt wallet.dat") + " " + _("on startup") + "\n"; diff --git a/src/core/init.cpp b/src/core/init.cpp index 7167e0f8..bfaaad21 100644 --- a/src/core/init.cpp +++ b/src/core/init.cpp @@ -380,7 +380,7 @@ std::string HelpMessage(HelpMessageMode mode) #ifdef ENABLE_WALLET strUsage += "\n" + _("Wallet options:") + "\n"; strUsage += " -disablewallet " + _("Do not load the wallet and disable wallet RPC calls") + "\n"; - strUsage += " -keypool= " + strprintf(_("Set key pool size to (default: %u)"), 100) + "\n"; + strUsage += " -keypool= " + strprintf(_("Set key pool size to (default: %u)"), 1) + "\n"; if (GetBoolArg("-help-debug", false)) strUsage += " -mintxfee= " + strprintf(_("Fees (in BTC/Kb) smaller than this are considered zero fee for transaction creation (default: %s)"), FormatMoney(CWallet::minTxFee.GetFeePerK())) + "\n"; strUsage += " -paytxfee= " + strprintf(_("Fee (in BTC/kB) to add to transactions you send (default: %s)"), FormatMoney(payTxFee.GetFeePerK())) + "\n"; diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 68d1da0b..6b3bfa77 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -3321,7 +3321,7 @@ bool CWallet::NewKeyPool() if (IsLocked()) return false; - int64_t nKeys = max(GetArg("-keypool", 100), (int64_t)0); + int64_t nKeys = max(GetArg("-keypool", mc_gState->m_NetworkParams->IsProtocolMultichain() ? 1 : 100), (int64_t)0); for (int i = 0; i < nKeys; i++) { int64_t nIndex = i+1; From 0ddc6d7765de2605979f37d40513a3f4276a50bd Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 22 May 2018 16:34:15 +0300 Subject: [PATCH 071/157] chainrewards --- src/rpc/rpcblockchain.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/rpcblockchain.cpp b/src/rpc/rpcblockchain.cpp index e24928d6..60789819 100644 --- a/src/rpc/rpcblockchain.cpp +++ b/src/rpc/rpcblockchain.cpp @@ -648,7 +648,7 @@ Value getblockchaininfo(const Array& params, bool fHelp) } chain_balance=(double)total_value/(double)COIN; } - obj.push_back(Pair("chainbalance", chain_balance)); + obj.push_back(Pair("chainrewards", chain_balance)); return obj; } From 923534add77a14e425f014c536036d122735c695 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 22 May 2018 17:10:11 +0300 Subject: [PATCH 072/157] Better param passing for pause/resume --- src/rpc/rpcserver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/rpcserver.cpp b/src/rpc/rpcserver.cpp index c8fa5c60..5e640147 100644 --- a/src/rpc/rpcserver.cpp +++ b/src/rpc/rpcserver.cpp @@ -365,7 +365,7 @@ uint32_t GetPausedServices(const char *str) c=*ptr; if( (c == ',') || (c ==0x00)) { - if(ptr > start) + if(ptr > start+4) { type=0; if(memcmp(start,"incoming", ptr-start) == 0)type = MC_NPS_INCOMING; From 8e60af8d4d02a8e1f94cf3cc6dc38f586538b937 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 22 May 2018 17:10:49 +0300 Subject: [PATCH 073/157] Chunk retrieval runtimeparams --- src/protocol/relay.cpp | 44 +++++++++++++++++++++------ src/wallet/chunkcollector.cpp | 57 +++++------------------------------ src/wallet/chunkcollector.h | 7 +++-- 3 files changed, 46 insertions(+), 62 deletions(-) diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index 3f87924d..fde4a240 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -9,13 +9,8 @@ uint32_t MultichainNextChunkQueryAttempt(uint32_t attempts) { - if(attempts < 4)return 0; - if(attempts < 8)return 60; - if(attempts < 12)return 3600; - if(attempts < 16)return 86400; - if(attempts < 20)return 86400*30; - if(attempts < 24)return 86400*365; - return 86400*365*10; + if(attempts < 2)return 0; + return (uint32_t)(int64_t)(pow(1.5,attempts-1)-1); } string mc_MsgTypeStr(uint32_t msg_type) @@ -338,7 +333,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) pRelayManager->DeleteRequest(item.first.request_id); } - max_total_size=(MC_CCW_TIMEOUT_REQUEST-2)*MC_CCW_MAX_MBS_PER_SECOND*1024*1024; + max_total_size=(collector->m_TimeoutRequest-MC_CCW_TIMEOUT_REQUEST_SHIFT)*MC_CCW_MAX_MBS_PER_SECOND*1024*1024; if(max_total_size > MAX_SIZE-OFFCHAIN_MSG_PADDING) { max_total_size=MAX_SIZE-OFFCHAIN_MSG_PADDING; @@ -462,7 +457,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) response=&(request->m_Responses[item.first.response_id]); - expiration=time_now+MC_CCW_TIMEOUT_REQUEST; + expiration=time_now+collector->m_TimeoutRequest; dest_expiration=expiration+response->m_MsgID.m_TimeStamp-request->m_MsgID.m_TimeStamp;// response->m_TimeDiff; ptrOut=&(payload[0]); *ptrOut=MC_RDT_EXPIRATION; @@ -541,7 +536,12 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) collect_subrow->m_State.m_Status -= MC_CCF_SELECTED; collect_subrow->m_State.m_Query=query_id; collect_subrow->m_State.m_QueryAttempts+=1; - collect_subrow->m_State.m_QueryTimeStamp=time_now+MC_CCW_TIMEOUT_QUERY; + collect_subrow->m_State.m_QueryTimeStamp=time_now+collector->m_TimeoutQuery; + collect_subrow->m_State.m_QuerySilenceTimestamp=time_now+collector->m_TimeoutRequest; + if(collect_subrow->m_State.m_QueryAttempts>1) + { + collect_subrow->m_State.m_QuerySilenceTimestamp=collect_subrow->m_State.m_QueryTimeStamp; + } collect_subrow->m_State.m_Status |= MC_CCF_UPDATED; } } @@ -555,7 +555,31 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) { if( (collect_row->m_State.m_Status & MC_CCF_DELETED ) == 0 ) { + int expired=0; if(collect_row->m_State.m_QueryTimeStamp <= time_now) + { + expired=1; + } + else + { + if(collect_row->m_State.m_QuerySilenceTimestamp <= time_now) + { + query=NULL; + if(!collect_row->m_State.m_Query.IsZero()) + { + query=pRelayManager->FindRequest(collect_row->m_State.m_Query); + } + if(query) + { + if(query->m_Responses.size() == 0) + { + expired=1; + } + } + } + } + + if(expired) { if(!collect_row->m_State.m_Request.IsZero()) { diff --git a/src/wallet/chunkcollector.cpp b/src/wallet/chunkcollector.cpp index 35711344..c0984549 100644 --- a/src/wallet/chunkcollector.cpp +++ b/src/wallet/chunkcollector.cpp @@ -63,6 +63,13 @@ void mc_ChunkCollector::Zero() m_MaxMemPoolSize=MC_CCW_DEFAULT_MEMPOOL_SIZE; m_AutoCommitDelay=MC_CCW_DEFAULT_AUTOCOMMIT_DELAY; + m_TimeoutRequest=(int)GetArg("-chunkrequesttimeout",MC_CCW_TIMEOUT_REQUEST); + if(m_TimeoutRequest <= MC_CCW_TIMEOUT_REQUEST_SHIFT) + { + m_TimeoutRequest=MC_CCW_TIMEOUT_REQUEST_SHIFT+1; + } + m_TimeoutQuery=(int)GetArg("-chunkquerytimeout",MC_CCW_TIMEOUT_QUERY); + m_Semaphore=NULL; m_LockedBy=0; @@ -391,56 +398,6 @@ int mc_ChunkCollector::Initialize(mc_ChunkDB *chunk_db,const char *name,uint32_t } } } -/* - ptr=(unsigned char*)m_DB->Read((char*)&collect_row+m_KeyOffset,m_KeySize,&value_len,MC_OPT_DB_DATABASE_SEEK_ON_READ,&err); - if(err) - { - return err; - } - - if(ptr) - { - ptr=(unsigned char*)m_DB->MoveNext(&err); - collect_row.Zero(); - if(ptr) - { - memcpy((char*)&collect_row,ptr,m_TotalDBSize); - } - while(ptr) - { - ptr=(unsigned char*)m_DB->MoveNext(&err); - if(err) - { - return MC_ERR_CORRUPTED; - } - collect_row.m_State.m_Status |= MC_CCF_INSERTED; - if(m_ChunkDB->GetChunkDef(&chunk_def,collect_row.m_ChunkDef.m_Hash,&(collect_row.m_ChunkDef.m_Entity),collect_row.m_TxID,collect_row.m_Vout) == MC_ERR_NOERROR) - { - collect_row.m_State.m_Status |= MC_CCF_DELETED; - } - m_MemPool->Add(&collect_row); - collect_row.Zero(); - if(ptr) - { - memcpy((char*)&collect_row,ptr,m_TotalDBSize); - } - } - } - else - { - err=m_DB->Write((char*)&collect_row+m_KeyOffset,m_KeySize,(char*)&collect_row+m_ValueOffset,m_ValueDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); - if(err) - { - return err; - } - err=m_DB->Commit(MC_OPT_DB_DATABASE_TRANSACTIONAL); - if(err) - { - return err; - } - } - } - */ Dump("Initialize"); diff --git a/src/wallet/chunkcollector.h b/src/wallet/chunkcollector.h index f06b03f0..ec8a7b57 100644 --- a/src/wallet/chunkcollector.h +++ b/src/wallet/chunkcollector.h @@ -21,10 +21,11 @@ #define MC_CCW_TIMEOUT_QUERY 35 #define MC_CCW_TIMEOUT_REQUEST 10 +#define MC_CCW_TIMEOUT_REQUEST_SHIFT 2 #define MC_CCW_MAX_CHUNKS_PER_QUERY 16 #define MC_CCW_DEFAULT_AUTOCOMMIT_DELAY 200 #define MC_CCW_WORST_RESPONSE_SCORE 1048576000 -#define MC_CCW_DEFAULT_MEMPOOL_SIZE 1000 +#define MC_CCW_DEFAULT_MEMPOOL_SIZE 10000 #define MC_CCW_MAX_MBS_PER_SECOND 8 @@ -47,7 +48,7 @@ typedef struct mc_ChunkEntityValue mc_OffchainMessageID m_Query; mc_OffchainMessageID m_Request; uint32_t m_QueryTimeStamp; - uint32_t m_Reserved2; + uint32_t m_QuerySilenceTimestamp; uint32_t m_RequestTimeStamp; uint32_t m_RequestPos; @@ -104,6 +105,8 @@ typedef struct mc_ChunkCollector int64_t m_NextAutoCommitTimestamp; int64_t m_NextTryTimestamp; int m_MaxMemPoolSize; + int m_TimeoutRequest; + int m_TimeoutQuery; char m_Name[MC_PRM_NETWORK_NAME_MAX_SIZE+1]; // Chain name char m_DBName[MC_DCT_DB_MAX_PATH]; // Full database name From 28f9ff7d99161fa6684f7a089b1e093e94617395 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 22 May 2018 19:11:41 +0300 Subject: [PATCH 074/157] Chunk delivery stats --- src/Makefile.am | 1 + src/protocol/relay.cpp | 39 +++++++++++++++++++++++++ src/rpc/rpcchunks.cpp | 54 +++++++++++++++++++++++++++++++++++ src/rpc/rpcdebug.cpp | 1 + src/rpc/rpchelp.cpp | 24 +++++++++++++++- src/rpc/rpclist.cpp | 2 ++ src/rpc/rpcserver.h | 2 ++ src/wallet/chunkcollector.cpp | 21 ++++++-------- src/wallet/chunkcollector.h | 20 +++++++++++-- 9 files changed, 149 insertions(+), 15 deletions(-) create mode 100644 src/rpc/rpcchunks.cpp diff --git a/src/Makefile.am b/src/Makefile.am index 9258b421..6305b149 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -193,6 +193,7 @@ libbitcoin_server_a_SOURCES = \ rpc/rpcrawdata.cpp \ rpc/rpcutils.cpp \ rpc/rpccache.cpp \ + rpc/rpcchunks.cpp \ rpc/rpcdebug.cpp \ rpc/rpchelp.cpp \ rpc/rpcblockchain.cpp \ diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index fde4a240..f8212700 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -130,16 +130,19 @@ int MultichainProcessChunkResponse(const CRelayResponsePair *response_pair,map < ptrOut+=size; if(chunk->m_Size != chunkOut->m_Size) { + for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Baddelivered+=k ? collect_row->m_ChunkDef.m_Size : 1; strError="Chunk info size mismatch"; goto exitlbl; } if(memcmp(chunk->m_Hash,chunkOut->m_Hash,sizeof(uint256))) { + for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Baddelivered+=k ? collect_row->m_ChunkDef.m_Size : 1; strError="Chunk info hash mismatch"; goto exitlbl; } if(memcmp(&(chunk->m_Entity),&(chunkOut->m_Entity),sizeof(mc_TxEntity))) { + for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Baddelivered+=k ? collect_row->m_ChunkDef.m_Size : 1; strError="Chunk info entity mismatch"; goto exitlbl; } @@ -152,6 +155,7 @@ int MultichainProcessChunkResponse(const CRelayResponsePair *response_pair,map < mc_gState->m_TmpBuffers->m_RpcHasher1->DoubleHash(ptrOut,sizeOut,&hash); if(memcmp(&hash,chunk->m_Hash,sizeof(uint256))) { + for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Baddelivered+=k ? collect_row->m_ChunkDef.m_Size : 1; strError="Chunk data hash mismatch"; goto exitlbl; } @@ -166,6 +170,7 @@ int MultichainProcessChunkResponse(const CRelayResponsePair *response_pair,map < } else { + for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Delivered+=k ? collect_row->m_ChunkDef.m_Size : 1; LogPrint("chunks","Retrieved chunk %s\n",(*(uint256*)(chunk->m_Hash)).ToString().c_str()); } collect_row->m_State.m_Status |= MC_CCF_DELETED; @@ -262,6 +267,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) vector payload; unsigned char buf[16]; int shift,count; + uint32_t size; unsigned char *ptrOut; mc_OffchainMessageID query_id,request_id; map query_to_delete; @@ -291,6 +297,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) { pRelayManager->DeleteRequest(collect_row->m_State.m_Request); collect_row->m_State.m_Request=0; + for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Undelivered+=k ? collect_row->m_ChunkDef.m_Size : 1; } } request=NULL; @@ -301,6 +308,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) { collect_row->m_State.m_Request=0; collect_row->m_State.m_RequestTimeStamp=0; + for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Undelivered+=k ? collect_row->m_ChunkDef.m_Size : 1; } } if(request) @@ -352,6 +360,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) { collect_row->m_State.m_Request=0; collect_row->m_State.m_RequestTimeStamp=0; + for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Undelivered+=k ? collect_row->m_ChunkDef.m_Size : 1; } } if(request) @@ -396,6 +405,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) collect_row->m_State.m_Query=0; collect_row->m_State.m_QueryNextAttempt=time_now+MultichainNextChunkQueryAttempt(collect_row->m_State.m_QueryAttempts); collect_row->m_State.m_Status |= MC_CCF_UPDATED; + for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Unresponded+=k ? collect_row->m_ChunkDef.m_Size : 1; } } if(query) @@ -486,6 +496,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) collect_subrow=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(chunk_row.first); collect_subrow->m_State.m_Request=request_id; collect_subrow->m_State.m_RequestTimeStamp=expiration; + for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Requested+=k ? collect_row->m_ChunkDef.m_Size : 1; // printf("T %d %d %s\n",chunk_row.first,collect_subrow->m_State.m_RequestPos,collect_subrow->m_State.m_Request.ToString().c_str()); } } @@ -543,6 +554,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) collect_subrow->m_State.m_QuerySilenceTimestamp=collect_subrow->m_State.m_QueryTimeStamp; } collect_subrow->m_State.m_Status |= MC_CCF_UPDATED; + for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Queried+=k ? collect_subrow->m_ChunkDef.m_Size : 1; } } last_row=row; @@ -585,6 +597,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) { pRelayManager->DeleteRequest(collect_row->m_State.m_Request); collect_row->m_State.m_Request=0; + for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Undelivered+=k ? collect_row->m_ChunkDef.m_Size : 1; } if(!collect_row->m_State.m_Query.IsZero()) { @@ -596,6 +609,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) collect_row->m_State.m_Query=0; collect_row->m_State.m_QueryNextAttempt=time_now+MultichainNextChunkQueryAttempt(collect_row->m_State.m_QueryAttempts); collect_row->m_State.m_Status |= MC_CCF_UPDATED; + for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Unresponded+=k ? collect_row->m_ChunkDef.m_Size : 1; } if(collect_row->m_State.m_QueryNextAttempt <= time_now) { @@ -639,6 +653,31 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) } } + for(int k=0;k<2;k++)collector->m_StatLast[k].Zero(); + for(row=0;rowm_MemPool->GetCount();row++) + { + collect_row=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(row); + if( (collect_row->m_State.m_Status & MC_CCF_DELETED ) == 0 ) + { + size=collect_row->m_ChunkDef.m_Size; + if(!collect_row->m_State.m_Request.IsZero()) + { + for(int k=0;k<2;k++)collector->m_StatLast[k].m_Requested+=k ? size : 1; + } + else + { + if(!collect_row->m_State.m_Query.IsZero()) + { + for(int k=0;k<2;k++)collector->m_StatLast[k].m_Queried+=k ? size : 1; + } + else + { + for(int k=0;k<2;k++)collector->m_StatLast[k].m_Sleeping+=k ? size : 1; + } + } + } + } + collector->UnLock(); if(not_processed < collector->m_MaxMemPoolSize/2) { diff --git a/src/rpc/rpcchunks.cpp b/src/rpc/rpcchunks.cpp new file mode 100644 index 00000000..5d7fdbc3 --- /dev/null +++ b/src/rpc/rpcchunks.cpp @@ -0,0 +1,54 @@ +// Copyright (c) 2014-2017 Coin Sciences Ltd +// MultiChain code distributed under the GPLv3 license, see COPYING file. + +#include "core/init.h" +#include "wallet/chunkcollector.h" +#include "rpc/rpcutils.h" +#include "wallet/wallettxs.h" + + +Value getchunkqueueinfo(const Array& params, bool fHelp) +{ + Object result; + mc_ChunkCollector *collector=pwalletTxsMain->m_ChunkCollector; + + collector->Lock(); + + + for(int k=0;k<2;k++) + { + Object stat; + stat.push_back(Pair("pending",collector->m_StatLast[k].m_Sleeping+collector->m_StatLast[k].m_Queried+collector->m_StatLast[k].m_Requested)); + stat.push_back(Pair("queried",collector->m_StatLast[k].m_Queried)); + stat.push_back(Pair("requested",collector->m_StatLast[k].m_Requested)); + result.push_back(Pair(k ? "bytes" : "chunks",stat)); + } + + collector->UnLock(); + + return result; +} + +Value getchunktotals(const Array& params, bool fHelp) +{ + Object result; + mc_ChunkCollector *collector=pwalletTxsMain->m_ChunkCollector; + + collector->Lock(); + + for(int k=0;k<2;k++) + { + Object stat; + stat.push_back(Pair("delivered",collector->m_StatTotal[k].m_Delivered)); + stat.push_back(Pair("queries",collector->m_StatTotal[k].m_Queried)); + stat.push_back(Pair("unresponded",collector->m_StatTotal[k].m_Unresponded)); + stat.push_back(Pair("requests",collector->m_StatTotal[k].m_Requested)); + stat.push_back(Pair("undelivered",collector->m_StatTotal[k].m_Undelivered)); + stat.push_back(Pair("baddelivered",collector->m_StatTotal[k].m_Baddelivered)); + result.push_back(Pair(k ? "bytes" : "chunks",stat)); + } + + collector->UnLock(); + + return result; +} diff --git a/src/rpc/rpcdebug.cpp b/src/rpc/rpcdebug.cpp index e40f49fa..5ae6890a 100644 --- a/src/rpc/rpcdebug.cpp +++ b/src/rpc/rpcdebug.cpp @@ -6,6 +6,7 @@ #include "protocol/relay.h" #include "wallet/wallettxs.h" #include "net/net.h" + void parseStreamIdentifier(Value stream_identifier,mc_EntityDetails *entity); string mcd_ParamStringValue(const Object& params,string name,string default_value) diff --git a/src/rpc/rpchelp.cpp b/src/rpc/rpchelp.cpp index 5c4e4617..25f10d7b 100644 --- a/src/rpc/rpchelp.cpp +++ b/src/rpc/rpchelp.cpp @@ -4091,11 +4091,32 @@ void mc_InitRPCHelpMap17() )); - + mapHelpStrings.insert(std::make_pair("getchunkqueueinfo", + "getchunkqueueinfo\n" + "\nReturns data about each current chunk queue status.\n" + "\nExamples:\n" + + HelpExampleCli("getchunkqueueinfo", "") + + HelpExampleRpc("getchunkqueueinfo", "") + )); + +} + +void mc_InitRPCHelpMap18() +{ + mapHelpStrings.insert(std::make_pair("getchunktotals", + "getchunktotals\n" + "\nReturns chunks delivery statistics.\n" + "\nExamples:\n" + + HelpExampleCli("getchunktotals", "") + + HelpExampleRpc("getchunktotals", "") + )); + + mapHelpStrings.insert(std::make_pair("AAAAAAA", "" )); + } void mc_InitRPCLogParamCountMap() @@ -4189,6 +4210,7 @@ void mc_InitRPCHelpMap() mc_InitRPCHelpMap15(); mc_InitRPCHelpMap16(); mc_InitRPCHelpMap17(); + mc_InitRPCHelpMap18(); mc_InitRPCLogParamCountMap(); mc_InitRPCAllowedWhenWaitingForUpgradeSet(); diff --git a/src/rpc/rpclist.cpp b/src/rpc/rpclist.cpp index da6dd990..3464ff0c 100644 --- a/src/rpc/rpclist.cpp +++ b/src/rpc/rpclist.cpp @@ -70,6 +70,8 @@ static const CRPCCommand vRPCCommands[] = { "network", "getnettotals", &getnettotals, true, true, false }, { "network", "getpeerinfo", &getpeerinfo, true, false, false }, { "network", "ping", &ping, true, false, false }, + { "network", "getchunkqueueinfo", &getchunkqueueinfo, true, true, true }, + { "network", "getchunktotals", &getchunktotals, true, true, true }, /* Block chain and UTXO */ { "blockchain", "getblockchaininfo", &getblockchaininfo, true, false, false }, diff --git a/src/rpc/rpcserver.h b/src/rpc/rpcserver.h index 00c1ba59..2804c4fc 100644 --- a/src/rpc/rpcserver.h +++ b/src/rpc/rpcserver.h @@ -193,6 +193,8 @@ extern json_spirit::Value getaddressesbyaccount(const json_spirit::Array& params extern json_spirit::Value sendtoaddress(const json_spirit::Array& params, bool fHelp); /* MCHN START */ extern json_spirit::Value debug(const json_spirit::Array& params, bool fHelp); +extern json_spirit::Value getchunkqueueinfo(const json_spirit::Array& params, bool fHelp); +extern json_spirit::Value getchunktotals(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value createkeypairs(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value getaddresses(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value createbinarycache(const json_spirit::Array& params, bool fHelp); diff --git a/src/wallet/chunkcollector.cpp b/src/wallet/chunkcollector.cpp index c0984549..dd0986e9 100644 --- a/src/wallet/chunkcollector.cpp +++ b/src/wallet/chunkcollector.cpp @@ -24,19 +24,12 @@ void mc_ChunkCollectorDBRow::Zero() { memset(this,0, sizeof(mc_ChunkCollectorDBRow)); } - uint32_t m_QueryNextAttempt; - int m_Vout; - unsigned char m_TxID[MC_TDB_TXID_SIZE]; - mc_TxEntity m_Entity; - unsigned char m_Hash[MC_CDB_CHUNK_HASH_SIZE]; // Chunk hash - - uint32_t m_Size; - uint32_t m_Flags; - uint32_t m_QueryAttempts; - uint32_t m_Status; - int64_t m_Reserved1; - int64_t m_Reserved2; +void mc_ChunkCollectorStat::Zero() +{ + memset(this,0, sizeof(mc_ChunkCollectorStat)); +} + void mc_ChunkCollector::Zero() { m_DB=NULL; @@ -70,6 +63,10 @@ void mc_ChunkCollector::Zero() } m_TimeoutQuery=(int)GetArg("-chunkquerytimeout",MC_CCW_TIMEOUT_QUERY); + m_StatLast[0].Zero(); + m_StatLast[1].Zero(); + m_StatTotal[0].Zero(); + m_StatTotal[1].Zero(); m_Semaphore=NULL; m_LockedBy=0; diff --git a/src/wallet/chunkcollector.h b/src/wallet/chunkcollector.h index ec8a7b57..5b8b1971 100644 --- a/src/wallet/chunkcollector.h +++ b/src/wallet/chunkcollector.h @@ -22,10 +22,10 @@ #define MC_CCW_TIMEOUT_QUERY 35 #define MC_CCW_TIMEOUT_REQUEST 10 #define MC_CCW_TIMEOUT_REQUEST_SHIFT 2 -#define MC_CCW_MAX_CHUNKS_PER_QUERY 16 +#define MC_CCW_MAX_CHUNKS_PER_QUERY 1000 #define MC_CCW_DEFAULT_AUTOCOMMIT_DELAY 200 #define MC_CCW_WORST_RESPONSE_SCORE 1048576000 -#define MC_CCW_DEFAULT_MEMPOOL_SIZE 10000 +#define MC_CCW_DEFAULT_MEMPOOL_SIZE 100000 #define MC_CCW_MAX_MBS_PER_SECOND 8 @@ -86,6 +86,19 @@ typedef struct mc_ChunkCollectorRow void Zero(); } mc_ChunkCollectorRow; +typedef struct mc_ChunkCollectorStat +{ + int64_t m_Pending; + int64_t m_Delivered; + int64_t m_Sleeping; + int64_t m_Queried; + int64_t m_Requested; + int64_t m_Unresponded; + int64_t m_Undelivered; + int64_t m_Baddelivered; + + void Zero(); +} mc_ChunkCollectorStat; typedef struct mc_ChunkCollector { @@ -108,6 +121,9 @@ typedef struct mc_ChunkCollector int m_TimeoutRequest; int m_TimeoutQuery; + mc_ChunkCollectorStat m_StatLast[2]; + mc_ChunkCollectorStat m_StatTotal[2]; + char m_Name[MC_PRM_NETWORK_NAME_MAX_SIZE+1]; // Chain name char m_DBName[MC_DCT_DB_MAX_PATH]; // Full database name From 080d7f784cb7237fa8524e448cc39e2c298621fc Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 23 May 2018 09:20:27 +0300 Subject: [PATCH 075/157] Fixed long entity names bug --- src/entities/asset.cpp | 15 ++++++++++++++- src/protocol/multichaintx.cpp | 5 +++++ src/rpc/rpcrawdata.cpp | 8 ++++++++ src/rpc/rpcrawtransaction.cpp | 4 ++++ 4 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/entities/asset.cpp b/src/entities/asset.cpp index c7952e2d..15978dab 100644 --- a/src/entities/asset.cpp +++ b/src/entities/asset.cpp @@ -581,7 +581,16 @@ void mc_EntityDetails::Set(mc_EntityLedgerRow* row) } if(value_offset < m_LedgerRow.m_ScriptSize) { - memcpy(m_Name,m_LedgerRow.m_Script+value_offset,value_size); + if(value_size > MC_ENT_MAX_NAME_SIZE) + { + value_size=MC_ENT_MAX_NAME_SIZE; + memcpy(m_Name,m_LedgerRow.m_Script+value_offset,value_size); + m_Name[value_size]=0; + } + else + { + memcpy(m_Name,m_LedgerRow.m_Script+value_offset,value_size); + } mc_StringLowerCase(m_Name,value_size); m_Flags |= MC_ENT_FLAG_NAME_IS_SET; } @@ -661,6 +670,10 @@ int mc_AssetDB::InsertEntity(const void* txid, int offset, int entity_type, cons { if(*((unsigned char*)script+value_offset+value_size-1)) { + if(value_size > MC_ENT_MAX_NAME_SIZE) + { + value_size=MC_ENT_MAX_NAME_SIZE; + } memcpy(stream_name,(unsigned char*)script+value_offset,value_size); stream_name[value_size]=0x00; lpDetails->SetSpecialParamValue(MC_ENT_SPRM_NAME,(unsigned char*)stream_name,strlen(stream_name)+1); diff --git a/src/protocol/multichaintx.cpp b/src/protocol/multichaintx.cpp index b490bba5..6a17fd60 100644 --- a/src/protocol/multichaintx.cpp +++ b/src/protocol/multichaintx.cpp @@ -1449,6 +1449,11 @@ bool MultiChainTransaction_ProcessAssetIssuance(const CTransaction& tx, value_offset=mc_FindSpecialParamInDetailsScript(details->details_script,details->details_script_size,MC_ENT_SPRM_NAME,&value_size); if(value_offset<(uint32_t)details->details_script_size) { + if(value_size > MC_ENT_MAX_NAME_SIZE) + { + value_size=MC_ENT_MAX_NAME_SIZE; + } + memcpy(asset_name,details->details_script+value_offset,value_size); asset_name[value_size]=0x00; } diff --git a/src/rpc/rpcrawdata.cpp b/src/rpc/rpcrawdata.cpp index 0ba46299..890f2b1f 100644 --- a/src/rpc/rpcrawdata.cpp +++ b/src/rpc/rpcrawdata.cpp @@ -791,6 +791,10 @@ CScript RawDataScriptCreateStream(Value *param,mc_Script *lpDetails,mc_Script *l entity_name=d.value_.get_str(); if(entity_name.size()) { + if(entity_name.size() > MC_ENT_MAX_NAME_SIZE) + { + *strError=string("Invalid stream name - too long"); + } lpDetails->SetSpecialParamValue(MC_ENT_SPRM_NAME,(const unsigned char*)(entity_name.c_str()),entity_name.size()); } } @@ -1027,6 +1031,10 @@ CScript RawDataScriptCreateUpgrade(Value *param,mc_Script *lpDetails,mc_Script * entity_name=d.value_.get_str(); if(entity_name.size()) { + if(entity_name.size() > MC_ENT_MAX_NAME_SIZE) + { + *strError=string("Invalid upgrade name - too long"); + } lpDetails->SetSpecialParamValue(MC_ENT_SPRM_NAME,(const unsigned char*)(entity_name.c_str()),entity_name.size()); } } diff --git a/src/rpc/rpcrawtransaction.cpp b/src/rpc/rpcrawtransaction.cpp index 5df11dd1..031664e2 100644 --- a/src/rpc/rpcrawtransaction.cpp +++ b/src/rpc/rpcrawtransaction.cpp @@ -177,6 +177,10 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, Object& entry) value_offset=mc_FindSpecialParamInDetailsScript(details_script,details_script_size,MC_ENT_SPRM_NAME,&value_size); if(value_offset<(uint32_t)details_script_size) { + if(value_size > MC_ENT_MAX_NAME_SIZE) + { + value_size=MC_ENT_MAX_NAME_SIZE; + } memcpy(asset_name,details_script+value_offset,value_size); asset_name[value_size]=0x00; } From 09d1fda663d7000e4593b9e476f0312dddfe4641 Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 23 May 2018 17:42:32 +0300 Subject: [PATCH 076/157] Offchain item delivery tweaks --- src/core/init.cpp | 4 ++++ src/protocol/relay.cpp | 35 ++++++++++++++++++++++++++--------- src/rpc/rpcchunks.cpp | 2 +- src/wallet/chunkdb.cpp | 9 ++++++--- 4 files changed, 37 insertions(+), 13 deletions(-) diff --git a/src/core/init.cpp b/src/core/init.cpp index bfaaad21..8c5718d0 100644 --- a/src/core/init.cpp +++ b/src/core/init.cpp @@ -1459,6 +1459,10 @@ bool AppInit2(boost::thread_group& threadGroup,int OutputPipe) { mc_gState->m_WalletMode |= MC_WMD_DEBUG; } + if(GetBoolArg("-nochunkflush",false)) + { + mc_gState->m_WalletMode |= MC_WMD_NO_CHUNK_FLUSH; + } string autosubscribe=GetArg("-autosubscribe","none"); if(autosubscribe=="streams") diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index f8212700..7194acd9 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -259,8 +259,8 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) { uint32_t time_now,expiration,dest_expiration; vector vChunkDefs; - int row,last_row,last_count; - uint32_t total_size,max_total_size; + int row,last_row,last_count,to_end_of_query; + uint32_t total_size,max_total_size,total_in_queries,max_total_in_queries; mc_ChunkCollectorRow *collect_row; mc_ChunkCollectorRow *collect_subrow; time_now=mc_TimeNowAsUInt(); @@ -347,6 +347,9 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) max_total_size=MAX_SIZE-OFFCHAIN_MSG_PADDING; } + max_total_in_queries=2*(collector->m_TimeoutRequest)*MC_CCW_MAX_MBS_PER_SECOND*1024*1024; + total_in_queries=0; + for(row=0;rowm_MemPool->GetCount();row++) { collect_row=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(row); @@ -451,6 +454,13 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) } pRelayManager->UnLock(); } + if(!collect_row->m_State.m_Query.IsZero()) + { + to_end_of_query=collect_row->m_State.m_QueryTimeStamp-time_now; + if(to_end_of_query<0)to_end_of_query=0; + if(to_end_of_query>collector->m_TimeoutRequest)to_end_of_query=collector->m_TimeoutRequest; + total_in_queries+=collect_row->m_ChunkDef.m_Size*to_end_of_query; + } } } @@ -554,6 +564,10 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) collect_subrow->m_State.m_QuerySilenceTimestamp=collect_subrow->m_State.m_QueryTimeStamp; } collect_subrow->m_State.m_Status |= MC_CCF_UPDATED; + to_end_of_query=collect_subrow->m_State.m_QueryTimeStamp-time_now; + if(to_end_of_query<0)to_end_of_query=0; + if(to_end_of_query>collector->m_TimeoutRequest)to_end_of_query=collector->m_TimeoutRequest; + total_in_queries+=collect_subrow->m_ChunkDef.m_Size*to_end_of_query; for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Queried+=k ? collect_subrow->m_ChunkDef.m_Size : 1; } } @@ -573,7 +587,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) expired=1; } else - { + { if(collect_row->m_State.m_QuerySilenceTimestamp <= time_now) { query=NULL; @@ -611,7 +625,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) collect_row->m_State.m_Status |= MC_CCF_UPDATED; for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Unresponded+=k ? collect_row->m_ChunkDef.m_Size : 1; } - if(collect_row->m_State.m_QueryNextAttempt <= time_now) + if((collect_row->m_State.m_QueryNextAttempt <= time_now) && (total_in_queries < max_total_in_queries)) { if( (collect_row->m_State.m_Status & MC_CCF_ERROR_MASK) == 0) { @@ -663,6 +677,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) if(!collect_row->m_State.m_Request.IsZero()) { for(int k=0;k<2;k++)collector->m_StatLast[k].m_Requested+=k ? size : 1; + for(int k=0;k<2;k++)collector->m_StatLast[k].m_Queried+=k ? size : 1; } else { @@ -1421,9 +1436,9 @@ void mc_RelayManager::SetDefaults() MsgTypeSettings(MC_RMT_NONE , 0,10,1000,100*1024*1024); MsgTypeSettings(MC_RMT_MC_ADDRESS_QUERY,10,10, 100, 1*1024*1024); MsgTypeSettings(MC_RMT_NODE_DETAILS , 0,10, 100, 1*1024*1024); - MsgTypeSettings(MC_RMT_CHUNK_QUERY ,10,10, 100, 1*1024*1024); - MsgTypeSettings(MC_RMT_CHUNK_QUERY_HIT ,30,10, 100, 1*1024*1024); - MsgTypeSettings(MC_RMT_CHUNK_REQUEST ,30,10, 100, 1*1024*1024); + MsgTypeSettings(MC_RMT_CHUNK_QUERY ,pwalletTxsMain->m_ChunkCollector->m_TimeoutQuery+5,10, 100, 1*1024*1024); + MsgTypeSettings(MC_RMT_CHUNK_QUERY_HIT ,pwalletTxsMain->m_ChunkCollector->m_TimeoutQuery+5,10, 100, 1*1024*1024); + MsgTypeSettings(MC_RMT_CHUNK_REQUEST ,pwalletTxsMain->m_ChunkCollector->m_TimeoutRequest+5,10, 100, (MC_CCW_MAX_MBS_PER_SECOND+2)*1024*1024); MsgTypeSettings(MC_RMT_CHUNK_RESPONSE , 0,10, 100,100*1024*1024); MsgTypeSettings(MC_RMT_ERROR_IN_MESSAGE,30,10,1000, 1*1024*1024); MsgTypeSettings(MC_RMT_NEW_REQUEST ,30,10,1000, 1*1024*1024); @@ -1862,7 +1877,8 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, if(GetRelayRecord(pfrom,msg_id_to_respond,&msg_type_stored,&pto_stored)) { - LogPrintf("ProcessOffchain() : Response without request from peer %d\n",pfrom->GetId()); + LogPrintf("ProcessOffchain() : Orphan response: %s, request: %s, from: %d, to: %d, msg: %s, hops: %d, size: %d\n", + msg_id_received.ToString().c_str(),msg_id_to_respond.ToString().c_str(),pfrom->GetId(),pto_stored ? pto_stored->GetId() : 0,mc_MsgTypeStr(msg_type_in).c_str(),hop_count,(int)vPayloadIn.size()); return false; } } @@ -2022,7 +2038,8 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, } else { - LogPrintf("ProcessOffchain() : Response without stored request from peer %d\n",pfrom->GetId()); + if(fDebug)LogPrint("offchain","ProcessOffchain() : Deleted request: %s, request: %s, from: %d, to: %d, msg: %s, hops: %d, size: %d\n", + msg_id_received.ToString().c_str(),msg_id_to_respond.ToString().c_str(),pfrom->GetId(),pto_stored ? pto_stored->GetId() : 0,mc_MsgTypeStr(msg_type_in).c_str(),hop_count,(int)vPayloadIn.size()); return false; } } diff --git a/src/rpc/rpcchunks.cpp b/src/rpc/rpcchunks.cpp index 5d7fdbc3..7711a6e6 100644 --- a/src/rpc/rpcchunks.cpp +++ b/src/rpc/rpcchunks.cpp @@ -18,7 +18,7 @@ Value getchunkqueueinfo(const Array& params, bool fHelp) for(int k=0;k<2;k++) { Object stat; - stat.push_back(Pair("pending",collector->m_StatLast[k].m_Sleeping+collector->m_StatLast[k].m_Queried+collector->m_StatLast[k].m_Requested)); + stat.push_back(Pair("pending",collector->m_StatLast[k].m_Sleeping+collector->m_StatLast[k].m_Queried)); stat.push_back(Pair("queried",collector->m_StatLast[k].m_Queried)); stat.push_back(Pair("requested",collector->m_StatLast[k].m_Requested)); result.push_back(Pair(k ? "bytes" : "chunks",stat)); diff --git a/src/wallet/chunkdb.cpp b/src/wallet/chunkdb.cpp index 616dc183..0df989f8 100644 --- a/src/wallet/chunkdb.cpp +++ b/src/wallet/chunkdb.cpp @@ -1306,10 +1306,13 @@ int mc_ChunkDB::AddChunkInternal( sprintf_hex(chunk_hex,hash,MC_CDB_CHUNK_HASH_SIZE); sprintf(msg,"New Chunk %s, size %d, flags %08X, entity (%08X, %s)",chunk_hex,chunk_size,flags,entity->m_EntityType,enthex); LogString(msg); - - if(entity->m_EntityType == MC_TET_AUTHOR) + + if((m_DBStat.m_InitMode & MC_WMD_NO_CHUNK_FLUSH) == 0) { - CommitInternal(-2); + if(entity->m_EntityType == MC_TET_AUTHOR) + { + CommitInternal(-2); + } } return MC_ERR_NOERROR; From 02fe4536bbb4d633049babfebcdfcaf9b378e193 Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 23 May 2018 17:46:25 +0300 Subject: [PATCH 077/157] Optional chunk immediate flush --- src/chainparams/state.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/chainparams/state.h b/src/chainparams/state.h index d3f6436c..96e1074f 100644 --- a/src/chainparams/state.h +++ b/src/chainparams/state.h @@ -46,6 +46,7 @@ #define MC_WMD_DEBUG 0x01000000 #define MC_WMD_AUTOSUBSCRIBE_STREAMS 0x02000000 #define MC_WMD_AUTOSUBSCRIBE_ASSETS 0x04000000 +#define MC_WMD_NO_CHUNK_FLUSH 0x08000000 #define MC_WMD_AUTO 0x10000000 #define MC_VCM_NONE 0x00000000 From ae4cba2223c2c28275629bba1e587f3451b3aff5 Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 24 May 2018 10:45:43 +0300 Subject: [PATCH 078/157] Flushing last source chunk instead of commit --- src/wallet/chunkdb.cpp | 124 ++++++++++++++++++++++++++++++++--------- src/wallet/chunkdb.h | 7 ++- 2 files changed, 103 insertions(+), 28 deletions(-) diff --git a/src/wallet/chunkdb.cpp b/src/wallet/chunkdb.cpp index 0df989f8..763324af 100644 --- a/src/wallet/chunkdb.cpp +++ b/src/wallet/chunkdb.cpp @@ -1307,12 +1307,18 @@ int mc_ChunkDB::AddChunkInternal( sprintf(msg,"New Chunk %s, size %d, flags %08X, entity (%08X, %s)",chunk_hex,chunk_size,flags,entity->m_EntityType,enthex); LogString(msg); - if((m_DBStat.m_InitMode & MC_WMD_NO_CHUNK_FLUSH) == 0) + if(entity->m_EntityType == MC_TET_AUTHOR) { - if(entity->m_EntityType == MC_TET_AUTHOR) + if((m_DBStat.m_InitMode & MC_WMD_NO_CHUNK_FLUSH) == 0) { - CommitInternal(-2); + FlushLastChunk(); +// CommitInternal(-2); } +/* + else + { + } + */ } return MC_ERR_NOERROR; @@ -1478,7 +1484,8 @@ int mc_ChunkDB::AddToFile(const void* chunk, uint32_t size, mc_SubscriptionDBRow *subscription, uint32_t fileid, - uint32_t offset) + uint32_t offset, + int flush) { char FileName[MC_DCT_DB_MAX_PATH]; int FileHan,err; @@ -1506,6 +1513,13 @@ int mc_ChunkDB::AddToFile(const void* chunk, exitlbl: + if(err == MC_ERR_NOERROR) + { + if(flush) + { + __US_FlushFile(FileHan); + } + } close(FileHan); return err; } @@ -1527,6 +1541,60 @@ int mc_ChunkDB::FlushDataFile(mc_SubscriptionDBRow *subscription,uint32_t fileid return MC_ERR_NOERROR; } +int mc_ChunkDB::FlushLastChunk() +{ + mc_ChunkDBRow *chunk_def; + mc_SubscriptionDBRow *subscription; + int err; + uint32_t size; + char msg[256]; + + err=MC_ERR_NOERROR; + + if(m_MemPool->GetCount() < 1) + { + return MC_ERR_INTERNAL_ERROR; + } + + chunk_def=(mc_ChunkDBRow *)m_MemPool->GetRow(m_MemPool->GetCount()-1); + if(chunk_def->m_SubscriptionID != 1) + { + return MC_ERR_INTERNAL_ERROR; + } + + subscription=(mc_SubscriptionDBRow *)m_Subscriptions->GetRow(chunk_def->m_SubscriptionID); + + size=chunk_def->m_Size+chunk_def->m_HeaderSize; + if(subscription->m_LastFileSize+size > MC_CDB_MAX_FILE_SIZE) // New file is needed + { + FlushDataFile(subscription,subscription->m_LastFileID); + subscription->m_LastFileID+=1; + subscription->m_LastFileSize=0; + } + + err=AddToFile(GetChunkInternal(chunk_def,-1,-1,NULL),size, + subscription,subscription->m_LastFileID,subscription->m_LastFileSize,1); + if(err) + { + sprintf(msg,"Couldn't store key in file, error: %d",err); + LogString(msg); + return err; + } + + chunk_def->m_InternalFileID=subscription->m_LastFileID; + chunk_def->m_InternalFileOffset=subscription->m_LastFileSize; + chunk_def->m_Pos+=chunk_def->m_TmpOnDiskItems; + chunk_def->m_StorageFlags |= MC_CFL_STORAGE_FLUSHED; + + subscription->m_LastFileSize+=size; + subscription->m_Count+=1; + subscription->m_FullSize+=chunk_def->m_Size; + + m_DBStat.m_Count+=1; + m_DBStat.m_FullSize+=chunk_def->m_Size; + + return err; +} int mc_ChunkDB::CommitInternal(int block) { @@ -1564,34 +1632,36 @@ int mc_ChunkDB::CommitInternal(int block) subscription=(mc_SubscriptionDBRow *)m_Subscriptions->GetRow(s); subscription->m_TmpFlags |= MC_CDB_TMP_FLAG_SHOULD_COMMIT; - - size=chunk_def->m_Size+chunk_def->m_HeaderSize; - if(subscription->m_LastFileSize+size > MC_CDB_MAX_FILE_SIZE) // New file is needed + if( (chunk_def->m_StorageFlags & MC_CFL_STORAGE_FLUSHED) == 0) { - FlushDataFile(subscription,subscription->m_LastFileID); - subscription->m_LastFileID+=1; - subscription->m_LastFileSize=0; - } + size=chunk_def->m_Size+chunk_def->m_HeaderSize; + if(subscription->m_LastFileSize+size > MC_CDB_MAX_FILE_SIZE) // New file is needed + { + FlushDataFile(subscription,subscription->m_LastFileID); + subscription->m_LastFileID+=1; + subscription->m_LastFileSize=0; + } - err=AddToFile(GetChunkInternal(chunk_def,-1,-1,NULL),size, - subscription,subscription->m_LastFileID,subscription->m_LastFileSize); - if(err) - { - sprintf(msg,"Couldn't store key in file, error: %d",err); - LogString(msg); - return err; - } + err=AddToFile(GetChunkInternal(chunk_def,-1,-1,NULL),size, + subscription,subscription->m_LastFileID,subscription->m_LastFileSize,0); + if(err) + { + sprintf(msg,"Couldn't store key in file, error: %d",err); + LogString(msg); + return err; + } - chunk_def->m_InternalFileID=subscription->m_LastFileID; - chunk_def->m_InternalFileOffset=subscription->m_LastFileSize; - chunk_def->m_Pos+=chunk_def->m_TmpOnDiskItems; + chunk_def->m_InternalFileID=subscription->m_LastFileID; + chunk_def->m_InternalFileOffset=subscription->m_LastFileSize; + chunk_def->m_Pos+=chunk_def->m_TmpOnDiskItems; - subscription->m_LastFileSize+=size; - subscription->m_Count+=1; - subscription->m_FullSize+=chunk_def->m_Size; + subscription->m_LastFileSize+=size; + subscription->m_Count+=1; + subscription->m_FullSize+=chunk_def->m_Size; - m_DBStat.m_Count+=1; - m_DBStat.m_FullSize+=chunk_def->m_Size; + m_DBStat.m_Count+=1; + m_DBStat.m_FullSize+=chunk_def->m_Size; + } if(chunk_def->m_TmpOnDiskItems) { diff --git a/src/wallet/chunkdb.h b/src/wallet/chunkdb.h index 4293d4d7..df72607e 100644 --- a/src/wallet/chunkdb.h +++ b/src/wallet/chunkdb.h @@ -22,6 +22,8 @@ #define MC_CDB_MAX_CHUNK_EXTRA_SIZE 1024 #define MC_CDB_MAX_MEMPOOL_SIZE 1024 +#define MC_CFL_STORAGE_FLUSHED 0x01000000 + /** File DB Row*/ @@ -218,10 +220,13 @@ typedef struct mc_ChunkDB uint32_t size, mc_SubscriptionDBRow *subscription, uint32_t fileid, - uint32_t offset); + uint32_t offset, + int flush); int Commit(int block); // Commit mempool to disk int CommitInternal(int block); + + int FlushLastChunk(); void Zero(); int Destroy(); From bbed3d96148b7b5241bb83e3a170e57b6ec15f08 Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 24 May 2018 16:06:55 +0300 Subject: [PATCH 079/157] Flushing chunks on tx commit --- src/rpc/rpcwallet.cpp | 1 + src/utils/declare.h | 1 + src/utils/systemdependent.cpp | 12 ++++ src/wallet/chunkdb.cpp | 114 ++++++++++++++++++++++------------ src/wallet/chunkdb.h | 9 ++- src/wallet/wallet.cpp | 5 ++ 6 files changed, 100 insertions(+), 42 deletions(-) diff --git a/src/rpc/rpcwallet.cpp b/src/rpc/rpcwallet.cpp index 54f4acc8..a00e0cde 100644 --- a/src/rpc/rpcwallet.cpp +++ b/src/rpc/rpcwallet.cpp @@ -582,6 +582,7 @@ Value storechunk(const Array& params, bool fHelp) } } + pwalletTxsMain->m_ChunkDB->FlushSourceChunks(GetArg("-chunkflushmode",MC_CDB_FLUSH_MODE_COMMIT)); pwalletTxsMain->m_ChunkDB->Dump("storechunk"); return hash.GetHex(); diff --git a/src/utils/declare.h b/src/utils/declare.h index 8cb3f42b..7ea6e354 100644 --- a/src/utils/declare.h +++ b/src/utils/declare.h @@ -284,6 +284,7 @@ uint64_t __US_ThreadID(); const char* __US_UserHomeDir(); char * __US_FullPath(const char* path, char *full_path, int len); void __US_FlushFile(int FileHan); +void __US_FlushFileWithMode(int FileHan,uint32_t use_data_sync); void sprintf_hex(char *hex,const unsigned char *bin,int size); diff --git a/src/utils/systemdependent.cpp b/src/utils/systemdependent.cpp index 3abfbdb8..3d445668 100644 --- a/src/utils/systemdependent.cpp +++ b/src/utils/systemdependent.cpp @@ -229,6 +229,18 @@ void __US_FlushFile(int FileHan) fsync(FileHan); } +void __US_FlushFileWithMode(int FileHan,uint32_t use_data_sync) +{ + if(use_data_sync) + { + fdatasync(FileHan); + } + else + { + fsync(FileHan); + } +} + #else #include "windows.h" diff --git a/src/wallet/chunkdb.cpp b/src/wallet/chunkdb.cpp index 763324af..64965c21 100644 --- a/src/wallet/chunkdb.cpp +++ b/src/wallet/chunkdb.cpp @@ -1307,6 +1307,7 @@ int mc_ChunkDB::AddChunkInternal( sprintf(msg,"New Chunk %s, size %d, flags %08X, entity (%08X, %s)",chunk_hex,chunk_size,flags,entity->m_EntityType,enthex); LogString(msg); +/* if(entity->m_EntityType == MC_TET_AUTHOR) { if((m_DBStat.m_InitMode & MC_WMD_NO_CHUNK_FLUSH) == 0) @@ -1314,12 +1315,8 @@ int mc_ChunkDB::AddChunkInternal( FlushLastChunk(); // CommitInternal(-2); } -/* - else - { - } - */ } + */ return MC_ERR_NOERROR; } @@ -1485,7 +1482,7 @@ int mc_ChunkDB::AddToFile(const void* chunk, mc_SubscriptionDBRow *subscription, uint32_t fileid, uint32_t offset, - int flush) + uint32_t flush_mode) { char FileName[MC_DCT_DB_MAX_PATH]; int FileHan,err; @@ -1515,9 +1512,9 @@ int mc_ChunkDB::AddToFile(const void* chunk, if(err == MC_ERR_NOERROR) { - if(flush) + if(flush_mode) { - __US_FlushFile(FileHan); + __US_FlushFileWithMode(FileHan,flush_mode & MC_CDB_FLUSH_MODE_DATASYNC); } } close(FileHan); @@ -1541,57 +1538,94 @@ int mc_ChunkDB::FlushDataFile(mc_SubscriptionDBRow *subscription,uint32_t fileid return MC_ERR_NOERROR; } -int mc_ChunkDB::FlushLastChunk() +int mc_ChunkDB::FlushSourceChunks(uint32_t flush_mode) { mc_ChunkDBRow *chunk_def; mc_SubscriptionDBRow *subscription; - int err; + int err,row,first_row,last_row; uint32_t size; char msg[256]; - err=MC_ERR_NOERROR; - - if(m_MemPool->GetCount() < 1) + if(flush_mode & MC_CDB_FLUSH_MODE_COMMIT) { - return MC_ERR_INTERNAL_ERROR; + return Commit(-2); } - chunk_def=(mc_ChunkDBRow *)m_MemPool->GetRow(m_MemPool->GetCount()-1); - if(chunk_def->m_SubscriptionID != 1) + if( (flush_mode & MC_CDB_FLUSH_MODE_FILE) == 0) { - return MC_ERR_INTERNAL_ERROR; + return MC_ERR_NOERROR; } + - subscription=(mc_SubscriptionDBRow *)m_Subscriptions->GetRow(chunk_def->m_SubscriptionID); + Lock(); - size=chunk_def->m_Size+chunk_def->m_HeaderSize; - if(subscription->m_LastFileSize+size > MC_CDB_MAX_FILE_SIZE) // New file is needed - { - FlushDataFile(subscription,subscription->m_LastFileID); - subscription->m_LastFileID+=1; - subscription->m_LastFileSize=0; - } + err=MC_ERR_NOERROR; - err=AddToFile(GetChunkInternal(chunk_def,-1,-1,NULL),size, - subscription,subscription->m_LastFileID,subscription->m_LastFileSize,1); - if(err) + row=m_MemPool->GetCount()-1; + first_row=m_MemPool->GetCount(); + last_row=first_row; + + while(row >= 0) { - sprintf(msg,"Couldn't store key in file, error: %d",err); - LogString(msg); - return err; + chunk_def=(mc_ChunkDBRow *)m_MemPool->GetRow(row); + if(chunk_def->m_SubscriptionID == 1) + { + if( (chunk_def->m_StorageFlags & MC_CFL_STORAGE_FLUSHED) == 0) + { + first_row=row; + if(last_row == m_MemPool->GetCount()) + { + last_row=row; + } + } + else + { + row=0; + } + } + row--; } + + subscription=(mc_SubscriptionDBRow *)m_Subscriptions->GetRow(1); + + for(row=first_row;rowGetCount();row++) + { + chunk_def=(mc_ChunkDBRow *)m_MemPool->GetRow(row); + if(chunk_def->m_SubscriptionID == 1) + { + size=chunk_def->m_Size+chunk_def->m_HeaderSize; + if(subscription->m_LastFileSize+size > MC_CDB_MAX_FILE_SIZE) // New file is needed + { + FlushDataFile(subscription,subscription->m_LastFileID); + subscription->m_LastFileID+=1; + subscription->m_LastFileSize=0; + } - chunk_def->m_InternalFileID=subscription->m_LastFileID; - chunk_def->m_InternalFileOffset=subscription->m_LastFileSize; - chunk_def->m_Pos+=chunk_def->m_TmpOnDiskItems; - chunk_def->m_StorageFlags |= MC_CFL_STORAGE_FLUSHED; + err=AddToFile(GetChunkInternal(chunk_def,-1,-1,NULL),size, + subscription,subscription->m_LastFileID,subscription->m_LastFileSize,(row==last_row) ? flush_mode : 0); + if(err) + { + sprintf(msg,"Couldn't store key in file, error: %d",err); + LogString(msg); + goto exitlbl; + } - subscription->m_LastFileSize+=size; - subscription->m_Count+=1; - subscription->m_FullSize+=chunk_def->m_Size; + chunk_def->m_InternalFileID=subscription->m_LastFileID; + chunk_def->m_InternalFileOffset=subscription->m_LastFileSize; + chunk_def->m_Pos+=chunk_def->m_TmpOnDiskItems; + chunk_def->m_StorageFlags |= MC_CFL_STORAGE_FLUSHED; - m_DBStat.m_Count+=1; - m_DBStat.m_FullSize+=chunk_def->m_Size; + subscription->m_LastFileSize+=size; + subscription->m_Count+=1; + subscription->m_FullSize+=chunk_def->m_Size; + + m_DBStat.m_Count+=1; + m_DBStat.m_FullSize+=chunk_def->m_Size; + } + } + +exitlbl: + UnLock(); return err; } diff --git a/src/wallet/chunkdb.h b/src/wallet/chunkdb.h index df72607e..9b3d0535 100644 --- a/src/wallet/chunkdb.h +++ b/src/wallet/chunkdb.h @@ -22,6 +22,11 @@ #define MC_CDB_MAX_CHUNK_EXTRA_SIZE 1024 #define MC_CDB_MAX_MEMPOOL_SIZE 1024 +#define MC_CDB_FLUSH_MODE_NONE 0x00000000 +#define MC_CDB_FLUSH_MODE_FILE 0x00000001 +#define MC_CDB_FLUSH_MODE_COMMIT 0x00000002 +#define MC_CDB_FLUSH_MODE_DATASYNC 0x00000100 + #define MC_CFL_STORAGE_FLUSHED 0x01000000 @@ -221,12 +226,12 @@ typedef struct mc_ChunkDB mc_SubscriptionDBRow *subscription, uint32_t fileid, uint32_t offset, - int flush); + uint32_t flush_mode); int Commit(int block); // Commit mempool to disk int CommitInternal(int block); - int FlushLastChunk(); + int FlushSourceChunks(uint32_t flush_mode); void Zero(); int Destroy(); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 6b3bfa77..fdccbaf3 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -3140,6 +3140,11 @@ bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, stri // Broadcast + if(mc_gState->m_WalletMode & MC_WMD_ADDRESS_TXS) + { + pwalletTxsMain->m_ChunkDB->FlushSourceChunks(GetArg("-chunkflushmode",MC_CDB_FLUSH_MODE_COMMIT)); + } + if (!wtxNew.AcceptToMemoryPoolReturnReason(false,true,reject_reason)) // MCHN { // This must not fail. The transaction has already been signed and recorded. From 1d3298fc8b150e8501bacb283144c204b7db057f Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 27 May 2018 10:45:37 +0300 Subject: [PATCH 080/157] Preallocation in chunk db data files --- src/entities/asset.h | 1 + src/wallet/chunkdb.cpp | 84 +++++++++++++++++++++++++++++++++++++----- src/wallet/chunkdb.h | 7 +++- 3 files changed, 80 insertions(+), 12 deletions(-) diff --git a/src/entities/asset.h b/src/entities/asset.h index ecb6ae29..c4d2a0e5 100644 --- a/src/entities/asset.h +++ b/src/entities/asset.h @@ -77,6 +77,7 @@ #define MC_ENT_SPRM_CHUNK_DATA 0x87 #define MC_ENT_SPRM_ITEM_COUNT 0x88 +#define MC_ENT_SPRM_FILE_END 0xFF #define MC_ENT_FLAG_OFFSET_IS_SET 0x00000001 #define MC_ENT_FLAG_NAME_IS_SET 0x00000010 diff --git a/src/wallet/chunkdb.cpp b/src/wallet/chunkdb.cpp index 64965c21..02a1554a 100644 --- a/src/wallet/chunkdb.cpp +++ b/src/wallet/chunkdb.cpp @@ -5,6 +5,7 @@ #include "wallet/chunkdb.h" #define MC_CDB_TMP_FLAG_SHOULD_COMMIT 0x00000001 +#define MC_CDB_FILE_PAGE_SIZE 0x00100000 unsigned char null_txid[MC_TDB_TXID_SIZE]; @@ -323,7 +324,7 @@ int mc_ChunkDB::RemoveEntityInternal(mc_TxEntity *entity) return MC_ERR_NOERROR; } - CommitInternal(-4); + CommitInternal(-4,0); memcpy(&subscription,old_subscription,sizeof(mc_SubscriptionDBRow)); subscription.m_Entity.m_EntityType |= MC_TET_DELETED; @@ -463,6 +464,10 @@ int mc_ChunkDB::RemoveEntityInternal(mc_TxEntity *entity) chunk_def.m_Pos=(uint32_t)mc_GetLE(buf+param_value_start,bytes); } break; + case MC_ENT_SPRM_FILE_END: + offset=buf_size; + file_offset=file_size; + break; } buf_offset=offset; } @@ -1118,7 +1123,7 @@ int mc_ChunkDB::AddChunkInternal( if( (m_ChunkData->m_Size + chunk_size + details_size + MC_CDB_MAX_CHUNK_EXTRA_SIZE > MC_CDB_MAX_CHUNK_DATA_POOL_SIZE) || (m_MemPool->GetCount() + 2 > MC_CDB_MAX_MEMPOOL_SIZE ) ) { - CommitInternal(-1); + CommitInternal(-1,0); } subscription=FindSubscription(entity); @@ -1477,6 +1482,7 @@ unsigned char *mc_ChunkDB::GetChunk(mc_ChunkDBRow *chunk_def, return ptr; } + int mc_ChunkDB::AddToFile(const void* chunk, uint32_t size, mc_SubscriptionDBRow *subscription, @@ -1486,6 +1492,12 @@ int mc_ChunkDB::AddToFile(const void* chunk, { char FileName[MC_DCT_DB_MAX_PATH]; int FileHan,err; + uint32_t tail_size,file_size,expected_end,new_file_size; + unsigned char tail[3]; + + tail[0]=0x00; + tail[1]=MC_ENT_SPRM_FILE_END; + tail[2]=0x00; SetFileName(FileName,subscription,fileid); err=MC_ERR_NOERROR; @@ -1496,6 +1508,21 @@ int mc_ChunkDB::AddToFile(const void* chunk, return MC_ERR_INTERNAL_ERROR; } + expected_end=offset+size+3; + tail_size=0; + if( (offset == 0) || ( (expected_end % MC_CDB_FILE_PAGE_SIZE) != (offset % MC_CDB_FILE_PAGE_SIZE) ) ) + { + file_size=lseek64(FileHan,0,SEEK_END); + new_file_size=((expected_end-1) / MC_CDB_FILE_PAGE_SIZE + 1) * MC_CDB_FILE_PAGE_SIZE; + if(new_file_size > file_size) + { + if(new_file_size > expected_end) + { + tail_size=new_file_size-expected_end; + } + } + } + if(lseek64(FileHan,offset,SEEK_SET) != offset) { err=MC_ERR_INTERNAL_ERROR; @@ -1507,6 +1534,32 @@ int mc_ChunkDB::AddToFile(const void* chunk, err=MC_ERR_INTERNAL_ERROR; return MC_ERR_INTERNAL_ERROR; } + + if(write(FileHan,tail,3) != 3) + { + err=MC_ERR_INTERNAL_ERROR; + return MC_ERR_INTERNAL_ERROR; + } + + if(tail_size) + { + unsigned char empty_buf[65536]; + uint32_t empty_buf_size=65536; + memset(empty_buf,0,empty_buf_size); + while(tail_size) + { + if(empty_buf_size>tail_size) + { + empty_buf_size=tail_size; + } + if(write(FileHan,empty_buf,empty_buf_size) != empty_buf_size) + { + err=MC_ERR_INTERNAL_ERROR; + return MC_ERR_INTERNAL_ERROR; + } + tail_size-=empty_buf_size; + } + } exitlbl: @@ -1521,7 +1574,7 @@ int mc_ChunkDB::AddToFile(const void* chunk, return err; } -int mc_ChunkDB::FlushDataFile(mc_SubscriptionDBRow *subscription,uint32_t fileid) +int mc_ChunkDB::FlushDataFile(mc_SubscriptionDBRow *subscription,uint32_t fileid,uint32_t flush_mode) { char FileName[MC_DCT_DB_MAX_PATH]; int FileHan; @@ -1533,7 +1586,7 @@ int mc_ChunkDB::FlushDataFile(mc_SubscriptionDBRow *subscription,uint32_t fileid { return MC_ERR_INTERNAL_ERROR; } - __US_FlushFile(FileHan); + __US_FlushFileWithMode(FileHan,flush_mode); close(FileHan); return MC_ERR_NOERROR; } @@ -1548,7 +1601,7 @@ int mc_ChunkDB::FlushSourceChunks(uint32_t flush_mode) if(flush_mode & MC_CDB_FLUSH_MODE_COMMIT) { - return Commit(-2); + return Commit(-2,flush_mode); } if( (flush_mode & MC_CDB_FLUSH_MODE_FILE) == 0) @@ -1596,7 +1649,7 @@ int mc_ChunkDB::FlushSourceChunks(uint32_t flush_mode) size=chunk_def->m_Size+chunk_def->m_HeaderSize; if(subscription->m_LastFileSize+size > MC_CDB_MAX_FILE_SIZE) // New file is needed { - FlushDataFile(subscription,subscription->m_LastFileID); + FlushDataFile(subscription,subscription->m_LastFileID,0); subscription->m_LastFileID+=1; subscription->m_LastFileSize=0; } @@ -1630,7 +1683,7 @@ int mc_ChunkDB::FlushSourceChunks(uint32_t flush_mode) return err; } -int mc_ChunkDB::CommitInternal(int block) +int mc_ChunkDB::CommitInternal(int block,uint32_t flush_mode) { int r,s; int err; @@ -1671,7 +1724,7 @@ int mc_ChunkDB::CommitInternal(int block) size=chunk_def->m_Size+chunk_def->m_HeaderSize; if(subscription->m_LastFileSize+size > MC_CDB_MAX_FILE_SIZE) // New file is needed { - FlushDataFile(subscription,subscription->m_LastFileID); + FlushDataFile(subscription,subscription->m_LastFileID,flush_mode); subscription->m_LastFileID+=1; subscription->m_LastFileSize=0; } @@ -1745,7 +1798,7 @@ int mc_ChunkDB::CommitInternal(int block) { subscription->m_TmpFlags=0; - FlushDataFile(subscription,subscription->m_LastFileID); + FlushDataFile(subscription,subscription->m_LastFileID,flush_mode); err=m_DB->Write((char*)subscription+m_KeyOffset,m_KeySize,(char*)subscription+m_ValueOffset,m_ValueSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); if(err) @@ -1795,7 +1848,18 @@ int mc_ChunkDB::Commit(int block) int err; Lock(); - err=CommitInternal(block); + err=CommitInternal(block,0); + UnLock(); + + return err; +} + +int mc_ChunkDB::Commit(int block,uint32_t flush_mode) +{ + int err; + + Lock(); + err=CommitInternal(block,flush_mode); UnLock(); return err; diff --git a/src/wallet/chunkdb.h b/src/wallet/chunkdb.h index 9b3d0535..d3ccc660 100644 --- a/src/wallet/chunkdb.h +++ b/src/wallet/chunkdb.h @@ -218,7 +218,8 @@ typedef struct mc_ChunkDB uint32_t fileid); int FlushDataFile(mc_SubscriptionDBRow *subscription, - uint32_t fileid); + uint32_t fileid, + uint32_t flush_mode); int AddToFile(const void *chunk, @@ -229,8 +230,10 @@ typedef struct mc_ChunkDB uint32_t flush_mode); int Commit(int block); // Commit mempool to disk - int CommitInternal(int block); + int Commit(int block,uint32_t flush_mode); // Commit mempool to disk + int CommitInternal(int block,uint32_t flush_mode); + int FlushSourceChunks(uint32_t flush_mode); void Zero(); From 280caa7a010831a5a2650923ac40d8d686b6aa1f Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 27 May 2018 12:51:11 +0300 Subject: [PATCH 081/157] Flushing chunks in raw transactions --- src/rpc/rpcrawtransaction.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/rpc/rpcrawtransaction.cpp b/src/rpc/rpcrawtransaction.cpp index 031664e2..c1e03415 100644 --- a/src/rpc/rpcrawtransaction.cpp +++ b/src/rpc/rpcrawtransaction.cpp @@ -1956,6 +1956,12 @@ Value sendrawtransaction(const Array& params, bool fHelp) bool fHaveMempool = mempool.exists(hashTx); bool fHaveChain = existingCoins && existingCoins->nHeight < 1000000000; bool fMissingInputs; + + if(mc_gState->m_WalletMode & MC_WMD_ADDRESS_TXS) + { + pwalletTxsMain->m_ChunkDB->FlushSourceChunks(GetArg("-chunkflushmode",MC_CDB_FLUSH_MODE_COMMIT)); + } + if (!fHaveMempool && !fHaveChain) { // push to local node and sync with wallets CValidationState state; From 9ed22e3c0a0af80a3c673cb4c311e12bd50a5b05 Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 27 May 2018 15:05:02 +0300 Subject: [PATCH 082/157] Sync options for LevelDB commit --- src/utils/dbwrapper.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/dbwrapper.cpp b/src/utils/dbwrapper.cpp index 106cff3c..46f3e847 100644 --- a/src/utils/dbwrapper.cpp +++ b/src/utils/dbwrapper.cpp @@ -624,7 +624,7 @@ int cs_Database::Commit(int Options) if(Options & MC_OPT_DB_DATABASE_TRANSACTIONAL) { - leveldb_write((leveldb_t*)m_DB, (leveldb_writeoptions_t*)m_WriteOptions, (leveldb_writebatch_t*)m_WriteBatch, &err); + leveldb_write((leveldb_t*)m_DB, (leveldb_writeoptions_t*)m_SyncOptions, (leveldb_writebatch_t*)m_WriteBatch, &err); leveldb_writebatch_clear((leveldb_writebatch_t*)m_WriteBatch); } From 8fe07d56088d19f64ee1787763fc04d24fc6be44 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 28 May 2018 11:28:36 +0300 Subject: [PATCH 083/157] Default 32 for max-std-op-returns-count --- src/chainparams/paramlist.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/chainparams/paramlist.h b/src/chainparams/paramlist.h index 8415db50..807a571e 100644 --- a/src/chainparams/paramlist.h +++ b/src/chainparams/paramlist.h @@ -226,7 +226,7 @@ static const mc_OneMultichainParam MultichainParamArray[] = "maxstdopreturnscount","", "Maximum size of standard transactions, in bytes."}, { "maxstdopreturnscount" , "max-std-op-returns-count" , - MC_PRM_UINT32 | MC_PRM_USER | MC_PRM_CLONE , -1, 10, 0, 1024, 0.0, 10006, 0, "-mc-maxstdopreturnscount", + MC_PRM_UINT32 | MC_PRM_USER | MC_PRM_CLONE , -1, 32, 0, 1024, 0.0, 10006, 0, "-mc-maxstdopreturnscount", "maxstdopreturnsize","", "Maximum number of OP_RETURN metadata outputs in standard transactions."}, { "maxstdopreturnsize" , "max-std-op-return-size" , From 2604b4a4827a5c9f8d88711ebb23fa2bdd1c00d8 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 28 May 2018 12:41:05 +0300 Subject: [PATCH 084/157] dbwrapper sync-on-commit option --- src/utils/dbwrapper.cpp | 9 ++++++++- src/utils/dbwrapper.h | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/utils/dbwrapper.cpp b/src/utils/dbwrapper.cpp index 46f3e847..f6897957 100644 --- a/src/utils/dbwrapper.cpp +++ b/src/utils/dbwrapper.cpp @@ -624,7 +624,14 @@ int cs_Database::Commit(int Options) if(Options & MC_OPT_DB_DATABASE_TRANSACTIONAL) { - leveldb_write((leveldb_t*)m_DB, (leveldb_writeoptions_t*)m_SyncOptions, (leveldb_writebatch_t*)m_WriteBatch, &err); + if(Options & MC_OPT_DB_DATABASE_SYNC_ON_COMMIT) + { + leveldb_write((leveldb_t*)m_DB, (leveldb_writeoptions_t*)m_SyncOptions, (leveldb_writebatch_t*)m_WriteBatch, &err); + } + else + { + leveldb_write((leveldb_t*)m_DB, (leveldb_writeoptions_t*)m_WriteOptions, (leveldb_writebatch_t*)m_WriteBatch, &err); + } leveldb_writebatch_clear((leveldb_writebatch_t*)m_WriteBatch); } diff --git a/src/utils/dbwrapper.h b/src/utils/dbwrapper.h index 790f1799..2c00e327 100644 --- a/src/utils/dbwrapper.h +++ b/src/utils/dbwrapper.h @@ -26,6 +26,7 @@ #define MC_OPT_DB_DATABASE_SEEK_ON_READ 0x00000010 #define MC_OPT_DB_DATABASE_DELAYED_OPEN 0x00000020 #define MC_OPT_DB_DATABASE_NEXT_ON_READ 0x00000040 +#define MC_OPT_DB_DATABASE_SYNC_ON_COMMIT 0x00000080 #define MC_OPT_DB_DATABASE_OPTION_MASK 0x000FFFFF #define MC_OPT_DB_DATABASE_LEVELDB 0x00100000 #define MC_OPT_DB_DATABASE_FSR_UTXOC_BLOCKS 0x00200000 From 9ffea56d2b39f0a48ccdec19b4a7e644e1b98340 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 28 May 2018 14:54:20 +0300 Subject: [PATCH 085/157] Source chunk DB recovery --- src/core/init.cpp | 1 + src/rpc/rpcrawtransaction.cpp | 2 +- src/rpc/rpcwallet.cpp | 3 +- src/rpc/rpcwalletutils.cpp | 1 - src/wallet/chunkdb.cpp | 234 +++++++++++++++++++++++++++++++++- src/wallet/chunkdb.h | 1 + src/wallet/wallet.cpp | 3 +- 7 files changed, 238 insertions(+), 7 deletions(-) diff --git a/src/core/init.cpp b/src/core/init.cpp index 8c5718d0..888bd7a7 100644 --- a/src/core/init.cpp +++ b/src/core/init.cpp @@ -496,6 +496,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += " -shortoutput " + _("Only show the node address (if connecting was successful) or an address in the wallet (if connect permissions must be granted by another node)") + "\n"; strUsage += " -bantx= " + _("Comma delimited list of banned transactions.") + "\n"; strUsage += " -lockblock= " + _("Blocks on branches without this block will be rejected") + "\n"; + strUsage += " -flushsourcechunks= " + _("Flush offchain items created by this node to disk immediately when created, default 1") + "\n"; strUsage += "\n" + _("MultiChain API response parameters") + "\n"; strUsage += " -hideknownopdrops= " + strprintf(_("Remove recognized MultiChain OP_DROP metadata from the responses to JSON_RPC calls (default: %u)"), 0) + "\n"; diff --git a/src/rpc/rpcrawtransaction.cpp b/src/rpc/rpcrawtransaction.cpp index c1e03415..3e352d22 100644 --- a/src/rpc/rpcrawtransaction.cpp +++ b/src/rpc/rpcrawtransaction.cpp @@ -1959,7 +1959,7 @@ Value sendrawtransaction(const Array& params, bool fHelp) if(mc_gState->m_WalletMode & MC_WMD_ADDRESS_TXS) { - pwalletTxsMain->m_ChunkDB->FlushSourceChunks(GetArg("-chunkflushmode",MC_CDB_FLUSH_MODE_COMMIT)); + pwalletTxsMain->m_ChunkDB->FlushSourceChunks(GetArg("-flushsourcechunks",true) ? (MC_CDB_FLUSH_MODE_FILE | MC_CDB_FLUSH_MODE_DATASYNC) : MC_CDB_FLUSH_MODE_NONE); } if (!fHaveMempool && !fHaveChain) { diff --git a/src/rpc/rpcwallet.cpp b/src/rpc/rpcwallet.cpp index a00e0cde..563fe76d 100644 --- a/src/rpc/rpcwallet.cpp +++ b/src/rpc/rpcwallet.cpp @@ -582,7 +582,8 @@ Value storechunk(const Array& params, bool fHelp) } } - pwalletTxsMain->m_ChunkDB->FlushSourceChunks(GetArg("-chunkflushmode",MC_CDB_FLUSH_MODE_COMMIT)); +// pwalletTxsMain->m_ChunkDB->FlushSourceChunks(GetArg("-chunkflushmode",MC_CDB_FLUSH_MODE_COMMIT)); + pwalletTxsMain->m_ChunkDB->FlushSourceChunks(GetArg("-flushsourcechunks",true) ? (MC_CDB_FLUSH_MODE_FILE | MC_CDB_FLUSH_MODE_DATASYNC) : MC_CDB_FLUSH_MODE_NONE); pwalletTxsMain->m_ChunkDB->Dump("storechunk"); return hash.GetHex(); diff --git a/src/rpc/rpcwalletutils.cpp b/src/rpc/rpcwalletutils.cpp index 73790595..77f87826 100644 --- a/src/rpc/rpcwalletutils.cpp +++ b/src/rpc/rpcwalletutils.cpp @@ -878,7 +878,6 @@ void AppendOffChainFormatData(uint32_t data_format, } } - pwalletTxsMain->m_ChunkDB->Dump("storechunk"); lpDetailsScript->SetChunkDefHash((unsigned char*)&hash,size); if(vChunkHashes) { diff --git a/src/wallet/chunkdb.cpp b/src/wallet/chunkdb.cpp index 02a1554a..3cd5a601 100644 --- a/src/wallet/chunkdb.cpp +++ b/src/wallet/chunkdb.cpp @@ -292,6 +292,219 @@ int mc_ChunkDB::AddEntity(mc_TxEntity* entity, uint32_t flags) return err; } +int mc_ChunkDB::SourceChunksRecovery() +{ + int err; + mc_SubscriptionDBRow *subscription; + mc_ChunkDBRow chunk_def; + char FileName[MC_DCT_DB_MAX_PATH]; + int FileHan; + unsigned char* buf; + int chunk_found=0; + + subscription=(mc_SubscriptionDBRow *)m_Subscriptions->GetRow(1); + + SetFileName(FileName,subscription,subscription->m_LastFileID); + FileHan=open(FileName,_O_BINARY | O_RDONLY, S_IRUSR | S_IWUSR); + if(FileHan<=0) + { + return MC_ERR_NOERROR; + } + int64_t file_size=0; + int64_t file_offset=subscription->m_LastFileSize; + int64_t buf_offset,buf_tail,buf_size,offset,read_offset; + uint32_t param_value_start; + size_t bytes; + int count=0; + buf_offset=0; + buf_tail=0; + file_size=lseek64(FileHan,0,SEEK_END); + lseek64(FileHan,file_offset,SEEK_SET); + m_TmpScript->Resize(MC_CDB_MAX_FILE_READ_BUFFER_SIZE,1); + buf=m_TmpScript->m_lpData; + chunk_def.Zero(); + chunk_def.m_SubscriptionID=subscription->m_SubscriptionID; + + while(file_offsetfile_size-file_offset) + { + buf_size=file_size-file_offset; + } + read_offset=file_offset; + if(read(FileHan,buf+buf_tail,buf_size) != buf_size) + { + err=MC_ERR_INTERNAL_ERROR; + } + if(err==MC_ERR_NOERROR) + { + file_offset+=buf_size; + buf_size+=buf_tail; + buf_tail=0; + buf_offset=0; + + while(buf_offset < buf_size) + { + offset=mc_GetParamFromDetailsScriptErr(buf,buf_size,buf_offset,¶m_value_start,&bytes,&err); + if(err) + { + buf_tail=buf_size-buf_offset; + if(param_value_startMC_CDB_MAX_FILE_READ_BUFFER_SIZE) + { + buf_tail=0; + file_offset+=param_value_start+bytes-buf_size; + if(file_offsetm_LastFileID; + chunk_def.m_InternalFileOffset=subscription->m_LastFileSize; + subscription->m_LastFileSize=read_offset+buf_offset; + chunk_def.m_HeaderSize=subscription->m_LastFileSize-chunk_def.m_InternalFileOffset-chunk_def.m_Size; + chunk_def.SwapPosBytes(); + err=m_DB->Write((char*)&chunk_def+m_KeyOffset,m_KeySize,(char*)&chunk_def+m_ValueOffset,m_ValueSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + chunk_def.SwapPosBytes(); + count++; + subscription->m_Count+=1; + subscription->m_FullSize+=chunk_def.m_Size; + m_DBStat.m_Count+=1; + m_DBStat.m_FullSize+=chunk_def.m_Size; + chunk_def.Zero(); + chunk_def.m_SubscriptionID=subscription->m_SubscriptionID; + } + if(err==MC_ERR_NOERROR) + { + chunk_found=1; + } + break; + case MC_ENT_SPRM_CHUNK_HASH: + memcpy(chunk_def.m_Hash,buf+param_value_start,bytes); + break; + case MC_ENT_SPRM_ITEM_COUNT: + if(bytes != sizeof(uint32_t)) + { + err=MC_ERR_CORRUPTED; + } + else + { + chunk_def.m_Pos=(uint32_t)mc_GetLE(buf+param_value_start,bytes); + } + break; + case MC_ENT_SPRM_CHUNK_SIZE: + if(bytes != sizeof(uint32_t)) + { + err=MC_ERR_CORRUPTED; + } + else + { + chunk_def.m_Size=(uint32_t)mc_GetLE(buf+param_value_start,bytes); + } + break; + case MC_ENT_SPRM_FILE_END: + chunk_def.m_InternalFileID=subscription->m_LastFileID; + chunk_def.m_InternalFileOffset=subscription->m_LastFileSize; + subscription->m_LastFileSize=read_offset+buf_offset; + chunk_def.m_HeaderSize=subscription->m_LastFileSize-chunk_def.m_InternalFileOffset-chunk_def.m_Size; + offset=buf_size; + read_offset=file_size; + break; + } + buf_offset=offset; + } + } + if(err==MC_ERR_NOERROR) + { + if(count >= 1000) + { + if(err == MC_ERR_NOERROR) + { + err=m_DB->Commit(MC_OPT_DB_DATABASE_TRANSACTIONAL); + count=0; + } + } + } + if(err) + { + buf_offset=buf_size; + read_offset=file_size; + } + } + } + else + { + read_offset=file_size; + } + } + + + if(FileHan>0) + { + close(FileHan); + } + + if(err == MC_ERR_NOERROR) + { + if(chunk_found) + { + if(chunk_def.m_HeaderSize == 0) + { + chunk_def.m_InternalFileID=subscription->m_LastFileID; + chunk_def.m_InternalFileOffset=subscription->m_LastFileSize; + subscription->m_LastFileSize=file_size; + chunk_def.m_HeaderSize=subscription->m_LastFileSize-chunk_def.m_InternalFileOffset-chunk_def.m_Size; + } + chunk_def.SwapPosBytes(); + err=m_DB->Write((char*)&chunk_def+m_KeyOffset,m_KeySize,(char*)&chunk_def+m_ValueOffset,m_ValueSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + chunk_def.SwapPosBytes(); + subscription->m_Count+=1; + subscription->m_FullSize+=chunk_def.m_Size; + m_DBStat.m_Count+=1; + m_DBStat.m_FullSize+=chunk_def.m_Size; + count++; + if(err == MC_ERR_NOERROR) + { + err=m_DB->Write((char*)subscription+m_KeyOffset,m_KeySize,(char*)subscription+m_ValueOffset,m_ValueSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + } + if(err == MC_ERR_NOERROR) + { + err=m_DB->Write((char*)&m_DBStat+m_KeyOffset,m_KeySize,(char*)&m_DBStat+m_ValueOffset,m_ValueSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + } + if(err == MC_ERR_NOERROR) + { + err=m_DB->Commit(MC_OPT_DB_DATABASE_TRANSACTIONAL | MC_OPT_DB_DATABASE_SYNC_ON_COMMIT); + } + } + } + + Dump("SourceChunkRecovery"); + return err; +} + int mc_ChunkDB::RemoveEntityInternal(mc_TxEntity *entity) { int err; @@ -578,7 +791,7 @@ int mc_ChunkDB::FindSubscription(const mc_TxEntity *entity,mc_SubscriptionDBRow int mc_ChunkDB::Initialize(const char *name,uint32_t mode) { - int err,value_len; + int err,value_len,new_db; char msg[256]; mc_SubscriptionDBRow subscription; @@ -624,8 +837,10 @@ int mc_ChunkDB::Initialize(const char *name,uint32_t mode) return err; } + new_db=1; if(ptr) { + new_db=0; memcpy((char*)&m_DBStat+m_ValueOffset,ptr,m_ValueSize); m_Subscriptions->SetCount(m_DBStat.m_LastSubscription+1); @@ -729,6 +944,11 @@ int mc_ChunkDB::Initialize(const char *name,uint32_t mode) sprintf(msg, "Initialized. Chunks: %d",m_DBStat.m_Count); LogString(msg); + if(new_db == 0) + { + SourceChunksRecovery(); + } + return err; } @@ -1596,9 +1816,10 @@ int mc_ChunkDB::FlushSourceChunks(uint32_t flush_mode) mc_ChunkDBRow *chunk_def; mc_SubscriptionDBRow *subscription; int err,row,first_row,last_row; + int full_commit_required; uint32_t size; char msg[256]; - + if(flush_mode & MC_CDB_FLUSH_MODE_COMMIT) { return Commit(-2,flush_mode); @@ -1609,6 +1830,7 @@ int mc_ChunkDB::FlushSourceChunks(uint32_t flush_mode) return MC_ERR_NOERROR; } + full_commit_required=0; Lock(); @@ -1652,6 +1874,7 @@ int mc_ChunkDB::FlushSourceChunks(uint32_t flush_mode) FlushDataFile(subscription,subscription->m_LastFileID,0); subscription->m_LastFileID+=1; subscription->m_LastFileSize=0; + full_commit_required=1; } err=AddToFile(GetChunkInternal(chunk_def,-1,-1,NULL),size, @@ -1680,6 +1903,11 @@ int mc_ChunkDB::FlushSourceChunks(uint32_t flush_mode) exitlbl: UnLock(); + if(full_commit_required) + { + return Commit(-2,flush_mode); + } + return err; } @@ -1815,7 +2043,7 @@ int mc_ChunkDB::CommitInternal(int block,uint32_t flush_mode) goto exitlbl; } - err=m_DB->Commit(MC_OPT_DB_DATABASE_TRANSACTIONAL); + err=m_DB->Commit(MC_OPT_DB_DATABASE_TRANSACTIONAL | MC_OPT_DB_DATABASE_SYNC_ON_COMMIT); if(err) { goto exitlbl; diff --git a/src/wallet/chunkdb.h b/src/wallet/chunkdb.h index d3ccc660..aac9a74b 100644 --- a/src/wallet/chunkdb.h +++ b/src/wallet/chunkdb.h @@ -161,6 +161,7 @@ typedef struct mc_ChunkDB int AddEntityInternal(mc_TxEntity *entity,uint32_t flags); int RemoveEntity(mc_TxEntity *entity); int RemoveEntityInternal(mc_TxEntity *entity); + int SourceChunksRecovery(); mc_SubscriptionDBRow *FindSubscription(const mc_TxEntity *entity); // Finds subscription int FindSubscription(const mc_TxEntity *entity,mc_SubscriptionDBRow *subscription); // Finds subscription diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index fdccbaf3..e63473f4 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -3142,7 +3142,8 @@ bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, stri if(mc_gState->m_WalletMode & MC_WMD_ADDRESS_TXS) { - pwalletTxsMain->m_ChunkDB->FlushSourceChunks(GetArg("-chunkflushmode",MC_CDB_FLUSH_MODE_COMMIT)); +// pwalletTxsMain->m_ChunkDB->FlushSourceChunks(GetArg("-chunkflushmode",MC_CDB_FLUSH_MODE_COMMIT)); + pwalletTxsMain->m_ChunkDB->FlushSourceChunks(GetArg("-flushsourcechunks",true) ? (MC_CDB_FLUSH_MODE_FILE | MC_CDB_FLUSH_MODE_DATASYNC) : MC_CDB_FLUSH_MODE_NONE); } if (!wtxNew.AcceptToMemoryPoolReturnReason(false,true,reject_reason)) // MCHN From 19b20215313e6fbb49706b5c951714c86fc04b98 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 28 May 2018 16:29:31 +0300 Subject: [PATCH 086/157] Counting pending chunks --- src/protocol/relay.cpp | 4 ++++ src/rpc/rpcchunks.cpp | 2 +- src/wallet/chunkcollector.cpp | 17 +++++++++++++++-- src/wallet/chunkcollector.h | 6 ++++-- 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index 7194acd9..9244b58d 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -691,6 +691,10 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) } } } + else + { + for(int k=0;k<2;k++)collector->m_StatLast[k].m_Delivered+=k ? size : 1; + } } collector->UnLock(); diff --git a/src/rpc/rpcchunks.cpp b/src/rpc/rpcchunks.cpp index 7711a6e6..2c098f56 100644 --- a/src/rpc/rpcchunks.cpp +++ b/src/rpc/rpcchunks.cpp @@ -18,7 +18,7 @@ Value getchunkqueueinfo(const Array& params, bool fHelp) for(int k=0;k<2;k++) { Object stat; - stat.push_back(Pair("pending",collector->m_StatLast[k].m_Sleeping+collector->m_StatLast[k].m_Queried)); + stat.push_back(Pair("pending",(k ? collector->m_TotalChunkSize : collector->m_TotalChunkCount) - collector->m_StatLast[k].m_Delivered)); stat.push_back(Pair("queried",collector->m_StatLast[k].m_Queried)); stat.push_back(Pair("requested",collector->m_StatLast[k].m_Requested)); result.push_back(Pair(k ? "bytes" : "chunks",stat)); diff --git a/src/wallet/chunkcollector.cpp b/src/wallet/chunkcollector.cpp index dd0986e9..71400d96 100644 --- a/src/wallet/chunkcollector.cpp +++ b/src/wallet/chunkcollector.cpp @@ -62,6 +62,8 @@ void mc_ChunkCollector::Zero() m_TimeoutRequest=MC_CCW_TIMEOUT_REQUEST_SHIFT+1; } m_TimeoutQuery=(int)GetArg("-chunkquerytimeout",MC_CCW_TIMEOUT_QUERY); + m_TotalChunkCount=0; + m_TotalChunkSize=0; m_StatLast[0].Zero(); m_StatLast[1].Zero(); @@ -229,7 +231,7 @@ int mc_ChunkCollector::ReadFromDB(mc_Buffer *mempool,int rows) err=MC_ERR_NOERROR; ptr=NULL; - row=0; + row=mempool->GetCount(); while(rowMoveNext(&err); @@ -375,6 +377,8 @@ int mc_ChunkCollector::Initialize(mc_ChunkDB *chunk_db,const char *name,uint32_t } if(err != MC_ERR_NOT_FOUND) { + m_TotalChunkCount=m_DBRow.m_TotalChunkCount; + m_TotalChunkSize=m_DBRow.m_TotalChunkSize; err=ReadFromDB(m_MemPool,m_MaxMemPoolSize); if(err) { @@ -439,6 +443,8 @@ int mc_ChunkCollector::InsertChunkInternal( if(mprow<0) { m_MemPool->Add(&collect_row); + m_TotalChunkCount++; + m_TotalChunkSize+=chunk_size; } return MC_ERR_NOERROR; @@ -596,10 +602,12 @@ int mc_ChunkCollector::CommitInternal() if(row->m_State.m_Status & MC_CCF_DELETED) { + m_TotalChunkCount--; + m_TotalChunkSize-=row->m_ChunkDef.m_Size; if(row->m_State.m_Status & MC_CCF_INSERTED) { commit_required=1; - DeleteDBRow(row); + DeleteDBRow(row); // m_DB->Delete((char*)row+m_KeyOffset,m_KeySize,MC_OPT_DB_DATABASE_TRANSACTIONAL); } } @@ -636,6 +644,11 @@ int mc_ChunkCollector::CommitInternal() if(commit_required) { + m_DBRow.Zero(); + m_DBRow.m_TotalChunkSize=m_TotalChunkSize; + m_DBRow.m_TotalChunkCount=m_TotalChunkCount; + m_DB->Write((char*)&m_DBRow+m_KeyDBOffset,m_KeyDBSize,(char*)&m_DBRow+m_ValueDBOffset,m_ValueDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + err=m_DB->Commit(MC_OPT_DB_DATABASE_TRANSACTIONAL); if(err) { diff --git a/src/wallet/chunkcollector.h b/src/wallet/chunkcollector.h index 5b8b1971..8bd26982 100644 --- a/src/wallet/chunkcollector.h +++ b/src/wallet/chunkcollector.h @@ -67,8 +67,8 @@ typedef struct mc_ChunkCollectorDBRow uint32_t m_Flags; uint32_t m_QueryAttempts; uint32_t m_Status; - int64_t m_Reserved1; - int64_t m_Reserved2; + int64_t m_TotalChunkSize; + int64_t m_TotalChunkCount; void Zero(); } mc_ChunkCollectorDBRow; @@ -120,6 +120,8 @@ typedef struct mc_ChunkCollector int m_MaxMemPoolSize; int m_TimeoutRequest; int m_TimeoutQuery; + int64_t m_TotalChunkSize; + int64_t m_TotalChunkCount; mc_ChunkCollectorStat m_StatLast[2]; mc_ChunkCollectorStat m_StatTotal[2]; From a95f323fd8c12ee6aa4460982969ba6eb117ec1b Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 28 May 2018 17:12:07 +0300 Subject: [PATCH 087/157] Fixed pending calcualtion in getchunkqueueinfo --- src/protocol/relay.cpp | 6 ++++-- src/rpc/rpcchunks.cpp | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index 9244b58d..cc0edafd 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -668,12 +668,14 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) } for(int k=0;k<2;k++)collector->m_StatLast[k].Zero(); + collector->m_StatLast[0].m_Pending=collector->m_TotalChunkCount; + collector->m_StatLast[1].m_Pending=collector->m_TotalChunkSize; for(row=0;rowm_MemPool->GetCount();row++) { collect_row=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(row); + size=collect_row->m_ChunkDef.m_Size; if( (collect_row->m_State.m_Status & MC_CCF_DELETED ) == 0 ) { - size=collect_row->m_ChunkDef.m_Size; if(!collect_row->m_State.m_Request.IsZero()) { for(int k=0;k<2;k++)collector->m_StatLast[k].m_Requested+=k ? size : 1; @@ -693,7 +695,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) } else { - for(int k=0;k<2;k++)collector->m_StatLast[k].m_Delivered+=k ? size : 1; + for(int k=0;k<2;k++)collector->m_StatLast[k].m_Pending-=k ? size : 1; } } diff --git a/src/rpc/rpcchunks.cpp b/src/rpc/rpcchunks.cpp index 2c098f56..c6ce2c45 100644 --- a/src/rpc/rpcchunks.cpp +++ b/src/rpc/rpcchunks.cpp @@ -18,7 +18,7 @@ Value getchunkqueueinfo(const Array& params, bool fHelp) for(int k=0;k<2;k++) { Object stat; - stat.push_back(Pair("pending",(k ? collector->m_TotalChunkSize : collector->m_TotalChunkCount) - collector->m_StatLast[k].m_Delivered)); + stat.push_back(Pair("pending",collector->m_StatLast[k].m_Pending)); stat.push_back(Pair("queried",collector->m_StatLast[k].m_Queried)); stat.push_back(Pair("requested",collector->m_StatLast[k].m_Requested)); result.push_back(Pair(k ? "bytes" : "chunks",stat)); From 35af8a2da6bcb8fb304b6b3695a1e99eed37cfcb Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 29 May 2018 17:33:28 +0300 Subject: [PATCH 088/157] Mempool limit for chunk delivery --- src/core/main.cpp | 15 ++++--- src/protocol/relay.cpp | 84 +++++++++++++++++++++++++++++++---- src/protocol/relay.h | 1 + src/rpc/rpcchunks.cpp | 1 + src/wallet/chunkcollector.cpp | 17 ++++++- src/wallet/chunkcollector.h | 4 +- 6 files changed, 104 insertions(+), 18 deletions(-) diff --git a/src/core/main.cpp b/src/core/main.cpp index 1d027c3b..b3f2f2a5 100644 --- a/src/core/main.cpp +++ b/src/core/main.cpp @@ -7312,18 +7312,19 @@ bool SendMessages(CNode* pto, bool fSendTrickle) } pto->fLastIgnoreIncoming=ignore_incoming; - if(MultichainNode_CollectChunks()) + if(pwalletTxsMain->m_ChunkCollector) { - if(pwalletTxsMain->m_ChunkCollector) - { - int64_t time_millis_now=GetTimeMillis(); + int64_t time_millis_now=GetTimeMillis(); - if(pwalletTxsMain->m_ChunkCollector->m_NextTryTimestamp < time_millis_now) + if(pwalletTxsMain->m_ChunkCollector->m_NextTryTimestamp < time_millis_now) + { + if(MultichainNode_CollectChunks()) { MultichainCollectChunks(pwalletTxsMain->m_ChunkCollector); + } + MultichainCollectChunksQueueStats(pwalletTxsMain->m_ChunkCollector); // if(fDebug)LogPrint("chunks", "Chunks to collect: %d\n", still_to_collect); - pwalletTxsMain->m_ChunkCollector->m_NextTryTimestamp=time_millis_now+100; - } + pwalletTxsMain->m_ChunkCollector->m_NextTryTimestamp=time_millis_now+100; } } /* MCHN END */ diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index cc0edafd..1620a28a 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -17,6 +17,10 @@ string mc_MsgTypeStr(uint32_t msg_type) { char *ptr; ptr=(char*)&msg_type; + if(msg_type < 0x01000000) + { + return strprintf("%08x",msg_type); + } return strprintf("%c%c%c%c",ptr[0],ptr[1],ptr[2],ptr[3]); } @@ -267,7 +271,6 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) vector payload; unsigned char buf[16]; int shift,count; - uint32_t size; unsigned char *ptrOut; mc_OffchainMessageID query_id,request_id; map query_to_delete; @@ -282,6 +285,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) CRelayRequestPairs request_pairs; int best_score,best_response,this_score,not_processed; + pRelayManager->CheckTime(); pRelayManager->InvalidateResponsesFromDisconnected(); collector->Lock(); @@ -667,6 +671,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) } } +/* for(int k=0;k<2;k++)collector->m_StatLast[k].Zero(); collector->m_StatLast[0].m_Pending=collector->m_TotalChunkCount; collector->m_StatLast[1].m_Pending=collector->m_TotalChunkSize; @@ -698,7 +703,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) for(int k=0;k<2;k++)collector->m_StatLast[k].m_Pending-=k ? size : 1; } } - + */ collector->UnLock(); if(not_processed < collector->m_MaxMemPoolSize/2) { @@ -713,6 +718,57 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) return not_processed; } +void MultichainCollectChunksQueueStats(mc_ChunkCollector* collector) +{ + int row; + mc_ChunkCollectorRow *collect_row; + uint32_t size; + + collector->Lock(); + + for(int k=0;k<2;k++)collector->m_StatLast[k].Zero(); + collector->m_StatLast[0].m_Pending=collector->m_TotalChunkCount; + collector->m_StatLast[1].m_Pending=collector->m_TotalChunkSize; + for(row=0;rowm_MemPool->GetCount();row++) + { + collect_row=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(row); + size=collect_row->m_ChunkDef.m_Size; + for(int k=0;k<2;k++)collector->m_StatLast[k].m_Undelivered+=k ? size : 1; + if( (collect_row->m_State.m_Status & MC_CCF_DELETED ) == 0 ) + { + if(!collect_row->m_State.m_Request.IsZero()) + { + for(int k=0;k<2;k++)collector->m_StatLast[k].m_Requested+=k ? size : 1; + for(int k=0;k<2;k++)collector->m_StatLast[k].m_Queried+=k ? size : 1; + } + else + { + if(!collect_row->m_State.m_Query.IsZero()) + { + for(int k=0;k<2;k++)collector->m_StatLast[k].m_Queried+=k ? size : 1; + } + else + { + for(int k=0;k<2;k++)collector->m_StatLast[k].m_Sleeping+=k ? size : 1; + } + } + } + else + { + for(int k=0;k<2;k++)collector->m_StatLast[k].m_Pending-=k ? size : 1; + } + } + + collector->UnLock(); + if(collector->m_MemPool->GetCount() > (int)(collector->m_MaxMemPoolSize*1.2)) + { + if(collector->m_NextAutoCommitTimestamp < GetTimeMillis()) + { + collector->Commit(); + collector->m_NextAutoCommitTimestamp=GetTimeMillis()+collector->m_AutoCommitDelay; + } + } +} void mc_RelayPayload_ChunkIDs(vector* payload,vector & vChunkDefs,int size) { @@ -1462,11 +1518,17 @@ void mc_RelayManager::CheckTime() { return; } - + + Lock(); for(map::iterator it = m_RelayRecords.begin(); it != m_RelayRecords.end();) { if(it->second.m_Timestamp < m_LastTime) { +/* + mc_OffchainMessageID msg_id=it->first.m_ID; + LogPrint("offchain","Offchain rrdl: %s, from: %d, to: %d, msg: %s, now: %d, exp: %d\n", + msg_id.ToString().c_str(),it->second.m_NodeFrom,it->first.m_NodeTo,mc_MsgTypeStr(it->second.m_MsgType).c_str(),m_LastTime,it->second.m_Timestamp); +*/ m_RelayRecords.erase(it++); } else @@ -1475,6 +1537,7 @@ void mc_RelayManager::CheckTime() } } m_LastTime=time_now; + UnLock(); } void mc_RelayManager::SetRelayRecord(CNode *pto,CNode *pfrom,uint32_t msg_type,mc_OffchainMessageID msg_id) @@ -1515,8 +1578,12 @@ void mc_RelayManager::SetRelayRecord(CNode *pto,CNode *pfrom,uint32_t msg_type,m value.m_Timestamp=(it->second).m_Timestamp; value.m_Count=(it->second).m_Count; it->second=value; - } -// printf("setrr: %d, ts: %u, nc: %u, mt: %d\n",pto_id,value.m_Timestamp,nonce,msg_type); + } +/* + LogPrint("offchain","Offchain rrst: %s, from: %d, to: %d, msg: %s, now: %d, exp: %d\n", + msg_id.ToString().c_str(),pfrom ? pfrom->GetId() : 0,pto ? pto->GetId() : 0,mc_MsgTypeStr(msg_type).c_str(),m_LastTime,value.m_Timestamp); +*/ + } int mc_RelayManager::GetRelayRecord(CNode *pfrom,mc_OffchainMessageID msg_id,uint32_t* msg_type,CNode **pto) @@ -1710,6 +1777,7 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, { uint32_t msg_type_in; uint32_t verify_flags; + int err; mc_OffchainMessageID msg_id_received; mc_OffchainMessageID msg_id_to_respond; vector vPayloadIn; @@ -1881,10 +1949,10 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, return state.DoS(100, error("ProcessOffchain() : This message should be response"),REJECT_INVALID, "bad-nonce"); } - if(GetRelayRecord(pfrom,msg_id_to_respond,&msg_type_stored,&pto_stored)) + if((err=GetRelayRecord(pfrom,msg_id_to_respond,&msg_type_stored,&pto_stored))) { - LogPrintf("ProcessOffchain() : Orphan response: %s, request: %s, from: %d, to: %d, msg: %s, hops: %d, size: %d\n", - msg_id_received.ToString().c_str(),msg_id_to_respond.ToString().c_str(),pfrom->GetId(),pto_stored ? pto_stored->GetId() : 0,mc_MsgTypeStr(msg_type_in).c_str(),hop_count,(int)vPayloadIn.size()); + LogPrintf("ProcessOffchain() : Orphan response: %s, request: %s, from: %d, to: %d, msg: %s, hops: %d, size: %d, error: %d\n", + msg_id_received.ToString().c_str(),msg_id_to_respond.ToString().c_str(),pfrom->GetId(),pto_stored ? pto_stored->GetId() : 0,mc_MsgTypeStr(msg_type_in).c_str(),hop_count,(int)vPayloadIn.size(),err); return false; } } diff --git a/src/protocol/relay.h b/src/protocol/relay.h index 4df4daab..3f0ad097 100644 --- a/src/protocol/relay.h +++ b/src/protocol/relay.h @@ -81,6 +81,7 @@ bool MultichainRelayResponse(uint32_t msg_type_stored, CNode *pto_stored, string& strError); int MultichainCollectChunks(mc_ChunkCollector* collector); +void MultichainCollectChunksQueueStats(mc_ChunkCollector* collector); diff --git a/src/rpc/rpcchunks.cpp b/src/rpc/rpcchunks.cpp index c6ce2c45..0b6aa28a 100644 --- a/src/rpc/rpcchunks.cpp +++ b/src/rpc/rpcchunks.cpp @@ -19,6 +19,7 @@ Value getchunkqueueinfo(const Array& params, bool fHelp) { Object stat; stat.push_back(Pair("pending",collector->m_StatLast[k].m_Pending)); + stat.push_back(Pair("processing",collector->m_StatLast[k].m_Undelivered)); stat.push_back(Pair("queried",collector->m_StatLast[k].m_Queried)); stat.push_back(Pair("requested",collector->m_StatLast[k].m_Requested)); result.push_back(Pair(k ? "bytes" : "chunks",stat)); diff --git a/src/wallet/chunkcollector.cpp b/src/wallet/chunkcollector.cpp index 71400d96..b6653c4e 100644 --- a/src/wallet/chunkcollector.cpp +++ b/src/wallet/chunkcollector.cpp @@ -223,6 +223,8 @@ int mc_ChunkCollector::ReadFromDB(mc_Buffer *mempool,int rows) mc_ChunkDBRow chunk_def; unsigned char *ptr; int row; + int64_t total_size=0; + if(rows <= 0) { return MC_ERR_NOERROR; @@ -251,6 +253,7 @@ int mc_ChunkCollector::ReadFromDB(mc_Buffer *mempool,int rows) mprow=mempool->Seek(&collect_row); if(mprow < 0) { + total_size+=collect_row.m_ChunkDef.m_Size; mempool->Add(&collect_row); row++; } @@ -261,6 +264,15 @@ int mc_ChunkCollector::ReadFromDB(mc_Buffer *mempool,int rows) } } + if(mempool->GetCount() < rows) + { + if(mempool->GetCount() <= m_TotalChunkCount) + { + m_TotalChunkCount=mempool->GetCount(); + m_TotalChunkSize=total_size; + } + } + memcpy(&m_LastDBRow,&m_DBRow,m_TotalDBSize); return MC_ERR_NOERROR; @@ -637,7 +649,10 @@ int mc_ChunkCollector::CommitInternal() */ if(!row->m_State.m_Query.IsZero() || (row->m_State.m_QueryNextAttempt <= time_now)) { - m_MemPoolNext->Add(row); + if(m_MemPoolNext->GetCount() < m_MaxMemPoolSize) + { + m_MemPoolNext->Add(row); + } } } } diff --git a/src/wallet/chunkcollector.h b/src/wallet/chunkcollector.h index 8bd26982..5ad154c3 100644 --- a/src/wallet/chunkcollector.h +++ b/src/wallet/chunkcollector.h @@ -19,13 +19,13 @@ #define MC_CCF_ERROR_MASK 0x00FF0000 #define MC_CCF_ALL 0xFFFFFFFF -#define MC_CCW_TIMEOUT_QUERY 35 +#define MC_CCW_TIMEOUT_QUERY 25 #define MC_CCW_TIMEOUT_REQUEST 10 #define MC_CCW_TIMEOUT_REQUEST_SHIFT 2 #define MC_CCW_MAX_CHUNKS_PER_QUERY 1000 #define MC_CCW_DEFAULT_AUTOCOMMIT_DELAY 200 #define MC_CCW_WORST_RESPONSE_SCORE 1048576000 -#define MC_CCW_DEFAULT_MEMPOOL_SIZE 100000 +#define MC_CCW_DEFAULT_MEMPOOL_SIZE 60000 #define MC_CCW_MAX_MBS_PER_SECOND 8 From 7c2c8be928404feb76cb13efd337ba3a7f088a89 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 29 May 2018 19:01:04 +0300 Subject: [PATCH 089/157] Fixed removing/adding queries to delivery mempool --- src/protocol/relay.cpp | 9 +++++++-- src/wallet/chunkcollector.cpp | 6 ++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index 1620a28a..936b098c 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -264,7 +264,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) uint32_t time_now,expiration,dest_expiration; vector vChunkDefs; int row,last_row,last_count,to_end_of_query; - uint32_t total_size,max_total_size,total_in_queries,max_total_in_queries; + uint32_t total_size,max_total_size,total_in_queries,max_total_in_queries,query_count; mc_ChunkCollectorRow *collect_row; mc_ChunkCollectorRow *collect_subrow; time_now=mc_TimeNowAsUInt(); @@ -353,6 +353,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) max_total_in_queries=2*(collector->m_TimeoutRequest)*MC_CCW_MAX_MBS_PER_SECOND*1024*1024; total_in_queries=0; + query_count=0; for(row=0;rowm_MemPool->GetCount();row++) { @@ -629,7 +630,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) collect_row->m_State.m_Status |= MC_CCF_UPDATED; for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Unresponded+=k ? collect_row->m_ChunkDef.m_Size : 1; } - if((collect_row->m_State.m_QueryNextAttempt <= time_now) && (total_in_queries < max_total_in_queries)) + if((collect_row->m_State.m_QueryNextAttempt <= time_now) && (total_in_queries < max_total_in_queries) && (query_countm_MaxMemPoolSize)) { if( (collect_row->m_State.m_Status & MC_CCF_ERROR_MASK) == 0) { @@ -639,6 +640,10 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) } } } + else + { + query_count++; + } } } row++; diff --git a/src/wallet/chunkcollector.cpp b/src/wallet/chunkcollector.cpp index b6653c4e..41a353e4 100644 --- a/src/wallet/chunkcollector.cpp +++ b/src/wallet/chunkcollector.cpp @@ -647,12 +647,10 @@ int mc_ChunkCollector::CommitInternal() row->m_State.m_Status |= MC_CCF_INSERTED; } */ - if(!row->m_State.m_Query.IsZero() || (row->m_State.m_QueryNextAttempt <= time_now)) + if(!row->m_State.m_Query.IsZero() || + ((row->m_State.m_QueryNextAttempt <= time_now) && (m_MemPoolNext->GetCount() < m_MaxMemPoolSize)) ) { - if(m_MemPoolNext->GetCount() < m_MaxMemPoolSize) - { m_MemPoolNext->Add(row); - } } } } From 4bc1cc5111b43f9cb65dd94f1185cbf916ee4a62 Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 30 May 2018 10:14:20 +0300 Subject: [PATCH 090/157] Fixed total chunk size calculation --- src/wallet/chunkcollector.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/wallet/chunkcollector.cpp b/src/wallet/chunkcollector.cpp index 41a353e4..1c12d21e 100644 --- a/src/wallet/chunkcollector.cpp +++ b/src/wallet/chunkcollector.cpp @@ -223,7 +223,6 @@ int mc_ChunkCollector::ReadFromDB(mc_Buffer *mempool,int rows) mc_ChunkDBRow chunk_def; unsigned char *ptr; int row; - int64_t total_size=0; if(rows <= 0) { @@ -253,7 +252,6 @@ int mc_ChunkCollector::ReadFromDB(mc_Buffer *mempool,int rows) mprow=mempool->Seek(&collect_row); if(mprow < 0) { - total_size+=collect_row.m_ChunkDef.m_Size; mempool->Add(&collect_row); row++; } @@ -269,7 +267,11 @@ int mc_ChunkCollector::ReadFromDB(mc_Buffer *mempool,int rows) if(mempool->GetCount() <= m_TotalChunkCount) { m_TotalChunkCount=mempool->GetCount(); - m_TotalChunkSize=total_size; + m_TotalChunkSize=0; + for(i=0;iGetCount();i++) + { + m_TotalChunkSize+=((mc_ChunkCollectorRow *)m_MemPool->GetRow(i))->m_ChunkDef.m_Size; + } } } From 7f09d944cd37c8a2fe1dbc1c729fd610a438b985 Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 30 May 2018 10:18:26 +0300 Subject: [PATCH 091/157] Fixed total chunk size calculation --- src/wallet/chunkcollector.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/wallet/chunkcollector.cpp b/src/wallet/chunkcollector.cpp index 1c12d21e..d14839e1 100644 --- a/src/wallet/chunkcollector.cpp +++ b/src/wallet/chunkcollector.cpp @@ -268,9 +268,9 @@ int mc_ChunkCollector::ReadFromDB(mc_Buffer *mempool,int rows) { m_TotalChunkCount=mempool->GetCount(); m_TotalChunkSize=0; - for(i=0;iGetCount();i++) + for(row=0;rowGetCount();row++) { - m_TotalChunkSize+=((mc_ChunkCollectorRow *)m_MemPool->GetRow(i))->m_ChunkDef.m_Size; + m_TotalChunkSize+=((mc_ChunkCollectorRow *)m_MemPool->GetRow(row))->m_ChunkDef.m_Size; } } } From 9667a0300cbeb8061a90faa72702398bc493f26d Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 30 May 2018 16:12:33 +0300 Subject: [PATCH 092/157] Chunk timeout help --- src/core/init.cpp | 2 ++ src/core/main.cpp | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/core/init.cpp b/src/core/init.cpp index 888bd7a7..405a5d17 100644 --- a/src/core/init.cpp +++ b/src/core/init.cpp @@ -496,6 +496,8 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += " -shortoutput " + _("Only show the node address (if connecting was successful) or an address in the wallet (if connect permissions must be granted by another node)") + "\n"; strUsage += " -bantx= " + _("Comma delimited list of banned transactions.") + "\n"; strUsage += " -lockblock= " + _("Blocks on branches without this block will be rejected") + "\n"; + strUsage += " -chunkquerytimeout= " + _("Timeout, after which undelivered chunk is moved to the end of the chunk queue, default 25s") + "\n"; + strUsage += " -chunkrequesttimeout= " + _("Timeout, after which chunk request is dropped and another source is tried, default 10s") + "\n"; strUsage += " -flushsourcechunks= " + _("Flush offchain items created by this node to disk immediately when created, default 1") + "\n"; strUsage += "\n" + _("MultiChain API response parameters") + "\n"; diff --git a/src/core/main.cpp b/src/core/main.cpp index b3f2f2a5..035bb443 100644 --- a/src/core/main.cpp +++ b/src/core/main.cpp @@ -7324,7 +7324,7 @@ bool SendMessages(CNode* pto, bool fSendTrickle) } MultichainCollectChunksQueueStats(pwalletTxsMain->m_ChunkCollector); // if(fDebug)LogPrint("chunks", "Chunks to collect: %d\n", still_to_collect); - pwalletTxsMain->m_ChunkCollector->m_NextTryTimestamp=time_millis_now+100; + pwalletTxsMain->m_ChunkCollector->m_NextTryTimestamp=time_millis_now+GetArg("-offchainrequestfreq",100); } } /* MCHN END */ From 2865f69694be0409fba7705328db20de2cab76cc Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 31 May 2018 12:31:51 +0300 Subject: [PATCH 093/157] Interval between chunk collections --- src/core/main.cpp | 2 +- src/wallet/chunkcollector.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/main.cpp b/src/core/main.cpp index 035bb443..32107e4e 100644 --- a/src/core/main.cpp +++ b/src/core/main.cpp @@ -7324,7 +7324,7 @@ bool SendMessages(CNode* pto, bool fSendTrickle) } MultichainCollectChunksQueueStats(pwalletTxsMain->m_ChunkCollector); // if(fDebug)LogPrint("chunks", "Chunks to collect: %d\n", still_to_collect); - pwalletTxsMain->m_ChunkCollector->m_NextTryTimestamp=time_millis_now+GetArg("-offchainrequestfreq",100); + pwalletTxsMain->m_ChunkCollector->m_NextTryTimestamp=time_millis_now+GetArg("-offchainrequestfreq",MC_CCW_TIMEOUT_BETWEEN_COLLECTS_MILLIS); } } /* MCHN END */ diff --git a/src/wallet/chunkcollector.h b/src/wallet/chunkcollector.h index 5ad154c3..e9f539f2 100644 --- a/src/wallet/chunkcollector.h +++ b/src/wallet/chunkcollector.h @@ -27,6 +27,7 @@ #define MC_CCW_WORST_RESPONSE_SCORE 1048576000 #define MC_CCW_DEFAULT_MEMPOOL_SIZE 60000 #define MC_CCW_MAX_MBS_PER_SECOND 8 +#define MC_CCW_TIMEOUT_BETWEEN_COLLECTS_MILLIS 100 typedef struct mc_ChunkEntityKey From 23241606c2178ccb63c0a0d67145e4737c4f6571 Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 31 May 2018 12:32:23 +0300 Subject: [PATCH 094/157] ignoremissing in getstreamkey/publishsummary --- src/rpc/rpchelp.cpp | 6 ++- src/rpc/rpcstreams.cpp | 100 ++++++++++++++++++++++++++++++++++------- src/rpc/rpcutils.cpp | 4 +- src/rpc/rpcutils.h | 3 +- 4 files changed, 92 insertions(+), 21 deletions(-) diff --git a/src/rpc/rpchelp.cpp b/src/rpc/rpchelp.cpp index 25f10d7b..1edbbdc0 100644 --- a/src/rpc/rpchelp.cpp +++ b/src/rpc/rpchelp.cpp @@ -3964,7 +3964,8 @@ void mc_InitRPCHelpMap17() " recursive - merge json sub-objects recursively\n" " noupdate - preserve first value for each key instead of taking the last\n" " omitnull - omit keys with null values\n" - " ignore - ignore items that cannot be included in summary (otherwise returns an error)\n" + " ignoreother - ignore items that cannot be included in summary (otherwise returns an error)\n" + " ignoremissing - ignore missing offchain items (otherwise returns an error)\n" " firstpublishersany - only summarize items by a publisher of first item with this key\n" " firstpublishersall - only summarize items by all publishers of first item with this key\n" "\nResult:\n" @@ -3987,7 +3988,8 @@ void mc_InitRPCHelpMap17() " recursive - merge json sub-objects recursively\n" " noupdate - preserve first value for each key instead of taking the last\n" " omitnull - omit keys with null values\n" - " ignore - ignore items that cannot be included in summary (otherwise returns an error)\n" + " ignoreother - ignore items that cannot be included in summary (otherwise returns an error)\n" + " ignoremissing - ignore missing offchain items (otherwise returns an error)\n" "\nResult:\n" "summary-object (object) Summary object for specific publisher.\n" "\nExamples:\n" diff --git a/src/rpc/rpcstreams.cpp b/src/rpc/rpcstreams.cpp index d6177565..a11bcb99 100644 --- a/src/rpc/rpcstreams.cpp +++ b/src/rpc/rpcstreams.cpp @@ -1350,9 +1350,14 @@ Value getstreamsummary(const Array& params, bool fPublisher) mode |= MC_VMM_RECURSIVE; found=true; } - if(inputStrings[j]=="ignore") + if( (inputStrings[j]=="ignore") || (inputStrings[j]=="ignoreother") ) { - mode |= MC_VMM_IGNORE; + mode |= MC_VMM_IGNORE_OTHER; + found=true; + } + if(inputStrings[j]=="ignoremissing") + { + mode |= MC_VMM_IGNORE_MISSING; found=true; } if(inputStrings[j]=="noupdate") @@ -1406,6 +1411,7 @@ Value getstreamsummary(const Array& params, bool fPublisher) Object empty_object; Object obj; int i,n,c,m,err,pcount; + bool available; err=MC_ERR_NOERROR; n=pwalletTxsMain->GetListSize(&entity,entStat.m_Generation,NULL); i=0; @@ -1476,32 +1482,94 @@ Value getstreamsummary(const Array& params, bool fPublisher) } } + available=true; + BOOST_FOREACH(const Pair& a, entry) + { + if(a.name_ == "offchain") + { + available=!a.value_.get_bool(); + } + } + + BOOST_FOREACH(const Pair& a, entry) { if(a.name_ == "data") { - if(i == 0) + if(!available) { -// if(a.value_.type() == obj_type) -/* + if(a.value_.type() != obj_type) { - result=empty_object; - } - - if( (i==0) && ((mode & MC_VMM_TAKE_FIRST) != 0) ) + available=true; + } + } + if(!available) + { + available=true; + BOOST_FOREACH(const Pair& b, a.value_.get_obj()) { - result=mc_MergeValues(&(a.value_),&result,mode,0,&err); + if(b.name_ == "available") + { + available=b.value_.get_bool(); + } + } + } + if(available) + { + if(i == 0) + { + // if(a.value_.type() == obj_type) + /* + { + result=empty_object; + } + + if( (i==0) && ((mode & MC_VMM_TAKE_FIRST) != 0) ) + { + result=mc_MergeValues(&(a.value_),&result,mode,0,&err); + } + else + { + result=mc_MergeValues(&result,&(a.value_),mode,0,&err); + } + */ + result=a.value_; } else { result=mc_MergeValues(&result,&(a.value_),mode,0,&err); - } - */ - result=a.value_; + } } else { - result=mc_MergeValues(&result,&(a.value_),mode,0,&err); + available=true; + BOOST_FOREACH(const Pair& b, a.value_.get_obj()) + { + if(b.name_ == "format") + { + available=false; + if(b.value_.get_str() != "json") + { + if(mode & MC_VMM_IGNORE_OTHER) + { + available=true; + } + else + { + err=MC_ERR_INVALID_PARAMETER_VALUE; + goto exitlbl; + } + } + + } + } + if( (mode & MC_VMM_IGNORE_MISSING) == 0) + { + if(!available) + { + throw JSONRPCError(RPC_NOT_ALLOWED, "Some items to be merged are missing (try using \'ignoremissing\')" ); + } + } } } } @@ -1521,7 +1589,7 @@ Value getstreamsummary(const Array& params, bool fPublisher) } else { - if( (mode & MC_VMM_IGNORE) == 0) + if( (mode & MC_VMM_IGNORE_OTHER) == 0) { err=MC_ERR_INVALID_PARAMETER_VALUE; } @@ -1533,7 +1601,7 @@ Value getstreamsummary(const Array& params, bool fPublisher) if(err) { - throw JSONRPCError(RPC_NOT_ALLOWED, "Some items to be merged are in the wrong format (try using \'ignore\')" ); + throw JSONRPCError(RPC_NOT_ALLOWED, "Some items to be merged are in the wrong format (try using \'ignoreother\')" ); } return result; diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index a36674eb..c71b16e0 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -4057,7 +4057,7 @@ Value mc_MergeValues(const Value *value1,const Value *value2,uint32_t mode,int l if(!value1_is_obj) { - if(mode & MC_VMM_IGNORE) + if(mode & MC_VMM_IGNORE_OTHER) { return *value2; } @@ -4068,7 +4068,7 @@ Value mc_MergeValues(const Value *value1,const Value *value2,uint32_t mode,int l { if(!value2_is_obj) { - if(mode & MC_VMM_IGNORE) + if(mode & MC_VMM_IGNORE_OTHER) { return *value1; } diff --git a/src/rpc/rpcutils.h b/src/rpc/rpcutils.h index 1fe1a683..8c54bad5 100644 --- a/src/rpc/rpcutils.h +++ b/src/rpc/rpcutils.h @@ -52,10 +52,11 @@ using namespace json_spirit; #define MC_VMM_MERGE_OBJECTS 0x00000001 #define MC_VMM_RECURSIVE 0x00000002 -#define MC_VMM_IGNORE 0x00000004 +#define MC_VMM_IGNORE_OTHER 0x00000004 #define MC_VMM_TAKE_FIRST 0x00000008 #define MC_VMM_TAKE_FIRST_FOR_FIELD 0x00000010 #define MC_VMM_OMIT_NULL 0x00000020 +#define MC_VMM_IGNORE_MISSING 0x00000040 #define MC_RFD_OPTION_NONE 0x00000000 #define MC_RFD_OPTION_INLINE 0x00000001 From e25eaa3a12c88a290a360bd16b5b1ec12997295a Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 31 May 2018 14:23:46 +0300 Subject: [PATCH 095/157] Fixed 10011/20003 switch --- src/chainparams/params.cpp | 31 +++++++++++++++++++++++++++++++ src/chainparams/state.h | 1 + 2 files changed, 32 insertions(+) diff --git a/src/chainparams/params.cpp b/src/chainparams/params.cpp index ffcf6b69..a8f14c95 100644 --- a/src/chainparams/params.cpp +++ b/src/chainparams/params.cpp @@ -1981,3 +1981,34 @@ int mc_Features::FixedIn1001020003() return ret; } + +int mc_Features::FixedIn1001120003() +{ + int ret=0; + if(mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) + { + return 1; + } + int protocol=mc_gState->m_NetworkParams->ProtocolVersion(); + + if(protocol) + { + if(protocol < 20000) + { + if(protocol >= 10011) + { + ret=1; + } + } + else + { + if(protocol >= 20003) + { + ret=1; + } + } + } + + return ret; +} + diff --git a/src/chainparams/state.h b/src/chainparams/state.h index 96e1074f..327913c7 100644 --- a/src/chainparams/state.h +++ b/src/chainparams/state.h @@ -143,6 +143,7 @@ typedef struct mc_Features int OffChainData(); int Chunks(); int FixedIn1001020003(); + int FixedIn1001120003(); } mc_Features; typedef struct mc_BlockHeaderInfo From c80282e804a48c1408c0d7d396ceb8bd4cb79037 Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 31 May 2018 15:12:17 +0300 Subject: [PATCH 096/157] Fixed anyone-can-send if stronger anyone-can-*=1 --- src/chainparams/globals.h | 1 + src/chainparams/params.h | 1 + src/permissions/permission.cpp | 2 +- src/utils/utilwrapper.cpp | 8 ++++++++ 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/chainparams/globals.h b/src/chainparams/globals.h index 8253f6fa..a6ee9324 100644 --- a/src/chainparams/globals.h +++ b/src/chainparams/globals.h @@ -39,6 +39,7 @@ int MCP_ANYONE_CAN_CONNECT=0; int MCP_ANYONE_CAN_SEND=0; int MCP_ANYONE_CAN_RECEIVE=0; int MCP_ANYONE_CAN_CREATE=0; +int MCP_ANYONE_CAN_ISSUE=0; int MCP_ANYONE_CAN_ACTIVATE=0; int64_t MCP_MINIMUM_PER_OUTPUT=0; int MCP_ALLOW_ARBITRARY_OUTPUTS=1; diff --git a/src/chainparams/params.h b/src/chainparams/params.h index e89a3109..ba1390c5 100644 --- a/src/chainparams/params.h +++ b/src/chainparams/params.h @@ -74,6 +74,7 @@ extern int MCP_ANYONE_CAN_CONNECT; extern int MCP_ANYONE_CAN_SEND; extern int MCP_ANYONE_CAN_RECEIVE; extern int MCP_ANYONE_CAN_CREATE; +extern int MCP_ANYONE_CAN_ISSUE; extern int MCP_ANYONE_CAN_ACTIVATE; extern int64_t MCP_MINIMUM_PER_OUTPUT; extern int MCP_ALLOW_ARBITRARY_OUTPUTS; diff --git a/src/permissions/permission.cpp b/src/permissions/permission.cpp index 85f88bc9..a8a97b95 100644 --- a/src/permissions/permission.cpp +++ b/src/permissions/permission.cpp @@ -1265,7 +1265,7 @@ int mc_Permissions::CanIssue(const void* lpEntity,const void* lpAddress) // if(lpEntity == NULL) if(mc_IsNullEntity(lpEntity)) { - if(mc_gState->m_NetworkParams->GetInt64Param("anyonecanissue")) + if(MCP_ANYONE_CAN_ISSUE) { return MC_PTP_ISSUE; } diff --git a/src/utils/utilwrapper.cpp b/src/utils/utilwrapper.cpp index c3843c79..7333cb0a 100644 --- a/src/utils/utilwrapper.cpp +++ b/src/utils/utilwrapper.cpp @@ -819,7 +819,15 @@ int mc_MultichainParams::SetGlobals() MCP_ANYONE_CAN_SEND=mc_gState->m_NetworkParams->GetInt64Param("anyonecansend"); MCP_ANYONE_CAN_RECEIVE=mc_gState->m_NetworkParams->GetInt64Param("anyonecanreceive"); MCP_ANYONE_CAN_CREATE=mc_gState->m_NetworkParams->GetInt64Param("anyonecancreate"); + MCP_ANYONE_CAN_ISSUE=mc_gState->m_NetworkParams->GetInt64Param("anyonecanissue"); MCP_ANYONE_CAN_ACTIVATE=mc_gState->m_NetworkParams->GetInt64Param("anyonecanactivate"); + if(mc_gState->m_Features->FixedIn1001120003()) + { + if(MCP_ANYONE_CAN_ADMIN)MCP_ANYONE_CAN_SEND=1; + if(MCP_ANYONE_CAN_ACTIVATE)MCP_ANYONE_CAN_SEND=1; + if(MCP_ANYONE_CAN_CREATE)MCP_ANYONE_CAN_SEND=1; + if(MCP_ANYONE_CAN_ISSUE)MCP_ANYONE_CAN_SEND=1; + } MCP_MINIMUM_PER_OUTPUT=mc_gState->m_NetworkParams->GetInt64Param("minimumperoutput"); MCP_ALLOW_MULTISIG_OUTPUTS=mc_gState->m_NetworkParams->GetInt64Param("allowmultisigoutputs"); MCP_ALLOW_P2SH_OUTPUTS=mc_gState->m_NetworkParams->GetInt64Param("allowp2shoutputs"); From fdacd9d1430eb1082176dc62451731c021e5ef08 Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 31 May 2018 15:52:47 +0300 Subject: [PATCH 097/157] Fixed long entity name bug on protocol level --- src/entities/asset.cpp | 4 ++++ src/protocol/multichaintx.cpp | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/src/entities/asset.cpp b/src/entities/asset.cpp index 15978dab..7aa14196 100644 --- a/src/entities/asset.cpp +++ b/src/entities/asset.cpp @@ -672,6 +672,10 @@ int mc_AssetDB::InsertEntity(const void* txid, int offset, int entity_type, cons { if(value_size > MC_ENT_MAX_NAME_SIZE) { + if(mc_gState->m_Features->FixedIn1001120003()) + { + return MC_ERR_ERROR_IN_SCRIPT; + } value_size=MC_ENT_MAX_NAME_SIZE; } memcpy(stream_name,(unsigned char*)script+value_offset,value_size); diff --git a/src/protocol/multichaintx.cpp b/src/protocol/multichaintx.cpp index 6a17fd60..58004add 100644 --- a/src/protocol/multichaintx.cpp +++ b/src/protocol/multichaintx.cpp @@ -1451,6 +1451,11 @@ bool MultiChainTransaction_ProcessAssetIssuance(const CTransaction& tx, { if(value_size > MC_ENT_MAX_NAME_SIZE) { + if(mc_gState->m_Features->FixedIn1001120003()) + { + reason="Metadata script rejected - entity name too long"; + return false; + } value_size=MC_ENT_MAX_NAME_SIZE; } From af9cdc3898fcf6450994f5c3516be72e2a10984e Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 3 Jun 2018 15:18:57 +0300 Subject: [PATCH 098/157] Fixed gettxoutdata bugs --- src/rpc/rpcutils.cpp | 7 +++---- src/rpc/rpcwallet.cpp | 6 +++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index c71b16e0..e9ecc04c 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -913,7 +913,6 @@ const unsigned char *GetChunkDataInRange(int64_t *out_size,unsigned char* hashes mc_ChunkDBRow chunk_def; int size,shift,chunk; unsigned char *ptr; - unsigned char *ptrEnd; size_t elem_size; int64_t total_size=0; unsigned char *elem; @@ -925,10 +924,9 @@ const unsigned char *GetChunkDataInRange(int64_t *out_size,unsigned char* hashes *out_size=0; ptr=hashes; - ptrEnd=ptr+MC_CDB_CHUNK_HASH_SIZE+16; for(chunk=0;chunk start+count) { read_size=start+count-total_size; } + read_size-=read_from; elem=pwalletTxsMain->m_ChunkDB->GetChunk(&chunk_def,0,-1,&elem_size); if(elem) { diff --git a/src/rpc/rpcwallet.cpp b/src/rpc/rpcwallet.cpp index 563fe76d..ea8eb14a 100644 --- a/src/rpc/rpcwallet.cpp +++ b/src/rpc/rpcwallet.cpp @@ -757,9 +757,13 @@ Value gettxoutdata(const Array& params, bool fHelp) if(elem == NULL) { elem=GetChunkDataInRange(&out_size,chunk_hashes,chunk_count,start,count); + if(elem == NULL) + { + throw JSONRPCError(RPC_INTERNAL_ERROR, "Couldn't retrieve data for this output"); + } + return OpReturnFormatEntry(elem,count,0,0,format,NULL); } } - return OpReturnFormatEntry(elem+start,count,0,0,format,NULL); } From 37964d54c2efde99c77dbe2566912d0a98e5e046 Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 3 Jun 2018 15:19:26 +0300 Subject: [PATCH 099/157] Changed defaults fox max-std-element-size --- src/chainparams/paramlist.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/chainparams/paramlist.h b/src/chainparams/paramlist.h index 807a571e..9678e79b 100644 --- a/src/chainparams/paramlist.h +++ b/src/chainparams/paramlist.h @@ -238,7 +238,7 @@ static const mc_OneMultichainParam MultichainParamArray[] = "maxstdelementsize","", "Maximum number of OP_DROPs per output in standard transactions."}, { "maxstdelementsize" , "max-std-element-size" , - MC_PRM_INT64 | MC_PRM_USER | MC_PRM_CLONE , -1, 8192, 128, 32768, 0.0, 10003, 0, "-mc-maxscriptelementsize", + MC_PRM_INT64 | MC_PRM_USER | MC_PRM_CLONE , -1, 40000, 128, 80000, 0.0, 10003, 0, "-mc-maxscriptelementsize", "maxstdopdropsize","", "Maximum size of data elements in standard transactions, in bytes."}, From 715e745ee0c81cdc158046b9ec94141ee00a2ef3 Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 3 Jun 2018 15:45:55 +0300 Subject: [PATCH 100/157] Chunk delivery tweaks --- src/core/main.cpp | 5 +++-- src/protocol/relay.cpp | 11 ++++++++++- src/protocol/relay.h | 2 +- src/rpc/rpcchunks.cpp | 4 ++-- src/wallet/chunkcollector.h | 2 +- 5 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/core/main.cpp b/src/core/main.cpp index 32107e4e..44a14973 100644 --- a/src/core/main.cpp +++ b/src/core/main.cpp @@ -7322,9 +7322,10 @@ bool SendMessages(CNode* pto, bool fSendTrickle) { MultichainCollectChunks(pwalletTxsMain->m_ChunkCollector); } - MultichainCollectChunksQueueStats(pwalletTxsMain->m_ChunkCollector); + pwalletTxsMain->m_ChunkCollector->m_NextTryTimestamp=time_millis_now+MultichainCollectChunksQueueStats(pwalletTxsMain->m_ChunkCollector); + // if(fDebug)LogPrint("chunks", "Chunks to collect: %d\n", still_to_collect); - pwalletTxsMain->m_ChunkCollector->m_NextTryTimestamp=time_millis_now+GetArg("-offchainrequestfreq",MC_CCW_TIMEOUT_BETWEEN_COLLECTS_MILLIS); +// pwalletTxsMain->m_ChunkCollector->m_NextTryTimestamp=time_millis_now+GetArg("-offchainrequestfreq",MC_CCW_TIMEOUT_BETWEEN_COLLECTS_MILLIS); } } /* MCHN END */ diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index 936b098c..8ddabea4 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -723,9 +723,10 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) return not_processed; } -void MultichainCollectChunksQueueStats(mc_ChunkCollector* collector) +int MultichainCollectChunksQueueStats(mc_ChunkCollector* collector) { int row; + int delay; mc_ChunkCollectorRow *collect_row; uint32_t size; @@ -773,6 +774,14 @@ void MultichainCollectChunksQueueStats(mc_ChunkCollector* collector) collector->m_NextAutoCommitTimestamp=GetTimeMillis()+collector->m_AutoCommitDelay; } } + + delay=collector->m_StatLast[0].m_Queried; + if(delay>MC_CCW_MAX_DELAY_BETWEEN_COLLECTS) + { + delay=MC_CCW_MAX_DELAY_BETWEEN_COLLECTS; + } + + return delay; } void mc_RelayPayload_ChunkIDs(vector* payload,vector & vChunkDefs,int size) diff --git a/src/protocol/relay.h b/src/protocol/relay.h index 3f0ad097..4d1aa1a4 100644 --- a/src/protocol/relay.h +++ b/src/protocol/relay.h @@ -81,7 +81,7 @@ bool MultichainRelayResponse(uint32_t msg_type_stored, CNode *pto_stored, string& strError); int MultichainCollectChunks(mc_ChunkCollector* collector); -void MultichainCollectChunksQueueStats(mc_ChunkCollector* collector); +int MultichainCollectChunksQueueStats(mc_ChunkCollector* collector); diff --git a/src/rpc/rpcchunks.cpp b/src/rpc/rpcchunks.cpp index 0b6aa28a..cbde85cc 100644 --- a/src/rpc/rpcchunks.cpp +++ b/src/rpc/rpcchunks.cpp @@ -19,7 +19,7 @@ Value getchunkqueueinfo(const Array& params, bool fHelp) { Object stat; stat.push_back(Pair("pending",collector->m_StatLast[k].m_Pending)); - stat.push_back(Pair("processing",collector->m_StatLast[k].m_Undelivered)); +// stat.push_back(Pair("processing",collector->m_StatLast[k].m_Undelivered)); stat.push_back(Pair("queried",collector->m_StatLast[k].m_Queried)); stat.push_back(Pair("requested",collector->m_StatLast[k].m_Requested)); result.push_back(Pair(k ? "bytes" : "chunks",stat)); @@ -40,10 +40,10 @@ Value getchunktotals(const Array& params, bool fHelp) for(int k=0;k<2;k++) { Object stat; - stat.push_back(Pair("delivered",collector->m_StatTotal[k].m_Delivered)); stat.push_back(Pair("queries",collector->m_StatTotal[k].m_Queried)); stat.push_back(Pair("unresponded",collector->m_StatTotal[k].m_Unresponded)); stat.push_back(Pair("requests",collector->m_StatTotal[k].m_Requested)); + stat.push_back(Pair("delivered",collector->m_StatTotal[k].m_Delivered)); stat.push_back(Pair("undelivered",collector->m_StatTotal[k].m_Undelivered)); stat.push_back(Pair("baddelivered",collector->m_StatTotal[k].m_Baddelivered)); result.push_back(Pair(k ? "bytes" : "chunks",stat)); diff --git a/src/wallet/chunkcollector.h b/src/wallet/chunkcollector.h index e9f539f2..644c48d8 100644 --- a/src/wallet/chunkcollector.h +++ b/src/wallet/chunkcollector.h @@ -27,7 +27,7 @@ #define MC_CCW_WORST_RESPONSE_SCORE 1048576000 #define MC_CCW_DEFAULT_MEMPOOL_SIZE 60000 #define MC_CCW_MAX_MBS_PER_SECOND 8 -#define MC_CCW_TIMEOUT_BETWEEN_COLLECTS_MILLIS 100 +#define MC_CCW_MAX_DELAY_BETWEEN_COLLECTS 1000 typedef struct mc_ChunkEntityKey From 18cb73ac0884130152f726712d78091fda4f6476 Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 3 Jun 2018 16:28:18 +0300 Subject: [PATCH 101/157] Change default for -maxsendbuffer --- src/core/init.cpp | 2 +- src/net/net.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/init.cpp b/src/core/init.cpp index 405a5d17..4b9ffc04 100644 --- a/src/core/init.cpp +++ b/src/core/init.cpp @@ -356,7 +356,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += " -listen " + _("Accept connections from outside (default: 1 if no -proxy or -connect)") + "\n"; strUsage += " -maxconnections= " + strprintf(_("Maintain at most connections to peers (default: %u)"), 125) + "\n"; strUsage += " -maxreceivebuffer= " + strprintf(_("Maximum per-connection receive buffer, *1000 bytes (default: %u)"), 5000) + "\n"; - strUsage += " -maxsendbuffer= " + strprintf(_("Maximum per-connection send buffer, *1000 bytes (default: %u)"), 1000) + "\n"; + strUsage += " -maxsendbuffer= " + strprintf(_("Maximum per-connection send buffer, *1000 bytes (default: %u)"), 100000) + "\n"; strUsage += " -onion= " + strprintf(_("Use separate SOCKS5 proxy to reach peers via Tor hidden services (default: %s)"), "-proxy") + "\n"; strUsage += " -onlynet= " + _("Only connect to nodes in network (ipv4, ipv6 or onion)") + "\n"; strUsage += " -permitbaremultisig " + strprintf(_("Relay non-P2SH multisig (default: %u)"), 1) + "\n"; diff --git a/src/net/net.cpp b/src/net/net.cpp index 0984c328..1ad98144 100644 --- a/src/net/net.cpp +++ b/src/net/net.cpp @@ -2173,7 +2173,7 @@ bool CAddrDB::Read(CAddrMan& addr) } unsigned int ReceiveFloodSize() { return 1000*GetArg("-maxreceivebuffer", 5*1000); } -unsigned int SendBufferSize() { return 1000*GetArg("-maxsendbuffer", 1*1000); } +unsigned int SendBufferSize() { return 1000*GetArg("-maxsendbuffer", 1*100000); } CNode::CNode(SOCKET hSocketIn, CAddress addrIn, std::string addrNameIn, bool fInboundIn) : ssSend(SER_NETWORK, INIT_PROTO_VERSION), setAddrKnown(5000) { From c3e745d2d9316048c9b79de1309d6a8823d3609b Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 4 Jun 2018 08:57:17 +0300 Subject: [PATCH 102/157] Flush file with sync mode for Windows --- src/utils/systemdependent.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/utils/systemdependent.cpp b/src/utils/systemdependent.cpp index 3d445668..c9d4c7bb 100644 --- a/src/utils/systemdependent.cpp +++ b/src/utils/systemdependent.cpp @@ -316,4 +316,10 @@ void __US_FlushFile(int FileHan) FlushFileBuffers(hFile); } +void __US_FlushFileWithMode(int FileHan,uint32_t use_data_sync) +{ + HANDLE hFile = (HANDLE)_get_osfhandle(FileHan); + FlushFileBuffers(hFile); +} + #endif From 03503e82c4e6c8eb9bc78c06fb8a73e9649ae059 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 4 Jun 2018 08:57:36 +0300 Subject: [PATCH 103/157] getchunktotals, responded field --- src/rpc/rpcchunks.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rpc/rpcchunks.cpp b/src/rpc/rpcchunks.cpp index cbde85cc..f990c46a 100644 --- a/src/rpc/rpcchunks.cpp +++ b/src/rpc/rpcchunks.cpp @@ -41,6 +41,7 @@ Value getchunktotals(const Array& params, bool fHelp) { Object stat; stat.push_back(Pair("queries",collector->m_StatTotal[k].m_Queried)); + stat.push_back(Pair("responded",collector->m_StatTotal[k].m_Queried-collector->m_StatTotal[k].m_Unresponded-collector->m_StatLast[k].m_Queried)); stat.push_back(Pair("unresponded",collector->m_StatTotal[k].m_Unresponded)); stat.push_back(Pair("requests",collector->m_StatTotal[k].m_Requested)); stat.push_back(Pair("delivered",collector->m_StatTotal[k].m_Delivered)); From 4bc4b89272763caedcb46e33f34f35a7547ea29e Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 4 Jun 2018 09:57:46 +0300 Subject: [PATCH 104/157] Fixed unicode items on Windows bug --- src/json/json_spirit_writer_template.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/json/json_spirit_writer_template.h b/src/json/json_spirit_writer_template.h index 10697ea4..af410317 100644 --- a/src/json/json_spirit_writer_template.h +++ b/src/json/json_spirit_writer_template.h @@ -113,7 +113,6 @@ namespace json_spirit result[ 9 ] = to_hex_char( part2 & 0x000F ); part2 >>= 4; result[ 8 ] = to_hex_char( part2 & 0x000F ); } - return result; } @@ -135,16 +134,17 @@ namespace json_spirit const wint_t unsigned_c( ( c >= 0 ) ? c : 256 + c ); - if( iswprint( unsigned_c ) ) + unsigned int codepoint=0; + unsigned int charlen,shift,j,mask; + charlen=utf8_len_and_mask(unsigned_c,&mask); + +// if( iswprint( unsigned_c ) ) + if(charlen == 1) { result += c; } else { - unsigned int codepoint=0; - unsigned int charlen,shift,j,mask; - - charlen=utf8_len_and_mask(unsigned_c,&mask); if( end - i >= charlen) { shift=6*(charlen-1); From ecb3046b42590462960c9b0fdf0039ed8dd23c96 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 4 Jun 2018 11:18:26 +0300 Subject: [PATCH 105/157] Offchain items tweaks --- src/rpc/rpchelp.cpp | 8 ++++---- src/rpc/rpcrawdata.cpp | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/rpc/rpchelp.cpp b/src/rpc/rpchelp.cpp index 1edbbdc0..2a5a7763 100644 --- a/src/rpc/rpchelp.cpp +++ b/src/rpc/rpchelp.cpp @@ -1396,7 +1396,7 @@ void mc_InitRPCHelpMap06() " or \n" "3. restrictions (object, optional) Stream restrictions\n" " {\n" - " \"restrict\" : \"restrictions\" (string, optional) Stream restrictions, comma delimited. Possible values: write,offchain,onchain\n" + " \"restrict\" : \"restrictions\" (string, optional) Stream restrictions, comma delimited. Possible values: write,offchainonly,onchainonly\n" " }\n" "4 custom-fields (object, optional) a json object with custom fields\n" " {\n" @@ -1443,7 +1443,7 @@ void mc_InitRPCHelpMap06() " or \n" "4. restrictions (object, optional) Stream restrictions\n" " {\n" - " \"restrict\" : \"restrictions\" (string, optional) Stream restrictions, comma delimited. Possible values: write,offchain,onchain\n" + " \"restrict\" : \"restrictions\" (string, optional) Stream restrictions, comma delimited. Possible values: write,offchainonly,onchainonly\n" " }\n" "5 custom-fields (object, optional) a json object with custom fields\n" " {\n" @@ -3480,7 +3480,7 @@ void mc_InitRPCHelpMap15() )); mapHelpStrings.insert(std::make_pair("unsubscribe", - "unsubscribe entity-identifier(s)\n" + "unsubscribe entity-identifier(s) ( purge )\n" "\nUnsubscribes from the stream.\n" "\nArguments:\n" "1. \"stream-identifier\" (string, required) Stream identifier - one of the following: stream txid, stream reference, stream name.\n" @@ -3767,7 +3767,7 @@ void mc_InitRPCHelpMap16() " \"create\" : \"stream\" (string, required) stream\n" " \"name\" : \"stream-name\" (string, optional) Stream name\n" " \"open\" : true|false (boolean, optional, default: false) If true, anyone can publish\n" - " \"restrict\" : \"restrictions\" (string, optional) Stream restrictions, comma delimited. Possible values: write,offchain,onchain\n" + " \"restrict\" : \"restrictions\" (string, optional) Stream restrictions, comma delimited. Possible values: write,offchainonly,onchainonly\n" " \"details\" : (object, optional) A json object with custom fields\n" " {\n" " \"param-name\": \"param-value\" (strings, required) The key is the parameter name, the value is parameter value\n" diff --git a/src/rpc/rpcrawdata.cpp b/src/rpc/rpcrawdata.cpp index 890f2b1f..4008e90b 100644 --- a/src/rpc/rpcrawdata.cpp +++ b/src/rpc/rpcrawdata.cpp @@ -740,9 +740,9 @@ bool RawDataParseRestrictParameter(const Value& param,uint32_t *restrict,uint32_ if(ptr > start) { match=0; - if(memcmp(start,"write", ptr-start) == 0){match = 1; *permissions |= MC_PTP_WRITE ;} - if(memcmp(start,"onchain", ptr-start) == 0){match = 1; *restrict |= MC_ENT_ENTITY_RESTRICTION_ONCHAIN;} - if(memcmp(start,"offchain", ptr-start) == 0){match = 1; *restrict |= MC_ENT_ENTITY_RESTRICTION_OFFCHAIN;} + if(( (ptr-start) == 5) && (memcmp(start,"write", ptr-start) == 0) ){match = 1; *permissions |= MC_PTP_WRITE ;} + if(( (ptr-start) == 11) && (memcmp(start,"onchainonly", ptr-start) == 0) ){match = 1; *restrict |= MC_ENT_ENTITY_RESTRICTION_ONCHAIN;} + if(( (ptr-start) == 12) && (memcmp(start,"offchainonly", ptr-start) == 0) ){match = 1; *restrict |= MC_ENT_ENTITY_RESTRICTION_OFFCHAIN;} if(match == 0) { From 4fa4b6d8f81a3b41fc1d2bdfab5d515a06dc417f Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 4 Jun 2018 13:04:26 +0300 Subject: [PATCH 106/157] Disallowed debug network functionality --- src/protocol/relay.cpp | 14 ++++++++++---- src/protocol/relay.h | 1 + src/rpc/rpcdebug.cpp | 2 ++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index 8ddabea4..aa028ab9 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -630,7 +630,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) collect_row->m_State.m_Status |= MC_CCF_UPDATED; for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Unresponded+=k ? collect_row->m_ChunkDef.m_Size : 1; } - if((collect_row->m_State.m_QueryNextAttempt <= time_now) && (total_in_queries < max_total_in_queries) && (query_countm_MaxMemPoolSize)) + if((collect_row->m_State.m_QueryNextAttempt <= time_now) && (total_in_queries < max_total_in_queries) && ((int)query_countm_MaxMemPoolSize)) { if( (collect_row->m_State.m_Status & MC_CCF_ERROR_MASK) == 0) { @@ -1846,10 +1846,10 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, switch(msg_type_in) { case MC_RMT_MC_ADDRESS_QUERY: - verify_flags |= MC_VRA_IS_NOT_RESPONSE; + verify_flags |= MC_VRA_IS_NOT_RESPONSE | MC_VRA_NOT_ALLOWED; break; case MC_RMT_NODE_DETAILS: - verify_flags |= MC_VRA_IS_RESPONSE | MC_VRA_SIGNATURE_ORIGIN; + verify_flags |= MC_VRA_IS_RESPONSE | MC_VRA_SIGNATURE_ORIGIN | MC_VRA_NOT_ALLOWED; break; case MC_RMT_CHUNK_QUERY: verify_flags |= MC_VRA_IS_NOT_RESPONSE; @@ -1866,12 +1866,18 @@ bool mc_RelayManager::ProcessRelay( CNode* pfrom, default: if(verify_flags & MC_VRA_MESSAGE_TYPE) { - LogPrintf("ProcessOffchain() : Unsupported relay message type %s\n",mc_MsgTypeStr(msg_type_in).c_str()); + LogPrintf("ProcessOffchain() : Unsupported offchain message type %s\n",mc_MsgTypeStr(msg_type_in).c_str()); return false; } break; } + if(verify_flags & MC_VRA_NOT_ALLOWED) + { + LogPrintf("ProcessOffchain() : Not allowed offchain message type %s\n",mc_MsgTypeStr(msg_type_in).c_str()); + return false; + } + if(verify_flags & MC_VRA_SINGLE_HOP) { if(hop_count) diff --git a/src/protocol/relay.h b/src/protocol/relay.h index 4d1aa1a4..82cfee02 100644 --- a/src/protocol/relay.h +++ b/src/protocol/relay.h @@ -36,6 +36,7 @@ #define MC_VRA_BROADCAST_ONCE 0x00800000 #define MC_VRA_SINGLE_HOP_BROADCAST 0x01000000 #define MC_VRA_TIMESTAMP 0x02000000 +#define MC_VRA_NOT_ALLOWED 0x80000000 #define MC_VRA_DEFAULT 0x03970000 diff --git a/src/rpc/rpcdebug.cpp b/src/rpc/rpcdebug.cpp index 5ae6890a..0231a4de 100644 --- a/src/rpc/rpcdebug.cpp +++ b/src/rpc/rpcdebug.cpp @@ -685,8 +685,10 @@ Value debug(const Array& params, bool fHelp) } request_id=pRelayManager->SendRequest(pto,request_type,0,payload); +/* printf("%s",request_str.c_str()); printf(". Request ID: %s\n",request_id.ToString().c_str()); + */ } uint32_t time_now; From 194d37a8336f0bf0ffefc414e3a33ad08bd574da Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 4 Jun 2018 13:57:38 +0300 Subject: [PATCH 107/157] Chunk file size - 128MB --- src/wallet/chunkdb.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wallet/chunkdb.h b/src/wallet/chunkdb.h index aac9a74b..4ba74ee4 100644 --- a/src/wallet/chunkdb.h +++ b/src/wallet/chunkdb.h @@ -16,7 +16,7 @@ #define MC_CDB_TYPE_SUBSCRIPTION 0x01000000 #define MC_CDB_TYPE_FILE 0x02000000 -#define MC_CDB_MAX_FILE_SIZE 0x40000000 // Maximal data file size, 1GB +#define MC_CDB_MAX_FILE_SIZE 0x08000000 // Maximal data file size, 1GB #define MC_CDB_MAX_CHUNK_DATA_POOL_SIZE 0x8000000 // Maximal size of chunk pool before commit, 128MB #define MC_CDB_MAX_FILE_READ_BUFFER_SIZE 0x0100000 // Maximal size of chunk pool before commit, 4MB #define MC_CDB_MAX_CHUNK_EXTRA_SIZE 1024 From b3e7bb7fc28b9e66d4ff75fb1ea5c13218fd00bf Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 4 Jun 2018 15:07:43 +0300 Subject: [PATCH 108/157] Parameter upgrade: maximum-chunk-size and maximum-chunk-count --- src/chainparams/globals.h | 4 ++-- src/chainparams/params.cpp | 16 ++++++++++++++++ src/core/main.cpp | 28 ++++++++++++++++++++++++++++ src/rpc/rpcmisc.cpp | 8 ++++++++ 4 files changed, 54 insertions(+), 2 deletions(-) diff --git a/src/chainparams/globals.h b/src/chainparams/globals.h index a6ee9324..ff0d52bf 100644 --- a/src/chainparams/globals.h +++ b/src/chainparams/globals.h @@ -26,8 +26,8 @@ int MAX_OP_RETURN_SHOWN=16384; int MAX_FORMATTED_DATA_DEPTH=100; unsigned int MAX_OP_RETURN_OP_DROP_COUNT=100000000; uint32_t JSON_NO_DOUBLE_FORMATTING=0; -int MAX_CHUNK_SIZE = 16777216; -int MAX_CHUNK_COUNT = 128; +int MAX_CHUNK_SIZE = 1048576; +int MAX_CHUNK_COUNT = 1024; int MCP_MAX_STD_OP_RETURN_COUNT=0; int64_t MCP_INITIAL_BLOCK_REWARD=0; diff --git a/src/chainparams/params.cpp b/src/chainparams/params.cpp index a8f14c95..49c604f5 100644 --- a/src/chainparams/params.cpp +++ b/src/chainparams/params.cpp @@ -264,6 +264,22 @@ int mc_MultichainParams::CanBeUpgradedByVersion(const char *param,int version,in } } + if(strcmp(param,"maximumchunksize") == 0) + { + if(version >= 20003) + { + return m_lpCoord[2 * index + 1]; + } + } + + if(strcmp(param,"maximumchunkcount") == 0) + { + if(version >= 20003) + { + return m_lpCoord[2 * index + 1]; + } + } + return 0; diff --git a/src/core/main.cpp b/src/core/main.cpp index 44a14973..175edd5f 100644 --- a/src/core/main.cpp +++ b/src/core/main.cpp @@ -409,6 +409,34 @@ int SetUpgradedParamValue(const mc_OneMultichainParam *param,int64_t value) pwalletMain->InitializeUnspentList(); } + if(strcmp(param->m_Name,"maximumchunksize") == 0) + { + int old_value=MAX_CHUNK_SIZE; + MAX_CHUNK_SIZE=(unsigned int)value; + + while(MAX_CHUNK_SIZE+OFFCHAIN_MSG_PADDING>MAX_SIZE) + { + MAX_SIZE *= 2; + } + + if(MAX_CHUNK_SIZE > old_value) + { + if(pwalletTxsMain) + { + if(pwalletTxsMain->m_ChunkBuffer) + { + mc_Delete(pwalletTxsMain->m_ChunkBuffer); + pwalletTxsMain->m_ChunkBuffer=(unsigned char*)mc_New(MAX_CHUNK_SIZE); + } + } + } + } + + if(strcmp(param->m_Name,"maximumchunkcount") == 0) + { + MAX_CHUNK_COUNT=value; + } + return MC_ERR_NOERROR; } diff --git a/src/rpc/rpcmisc.cpp b/src/rpc/rpcmisc.cpp index 73971832..8f11ae37 100644 --- a/src/rpc/rpcmisc.cpp +++ b/src/rpc/rpcmisc.cpp @@ -712,6 +712,14 @@ Value getblockchainparams(const json_spirit::Array& params, bool fHelp) { param_value=(int)MAX_SCRIPT_ELEMENT_SIZE; } + if(strcmp("maximumchunksize",(mc_gState->m_NetworkParams->m_lpParams+i)->m_Name) == 0) + { + param_value=(int)MAX_CHUNK_SIZE; + } + if(strcmp("maximumchunkcount",(mc_gState->m_NetworkParams->m_lpParams+i)->m_Name) == 0) + { + param_value=(int)MAX_CHUNK_COUNT; + } } if(!hidden) From 12cc667d2397b9ff18efab356392d812326ebf71 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 5 Jun 2018 13:48:46 +0300 Subject: [PATCH 109/157] Available field outside data object and avoiding protocol checks in case of chunk parma upgrades --- src/protocol/multichainscript.cpp | 30 +++++++++++++---- src/protocol/multichainscript.h | 2 ++ src/protocol/multichaintx.cpp | 4 +-- src/rpc/rpcutils.cpp | 53 ++++++++++++++++++++----------- src/rpc/rpcutils.h | 1 + src/rpc/rpcwalletutils.cpp | 11 +++++++ 6 files changed, 73 insertions(+), 28 deletions(-) diff --git a/src/protocol/multichainscript.cpp b/src/protocol/multichainscript.cpp index a9dd21a5..01251313 100644 --- a/src/protocol/multichainscript.cpp +++ b/src/protocol/multichainscript.cpp @@ -2354,7 +2354,7 @@ int mc_Script::SetDataFormat(const uint32_t format) return MC_ERR_NOERROR; } -int mc_Script::GetChunkDef(uint32_t *format,unsigned char** hashes,int *chunk_count,int64_t *total_size) +int mc_Script::GetChunkDef(uint32_t *format,unsigned char** hashes,int *chunk_count,int64_t *total_size,int check_sizes) { unsigned char *ptr; unsigned char *ptrEnd; @@ -2431,9 +2431,12 @@ int mc_Script::GetChunkDef(uint32_t *format,unsigned char** hashes,int *chunk_co ptr+=shift; - if(count > MAX_CHUNK_COUNT) + if(check_sizes) { - return MC_ERR_ERROR_IN_SCRIPT; + if(count > MAX_CHUNK_COUNT) + { + return MC_ERR_ERROR_IN_SCRIPT; + } } if(chunk_count) @@ -2460,9 +2463,12 @@ int mc_Script::GetChunkDef(uint32_t *format,unsigned char** hashes,int *chunk_co return MC_ERR_ERROR_IN_SCRIPT; } - if(size > MAX_CHUNK_SIZE) + if(check_sizes) { - return MC_ERR_ERROR_IN_SCRIPT; + if(size > MAX_CHUNK_SIZE) + { + return MC_ERR_ERROR_IN_SCRIPT; + } } ptr+=shift; @@ -2483,6 +2489,11 @@ int mc_Script::GetChunkDef(uint32_t *format,unsigned char** hashes,int *chunk_co return MC_ERR_NOERROR; } +int mc_Script::GetChunkDef(uint32_t *format,unsigned char** hashes,int *chunk_count,int64_t *total_size) +{ + return GetChunkDef(format,hashes,chunk_count,total_size,0); +} + int mc_Script::SetChunkDefHeader(const uint32_t format,int chunk_count) { int err,shift; @@ -2537,7 +2548,7 @@ int mc_Script::ExtractAndDeleteDataFormat(uint32_t *format) return ExtractAndDeleteDataFormat(format,NULL,NULL,NULL); } -int mc_Script::ExtractAndDeleteDataFormat(uint32_t *format,unsigned char** hashes,int *chunk_count,int64_t *total_size) +int mc_Script::ExtractAndDeleteDataFormat(uint32_t *format,unsigned char** hashes,int *chunk_count,int64_t *total_size,int check_sizes) { int elem,err; @@ -2601,7 +2612,7 @@ int mc_Script::ExtractAndDeleteDataFormat(uint32_t *format,unsigned char** hashe elem=m_NumElements-2; SetElement(elem); - while( (elem >= 0 ) && ((err=GetChunkDef(format,hashes,chunk_count,total_size)) == MC_ERR_NOERROR) ) + while( (elem >= 0 ) && ((err=GetChunkDef(format,hashes,chunk_count,total_size,check_sizes)) == MC_ERR_NOERROR) ) { m_Restrictions |= MC_ENT_ENTITY_RESTRICTION_OFFCHAIN; DeleteElement(elem); @@ -2632,6 +2643,11 @@ int mc_Script::ExtractAndDeleteDataFormat(uint32_t *format,unsigned char** hashe return MC_ERR_NOERROR; } +int mc_Script::ExtractAndDeleteDataFormat(uint32_t *format,unsigned char** hashes,int *chunk_count,int64_t *total_size) +{ + return ExtractAndDeleteDataFormat(format,hashes,chunk_count,total_size,0); +} + int mc_Script::DeleteDuplicatesInRange(int from,int to) { int *len0,*len1,*len2,*len3,*len4; diff --git a/src/protocol/multichainscript.h b/src/protocol/multichainscript.h index b7b918fb..9bd48084 100644 --- a/src/protocol/multichainscript.h +++ b/src/protocol/multichainscript.h @@ -107,11 +107,13 @@ typedef struct mc_Script int SetDataFormat(const uint32_t format); int GetChunkDef(uint32_t *format,unsigned char** hashes,int *chunk_count,int64_t *total_size); + int GetChunkDef(uint32_t *format,unsigned char** hashes,int *chunk_count,int64_t *total_size,int check_sizes); int SetChunkDefHeader(const uint32_t format,int chunk_count); int SetChunkDefHash(unsigned char *hash,int size); int ExtractAndDeleteDataFormat(uint32_t *format); int ExtractAndDeleteDataFormat(uint32_t *format,unsigned char** hashes,int *chunk_count,int64_t *total_size); + int ExtractAndDeleteDataFormat(uint32_t *format,unsigned char** hashes,int *chunk_count,int64_t *total_size,int check_sizes); int DeleteDuplicatesInRange(int from,int to); diff --git a/src/protocol/multichaintx.cpp b/src/protocol/multichaintx.cpp index 58004add..705169e2 100644 --- a/src/protocol/multichaintx.cpp +++ b/src/protocol/multichaintx.cpp @@ -568,7 +568,7 @@ void MultiChainTransaction_FillAdminPermissionsBeforeTx(const CTransaction& tx, bool MultiChainTransaction_VerifyAndDeleteDataFormatElements(string& reason,int64_t *total_size) { - if(mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(NULL,NULL,NULL,total_size)) + if(mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(NULL,NULL,NULL,total_size,1)) { reason="Error in data format script"; return false; @@ -590,7 +590,6 @@ bool MultiChainTransaction_CheckOpReturnScript(const CTransaction& tx, int64_t total_offchain_size; total_offchain_size=0; -// mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(NULL); // Format scripts are eliminated for protocol checks if(!MultiChainTransaction_VerifyAndDeleteDataFormatElements(reason,&total_offchain_size)) { return false; @@ -858,7 +857,6 @@ bool MultiChainTransaction_CheckEntityItem(const CTransaction& tx, unsigned char short_txid[MC_AST_SHORT_TXID_SIZE]; mc_EntityDetails entity; -// mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(NULL); // Format scripts are eliminated for protocol checks if(!MultiChainTransaction_VerifyAndDeleteDataFormatElements(reason,NULL)) { return false; diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index e9ecc04c..659a526b 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -1028,12 +1028,13 @@ uint32_t GetFormattedData(mc_Script *lpScript,const unsigned char **elem,int64_t return status; } +/* if(size > MAX_CHUNK_SIZE) { status |= MC_OST_ERROR_SCRIPT; return status; } - +*/ ptr+=shift; if(pwalletTxsMain->m_ChunkDB->GetChunkDef(&chunk_def,ptr,NULL,NULL,-1) == MC_ERR_NOERROR) { @@ -1119,19 +1120,10 @@ string OffChainError(uint32_t status,int *errorCode) return error_str; } -Value OpReturnFormatEntry(const unsigned char *elem,int64_t elem_size,uint256 txid, int vout, uint32_t format, string *format_text_out,uint32_t status) +bool AvailableFromStatus(uint32_t status) { - string metadata=""; - Object metadata_object; - Value metadata_value; - bool available;//,offchain; - string status_str,error_str; - int errorCode; - int err; - + bool available; available=false; -// offchain=true; - status_str=""; if( status == MC_OST_UNDEFINED ) { @@ -1140,16 +1132,13 @@ Value OpReturnFormatEntry(const unsigned char *elem,int64_t elem_size,uint256 tx if( (status & MC_OST_STORAGE_MASK) == MC_OST_ON_CHAIN ) { - status_str="on-chain"; available=true; -// offchain=false; } if( (status & MC_OST_STORAGE_MASK) == MC_OST_OFF_CHAIN ) { if( (status & MC_OST_STATUS_MASK) == MC_OST_RETRIEVED ) { - status_str="retrieved"; available=true; } @@ -1159,6 +1148,21 @@ Value OpReturnFormatEntry(const unsigned char *elem,int64_t elem_size,uint256 tx } } + return available; +} + +Value OpReturnFormatEntry(const unsigned char *elem,int64_t elem_size,uint256 txid, int vout, uint32_t format, string *format_text_out,uint32_t status) +{ + string metadata=""; + Object metadata_object; + Value metadata_value; + bool available;//,offchain; + string error_str; + int errorCode; + int err; + + available=AvailableFromStatus(status); + if(status & MC_OST_ERROR_MASK) { error_str=OffChainError(status,&errorCode); @@ -1215,7 +1219,7 @@ Value OpReturnFormatEntry(const unsigned char *elem,int64_t elem_size,uint256 tx metadata_object.push_back(Pair("vout", vout)); metadata_object.push_back(Pair("format", OpReturnFormatToText(format))); metadata_object.push_back(Pair("size", elem_size)); -// metadata_object.push_back(Pair("offchain", offchain)); +/* if( ( status & MC_OST_CONTROL_NO_DATA ) == 0) { if(status & MC_OST_ERROR_MASK) @@ -1223,8 +1227,8 @@ Value OpReturnFormatEntry(const unsigned char *elem,int64_t elem_size,uint256 tx metadata_object.push_back(Pair("error", error_str)); } metadata_object.push_back(Pair("available", available)); -// metadata_object.push_back(Pair("status", status_str)); } + */ return metadata_object; } @@ -1396,7 +1400,20 @@ Value DataItemEntry(const CTransaction& tx,int n,set & already_seen,uin { entry.push_back(Pair("key", keys[0])); } - entry.push_back(Pair("data", format_item_value)); + entry.push_back(Pair("data", format_item_value)); + + if( ( retrieve_status & MC_OST_CONTROL_NO_DATA ) == 0) + { + if(retrieve_status & MC_OST_ERROR_MASK) + { + string error_str; + int errorCode; + error_str=OffChainError(retrieve_status,&errorCode); + entry.push_back(Pair("error", error_str)); + } + entry.push_back(Pair("available", AvailableFromStatus(retrieve_status))); + } + entry.push_back(Pair("offchain", (retrieve_status & MC_OST_STORAGE_MASK) == MC_OST_OFF_CHAIN)); if(retrieve_status & MC_OST_CONTROL_NO_DATA) { diff --git a/src/rpc/rpcutils.h b/src/rpc/rpcutils.h index 8c54bad5..14f83057 100644 --- a/src/rpc/rpcutils.h +++ b/src/rpc/rpcutils.h @@ -142,6 +142,7 @@ Value mc_ExtractDetailsJSONObject(const unsigned char *script,uint32_t total); void AppendOffChainFormatData(uint32_t data_format,uint32_t out_options,mc_Script *lpDetailsScript,vector& vValue,vector* vChunkHashes,int *errorCode,string *strError); int mc_BinaryCacheFile(string id,int mode); void mc_RemoveBinaryCacheFile(string id); +bool AvailableFromStatus(uint32_t status); string OffChainError(uint32_t status,int *errorCode); bool RawDataParseRestrictParameter(const Value& param,uint32_t *restrict,uint32_t *permissions,string *strError); diff --git a/src/rpc/rpcwalletutils.cpp b/src/rpc/rpcwalletutils.cpp index 77f87826..ef97ba49 100644 --- a/src/rpc/rpcwalletutils.cpp +++ b/src/rpc/rpcwalletutils.cpp @@ -603,6 +603,17 @@ Object StreamItemEntry(const CWalletTx& wtx,int first_output,const unsigned char entry.push_back(Pair("key", keys[0])); } entry.push_back(Pair("data", format_item_value)); + if( ( retrieve_status & MC_OST_CONTROL_NO_DATA ) == 0) + { + if(retrieve_status & MC_OST_ERROR_MASK) + { + string error_str; + int errorCode; + error_str=OffChainError(retrieve_status,&errorCode); + entry.push_back(Pair("error", error_str)); + } + entry.push_back(Pair("available", AvailableFromStatus(retrieve_status))); + } entry.push_back(Pair("offchain", (retrieve_status & MC_OST_STORAGE_MASK) == MC_OST_OFF_CHAIN)); if(verbose) From 07c3878a4c0e7ec16e110b997b36ebd823171a46 Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 6 Jun 2018 10:11:15 +0300 Subject: [PATCH 110/157] Help message for new impot rescan parameters --- src/rpc/rpchelp.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/rpc/rpchelp.cpp b/src/rpc/rpchelp.cpp index 2a5a7763..4c7b1bf8 100644 --- a/src/rpc/rpchelp.cpp +++ b/src/rpc/rpchelp.cpp @@ -2203,7 +2203,8 @@ void mc_InitRPCHelpMap10() " or\n" "1. address(es) (array, optional) A json array of addresses \n" "2. \"label\" (string, optional, default=\"\") An optional label\n" - "3. rescan (boolean, optional, default=true) Rescan the wallet for transactions\n" + "3. rescan (boolean or integer, optional, default=true) Rescan the wallet for transactions. \n" + " If integer rescan from block, if negative - from the end.\n" "\nNote: This call can take minutes to complete if rescan is true.\n" "\nResult:\n" "\nExamples:\n" @@ -2223,7 +2224,8 @@ void mc_InitRPCHelpMap10() " or\n" "1. privkey(s) (array, optional) A json array of private keys \n" "2. \"label\" (string, optional, default=\"\") An optional label\n" - "3. rescan (boolean, optional, default=true) Rescan the wallet for transactions\n" + "3. rescan (boolean or integer, optional, default=true) Rescan the wallet for transactions. \n" + " If integer rescan from block, if negative - from the end.\n" "\nNote: This call can take minutes to complete if rescan is true.\n" "\nResult:\n" "\nExamples:\n" @@ -2238,10 +2240,11 @@ void mc_InitRPCHelpMap10() )); mapHelpStrings.insert(std::make_pair("importwallet", - "importwallet \"filename\"\n" + "importwallet \"filename\" ( rescan )\n" "\nImports keys from a wallet dump file (see dumpwallet).\n" "\nArguments:\n" "1. \"filename\" (string, required) The wallet file\n" + "2. rescan (integer, optional, default=0) Rescan from block, if negative - from the end.\n" "\nExamples:\n" "\nDump the wallet\n" + HelpExampleCli("dumpwallet", "\"test\"") + From e5118f9b5bb80dd221c4b4c2cd75219e9d5bfd55 Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 6 Jun 2018 15:00:48 +0300 Subject: [PATCH 111/157] -rpcallowdebug --- src/rpc/rpcdebug.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/rpc/rpcdebug.cpp b/src/rpc/rpcdebug.cpp index 0231a4de..20bc773c 100644 --- a/src/rpc/rpcdebug.cpp +++ b/src/rpc/rpcdebug.cpp @@ -361,6 +361,11 @@ Value debug(const Array& params, bool fHelp) if (fHelp || params.size() < 2 || params.size() > 4) throw runtime_error("Help message not found\n"); + if(!GetBoolArg("-rpcallowdebug",false)) + { + throw JSONRPCError(RPC_NOT_ALLOWED, "API is not allowed"); + } + uint32_t request_type=MC_RMT_NONE; int timeout=5; vector payload; From 2a46ee69a9cfa68e081972dde1b0877b58268776 Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 6 Jun 2018 15:02:26 +0300 Subject: [PATCH 112/157] Moved available field --- src/rpc/rpcutils.cpp | 6 +++--- src/rpc/rpcwalletutils.cpp | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index 659a526b..80817d35 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -1400,10 +1400,9 @@ Value DataItemEntry(const CTransaction& tx,int n,set & already_seen,uin { entry.push_back(Pair("key", keys[0])); } - entry.push_back(Pair("data", format_item_value)); - if( ( retrieve_status & MC_OST_CONTROL_NO_DATA ) == 0) { + entry.push_back(Pair("available", AvailableFromStatus(retrieve_status))); if(retrieve_status & MC_OST_ERROR_MASK) { string error_str; @@ -1411,8 +1410,9 @@ Value DataItemEntry(const CTransaction& tx,int n,set & already_seen,uin error_str=OffChainError(retrieve_status,&errorCode); entry.push_back(Pair("error", error_str)); } - entry.push_back(Pair("available", AvailableFromStatus(retrieve_status))); } + entry.push_back(Pair("data", format_item_value)); + entry.push_back(Pair("offchain", (retrieve_status & MC_OST_STORAGE_MASK) == MC_OST_OFF_CHAIN)); if(retrieve_status & MC_OST_CONTROL_NO_DATA) diff --git a/src/rpc/rpcwalletutils.cpp b/src/rpc/rpcwalletutils.cpp index ef97ba49..3f287269 100644 --- a/src/rpc/rpcwalletutils.cpp +++ b/src/rpc/rpcwalletutils.cpp @@ -602,9 +602,9 @@ Object StreamItemEntry(const CWalletTx& wtx,int first_output,const unsigned char { entry.push_back(Pair("key", keys[0])); } - entry.push_back(Pair("data", format_item_value)); if( ( retrieve_status & MC_OST_CONTROL_NO_DATA ) == 0) { + entry.push_back(Pair("available", AvailableFromStatus(retrieve_status))); if(retrieve_status & MC_OST_ERROR_MASK) { string error_str; @@ -612,8 +612,8 @@ Object StreamItemEntry(const CWalletTx& wtx,int first_output,const unsigned char error_str=OffChainError(retrieve_status,&errorCode); entry.push_back(Pair("error", error_str)); } - entry.push_back(Pair("available", AvailableFromStatus(retrieve_status))); } + entry.push_back(Pair("data", format_item_value)); entry.push_back(Pair("offchain", (retrieve_status & MC_OST_STORAGE_MASK) == MC_OST_OFF_CHAIN)); if(verbose) From e418ab1f7a20822a4a7f0d36ba72fff7d3091716 Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 6 Jun 2018 15:54:51 +0300 Subject: [PATCH 113/157] Splitting query records to mitigate latency influence --- src/protocol/relay.cpp | 6 ++++++ src/wallet/chunkcollector.h | 1 + 2 files changed, 7 insertions(+) diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index aa028ab9..faac65a3 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -346,11 +346,17 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) } max_total_size=(collector->m_TimeoutRequest-MC_CCW_TIMEOUT_REQUEST_SHIFT)*MC_CCW_MAX_MBS_PER_SECOND*1024*1024; + max_total_size/=MC_CCW_QUERY_SPLIT; if(max_total_size > MAX_SIZE-OFFCHAIN_MSG_PADDING) { max_total_size=MAX_SIZE-OFFCHAIN_MSG_PADDING; } + if(max_total_size < MAX_CHUNK_SIZE + sizeof(mc_ChunkEntityKey)) + { + max_total_size = MAX_CHUNK_SIZE + sizeof(mc_ChunkEntityKey); + } + max_total_in_queries=2*(collector->m_TimeoutRequest)*MC_CCW_MAX_MBS_PER_SECOND*1024*1024; total_in_queries=0; query_count=0; diff --git a/src/wallet/chunkcollector.h b/src/wallet/chunkcollector.h index 644c48d8..ebf31619 100644 --- a/src/wallet/chunkcollector.h +++ b/src/wallet/chunkcollector.h @@ -28,6 +28,7 @@ #define MC_CCW_DEFAULT_MEMPOOL_SIZE 60000 #define MC_CCW_MAX_MBS_PER_SECOND 8 #define MC_CCW_MAX_DELAY_BETWEEN_COLLECTS 1000 +#define MC_CCW_QUERY_SPLIT 4 typedef struct mc_ChunkEntityKey From 69280ced4496d8a6726cd8b3b299f5a6c4dd701e Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 7 Jun 2018 09:01:48 +0300 Subject: [PATCH 114/157] Sending not more than one max-size chunk per query --- src/protocol/relay.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index faac65a3..45ff9a77 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -218,7 +218,7 @@ int MultichainResponseScore(mc_RelayResponse *response,mc_ChunkCollectorRow *col total_size=itdld->second; } - if(total_size + collect_row->m_ChunkDef.m_Size + sizeof(mc_ChunkEntityKey) >= max_total_size) + if(total_size + collect_row->m_ChunkDef.m_Size + sizeof(mc_ChunkEntityKey) > max_total_size) { return MC_CCW_WORST_RESPONSE_SCORE; } @@ -344,7 +344,8 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) MultichainProcessChunkResponse(&(item.first),&(item.second.m_Pairs),collector); pRelayManager->DeleteRequest(item.first.request_id); } - + +/* max_total_size=(collector->m_TimeoutRequest-MC_CCW_TIMEOUT_REQUEST_SHIFT)*MC_CCW_MAX_MBS_PER_SECOND*1024*1024; max_total_size/=MC_CCW_QUERY_SPLIT; if(max_total_size > MAX_SIZE-OFFCHAIN_MSG_PADDING) @@ -356,6 +357,8 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) { max_total_size = MAX_CHUNK_SIZE + sizeof(mc_ChunkEntityKey); } + */ + max_total_size=MAX_CHUNK_SIZE + sizeof(mc_ChunkEntityKey); max_total_in_queries=2*(collector->m_TimeoutRequest)*MC_CCW_MAX_MBS_PER_SECOND*1024*1024; total_in_queries=0; From 7b7baa400fafd9a91454e589338897b879ab4ca4 Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 7 Jun 2018 09:21:21 +0300 Subject: [PATCH 115/157] Fixed query/destination limits --- src/protocol/relay.cpp | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index 45ff9a77..c40e247e 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -264,7 +264,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) uint32_t time_now,expiration,dest_expiration; vector vChunkDefs; int row,last_row,last_count,to_end_of_query; - uint32_t total_size,max_total_size,total_in_queries,max_total_in_queries,query_count; + uint32_t total_size,max_total_query_size,max_total_destination_size,total_in_queries,max_total_in_queries,query_count; mc_ChunkCollectorRow *collect_row; mc_ChunkCollectorRow *collect_subrow; time_now=mc_TimeNowAsUInt(); @@ -345,20 +345,21 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) pRelayManager->DeleteRequest(item.first.request_id); } -/* - max_total_size=(collector->m_TimeoutRequest-MC_CCW_TIMEOUT_REQUEST_SHIFT)*MC_CCW_MAX_MBS_PER_SECOND*1024*1024; - max_total_size/=MC_CCW_QUERY_SPLIT; - if(max_total_size > MAX_SIZE-OFFCHAIN_MSG_PADDING) + + max_total_destination_size=(collector->m_TimeoutRequest-MC_CCW_TIMEOUT_REQUEST_SHIFT)*MC_CCW_MAX_MBS_PER_SECOND*1024*1024; +// max_total_size/=MC_CCW_QUERY_SPLIT; + if(max_total_destination_size > MAX_SIZE-OFFCHAIN_MSG_PADDING) { - max_total_size=MAX_SIZE-OFFCHAIN_MSG_PADDING; + max_total_destination_size=MAX_SIZE-OFFCHAIN_MSG_PADDING; } +/* if(max_total_size < MAX_CHUNK_SIZE + sizeof(mc_ChunkEntityKey)) { max_total_size = MAX_CHUNK_SIZE + sizeof(mc_ChunkEntityKey); } */ - max_total_size=MAX_CHUNK_SIZE + sizeof(mc_ChunkEntityKey); + max_total_query_size=MAX_CHUNK_SIZE + sizeof(mc_ChunkEntityKey); max_total_in_queries=2*(collector->m_TimeoutRequest)*MC_CCW_MAX_MBS_PER_SECOND*1024*1024; total_in_queries=0; @@ -431,7 +432,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) best_score=MC_CCW_WORST_RESPONSE_SCORE; for(int i=0;i<(int)query->m_Responses.size();i++) { - this_score=MultichainResponseScore(&(query->m_Responses[i]),collect_row,destination_loads,max_total_size); + this_score=MultichainResponseScore(&(query->m_Responses[i]),collect_row,destination_loads,max_total_destination_size); if(this_score < best_score) { best_score=this_score; @@ -520,7 +521,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) collect_subrow=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(chunk_row.first); collect_subrow->m_State.m_Request=request_id; collect_subrow->m_State.m_RequestTimeStamp=expiration; - for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Requested+=k ? collect_row->m_ChunkDef.m_Size : 1; + for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Requested+=k ? collect_subrow->m_ChunkDef.m_Size : 1; // printf("T %d %d %s\n",chunk_row.first,collect_subrow->m_State.m_RequestPos,collect_subrow->m_State.m_Request.ToString().c_str()); } } @@ -538,7 +539,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) collect_row=(mc_ChunkCollectorRow *)collector->m_MemPool->GetRow(row); } - if( (collect_row == NULL)|| (last_count >= MC_CCW_MAX_CHUNKS_PER_QUERY) || (total_size+collect_row->m_ChunkDef.m_Size + sizeof(mc_ChunkEntityKey)> max_total_size) ) + if( (collect_row == NULL)|| (last_count >= MC_CCW_MAX_CHUNKS_PER_QUERY) || (total_size+collect_row->m_ChunkDef.m_Size + sizeof(mc_ChunkEntityKey)> max_total_query_size) ) { if(last_count) { From 05fd65a49547074ddfdc73c83bc3afa3af5227fe Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 7 Jun 2018 09:59:08 +0300 Subject: [PATCH 116/157] Fixed number of simlultaneous queries --- src/protocol/relay.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index c40e247e..5facb707 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -348,12 +348,12 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) max_total_destination_size=(collector->m_TimeoutRequest-MC_CCW_TIMEOUT_REQUEST_SHIFT)*MC_CCW_MAX_MBS_PER_SECOND*1024*1024; // max_total_size/=MC_CCW_QUERY_SPLIT; +/* if(max_total_destination_size > MAX_SIZE-OFFCHAIN_MSG_PADDING) { max_total_destination_size=MAX_SIZE-OFFCHAIN_MSG_PADDING; } -/* if(max_total_size < MAX_CHUNK_SIZE + sizeof(mc_ChunkEntityKey)) { max_total_size = MAX_CHUNK_SIZE + sizeof(mc_ChunkEntityKey); @@ -361,7 +361,8 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) */ max_total_query_size=MAX_CHUNK_SIZE + sizeof(mc_ChunkEntityKey); - max_total_in_queries=2*(collector->m_TimeoutRequest)*MC_CCW_MAX_MBS_PER_SECOND*1024*1024; + max_total_in_queries=(collector->m_TimeoutRequest-MC_CCW_TIMEOUT_REQUEST_SHIFT)*MC_CCW_MAX_MBS_PER_SECOND*1024*1024; + max_total_in_queries*=max_total_destination_size; total_in_queries=0; query_count=0; From 4dd3e536a9f4f599168bf7fef582920fa297c03e Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 7 Jun 2018 10:09:09 +0300 Subject: [PATCH 117/157] Fixed query limit calculation --- src/protocol/relay.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index 5facb707..65b7c293 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -134,19 +134,19 @@ int MultichainProcessChunkResponse(const CRelayResponsePair *response_pair,map < ptrOut+=size; if(chunk->m_Size != chunkOut->m_Size) { - for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Baddelivered+=k ? collect_row->m_ChunkDef.m_Size : 1; + for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Baddelivered+=k ? chunk->m_Size : 1; strError="Chunk info size mismatch"; goto exitlbl; } if(memcmp(chunk->m_Hash,chunkOut->m_Hash,sizeof(uint256))) { - for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Baddelivered+=k ? collect_row->m_ChunkDef.m_Size : 1; + for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Baddelivered+=k ? chunk->m_Size : 1; strError="Chunk info hash mismatch"; goto exitlbl; } if(memcmp(&(chunk->m_Entity),&(chunkOut->m_Entity),sizeof(mc_TxEntity))) { - for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Baddelivered+=k ? collect_row->m_ChunkDef.m_Size : 1; + for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Baddelivered+=k ? chunk->m_Size : 1; strError="Chunk info entity mismatch"; goto exitlbl; } @@ -362,7 +362,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) max_total_query_size=MAX_CHUNK_SIZE + sizeof(mc_ChunkEntityKey); max_total_in_queries=(collector->m_TimeoutRequest-MC_CCW_TIMEOUT_REQUEST_SHIFT)*MC_CCW_MAX_MBS_PER_SECOND*1024*1024; - max_total_in_queries*=max_total_destination_size; + max_total_in_queries*=collector->m_TimeoutRequest; total_in_queries=0; query_count=0; From fef5f6d3968c47d7ffcfc98bc2ad6e0095f6d2f5 Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 7 Jun 2018 15:16:32 +0300 Subject: [PATCH 118/157] Fixed JSON rounding issue --- src/chainparams/globals.h | 1 + src/core/init.cpp | 3 + src/json/json_spirit_writer_template.h | 167 +++++++++++++++++++++++-- src/multichain/multichain-cli.cpp | 2 + 4 files changed, 164 insertions(+), 9 deletions(-) diff --git a/src/chainparams/globals.h b/src/chainparams/globals.h index ff0d52bf..b0cdc6a9 100644 --- a/src/chainparams/globals.h +++ b/src/chainparams/globals.h @@ -26,6 +26,7 @@ int MAX_OP_RETURN_SHOWN=16384; int MAX_FORMATTED_DATA_DEPTH=100; unsigned int MAX_OP_RETURN_OP_DROP_COUNT=100000000; uint32_t JSON_NO_DOUBLE_FORMATTING=0; +int JSON_DOUBLE_DECIMAL_DIGITS=-1; int MAX_CHUNK_SIZE = 1048576; int MAX_CHUNK_COUNT = 1024; diff --git a/src/core/init.cpp b/src/core/init.cpp index 4b9ffc04..3baf51a4 100644 --- a/src/core/init.cpp +++ b/src/core/init.cpp @@ -67,6 +67,7 @@ mc_WalletTxs* pwalletTxsMain = NULL; #endif mc_RelayManager* pRelayManager = NULL; bool fFeeEstimatesInitialized = false; +extern int JSON_DOUBLE_DECIMAL_DIGITS; #ifdef WIN32 // Win32 LevelDB doesn't use filedescriptors, and the ones used for @@ -505,6 +506,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += " -maxshowndata= " + strprintf(_("The maximum number of bytes to show in the data field of API responses. (default: %u)"), MAX_OP_RETURN_SHOWN) + "\n"; strUsage += " " + _("Pieces of data larger than this will be returned as an object with txid, vout and size fields, for use with the gettxoutdata command.") + "\n"; strUsage += " -v1apicompatible= " + strprintf(_("JSON_RPC calls responses compatible with MultiChain 1.0 (default: %u)"), 0) + "\n"; + strUsage += " -apidecimaldigits= " + _("maximal number of decimal digits in API output (default: auto)") + "\n"; strUsage += "\n" + _("Wallet optimization options:") + "\n"; strUsage += " -autocombineminconf " + _("Only automatically combine outputs with at least this number of confirmations, default 1") + "\n"; @@ -1762,6 +1764,7 @@ bool AppInit2(boost::thread_group& threadGroup,int OutputPipe) if (fServer) { + JSON_DOUBLE_DECIMAL_DIGITS=GetArg("-apidecimaldigits",-1); uiInterface.InitMessage.connect(SetRPCWarmupStatus); StartRPCThreads(); } diff --git a/src/json/json_spirit_writer_template.h b/src/json/json_spirit_writer_template.h index af410317..e15c13f9 100644 --- a/src/json/json_spirit_writer_template.h +++ b/src/json/json_spirit_writer_template.h @@ -13,6 +13,7 @@ #include extern uint32_t JSON_NO_DOUBLE_FORMATTING; +extern int JSON_DOUBLE_DECIMAL_DIGITS; namespace json_spirit @@ -237,8 +238,114 @@ namespace json_spirit os_ << value.get_int64(); } } - + + int output_double_precision( const double& value ) + { + int p=14; + if(JSON_DOUBLE_DECIMAL_DIGITS >= 0) + { + p=JSON_DOUBLE_DECIMAL_DIGITS; + } + + char fp[20]; + char sp[40]; + sprintf(fp,"%%0.%df",p); + sprintf(sp,fp,value); + char *tp=sp+strlen(sp)-1; + while( (tp>sp) && (p>=0) && *tp=='0') + { + p--; + tp--; + } +// printf("%0.16f %s %s %c %d %d\n",value,fp,sp,*tp,(tp-sp),p); + if( (tp == sp) || (*tp == '.') ) + { + p=0; + } + return p; + } + void output_double( const double& value ) + { + if(JSON_NO_DOUBLE_FORMATTING) + { + int p=14; + if(JSON_DOUBLE_DECIMAL_DIGITS >= 0) + { + p=JSON_DOUBLE_DECIMAL_DIGITS; + } + os_ << std::showpoint << std::fixed << std::setprecision(p) << value; + return; + } + double a=fabs(value); + double e=0.0; + int z=0; + double f=0.; + int j=0; + if(a > 0) + { + e=log10(a); + } + if(e < -4) + { + f=a*1.e+9; + j=(int)f; + if(j) + { + if( (f-j) < 0.0001) + { + z=1; + } + } + } + int k=(int)e; + if(e 9) + { + z=0; + } + + if( ((e < -4.) || (e > 12.)) && (z == 0)) + { + if(p > 0) + { + os_ << std::showpoint << std::fixed << std::setprecision(p) << v; + } + else + { + os_ << (int)v; + } + os_ << "e" << ((e>=0) ? "+" : "") << k; + } + else + { + int pfull=output_double_precision(value); + if(pfull + k <= 14) + { + p=pfull; + } + else + { + p-=k; + } + if(p > 0) + { + os_ << std::showpoint << std::fixed << std::setprecision(p) << value; + } + else + { + os_ << (int64_t)value; + } + } + } + + void output_double_old( const double& value ) { if(JSON_NO_DOUBLE_FORMATTING) { @@ -277,19 +384,61 @@ namespace json_spirit double t=fabs(v)*pow(10.,p); int64_t n=(int64_t)(t+0.5); int64_t m=(int64_t)(t/10+0.5); - while( (p>0) && (n == m*10)) + + if(JSON_DOUBLE_DECIMAL_DIGITS < 0) { - p--; - t/=10.; - n=m; - m=(int64_t)(t/10.+0.5); + while( (p>0) && (n == m*10)) + { + p--; + t/=10.; + n=m; + m=(int64_t)(t/10.+0.5); + } + if(p-k > 9) + { + z=0; + } } - - if(p-k > 9) + else { - z=0; + p=JSON_DOUBLE_DECIMAL_DIGITS; + if(p>16) + { + p=16; + } + if(p-k > 9) + { + z=0; + } + if(p) + { + double tv=value; + int ap=k; + if( ((e < -4.) || (e > 12.)) && (z == 0)) + { + tv=v; + ap=0; + } + char fp[20]; + char sp[40]; + sprintf(fp,"%%0.%df",p); + sprintf(sp,fp,tv); + char *tp=sp+strlen(sp)-1; + while( (tp>sp) && (p>=0) && *tp=='0') + { + p--; + tp--; + } + printf("%d %s %s %c %d %d\n",k,fp,sp,*tp,(tp-sp),p); + if( (tp == sp) || (*tp == '.') ) + { + p=0; + } + p+=ap; + } } + if( ((e < -4.) || (e > 12.)) && (z == 0)) { if(p > 0) diff --git a/src/multichain/multichain-cli.cpp b/src/multichain/multichain-cli.cpp index 22b0fb19..e97ef96a 100644 --- a/src/multichain/multichain-cli.cpp +++ b/src/multichain/multichain-cli.cpp @@ -28,6 +28,7 @@ using namespace json_spirit; static const int CONTINUE_EXECUTION=-1; extern unsigned int JSON_NO_DOUBLE_FORMATTING; +extern int JSON_DOUBLE_DECIMAL_DIGITS; std::string HelpMessageCli() { @@ -242,6 +243,7 @@ Object CallRPC(const string& strMethod, const Array& params) JSON_NO_DOUBLE_FORMATTING=1; string strRequest = JSONRPCRequest(strMethod, params, 1); JSON_NO_DOUBLE_FORMATTING=0; + JSON_DOUBLE_DECIMAL_DIGITS=GetArg("-apidecimaldigits",-1); string strPost = HTTPPost(strRequest, mapRequestHeaders); stream << strPost << std::flush; From c9a15cec3d637c35d93597baef361cb741edd11e Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 10 Jun 2018 10:03:10 +0300 Subject: [PATCH 119/157] Fixed available calculation in getstreamkey/publisher summary --- src/rpc/rpcstreams.cpp | 44 +++++++++++------------------------------- 1 file changed, 11 insertions(+), 33 deletions(-) diff --git a/src/rpc/rpcstreams.cpp b/src/rpc/rpcstreams.cpp index a11bcb99..c07de87f 100644 --- a/src/rpc/rpcstreams.cpp +++ b/src/rpc/rpcstreams.cpp @@ -1490,49 +1490,27 @@ Value getstreamsummary(const Array& params, bool fPublisher) available=!a.value_.get_bool(); } } + + if(!available) + { + BOOST_FOREACH(const Pair& a, entry) + { + if(a.name_ == "available") + { + available=a.value_.get_bool(); + } + } + } BOOST_FOREACH(const Pair& a, entry) { if(a.name_ == "data") { - if(!available) - { - if(a.value_.type() != obj_type) - { - available=true; - } - } - if(!available) - { - available=true; - BOOST_FOREACH(const Pair& b, a.value_.get_obj()) - { - if(b.name_ == "available") - { - available=b.value_.get_bool(); - } - } - } if(available) { if(i == 0) { - // if(a.value_.type() == obj_type) - /* - { - result=empty_object; - } - - if( (i==0) && ((mode & MC_VMM_TAKE_FIRST) != 0) ) - { - result=mc_MergeValues(&(a.value_),&result,mode,0,&err); - } - else - { - result=mc_MergeValues(&result,&(a.value_),mode,0,&err); - } - */ result=a.value_; } else From 81d0417872cb8e100ef24bf5157cd84b208c2da9 Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 10 Jun 2018 10:03:31 +0300 Subject: [PATCH 120/157] Fixed floating point rounding --- src/core/init.cpp | 2 +- src/json/json_spirit_writer_template.h | 29 ++++++++++++-------------- src/multichain/multichain-cli.cpp | 4 ++-- 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/src/core/init.cpp b/src/core/init.cpp index 3baf51a4..baf43e92 100644 --- a/src/core/init.cpp +++ b/src/core/init.cpp @@ -506,7 +506,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += " -maxshowndata= " + strprintf(_("The maximum number of bytes to show in the data field of API responses. (default: %u)"), MAX_OP_RETURN_SHOWN) + "\n"; strUsage += " " + _("Pieces of data larger than this will be returned as an object with txid, vout and size fields, for use with the gettxoutdata command.") + "\n"; strUsage += " -v1apicompatible= " + strprintf(_("JSON_RPC calls responses compatible with MultiChain 1.0 (default: %u)"), 0) + "\n"; - strUsage += " -apidecimaldigits= " + _("maximal number of decimal digits in API output (default: auto)") + "\n"; +// strUsage += " -apidecimaldigits= " + _("maximal number of decimal digits in API output (default: auto)") + "\n"; strUsage += "\n" + _("Wallet optimization options:") + "\n"; strUsage += " -autocombineminconf " + _("Only automatically combine outputs with at least this number of confirmations, default 1") + "\n"; diff --git a/src/json/json_spirit_writer_template.h b/src/json/json_spirit_writer_template.h index e15c13f9..4e37ae31 100644 --- a/src/json/json_spirit_writer_template.h +++ b/src/json/json_spirit_writer_template.h @@ -239,13 +239,9 @@ namespace json_spirit } } - int output_double_precision( const double& value ) + int output_double_precision( const double& value, int max_p ) { - int p=14; - if(JSON_DOUBLE_DECIMAL_DIGITS >= 0) - { - p=JSON_DOUBLE_DECIMAL_DIGITS; - } + int p=max_p; char fp[20]; char sp[40]; @@ -267,14 +263,15 @@ namespace json_spirit void output_double( const double& value ) { + int max_p=14; + if(JSON_DOUBLE_DECIMAL_DIGITS >= 0) + { + max_p=JSON_DOUBLE_DECIMAL_DIGITS; + } + if(JSON_NO_DOUBLE_FORMATTING) { - int p=14; - if(JSON_DOUBLE_DECIMAL_DIGITS >= 0) - { - p=JSON_DOUBLE_DECIMAL_DIGITS; - } - os_ << std::showpoint << std::fixed << std::setprecision(p) << value; + os_ << std::showpoint << std::fixed << std::setprecision(max_p) << value; return; } double a=fabs(value); @@ -305,8 +302,8 @@ namespace json_spirit } double v=value/pow(10.,k); - int p=output_double_precision(v); - if(p-k > 9) + int p=output_double_precision(v,max_p); + if(p-k > max_p) { z=0; } @@ -325,8 +322,8 @@ namespace json_spirit } else { - int pfull=output_double_precision(value); - if(pfull + k <= 14) + int pfull=output_double_precision(value,max_p); + if(pfull + k <= max_p) { p=pfull; } diff --git a/src/multichain/multichain-cli.cpp b/src/multichain/multichain-cli.cpp index e97ef96a..221f23e2 100644 --- a/src/multichain/multichain-cli.cpp +++ b/src/multichain/multichain-cli.cpp @@ -240,9 +240,9 @@ Object CallRPC(const string& strMethod, const Array& params) map mapRequestHeaders; mapRequestHeaders["Authorization"] = string("Basic ") + strUserPass64; // Send request - JSON_NO_DOUBLE_FORMATTING=1; +// JSON_NO_DOUBLE_FORMATTING=1; string strRequest = JSONRPCRequest(strMethod, params, 1); - JSON_NO_DOUBLE_FORMATTING=0; +// JSON_NO_DOUBLE_FORMATTING=0; JSON_DOUBLE_DECIMAL_DIGITS=GetArg("-apidecimaldigits",-1); string strPost = HTTPPost(strRequest, mapRequestHeaders); stream << strPost << std::flush; From df55efa2553a086b223f1400e63892fffa20e004 Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 10 Jun 2018 13:40:02 +0300 Subject: [PATCH 121/157] Allowed write permissions for streams with open=true --- src/entities/asset.cpp | 9 +++++++-- src/entities/asset.h | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/entities/asset.cpp b/src/entities/asset.cpp index 7aa14196..1f75f5b4 100644 --- a/src/entities/asset.cpp +++ b/src/entities/asset.cpp @@ -539,6 +539,8 @@ void mc_EntityDetails::Set(mc_EntityLedgerRow* row) m_Permissions=0; m_Restrictions=0; + m_ScriptPermissions=0; + switch(m_LedgerRow.m_EntityType) { case MC_ENT_TYPE_ASSET: @@ -598,14 +600,17 @@ void mc_EntityDetails::Set(mc_EntityLedgerRow* row) value_offset=mc_FindSpecialParamInDetailsScript(m_LedgerRow.m_Script,m_LedgerRow.m_ScriptSize,MC_ENT_SPRM_PERMISSIONS,&value_size); if(value_offset <= m_LedgerRow.m_ScriptSize) { +/* if(m_Permissions & MC_PTP_WRITE) { m_Permissions -= MC_PTP_WRITE; } + */ m_Permissions |= MC_PTP_SPECIFIED; if((value_size>0) && (value_size<=4)) { - m_Permissions |= (uint32_t)mc_GetLE(m_LedgerRow.m_Script+value_offset,value_size); + m_ScriptPermissions=(uint32_t)mc_GetLE(m_LedgerRow.m_Script+value_offset,value_size); + m_Permissions |= m_ScriptPermissions; } } @@ -1778,7 +1783,7 @@ int mc_EntityDetails::AnyoneCanWrite() { if(mc_gState->m_Features->OffChainData()) { - if(m_Permissions & MC_PTP_WRITE) + if(m_ScriptPermissions & MC_PTP_WRITE) { return 0; } diff --git a/src/entities/asset.h b/src/entities/asset.h index c4d2a0e5..0c512cc5 100644 --- a/src/entities/asset.h +++ b/src/entities/asset.h @@ -155,6 +155,7 @@ typedef struct mc_EntityDetails char m_Name[MC_ENT_MAX_NAME_SIZE+6]; // Entity name uint32_t m_Flags; uint32_t m_Permissions; + uint32_t m_ScriptPermissions; uint32_t m_Restrictions; unsigned char m_Reserved[36]; mc_EntityLedgerRow m_LedgerRow; From 23b2e5eb0cb952ae4ed4cbb4a6168b26190550f9 Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 10 Jun 2018 13:40:33 +0300 Subject: [PATCH 122/157] onchain/offchain instead of onchainonly/offchainonly --- src/rpc/rpchelp.cpp | 6 +++--- src/rpc/rpcrawdata.cpp | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/rpc/rpchelp.cpp b/src/rpc/rpchelp.cpp index 4c7b1bf8..639eec72 100644 --- a/src/rpc/rpchelp.cpp +++ b/src/rpc/rpchelp.cpp @@ -1396,7 +1396,7 @@ void mc_InitRPCHelpMap06() " or \n" "3. restrictions (object, optional) Stream restrictions\n" " {\n" - " \"restrict\" : \"restrictions\" (string, optional) Stream restrictions, comma delimited. Possible values: write,offchainonly,onchainonly\n" + " \"restrict\" : \"restrictions\" (string, optional) Stream restrictions, comma delimited. Possible values: write,offchain,onchain\n" " }\n" "4 custom-fields (object, optional) a json object with custom fields\n" " {\n" @@ -1443,7 +1443,7 @@ void mc_InitRPCHelpMap06() " or \n" "4. restrictions (object, optional) Stream restrictions\n" " {\n" - " \"restrict\" : \"restrictions\" (string, optional) Stream restrictions, comma delimited. Possible values: write,offchainonly,onchainonly\n" + " \"restrict\" : \"restrictions\" (string, optional) Stream restrictions, comma delimited. Possible values: write,offchain,onchain\n" " }\n" "5 custom-fields (object, optional) a json object with custom fields\n" " {\n" @@ -3770,7 +3770,7 @@ void mc_InitRPCHelpMap16() " \"create\" : \"stream\" (string, required) stream\n" " \"name\" : \"stream-name\" (string, optional) Stream name\n" " \"open\" : true|false (boolean, optional, default: false) If true, anyone can publish\n" - " \"restrict\" : \"restrictions\" (string, optional) Stream restrictions, comma delimited. Possible values: write,offchainonly,onchainonly\n" + " \"restrict\" : \"restrictions\" (string, optional) Stream restrictions, comma delimited. Possible values: write,offchain,onchain\n" " \"details\" : (object, optional) A json object with custom fields\n" " {\n" " \"param-name\": \"param-value\" (strings, required) The key is the parameter name, the value is parameter value\n" diff --git a/src/rpc/rpcrawdata.cpp b/src/rpc/rpcrawdata.cpp index 4008e90b..87ced8fb 100644 --- a/src/rpc/rpcrawdata.cpp +++ b/src/rpc/rpcrawdata.cpp @@ -741,8 +741,8 @@ bool RawDataParseRestrictParameter(const Value& param,uint32_t *restrict,uint32_ { match=0; if(( (ptr-start) == 5) && (memcmp(start,"write", ptr-start) == 0) ){match = 1; *permissions |= MC_PTP_WRITE ;} - if(( (ptr-start) == 11) && (memcmp(start,"onchainonly", ptr-start) == 0) ){match = 1; *restrict |= MC_ENT_ENTITY_RESTRICTION_ONCHAIN;} - if(( (ptr-start) == 12) && (memcmp(start,"offchainonly", ptr-start) == 0) ){match = 1; *restrict |= MC_ENT_ENTITY_RESTRICTION_OFFCHAIN;} + if(( (ptr-start) == 7) && (memcmp(start,"onchain", ptr-start) == 0) ){match = 1; *restrict |= MC_ENT_ENTITY_RESTRICTION_ONCHAIN;} + if(( (ptr-start) == 8) && (memcmp(start,"offchain", ptr-start) == 0) ){match = 1; *restrict |= MC_ENT_ENTITY_RESTRICTION_OFFCHAIN;} if(match == 0) { From 34343494e4ceb4eec4537d07f3618b12d7abdbaf Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 10 Jun 2018 13:41:02 +0300 Subject: [PATCH 123/157] Updated versions +10010,10011 --- src/version/version.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version/version.cpp b/src/version/version.cpp index e8749e24..8f8f2d9c 100644 --- a/src/version/version.cpp +++ b/src/version/version.cpp @@ -41,7 +41,7 @@ int mc_State::VersionInfo(int version) } if(version < 10002)return 10002; // first version if(version < 10004)return -10000201; // last build supporting this version (negative) - if(version < 10010)return -this_build; // supported by this version + if(version < 10012)return -this_build; // supported by this version if(version < 20001)return 20001; // next version if(version < this_protocol+1)return -this_build; // supported by this version From 1be69d83b82316c6f617ca291df6363fcac874dd Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 10 Jun 2018 13:52:32 +0300 Subject: [PATCH 124/157] getchunkqueuetotals/getchunkqueueinfo tweaks --- src/rpc/rpcchunks.cpp | 8 ++++---- src/rpc/rpchelp.cpp | 8 ++++---- src/rpc/rpclist.cpp | 2 +- src/rpc/rpcserver.h | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/rpc/rpcchunks.cpp b/src/rpc/rpcchunks.cpp index f990c46a..d7c0a6eb 100644 --- a/src/rpc/rpcchunks.cpp +++ b/src/rpc/rpcchunks.cpp @@ -18,10 +18,10 @@ Value getchunkqueueinfo(const Array& params, bool fHelp) for(int k=0;k<2;k++) { Object stat; - stat.push_back(Pair("pending",collector->m_StatLast[k].m_Pending)); + stat.push_back(Pair("waiting",collector->m_StatLast[k].m_Pending-collector->m_StatLast[k].m_Queried)); // stat.push_back(Pair("processing",collector->m_StatLast[k].m_Undelivered)); - stat.push_back(Pair("queried",collector->m_StatLast[k].m_Queried)); - stat.push_back(Pair("requested",collector->m_StatLast[k].m_Requested)); + stat.push_back(Pair("querying",collector->m_StatLast[k].m_Queried-collector->m_StatLast[k].m_Requested)); + stat.push_back(Pair("retrieving",collector->m_StatLast[k].m_Requested)); result.push_back(Pair(k ? "bytes" : "chunks",stat)); } @@ -30,7 +30,7 @@ Value getchunkqueueinfo(const Array& params, bool fHelp) return result; } -Value getchunktotals(const Array& params, bool fHelp) +Value getchunkqueuetotals(const Array& params, bool fHelp) { Object result; mc_ChunkCollector *collector=pwalletTxsMain->m_ChunkCollector; diff --git a/src/rpc/rpchelp.cpp b/src/rpc/rpchelp.cpp index 639eec72..398a7622 100644 --- a/src/rpc/rpchelp.cpp +++ b/src/rpc/rpchelp.cpp @@ -4108,12 +4108,12 @@ void mc_InitRPCHelpMap17() void mc_InitRPCHelpMap18() { - mapHelpStrings.insert(std::make_pair("getchunktotals", - "getchunktotals\n" + mapHelpStrings.insert(std::make_pair("getchunkqueuetotals", + "getchunkqueuetotals\n" "\nReturns chunks delivery statistics.\n" "\nExamples:\n" - + HelpExampleCli("getchunktotals", "") - + HelpExampleRpc("getchunktotals", "") + + HelpExampleCli("getchunkqueuetotals", "") + + HelpExampleRpc("getchunkqueuetotals", "") )); diff --git a/src/rpc/rpclist.cpp b/src/rpc/rpclist.cpp index 3464ff0c..abffe3ff 100644 --- a/src/rpc/rpclist.cpp +++ b/src/rpc/rpclist.cpp @@ -71,7 +71,7 @@ static const CRPCCommand vRPCCommands[] = { "network", "getpeerinfo", &getpeerinfo, true, false, false }, { "network", "ping", &ping, true, false, false }, { "network", "getchunkqueueinfo", &getchunkqueueinfo, true, true, true }, - { "network", "getchunktotals", &getchunktotals, true, true, true }, + { "network", "getchunkqueuetotals", &getchunkqueuetotals, true, true, true }, /* Block chain and UTXO */ { "blockchain", "getblockchaininfo", &getblockchaininfo, true, false, false }, diff --git a/src/rpc/rpcserver.h b/src/rpc/rpcserver.h index 2804c4fc..d53a66cc 100644 --- a/src/rpc/rpcserver.h +++ b/src/rpc/rpcserver.h @@ -194,7 +194,7 @@ extern json_spirit::Value sendtoaddress(const json_spirit::Array& params, bool f /* MCHN START */ extern json_spirit::Value debug(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value getchunkqueueinfo(const json_spirit::Array& params, bool fHelp); -extern json_spirit::Value getchunktotals(const json_spirit::Array& params, bool fHelp); +extern json_spirit::Value getchunkqueuetotals(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value createkeypairs(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value getaddresses(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value createbinarycache(const json_spirit::Array& params, bool fHelp); From f79f0dd41b7a4d53a40ffb5263688e8bddb3d62f Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 10 Jun 2018 15:25:14 +0300 Subject: [PATCH 125/157] Setting max-std-element-size for old versions --- src/chainparams/params.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/chainparams/params.cpp b/src/chainparams/params.cpp index 49c604f5..52d5b522 100644 --- a/src/chainparams/params.cpp +++ b/src/chainparams/params.cpp @@ -418,6 +418,7 @@ int mc_MultichainParams::Create(const char* name,int version) mc_OneMultichainParam *param; char *ptrData; int num_sets; + int64_t override_int64; uint32_t network_port=MC_DEFAULT_NETWORK_PORT; uint32_t rpc_port=MC_DEFAULT_RPC_PORT; @@ -496,6 +497,14 @@ int mc_MultichainParams::Create(const char* name,int version) case MC_PRM_INT64: size=8; mc_PutLE(ptrData,&(param->m_DefaultIntegerValue),8); + if(strcmp(param->m_Name,"maxstdelementsize") == 0) + { + if(version<20003) + { + override_int64=8192; + mc_PutLE(ptrData,&override_int64,8); + } + } break; case MC_PRM_DOUBLE: size=8; From 3d1c3d9fe94669b1618daeb0a19c749e516eae9b Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 11 Jun 2018 14:44:08 +0300 Subject: [PATCH 126/157] Fixed stopping source recovery scan --- src/wallet/chunkdb.cpp | 6 +++--- src/wallet/chunkdb.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/wallet/chunkdb.cpp b/src/wallet/chunkdb.cpp index 3cd5a601..107d5fb4 100644 --- a/src/wallet/chunkdb.cpp +++ b/src/wallet/chunkdb.cpp @@ -431,7 +431,7 @@ int mc_ChunkDB::SourceChunksRecovery() subscription->m_LastFileSize=read_offset+buf_offset; chunk_def.m_HeaderSize=subscription->m_LastFileSize-chunk_def.m_InternalFileOffset-chunk_def.m_Size; offset=buf_size; - read_offset=file_size; + file_offset=file_size; break; } buf_offset=offset; @@ -451,13 +451,13 @@ int mc_ChunkDB::SourceChunksRecovery() if(err) { buf_offset=buf_size; - read_offset=file_size; + file_offset=file_size; } } } else { - read_offset=file_size; + file_offset=file_size; } } diff --git a/src/wallet/chunkdb.h b/src/wallet/chunkdb.h index 4ba74ee4..2d95f973 100644 --- a/src/wallet/chunkdb.h +++ b/src/wallet/chunkdb.h @@ -18,7 +18,7 @@ #define MC_CDB_MAX_FILE_SIZE 0x08000000 // Maximal data file size, 1GB #define MC_CDB_MAX_CHUNK_DATA_POOL_SIZE 0x8000000 // Maximal size of chunk pool before commit, 128MB -#define MC_CDB_MAX_FILE_READ_BUFFER_SIZE 0x0100000 // Maximal size of chunk pool before commit, 4MB +#define MC_CDB_MAX_FILE_READ_BUFFER_SIZE 0x0100000 // Maximal size of chunk pool before commit, 1MB #define MC_CDB_MAX_CHUNK_EXTRA_SIZE 1024 #define MC_CDB_MAX_MEMPOOL_SIZE 1024 From d4fb0d717b2231fd0e567adf2d1f82a75172a739 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 11 Jun 2018 16:56:26 +0300 Subject: [PATCH 127/157] Fixed source recovery --- src/wallet/chunkdb.cpp | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/wallet/chunkdb.cpp b/src/wallet/chunkdb.cpp index 107d5fb4..04649e88 100644 --- a/src/wallet/chunkdb.cpp +++ b/src/wallet/chunkdb.cpp @@ -301,6 +301,7 @@ int mc_ChunkDB::SourceChunksRecovery() int FileHan; unsigned char* buf; int chunk_found=0; + char msg[256]; subscription=(mc_SubscriptionDBRow *)m_Subscriptions->GetRow(1); @@ -325,6 +326,11 @@ int mc_ChunkDB::SourceChunksRecovery() chunk_def.Zero(); chunk_def.m_SubscriptionID=subscription->m_SubscriptionID; + err=0; + + sprintf(msg,"Starting source recovery, last file: %d, file_size: %ld, file offset: %ld",subscription->m_LastFileID,file_size,file_offset); + LogString(msg); + while(file_offsetm_LastFileSize; subscription->m_LastFileSize=read_offset+buf_offset; chunk_def.m_HeaderSize=subscription->m_LastFileSize-chunk_def.m_InternalFileOffset-chunk_def.m_Size; + sprintf(msg,"Found end marker at: %ld",read_offset+buf_offset); + LogString(msg); offset=buf_size; file_offset=file_size; break; @@ -450,6 +458,8 @@ int mc_ChunkDB::SourceChunksRecovery() } if(err) { + sprintf(msg,"Error %d on file offset %ld",err,read_offset+buf_offset); + LogString(msg); buf_offset=buf_size; file_offset=file_size; } @@ -457,6 +467,8 @@ int mc_ChunkDB::SourceChunksRecovery() } else { + sprintf(msg,"Read error %d on file offset %ld",err,read_offset+buf_offset); + LogString(msg); file_offset=file_size; } } @@ -501,6 +513,23 @@ int mc_ChunkDB::SourceChunksRecovery() } } + if(err) + { + sprintf(msg,"Source recovery completed with error %d",err); + } + else + { + if(chunk_found) + { + sprintf(msg,"Source recovery completed, chunks recovered"); + } + else + { + sprintf(msg,"Source recovery completed, no recovery needed"); + } + } + LogString(msg); + Dump("SourceChunkRecovery"); return err; } From 54f33bdce3965f924cc7c23e192bafc596996799 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 11 Jun 2018 17:32:27 +0300 Subject: [PATCH 128/157] Deleting file on Windows --- src/rpc/rpcutils.cpp | 2 +- src/utils/declare.h | 1 + src/utils/systemdependent.cpp | 11 +++++++++++ src/utils/utilwrapper.cpp | 2 +- 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index 80817d35..466839f8 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -4205,6 +4205,6 @@ void mc_RemoveBinaryCacheFile(string id) mc_GetFullFileName(mc_gState->m_Params->NetworkName(),str_file_name.c_str(),"",MC_FOM_RELATIVE_TO_DATADIR,file_name); - unlink(file_name); + __US_DeleteFile(file_name); } diff --git a/src/utils/declare.h b/src/utils/declare.h index 7ea6e354..e379edfe 100644 --- a/src/utils/declare.h +++ b/src/utils/declare.h @@ -285,6 +285,7 @@ const char* __US_UserHomeDir(); char * __US_FullPath(const char* path, char *full_path, int len); void __US_FlushFile(int FileHan); void __US_FlushFileWithMode(int FileHan,uint32_t use_data_sync); +int __US_DeleteFile(const char *file_name); void sprintf_hex(char *hex,const unsigned char *bin,int size); diff --git a/src/utils/systemdependent.cpp b/src/utils/systemdependent.cpp index c9d4c7bb..27933160 100644 --- a/src/utils/systemdependent.cpp +++ b/src/utils/systemdependent.cpp @@ -241,6 +241,11 @@ void __US_FlushFileWithMode(int FileHan,uint32_t use_data_sync) } } +int __US_DeleteFile(const char *file_name) +{ + return unlink(file_name); +} + #else #include "windows.h" @@ -322,4 +327,10 @@ void __US_FlushFileWithMode(int FileHan,uint32_t use_data_sync) FlushFileBuffers(hFile); } +int __US_DeleteFile(const char *file_name) +{ + return (int)DeleteFile(file_name); +} + + #endif diff --git a/src/utils/utilwrapper.cpp b/src/utils/utilwrapper.cpp index 7333cb0a..66ff33bb 100644 --- a/src/utils/utilwrapper.cpp +++ b/src/utils/utilwrapper.cpp @@ -504,7 +504,7 @@ FILE *mc_OpenFile(const char *network_name,const char *filename, const char *ext int mc_RemoveFile(const char *network_name,const char *filename, const char *extension,int options) { - return unlink(mc_GetFullFileName(network_name,filename,extension,options).c_str()); + return __US_DeleteFile(mc_GetFullFileName(network_name,filename,extension,options).c_str()); } From aaffafb7dc37a80f6a54131165b9f547204136f1 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 11 Jun 2018 18:20:02 +0300 Subject: [PATCH 129/157] Fixed deleting of binary cache files on Windows --- src/rpc/rpccache.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/rpc/rpccache.cpp b/src/rpc/rpccache.cpp index f98aacc6..87d1dd9f 100644 --- a/src/rpc/rpccache.cpp +++ b/src/rpc/rpccache.cpp @@ -106,6 +106,8 @@ Value deletebinarycache(const Array& params, bool fHelp) throw JSONRPCError(RPC_INVALID_PARAMETER, "Binary cache item with this identifier not found"); } + close(fHan); + mc_RemoveBinaryCacheFile(params[0].get_str()); return Value::null; From 5e0f013e6164d69b2d4643927d9ff675c367512c Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 11 Jun 2018 21:46:13 +0300 Subject: [PATCH 130/157] listreamitems API tweaks --- src/rpc/rpcutils.cpp | 2 +- src/rpc/rpcwalletutils.cpp | 2 +- src/wallet/chunkdb.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index 466839f8..3871b525 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -1400,6 +1400,7 @@ Value DataItemEntry(const CTransaction& tx,int n,set & already_seen,uin { entry.push_back(Pair("key", keys[0])); } + entry.push_back(Pair("offchain", (retrieve_status & MC_OST_STORAGE_MASK) == MC_OST_OFF_CHAIN)); if( ( retrieve_status & MC_OST_CONTROL_NO_DATA ) == 0) { entry.push_back(Pair("available", AvailableFromStatus(retrieve_status))); @@ -1414,7 +1415,6 @@ Value DataItemEntry(const CTransaction& tx,int n,set & already_seen,uin entry.push_back(Pair("data", format_item_value)); - entry.push_back(Pair("offchain", (retrieve_status & MC_OST_STORAGE_MASK) == MC_OST_OFF_CHAIN)); if(retrieve_status & MC_OST_CONTROL_NO_DATA) { if((retrieve_status & MC_OST_STORAGE_MASK) == MC_OST_OFF_CHAIN) diff --git a/src/rpc/rpcwalletutils.cpp b/src/rpc/rpcwalletutils.cpp index 3f287269..bb10546a 100644 --- a/src/rpc/rpcwalletutils.cpp +++ b/src/rpc/rpcwalletutils.cpp @@ -602,6 +602,7 @@ Object StreamItemEntry(const CWalletTx& wtx,int first_output,const unsigned char { entry.push_back(Pair("key", keys[0])); } + entry.push_back(Pair("offchain", (retrieve_status & MC_OST_STORAGE_MASK) == MC_OST_OFF_CHAIN)); if( ( retrieve_status & MC_OST_CONTROL_NO_DATA ) == 0) { entry.push_back(Pair("available", AvailableFromStatus(retrieve_status))); @@ -614,7 +615,6 @@ Object StreamItemEntry(const CWalletTx& wtx,int first_output,const unsigned char } } entry.push_back(Pair("data", format_item_value)); - entry.push_back(Pair("offchain", (retrieve_status & MC_OST_STORAGE_MASK) == MC_OST_OFF_CHAIN)); if(verbose) { diff --git a/src/wallet/chunkdb.cpp b/src/wallet/chunkdb.cpp index 04649e88..9a771f16 100644 --- a/src/wallet/chunkdb.cpp +++ b/src/wallet/chunkdb.cpp @@ -326,7 +326,7 @@ int mc_ChunkDB::SourceChunksRecovery() chunk_def.Zero(); chunk_def.m_SubscriptionID=subscription->m_SubscriptionID; - err=0; + err=MC_ERR_NOERROR; sprintf(msg,"Starting source recovery, last file: %d, file_size: %ld, file offset: %ld",subscription->m_LastFileID,file_size,file_offset); LogString(msg); From 3d0eebef4dcc1492d51c8a8fdc99e4f9974ef350 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 12 Jun 2018 23:16:21 +0300 Subject: [PATCH 131/157] Fixed number of queries issue --- src/protocol/relay.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index 65b7c293..272a3072 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -583,7 +583,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) to_end_of_query=collect_subrow->m_State.m_QueryTimeStamp-time_now; if(to_end_of_query<0)to_end_of_query=0; if(to_end_of_query>collector->m_TimeoutRequest)to_end_of_query=collector->m_TimeoutRequest; - total_in_queries+=collect_subrow->m_ChunkDef.m_Size*to_end_of_query; + total_in_queries+=(collect_subrow->m_ChunkDef.m_Size+ sizeof(mc_ChunkEntityKey))*to_end_of_query; for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Queried+=k ? collect_subrow->m_ChunkDef.m_Size : 1; } } @@ -648,6 +648,7 @@ int MultichainCollectChunks(mc_ChunkCollector* collector) collect_row->m_State.m_Status |= MC_CCF_SELECTED; last_count++; total_size+=collect_row->m_ChunkDef.m_Size + sizeof(mc_ChunkEntityKey); + query_count++; } } } From b179bd3707ee90931d363e26e4a6498f57e6eed9 Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 14 Jun 2018 10:09:11 +0300 Subject: [PATCH 132/157] Fixed stack smasching on asset ref decoding --- src/rpc/rpcutils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index 3871b525..1afa1525 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -75,7 +75,7 @@ bool AssetRefDecode(unsigned char *bin, const char* string, const size_t stringL return false; mc_PutLE(bin+0,&blockNum,4); - mc_PutLE(bin+4,&txOffset,8); + mc_PutLE(bin+4,&txOffset,4); bin[8]=(unsigned char)(txIDPrefixInteger%256); bin[9]=(unsigned char)(txIDPrefixInteger/256); From 60780f3ee58c90df355b2eec04e0d314c81a22b4 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 18 Jun 2018 17:23:04 +0300 Subject: [PATCH 133/157] Logging RPC request IDs --- src/multichain/multichain-cli.cpp | 7 ++++++- src/rpc/rpcserver.cpp | 34 +++++++++++++++++++------------ src/rpc/rpcserver.h | 2 +- 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/src/multichain/multichain-cli.cpp b/src/multichain/multichain-cli.cpp index 221f23e2..64c35a6b 100644 --- a/src/multichain/multichain-cli.cpp +++ b/src/multichain/multichain-cli.cpp @@ -241,7 +241,12 @@ Object CallRPC(const string& strMethod, const Array& params) mapRequestHeaders["Authorization"] = string("Basic ") + strUserPass64; // Send request // JSON_NO_DOUBLE_FORMATTING=1; - string strRequest = JSONRPCRequest(strMethod, params, 1); + + int32_t id_nonce; + id_nonce=mc_RandomInRange(10000000,99999999); + Value req_id=strprintf("%08d-%u",id_nonce,mc_TimeNowAsUInt()); + + string strRequest = JSONRPCRequest(strMethod, params, req_id); // JSON_NO_DOUBLE_FORMATTING=0; JSON_DOUBLE_DECIMAL_DIGITS=GetArg("-apidecimaldigits",-1); string strPost = HTTPPost(strRequest, mapRequestHeaders); diff --git a/src/rpc/rpcserver.cpp b/src/rpc/rpcserver.cpp index 5e640147..42870ee5 100644 --- a/src/rpc/rpcserver.cpp +++ b/src/rpc/rpcserver.cpp @@ -73,11 +73,19 @@ string JSONRPCRequestForLog(const string& strMethod, const Array& params, const { request.push_back(Pair("params", params)); } -/* request.push_back(Pair("id", id)); +/* request.push_back(Pair("chain_name", string(mc_gState->m_Params->NetworkName()))); */ - return write_string(Value(request), false) + "\n"; + return write_string(Value(request), false);// + "\n"; +} + +string JSONRPCMethodIDForLog(const string& strMethod, const Value& id) +{ + Object request; + request.push_back(Pair("method", strMethod)); + request.push_back(Pair("id", id)); + return write_string(Value(request), false);// + "\n"; } @@ -979,14 +987,14 @@ static Object JSONRPCExecOne(const Value& req) try { jreq.parse(req); - Value result = tableRPC.execute(jreq.strMethod, jreq.params); + Value result = tableRPC.execute(jreq.strMethod, jreq.params,jreq.id); rpc_result = JSONRPCReplyObj(result, Value::null, jreq.id); } catch (Object& objError) { /* MCHN START */ mc_gState->m_WalletMode=wallet_mode; - if(fDebug)LogPrint("mcapi","mcapi: API request failure A\n"); + if(fDebug)LogPrint("mcapi","mcapi: API request failure A: %s\n",JSONRPCMethodIDForLog(jreq.strMethod,jreq.id).c_str()); /* MCHN END */ rpc_result = JSONRPCReplyObj(Value::null, objError, jreq.id); } @@ -994,7 +1002,7 @@ static Object JSONRPCExecOne(const Value& req) { /* MCHN START */ mc_gState->m_WalletMode=wallet_mode; - if(fDebug)LogPrint("mcapi","mcapi: API request failure B\n"); + if(fDebug)LogPrint("mcapi","mcapi: API request failure B: %s\n",JSONRPCMethodIDForLog(jreq.strMethod,jreq.id).c_str()); /* MCHN END */ rpc_result = JSONRPCReplyObj(Value::null, JSONRPCError(RPC_PARSE_ERROR, e.what()), jreq.id); @@ -1061,7 +1069,7 @@ static bool HTTPReq_JSONRPC(AcceptedConnection *conn, if (valRequest.type() == obj_type) { jreq.parse(valRequest); - Value result = tableRPC.execute(jreq.strMethod, jreq.params); + Value result = tableRPC.execute(jreq.strMethod, jreq.params,jreq.id); // Send reply strReply = JSONRPCReply(result, Value::null, jreq.id); @@ -1078,7 +1086,7 @@ static bool HTTPReq_JSONRPC(AcceptedConnection *conn, { /* MCHN START */ mc_gState->m_WalletMode=wallet_mode; - if(fDebug)LogPrint("mcapi","mcapi: API request failure: %s, code: %d\n",jreq.strMethod.c_str(),find_value(objError, "code").get_int()); + if(fDebug)LogPrint("mcapi","mcapi: API request failure: %s, code: %d\n",JSONRPCMethodIDForLog(jreq.strMethod,jreq.id).c_str(),find_value(objError, "code").get_int()); // if(fDebug)LogPrint("mcapi","mcapi: API request failure C\n"); /* MCHN END */ @@ -1089,7 +1097,7 @@ static bool HTTPReq_JSONRPC(AcceptedConnection *conn, { /* MCHN START */ mc_gState->m_WalletMode=wallet_mode; - if(fDebug)LogPrint("mcapi","mcapi: API request failure D\n"); + if(fDebug)LogPrint("mcapi","mcapi: API request failure D: %s\n",JSONRPCMethodIDForLog(jreq.strMethod,jreq.id).c_str()); /* MCHN END */ ErrorReply(conn->stream(), JSONRPCError(RPC_PARSE_ERROR, e.what()), jreq.id); return false; @@ -1134,7 +1142,7 @@ void ServiceConnection(AcceptedConnection *conn) } } -json_spirit::Value CRPCTable::execute(const std::string &strMethod, const json_spirit::Array ¶ms) const +json_spirit::Value CRPCTable::execute(const std::string &strMethod, const json_spirit::Array ¶ms, const Value& req_id) const { // Find method const CRPCCommand *pcmd = tableRPC[strMethod]; @@ -1190,7 +1198,7 @@ json_spirit::Value CRPCTable::execute(const std::string &strMethod, const json_s // Execute if(fDebug) { - string strRequest = JSONRPCRequestForLog(strMethod, params, 1); + string strRequest = JSONRPCRequestForLog(strMethod, params, req_id); LogPrint("mcapi","mcapi: API request: %s\n",strRequest.c_str()); } @@ -1233,7 +1241,7 @@ json_spirit::Value CRPCTable::execute(const std::string &strMethod, const json_s strResult=JSONRPCReply(result, Value::null, 1); if(strcmp(strResultNone.c_str(),strResult.c_str())) { - string strRequestBad = JSONRPCRequestForLog(strMethod, params, 1); + string strRequestBad = JSONRPCRequestForLog(strMethod, params, req_id); if(fDebug)LogPrint("walletcompare","walletcompare: ERROR: Result mismatch on API request: %s\n",strRequestBad.c_str()); if(fDebug)LogPrint("walletcompare","walletcompare: %s\n",strResultNone.c_str()); if(fDebug)LogPrint("walletcompare","walletcompare: %s\n",strResult.c_str()); @@ -1256,13 +1264,13 @@ json_spirit::Value CRPCTable::execute(const std::string &strMethod, const json_s #endif // !ENABLE_WALLET } /* MCHN START */ - if(fDebug)LogPrint("mcapi","mcapi: API request successful: %s\n",strMethod.c_str()); + if(fDebug)LogPrint("mcapi","mcapi: API request successful: %s\n",JSONRPCMethodIDForLog(strMethod,req_id).c_str()); /* MCHN END */ return result; } catch (std::exception& e) { - if(fDebug)LogPrint("mcapi","mcapi: API request failure: %s\n",strMethod.c_str()); + if(fDebug)LogPrint("mcapi","mcapi: API request failure: %s\n",JSONRPCMethodIDForLog(strMethod,req_id).c_str());//strMethod.c_str()); if(strcmp(e.what(),"Help message not found\n") == 0) { throw JSONRPCError(RPC_MISC_ERROR, mc_RPCHelpString(strMethod).c_str()); diff --git a/src/rpc/rpcserver.h b/src/rpc/rpcserver.h index d53a66cc..ce2010b8 100644 --- a/src/rpc/rpcserver.h +++ b/src/rpc/rpcserver.h @@ -114,7 +114,7 @@ class CRPCTable * @throws an exception (json_spirit::Value) when an error happens. */ void initialize(); - json_spirit::Value execute(const std::string &method, const json_spirit::Array ¶ms) const; + json_spirit::Value execute(const std::string &method, const json_spirit::Array ¶ms, const json_spirit::Value& req_id) const; }; extern CRPCTable tableRPC; From 025784061a25f3234f96ada06bcb8a923191df41 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 9 Jul 2018 12:09:32 +0300 Subject: [PATCH 134/157] Fixed autosubscribe for offchain items --- src/wallet/wallettxs.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/wallet/wallettxs.cpp b/src/wallet/wallettxs.cpp index af99bcbb..c7c2d036 100644 --- a/src/wallet/wallettxs.cpp +++ b/src/wallet/wallettxs.cpp @@ -2800,7 +2800,13 @@ int mc_WalletTxs::AddTx(mc_TxImport *import,const CWalletTx& tx,int block,CDiskT entity.m_EntityType=MC_TET_STREAM_PUBLISHER | MC_TET_CHAINPOS; m_Database->AddEntity(imp,&entity,0); entity.m_EntityType=MC_TET_STREAM_PUBLISHER | MC_TET_TIMERECEIVED; - m_Database->AddEntity(imp,&entity,0); + m_Database->AddEntity(imp,&entity,0); + + if(mc_gState->m_Features->Chunks()) + { + entity.m_EntityType=MC_TET_STREAM; + m_ChunkDB->AddEntity(&entity,0); + } } } From 47cebfb23b6bf40e2706fdaef98d02f052119caa Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 16 Jul 2018 14:38:28 +0300 Subject: [PATCH 135/157] Fixed txcount in getwalletinfo --- src/rpc/rpcwallet.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/rpcwallet.cpp b/src/rpc/rpcwallet.cpp index ea8eb14a..630639b1 100644 --- a/src/rpc/rpcwallet.cpp +++ b/src/rpc/rpcwallet.cpp @@ -2169,7 +2169,7 @@ Value getwalletinfo(const Array& params, bool fHelp) obj.push_back(Pair("walletdbversion", mc_gState->GetWalletDBVersion())); if(mc_gState->m_WalletMode & MC_WMD_ADDRESS_TXS) { - obj.push_back(Pair("txcount", (int)pwalletTxsMain->m_Database->m_DBStat.m_Count+(int)pwalletTxsMain->m_UnconfirmedSends.size())); + obj.push_back(Pair("txcount",(int)pwalletTxsMain->m_Database->m_DBStat.m_Count+pwalletTxsMain->m_Database->m_RawMemPools[0]->GetCount())); } else { From a4822f27c2f1a45a8c9da98a228de1c25bacee33 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 17 Jul 2018 16:05:12 +0300 Subject: [PATCH 136/157] Fixed Mac compilation error --- src/protocol/relay.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index 272a3072..5540e225 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -1546,7 +1546,7 @@ void mc_RelayManager::CheckTime() } Lock(); - for(map::iterator it = m_RelayRecords.begin(); it != m_RelayRecords.end();) + for(map::iterator it = m_RelayRecords.begin(); it != m_RelayRecords.end();) { if(it->second.m_Timestamp < m_LastTime) { @@ -1623,7 +1623,7 @@ int mc_RelayManager::GetRelayRecord(CNode *pfrom,mc_OffchainMessageID msg_id,uin } // printf("getrr: %d, ts: %u, nc: %u\n",pfrom_id,timestamp,nonce); const mc_RelayRecordKey key=mc_RelayRecordKey(msg_id,pfrom_id); - map::iterator it = m_RelayRecords.find(key); + map::iterator it = m_RelayRecords.find(key); if (it == m_RelayRecords.end()) { return MC_ERR_NOT_FOUND; From caa4d73cafef76910efea5f2911d2c4b316adce1 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 17 Jul 2018 17:10:09 +0300 Subject: [PATCH 137/157] Fixed Mac compilation issue --- src/rpc/rpcdebug.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/rpcdebug.cpp b/src/rpc/rpcdebug.cpp index 20bc773c..64451723 100644 --- a/src/rpc/rpcdebug.cpp +++ b/src/rpc/rpcdebug.cpp @@ -573,7 +573,7 @@ Value debug(const Array& params, bool fHelp) chunk_found=pwalletTxsMain->m_ChunkDB->GetChunk(&chunk_def,0,-1,&chunk_bytes); if(chunk_found) { - chunk_obj.push_back(Pair("size",chunk_bytes)); + chunk_obj.push_back(Pair("size",(int)chunk_bytes)); chunk_obj.push_back(Pair("data",HexStr(chunk_found,chunk_found+chunk_bytes))); } else From 473edb8d2939702a5174ae3bd58437ed906e4275 Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 18 Jul 2018 14:36:21 +0300 Subject: [PATCH 138/157] Fixed unicode representation of non-printable characters --- src/json/json_spirit_writer_template.h | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/json/json_spirit_writer_template.h b/src/json/json_spirit_writer_template.h index 4e37ae31..21bc0920 100644 --- a/src/json/json_spirit_writer_template.h +++ b/src/json/json_spirit_writer_template.h @@ -21,6 +21,7 @@ namespace json_spirit inline unsigned int utf8_len_and_mask(unsigned char c,unsigned int *mask) { + if(c<0x20){*mask=0x00;return 0;} if(c<0x80){*mask=0x7F;return 1;} if(c<0xC0){*mask=0x00;return 0;} if(c<0xE0){*mask=0x1F;return 2;} @@ -146,18 +147,24 @@ namespace json_spirit } else { - if( end - i >= charlen) + if(charlen) { - shift=6*(charlen-1); - codepoint |= ( unsigned_c & mask) << shift; - for(j=1;j= charlen) { - shift-=6; - codepoint |= ( *( ++i ) & 0x3F) << shift; + shift=6*(charlen-1); + codepoint |= ( unsigned_c & mask) << shift; + for(j=1;j( codepoint ); } - result += codepoint_to_string< String_type >( codepoint ); } -// result += non_printable_to_string< String_type >( unsigned_c ); + else + { + result += non_printable_to_string< String_type >( unsigned_c ); + } } } From d93d29be8477d2d13fab157a964ded751b8cbe95 Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 19 Jul 2018 13:48:41 +0300 Subject: [PATCH 139/157] Boolean rescan for importwallet --- src/rpc/rpcdump.cpp | 26 ++++++++++++++------------ src/rpc/rpchelp.cpp | 3 ++- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/rpc/rpcdump.cpp b/src/rpc/rpcdump.cpp index dedc3881..aec2c379 100644 --- a/src/rpc/rpcdump.cpp +++ b/src/rpc/rpcdump.cpp @@ -408,25 +408,27 @@ Value importwallet(const Array& params, bool fHelp) pwalletMain->nTimeFirstKey = nTimeBegin; /* MCHN START */ - if(mc_gState->m_WalletMode & MC_WMD_ADDRESS_TXS) + if(fRescan) { - if(start_block) + if(mc_gState->m_WalletMode & MC_WMD_ADDRESS_TXS) { - LogPrintf("Rescanning last %i blocks\n", chainActive.Height()-start_block+1); + if(start_block) + { + LogPrintf("Rescanning last %i blocks\n", chainActive.Height()-start_block+1); + } + else + { + LogPrintf("Rescanning all %i blocks\n", chainActive.Height()); + } + pwalletMain->ScanForWalletTransactions(chainActive[start_block],false,true); } else { - LogPrintf("Rescanning all %i blocks\n", chainActive.Height()); + LogPrintf("Rescanning last %i blocks\n", chainActive.Height() - pindex->nHeight + 1); + pwalletMain->ScanForWalletTransactions(pindex,false,true); } - pwalletMain->ScanForWalletTransactions(chainActive[start_block],false,true); - pwalletMain->MarkDirty(); - } - else - { - LogPrintf("Rescanning last %i blocks\n", chainActive.Height() - pindex->nHeight + 1); - pwalletMain->ScanForWalletTransactions(pindex,false,true); - pwalletMain->MarkDirty(); } + pwalletMain->MarkDirty(); if (!fGood) throw JSONRPCError(RPC_WALLET_ERROR, "Error adding some keys to wallet"); diff --git a/src/rpc/rpchelp.cpp b/src/rpc/rpchelp.cpp index 398a7622..fa2d69d3 100644 --- a/src/rpc/rpchelp.cpp +++ b/src/rpc/rpchelp.cpp @@ -2244,7 +2244,8 @@ void mc_InitRPCHelpMap10() "\nImports keys from a wallet dump file (see dumpwallet).\n" "\nArguments:\n" "1. \"filename\" (string, required) The wallet file\n" - "2. rescan (integer, optional, default=0) Rescan from block, if negative - from the end.\n" + "2. rescan (boolean or integer, optional, default=true) Rescan the wallet for transactions. \n" + " If integer rescan from block, if negative - from the end.\n" "\nExamples:\n" "\nDump the wallet\n" + HelpExampleCli("dumpwallet", "\"test\"") + From e9d4a5e9d6edaf89281ded3587171f7111b123d6 Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 19 Jul 2018 14:13:13 +0300 Subject: [PATCH 140/157] Fixed fdatasync for Mac compilation --- src/utils/systemdependent.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/utils/systemdependent.cpp b/src/utils/systemdependent.cpp index 27933160..b8921862 100644 --- a/src/utils/systemdependent.cpp +++ b/src/utils/systemdependent.cpp @@ -233,7 +233,11 @@ void __US_FlushFileWithMode(int FileHan,uint32_t use_data_sync) { if(use_data_sync) { +#ifdef MAC_OSX + fcntl(FileHan, F_FULLFSYNC, 0); +#else fdatasync(FileHan); +#endif } else { From 2a1fc7a995b3bab7430b08abc07f50b2ec5bc303 Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 19 Jul 2018 15:17:14 +0300 Subject: [PATCH 141/157] Boolean verbose parameter for getrawtransaction --- src/rpc/rpcrawtransaction.cpp | 3 ++- src/rpc/rpcutils.h | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/rpc/rpcrawtransaction.cpp b/src/rpc/rpcrawtransaction.cpp index 3e352d22..69525455 100644 --- a/src/rpc/rpcrawtransaction.cpp +++ b/src/rpc/rpcrawtransaction.cpp @@ -504,7 +504,8 @@ Value getrawtransaction(const Array& params, bool fHelp) bool fVerbose = false; if (params.size() > 1) - fVerbose = (params[1].get_int() != 0); + fVerbose=paramtobool(params[1],false); +// fVerbose = (params[1].get_int() != 0); CTransaction tx; uint256 hashBlock = 0; diff --git a/src/rpc/rpcutils.h b/src/rpc/rpcutils.h index 14f83057..f80c34a1 100644 --- a/src/rpc/rpcutils.h +++ b/src/rpc/rpcutils.h @@ -129,6 +129,7 @@ bool AssetCompareByRef(Value a,Value b); Array AssetArrayFromAmounts(mc_Buffer *asset_amounts,int issue_asset_id,uint256 hash,int show_type); void ParseRawAction(string action,bool& lock_it, bool& sign_it,bool& send_it); bool paramtobool(Value param); +bool paramtobool(Value param,bool strict); int paramtoint(Value param,bool check_for_min,int min_value,string error_message); int64_t paramtoint64(Value param,bool check_for_min,int64_t min_value,string error_message); int ParseBlockIdentifier(Value blockset_identifier); From bef04be3bf3c9d61e1a37d2128075bf335ad85a9 Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 26 Jul 2018 18:55:40 +0300 Subject: [PATCH 142/157] Flushing offchain queue on subscribe/unsubscribe --- src/rpc/rpcstreams.cpp | 7 +- src/wallet/chunkcollector.cpp | 144 +++++++++++++++++++++++++++++++--- src/wallet/chunkcollector.h | 4 +- src/wallet/wallettxs.cpp | 5 ++ 4 files changed, 149 insertions(+), 11 deletions(-) diff --git a/src/rpc/rpcstreams.cpp b/src/rpc/rpcstreams.cpp index c07de87f..ea2377a2 100644 --- a/src/rpc/rpcstreams.cpp +++ b/src/rpc/rpcstreams.cpp @@ -889,8 +889,13 @@ Value unsubscribe(const Array& params, bool fHelp) inputEntities.push_back(entity_to_subscribe); } + int32_t buf_mode=MC_BUF_MODE_DEFAULT; + if(inputStrings.size() > 1) + { + buf_mode=MC_BUF_MODE_MAP; + } mc_Buffer *streams=mc_gState->m_TmpBuffers->m_RpcBuffer1; - streams->Initialize(sizeof(mc_TxEntity),sizeof(mc_TxEntity),MC_BUF_MODE_DEFAULT); + streams->Initialize(sizeof(mc_TxEntity),sizeof(mc_TxEntity),buf_mode); bool fNewFound=false; diff --git a/src/wallet/chunkcollector.cpp b/src/wallet/chunkcollector.cpp index d14839e1..5063e4aa 100644 --- a/src/wallet/chunkcollector.cpp +++ b/src/wallet/chunkcollector.cpp @@ -434,7 +434,125 @@ int mc_ChunkCollector::InsertChunk( return err; } + +int mc_ChunkCollector::Unsubscribe(mc_Buffer* lpEntities) +{ + Lock(); + + int i,err; + unsigned char *ptr; + mc_ChunkCollectorRow *row; + mc_ChunkCollectorRow collect_row; + int try_again,commit_required; + mc_TxEntity entity; + + commit_required=0; + for(i=0;iGetCount();i++) + { + row=(mc_ChunkCollectorRow *)m_MemPool->GetRow(i); + memcpy(&entity,&row->m_ChunkDef.m_Entity,sizeof(mc_TxEntity)); + entity.m_EntityType |= MC_TET_CHAINPOS; + if(lpEntities->Seek(&entity) >= 0) + { + row->m_State.m_Status |= MC_CCF_DELETED; + commit_required=1; + } + } + if(commit_required) + { + CommitInternal(0); + } + + if(m_MemPool == m_MemPool1) + { + m_MemPoolNext=m_MemPool2; + } + else + { + m_MemPoolNext=m_MemPool1; + } + + + try_again=1; + m_DBRow.Zero(); + if(m_DB) + { + while(try_again) + { + m_MemPoolNext->Clear(); + try_again=0; + err=SeekDB(&m_DBRow); + + ptr=(unsigned char*)m_DB->MoveNext(&err); + while(ptr) + { + if(err) + { + return MC_ERR_CORRUPTED; + } + memcpy((char*)&m_DBRow,ptr,m_TotalDBSize); + ptr=(unsigned char*)m_DB->MoveNext(&err); + GetDBRow(&collect_row); + memcpy(&entity,&collect_row.m_ChunkDef.m_Entity,sizeof(mc_TxEntity)); + entity.m_EntityType |= MC_TET_CHAINPOS; + if(lpEntities->Seek(&entity) >= 0) + { + m_MemPoolNext->Add(&collect_row); + } + else + { + memcpy(&m_LastDBRow,&m_DBRow,m_TotalDBSize); + } + if( (m_MemPoolNext->GetCount() >= 2*m_MaxMemPoolSize) || (ptr == NULL) ) + { + if(ptr) + { + try_again=1; + ptr=NULL; + memcpy(&m_DBRow,&m_LastDBRow,m_TotalDBSize); + } + if(m_MemPoolNext->GetCount()) + { + commit_required=1; + for(i=0;iGetCount();i++) + { + row=(mc_ChunkCollectorRow *)m_MemPoolNext->GetRow(i); + m_TotalChunkCount--; + m_TotalChunkSize-=row->m_ChunkDef.m_Size; + if(row->m_State.m_Status & MC_CCF_INSERTED) + { + DeleteDBRow(row); + } + } + m_DBRow.Zero(); + m_DBRow.m_TotalChunkSize=m_TotalChunkSize; + m_DBRow.m_TotalChunkCount=m_TotalChunkCount; + m_DB->Write((char*)&m_DBRow+m_KeyDBOffset,m_KeyDBSize,(char*)&m_DBRow+m_ValueDBOffset,m_ValueDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + + err=m_DB->Commit(MC_OPT_DB_DATABASE_TRANSACTIONAL); + if(err) + { + return err; + } + } + } + } + } + } + + m_MemPoolNext->Clear(); + + if(commit_required) + { + CommitInternal(1); + } + + UnLock(); + + return MC_ERR_NOERROR; +} + int mc_ChunkCollector::InsertChunkInternal( const unsigned char *hash, const mc_TxEntity *entity, @@ -459,6 +577,11 @@ int mc_ChunkCollector::InsertChunkInternal( m_MemPool->Add(&collect_row); m_TotalChunkCount++; m_TotalChunkSize+=chunk_size; + + if(m_MemPoolNext->GetCount() >= 2*m_MaxMemPoolSize) + { + CommitInternal(0); + } } return MC_ERR_NOERROR; @@ -573,13 +696,13 @@ int mc_ChunkCollector::Commit() int err; Lock(); - err=CommitInternal(); + err=CommitInternal(1); UnLock(); return err; } -int mc_ChunkCollector::CommitInternal() +int mc_ChunkCollector::CommitInternal(int fill_mempool) { int i; mc_ChunkCollectorRow *row; @@ -650,7 +773,7 @@ int mc_ChunkCollector::CommitInternal() } */ if(!row->m_State.m_Query.IsZero() || - ((row->m_State.m_QueryNextAttempt <= time_now) && (m_MemPoolNext->GetCount() < m_MaxMemPoolSize)) ) + ((fill_mempool != 0) && (row->m_State.m_QueryNextAttempt <= time_now) && (m_MemPoolNext->GetCount() < m_MaxMemPoolSize)) ) { m_MemPoolNext->Add(row); } @@ -671,16 +794,19 @@ int mc_ChunkCollector::CommitInternal() } } - if(m_MemPoolNext->GetCount() < m_MaxMemPoolSize) + if(fill_mempool) { - m_LastDBRow.Zero(); - err=SeekDB(&m_LastDBRow); - if(err == MC_ERR_NOERROR) + if(m_MemPoolNext->GetCount() < m_MaxMemPoolSize) { - ReadFromDB(m_MemPoolNext,m_MaxMemPoolSize); + m_LastDBRow.Zero(); + err=SeekDB(&m_LastDBRow); + if(err == MC_ERR_NOERROR) + { + ReadFromDB(m_MemPoolNext,m_MaxMemPoolSize); + } } } - + m_MemPool->Clear(); m_MemPool=m_MemPoolNext; diff --git a/src/wallet/chunkcollector.h b/src/wallet/chunkcollector.h index ebf31619..ee2c3e2f 100644 --- a/src/wallet/chunkcollector.h +++ b/src/wallet/chunkcollector.h @@ -188,7 +188,9 @@ typedef struct mc_ChunkCollector int FillMarkPoolByFlag(uint32_t flag, uint32_t not_flag); int Commit(); - int CommitInternal(); + int CommitInternal(int fill_mempool); + + int Unsubscribe(mc_Buffer* lpEntities); void Zero(); int Destroy(); diff --git a/src/wallet/wallettxs.cpp b/src/wallet/wallettxs.cpp index c7c2d036..bf819439 100644 --- a/src/wallet/wallettxs.cpp +++ b/src/wallet/wallettxs.cpp @@ -1334,6 +1334,11 @@ int mc_WalletTxs::Unsubscribe(mc_Buffer* lpEntities,bool purge) } if(fDebug)LogPrint("wallet","wtxs: Unsubscribed from %d entities\n",lpEntities->GetCount()); m_Database->UnLock(); + + if(m_ChunkCollector) + { + m_ChunkCollector->Unsubscribe(lpEntities); + } return err; } From 7c9fbd0cc5ea8bb534e62d5340e6172191bcc140 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 31 Jul 2018 15:12:59 +0300 Subject: [PATCH 143/157] Random seed, fixing Mac sempahore issue --- src/multichain/multichaind.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/multichain/multichaind.cpp b/src/multichain/multichaind.cpp index 93d63b49..c9560b10 100644 --- a/src/multichain/multichaind.cpp +++ b/src/multichain/multichaind.cpp @@ -80,6 +80,8 @@ bool AppInit(int argc, char* argv[]) mc_gState->m_Params->Parse(argc, argv, MC_ETP_DAEMON); mc_CheckDataDirInConfFile(); + mc_RandomSeed(mc_TimeNowAsUInt()); + if(mc_gState->m_Params->NetworkName()) { if(strlen(mc_gState->m_Params->NetworkName()) > MC_PRM_NETWORK_NAME_MAX_SIZE) From 4ec9a04c32f1a4a2f7b08ea18e3a1921f89a7780 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 31 Jul 2018 15:38:29 +0300 Subject: [PATCH 144/157] boost downgrading instructions for MacOS compilation --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index c3a7db25..8e1bd8ef 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,12 @@ Install dependencies Install brew (follow instructions on brew.sh) brew install autoconf automake berkeley-db4 libtool boost openssl pkg-config rename +on MacOS High Sierra + + brew uninstall boost + brew install boost@1.57 + brew link boost@1.57 --force + Prepare for static linking -------------------------- Apple does not support statically linked binaries as [documented here](https://developer.apple.com/library/content/qa/qa1118/_index.html), however, it is convenient for end-users to launch a binary without having to first install brew, a third-party system designed for developers. @@ -111,6 +117,8 @@ The default brew cookbook for berkeley-db and boost builds static libraries, but In 'def configure_args' change 'shared' to 'no-shared' brew install openssl --force + + Compile MultiChain for Mac (64-bit) -------------------------- From 0168cea94c56ff57126685531869275f36d6fcc7 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 31 Jul 2018 15:40:28 +0300 Subject: [PATCH 145/157] Fixed help message for verbose parameter in getrawtransaction --- src/rpc/rpchelp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/rpchelp.cpp b/src/rpc/rpchelp.cpp index fa2d69d3..e803bb4f 100644 --- a/src/rpc/rpchelp.cpp +++ b/src/rpc/rpchelp.cpp @@ -1068,7 +1068,7 @@ void mc_InitRPCHelpMap05() "\nArguments:\n" "1. \"txid\" (string, required) The transaction id\n" - "2. verbose (numeric, optional, default=0) If 0, return a string, other return a json object\n" + "2. verbose (numeric or boolean, optional, default=0(false)) If 0, return a string, other return a json object\n" "\nResult (if verbose is not set or set to 0):\n" "\"data\" (string) The serialized, hex-encoded data for 'txid'\n" From a08285bec48cd576707132861a8ec2780d5d6752 Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 2 Aug 2018 09:55:01 +0300 Subject: [PATCH 146/157] Avoiding followon scanning when unnecessary --- src/rpc/rpcutils.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index 1afa1525..d16262ce 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -1562,7 +1562,9 @@ Object AssetEntry(const unsigned char *txid,int64_t quantity,uint32_t output_lev Array issues; int64_t total=0; - if(( (output_level & 0x0020) !=0 )|| mc_gState->m_Assets->HasFollowOns(txid)) + if(( (output_level & 0x0020) !=0 ) || // issuers + // For listassets with followons + ( (mc_gState->m_Assets->HasFollowOns(txid) != 0) && (quantity < 0) && ( (output_level & 0x00C0) == 0) )) { int64_t qty; mc_Buffer *followons; From 61cb4406cf6e29426ee65a154c2812c0d7f14300 Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 2 Aug 2018 11:35:06 +0300 Subject: [PATCH 147/157] Fixed watch only flag in getbalance --- src/rpc/rpcwallet.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/rpc/rpcwallet.cpp b/src/rpc/rpcwallet.cpp index 630639b1..277829a9 100644 --- a/src/rpc/rpcwallet.cpp +++ b/src/rpc/rpcwallet.cpp @@ -1002,9 +1002,15 @@ Value getbalance(const Array& params, bool fHelp) { CTxOut txout; out.GetHashAndTxOut(txout); - if(out.IsTrustedNoDepth() || (out.nDepth >= nMinDepth)) + + isminetype fIsMine=pwalletMain->IsMine(txout); + + if(fIsMine & filter) { - nBalance+=txout.nValue; + if(out.IsTrustedNoDepth() || (out.nDepth >= nMinDepth)) + { + nBalance+=txout.nValue; + } } } } From 4635a60feedfec6e99eada98d47eae0c2d091158 Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 2 Aug 2018 18:13:21 +0300 Subject: [PATCH 148/157] liststreamqueryitems preparations --- src/chainparams/state.h | 8 ++ src/rpc/rpcclient.cpp | 2 + src/rpc/rpclist.cpp | 1 + src/rpc/rpcserver.h | 1 + src/rpc/rpcstreams.cpp | 242 ++++++++++++++++++++++++++++++++++--- src/rpc/rpcwallet.h | 23 +++- src/rpc/rpcwalletutils.cpp | 233 ++++++++++++++++++++++++++++++++++- 7 files changed, 489 insertions(+), 21 deletions(-) diff --git a/src/chainparams/state.h b/src/chainparams/state.h index 327913c7..7ba26417 100644 --- a/src/chainparams/state.h +++ b/src/chainparams/state.h @@ -176,6 +176,8 @@ typedef struct mc_TmpBuffers mc_Buffer *m_RpcABNoMapBuffer1; mc_Buffer *m_RpcABNoMapBuffer2; mc_Buffer *m_RpcEntityRows; + mc_Buffer *m_RpcEntityRowsToMerge; + mc_Buffer *m_RpcEntityRowsFull; mc_SHA256 *m_RpcHasher1; mc_Script *m_RpcChunkScript1; mc_Script *m_RelayTmpBuffer; @@ -197,6 +199,10 @@ typedef struct mc_TmpBuffers mc_InitABufferDefault(m_RpcABNoMapBuffer2); m_RpcEntityRows=new mc_Buffer; m_RpcEntityRows->Initialize(MC_TDB_ENTITY_KEY_SIZE,MC_TDB_ROW_SIZE,MC_BUF_MODE_DEFAULT); + m_RpcEntityRowsToMerge=new mc_Buffer; + m_RpcEntityRowsToMerge->Initialize(MC_TDB_ENTITY_KEY_SIZE,MC_TDB_ROW_SIZE,MC_BUF_MODE_DEFAULT); + m_RpcEntityRowsFull=new mc_Buffer; + m_RpcEntityRowsFull->Initialize(MC_TDB_ENTITY_KEY_SIZE,MC_TDB_ROW_SIZE,MC_BUF_MODE_DEFAULT); m_RpcHasher1=new mc_SHA256(); m_RpcChunkScript1=new mc_Script(); m_RelayTmpBuffer=new mc_Script(); @@ -214,6 +220,8 @@ typedef struct mc_TmpBuffers delete m_RpcABNoMapBuffer1; delete m_RpcABNoMapBuffer2; delete m_RpcEntityRows; + delete m_RpcEntityRowsToMerge; + delete m_RpcEntityRowsFull; delete m_RpcHasher1; delete m_RpcChunkScript1; delete m_RelayTmpBuffer; diff --git a/src/rpc/rpcclient.cpp b/src/rpc/rpcclient.cpp index 5d49d62a..a5512c9b 100644 --- a/src/rpc/rpcclient.cpp +++ b/src/rpc/rpcclient.cpp @@ -196,6 +196,8 @@ static const CRPCConvertParam vRPCConvertParams[] = { "liststreampublishers", 3 }, { "liststreampublishers", 4 }, { "liststreampublishers", 5 }, + { "liststreamqueryitems", 1 }, + { "liststreamqueryitems", 2 }, { "liststreamkeyitems", 2 }, { "liststreamkeyitems", 3 }, { "liststreamkeyitems", 4 }, diff --git a/src/rpc/rpclist.cpp b/src/rpc/rpclist.cpp index abffe3ff..294e74df 100644 --- a/src/rpc/rpclist.cpp +++ b/src/rpc/rpclist.cpp @@ -240,6 +240,7 @@ static const CRPCCommand vRPCCommands[] = { "wallet", "getstreamitem", &getstreamitem, false, false, true }, { "wallet", "liststreamtxitems", &liststreamtxitems, false, false, true }, { "wallet", "liststreamitems", &liststreamitems, false, false, true }, + { "wallet", "liststreamqueryitems", &liststreamqueryitems, false, false, true }, { "wallet", "liststreamkeyitems", &liststreamkeyitems, false, false, true }, { "wallet", "liststreampublisheritems",&liststreampublisheritems,false, false, true }, { "wallet", "liststreamkeys", &liststreamkeys, false, false, true }, diff --git a/src/rpc/rpcserver.h b/src/rpc/rpcserver.h index ce2010b8..e03f08a1 100644 --- a/src/rpc/rpcserver.h +++ b/src/rpc/rpcserver.h @@ -260,6 +260,7 @@ extern json_spirit::Value getstreamitem(const json_spirit::Array& params, bool f extern json_spirit::Value liststreamtxitems(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value liststreamitems(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value liststreamkeyitems(const json_spirit::Array& params, bool fHelp); +extern json_spirit::Value liststreamqueryitems(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value liststreampublisheritems(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value liststreamkeys(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value liststreampublishers(const json_spirit::Array& params, bool fHelp); diff --git a/src/rpc/rpcstreams.cpp b/src/rpc/rpcstreams.cpp index ea2377a2..3fa0350a 100644 --- a/src/rpc/rpcstreams.cpp +++ b/src/rpc/rpcstreams.cpp @@ -10,6 +10,15 @@ #include "json/json_spirit_reader_template.h" #include "json/json_spirit_writer_template.h" +#define MC_QPR_MAX_UNCHECKED_TX_LIST_SIZE 1048576 +#define MC_QPR_MAX_MERGED_TX_LIST_SIZE 1024 +#define MC_QPR_MAX_DIRTY_TX_LIST_SIZE 256 +#define MC_QPR_MAX_SECONDARY_TX_LIST_SIZE 1048576 +#define MC_QPR_TX_CHECK_COST 100 +#define MC_QPR_MAX_TX_PER_BLOCK 4 + + + Value createupgradefromcmd(const Array& params, bool fHelp); void parseStreamIdentifier(Value stream_identifier,mc_EntityDetails *entity) @@ -988,7 +997,7 @@ Value liststreamtxitems(const Array& params, bool fHelp) int stream_output; while(first_output < (int)wtx.vout.size()) { - Object entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,verbose,NULL,NULL,&stream_output); + Object entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,verbose,NULL,&stream_output); if(stream_output < (int)wtx.vout.size()) { @@ -1116,7 +1125,7 @@ Value liststreamitems(const Array& params, bool fHelp) uint256 hash; int first_output=mc_GetHashAndFirstOutput(lpEntTx,&hash); const CWalletTx& wtx=pwalletTxsMain->GetWalletTx(hash,NULL,NULL); - Object entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,verbose,NULL,NULL,NULL); + Object entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,verbose,NULL,NULL); if(entry.size()) { retArray.push_back(entry); @@ -1236,7 +1245,7 @@ Value liststreamblockitems(const Array& params, bool fHelp) int stream_output; while(first_output < (int)wtx.vout.size()) { - Object entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,verbose,NULL,NULL,&stream_output); + Object entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,verbose,NULL,&stream_output); if(entry.size()) { retArray.push_back(entry); @@ -1327,14 +1336,17 @@ Value getstreamsummary(const Array& params, bool fPublisher) bool fFirstPublisher=false; bool fFirstPublisherAll=false; string key_string=params[1].get_str(); - const char *key_ptr=key_string.c_str(); + vector conditions; + if(fPublisher) { - getSubKeyEntityFromPublisher(params[1].get_str(),entStat,&entity); + getSubKeyEntityFromPublisher(params[1].get_str(),entStat,&entity); + conditions.push_back(mc_QueryCondition(MC_QCT_PUBLISHER,params[1].get_str())); } else { getSubKeyEntityFromKey(params[1].get_str(),entStat,&entity); + conditions.push_back(mc_QueryCondition(MC_QCT_KEY,params[1].get_str())); } set setFirstPublishers; @@ -1441,6 +1453,8 @@ Value getstreamsummary(const Array& params, bool fPublisher) int first_output=mc_GetHashAndFirstOutput(lpEntTx,&hash); const CWalletTx& wtx=pwalletTxsMain->GetWalletTx(hash,NULL,NULL); Object entry; + entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,false,&conditions,NULL); +/* if(fPublisher) { entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,false,NULL,&key_ptr,NULL); @@ -1449,7 +1463,7 @@ Value getstreamsummary(const Array& params, bool fPublisher) { entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,false,&key_ptr,NULL,NULL); } - +*/ if(fFirstPublisher) { pcount=0; @@ -1678,10 +1692,12 @@ Value liststreamkeyitems(const Array& params, bool fHelp) } string key_string=params[1].get_str(); - const char *key_ptr=key_string.c_str(); getSubKeyEntityFromKey(params[1].get_str(),entStat,&entity); - + vector conditions; + + conditions.push_back(mc_QueryCondition(MC_QCT_KEY,params[1].get_str())); + mc_Buffer *entity_rows=mc_gState->m_TmpBuffers->m_RpcEntityRows; entity_rows->Clear(); @@ -1697,7 +1713,7 @@ Value liststreamkeyitems(const Array& params, bool fHelp) uint256 hash; int first_output=mc_GetHashAndFirstOutput(lpEntTx,&hash); const CWalletTx& wtx=pwalletTxsMain->GetWalletTx(hash,NULL,NULL); - Object entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,verbose,&key_ptr,NULL,NULL); + Object entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,verbose,&conditions,NULL); if(entry.size()) { retArray.push_back(entry); @@ -1781,9 +1797,12 @@ Value liststreampublisheritems(const Array& params, bool fHelp) } string key_string=params[1].get_str(); - const char *key_ptr=key_string.c_str(); getSubKeyEntityFromPublisher(params[1].get_str(),entStat,&entity); + vector conditions; + + conditions.push_back(mc_QueryCondition(MC_QCT_PUBLISHER,params[1].get_str())); + mc_Buffer *entity_rows=mc_gState->m_TmpBuffers->m_RpcEntityRows; entity_rows->Clear(); @@ -1799,7 +1818,7 @@ Value liststreampublisheritems(const Array& params, bool fHelp) uint256 hash; int first_output=mc_GetHashAndFirstOutput(lpEntTx,&hash); const CWalletTx& wtx=pwalletTxsMain->GetWalletTx(hash,NULL,NULL); - Object entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,verbose,NULL,&key_ptr,NULL); + Object entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,verbose,&conditions,NULL); if(entry.size()) { retArray.push_back(entry); @@ -1826,8 +1845,6 @@ Value liststreammap_operation(mc_TxEntity *parent_entity,vector& in mc_TxEntityRow erow; uint160 stream_subkey_hash; int row,enitity_count; - const char **given_key; - const char **given_publisher; entity_rows->Clear(); enitity_count=inputEntities.size(); @@ -1879,18 +1896,17 @@ Value liststreammap_operation(mc_TxEntity *parent_entity,vector& in { shift=1; } - const char *key_ptr=key_string.c_str(); - given_key=NULL; - given_publisher=NULL; + vector conditions; + if((parent_entity->m_EntityType & MC_TET_TYPE_MASK) == MC_TET_STREAM_PUBLISHER) { all_entry.push_back(Pair("publisher", key_string)); - given_publisher=&key_ptr; + conditions.push_back(mc_QueryCondition(MC_QCT_PUBLISHER,key_string)); } else { all_entry.push_back(Pair("key", key_string)); - given_key=&key_ptr; + conditions.push_back(mc_QueryCondition(MC_QCT_KEY,key_string)); } all_entry.push_back(Pair("items", total)); all_entry.push_back(Pair("confirmed", confirmed)); @@ -1915,7 +1931,7 @@ Value liststreammap_operation(mc_TxEntity *parent_entity,vector& in Value item_value; - item_value=StreamItemEntry(wtx,first_output,parent_entity->m_EntityID,true,given_key,given_publisher,NULL); + item_value=StreamItemEntry(wtx,first_output,parent_entity->m_EntityID,true,&conditions,NULL); if(row == 1) { all_entry.push_back(Pair("first", item_value)); @@ -2064,3 +2080,191 @@ Value liststreampublishers(const Array& params, bool fHelp) return liststreamkeys_or_publishers(params,true); } +int GetAndQueryDirtyList(vector& conditions, mc_EntityDetails *stream_entity,bool fLocalOrdering,mc_Buffer *entity_rows) +{ + int i,row,out_row; + int conditions_count=(int)conditions.size(); + int conditions_used=0; + int max_size=0; + int clean_count,dirty_count,last_state; + vector vConditionEntities; + vector vConditionListSizes; + vector vConditionMerged; + mc_TxEntityStat entStat; + bool merge_lists=true; + + vConditionEntities.resize(conditions_count+1); + vConditionListSizes.resize(conditions_count+1); + vConditionMerged.resize(conditions_count+1); + + entStat.Zero(); + memcpy(&entStat,stream_entity->GetTxID()+MC_AST_SHORT_TXID_OFFSET,MC_AST_SHORT_TXID_SIZE); + entStat.m_Entity.m_EntityType=MC_TET_STREAM; + if(fLocalOrdering) + { + entStat.m_Entity.m_EntityType |= MC_TET_TIMERECEIVED; + } + else + { + entStat.m_Entity.m_EntityType |= MC_TET_CHAINPOS; + } + if(!pwalletTxsMain->FindEntity(&entStat)) + { + throw JSONRPCError(RPC_NOT_SUBSCRIBED, "Not subscribed to this stream"); + } + + for(i=0;i<=conditions_count;i++) + { + vConditionEntities[i].Zero(); + vConditionListSizes[i]=-1; + vConditionMerged[i]=0; + + entStat.m_Entity.m_EntityType &= MC_TET_ORDERMASK; + if(iGetListSize(&vConditionEntities[i],entStat.m_Generation,NULL); + if(vConditionListSizes[i]>max_size) + { + max_size=vConditionListSizes[i]; + } + } + } + + clean_count=0; + dirty_count=0; + + while(merge_lists) + { + int min_size=max_size+1; + int min_condition=conditions_count; + conditions_count=false; + for(i=0;i<=conditions_count;i++) + { + if(vConditionMerged[i] == 0) + { + if(vConditionListSizes[i]<=min_size) + { + min_size=vConditionListSizes[i]; + min_condition=i; + } + } + } + + if(min_condition MC_QPR_MAX_UNCHECKED_TX_LIST_SIZE) + { + throw JSONRPCError(RPC_NOT_SUBSCRIBED, "Not subscribed to this stream"); + } + pwalletTxsMain->GetList(&vConditionEntities[min_condition],entStat.m_Generation,1,min_size,entity_rows); + conditions_used++; + clean_count=0; + dirty_count=0; + for(row=0;rowGetCount();row++) + { + mc_TxEntityRow *lpEntTx; + lpEntTx=(mc_TxEntityRow*)entity_rows->GetRow(row); + if( (lpEntTx->m_Flags & MC_TFL_IS_EXTENSION) == 0 ) + { + clean_count++; + } + if(!fLocalOrdering) + { + if(lpEntTx->m_Block == -1) + { + lpEntTx->m_Block=chainActive.Height()+1; + } + } + } + } + else + { + merge_lists=false; + } + } + } + + last_state=-2; + clean_count=0; + dirty_count=0; + out_row=0; + for(row=0;rowGetCount();row++) + { + mc_TxEntityRow *lpEntTx; + lpEntTx=(mc_TxEntityRow*)entity_rows->GetRow(row); + if(lpEntTx->m_Flags & MC_TFL_IS_EXTENSION) + { + lpEntTx->m_Generation=last_state; + } + else + { + switch(lpEntTx->m_Generation) + { + case -1: + break; + case -2: + dirty_count++; + break; + default: + if(conditions_used < conditions_count) + { + lpEntTx->m_Generation=-2; + dirty_count++; + } + else + { + clean_count++; + } + break; + } + last_state=lpEntTx->m_Generation; + } + if(lpEntTx->m_Generation != -1) + { + if(out_row < row) + { + memcpy(entity_rows->GetRow(out_row),lpEntTx,entity_rows->m_Size); + out_row++; + } + } + } + + entity_rows->SetCount(out_row); + + return dirty_count; +} + +Value liststreamqueryitems(const Array& params, bool fHelp) +{ + if (fHelp || params.size() < 2 || params.size() > 6) + throw runtime_error("Help message not found\n"); + + if((mc_gState->m_WalletMode & MC_WMD_TXS) == 0) + { + throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported with this wallet version. For full streams functionality, run \"multichaind -walletdbversion=2 -rescan\" "); + } + + return Value::null; +} diff --git a/src/rpc/rpcwallet.h b/src/rpc/rpcwallet.h index adf4cf49..b212d645 100644 --- a/src/rpc/rpcwallet.h +++ b/src/rpc/rpcwallet.h @@ -23,12 +23,33 @@ #include "wallet/chunkdb.h" #include "rpc/rpcutils.h" +#define MC_QCT_KEY 1 +#define MC_QCT_PUBLISHER 2 + + +typedef struct mc_QueryCondition +{ + uint32_t m_Type; + string m_Value; + bool m_TmpMatch; + + mc_QueryCondition(int type, string value) + { + m_Type=type; + m_Value=value; + m_TmpMatch=false; + } +} mc_QueryCondition; + + + + void SendMoneyToSeveralAddresses(const std::vector addresses, CAmount nValue, CWalletTx& wtxNew,mc_Script *dropscript,CScript scriptOpReturn,const std::vector& fromaddresses); vector ParseAddresses(string param, bool create_full_list, bool allow_scripthash); void FindAddressesWithPublishPermission(std::vector& fromaddresses,mc_EntityDetails *stream_entity); set ParseAddresses(Value param, isminefilter filter); bool CBitcoinAddressFromTxEntity(CBitcoinAddress &address,mc_TxEntity *lpEntity); -Object StreamItemEntry(const CWalletTx& wtx,int first_output,const unsigned char *stream_id, bool verbose, const char** given_key,const char ** given_publisher,int *output); +Object StreamItemEntry(const CWalletTx& wtx,int first_output,const unsigned char *stream_id, bool verbose, vector *given_conditions,int *output); Object TxOutEntry(const CTxOut& TxOutIn,int vout,const CTxIn& TxIn,uint256 hash,mc_Buffer *amounts,mc_Script *lpScript); void WalletTxToJSON(const CWalletTx& wtx, Object& entry,bool skipWalletConflicts = false, int vout = -1); void MinimalWalletTxToJSON(const CWalletTx& wtx, Object& entry); diff --git a/src/rpc/rpcwalletutils.cpp b/src/rpc/rpcwalletutils.cpp index bb10546a..74804c38 100644 --- a/src/rpc/rpcwalletutils.cpp +++ b/src/rpc/rpcwalletutils.cpp @@ -430,7 +430,7 @@ bool CBitcoinAddressFromTxEntity(CBitcoinAddress &address,mc_TxEntity *lpEntity) return false; } -Object StreamItemEntry(const CWalletTx& wtx,int first_output,const unsigned char *stream_id, bool verbose, const char** given_key,const char ** given_publisher,int *output) +Object StreamItemEntry1(const CWalletTx& wtx,int first_output,const unsigned char *stream_id, bool verbose, const char** given_key,const char ** given_publisher,int *output) { Object entry; Array publishers; @@ -628,6 +628,237 @@ Object StreamItemEntry(const CWalletTx& wtx,int first_output,const unsigned char return entry; } +Object StreamItemEntry(const CWalletTx& wtx,int first_output,const unsigned char *stream_id, bool verbose, vector *given_conditions,int *output) +{ + Object entry; + Array publishers; + set publishers_set; + Array keys; + int stream_output; + const unsigned char *ptr; + unsigned char item_key[MC_ENT_MAX_ITEM_KEY_SIZE+1]; + int item_key_size; +// Value item_value; + uint32_t format; + unsigned char *chunk_hashes; + int chunk_count; + int64_t total_chunk_size,out_size; + uint32_t retrieve_status=0; + Value format_item_value; + string format_text_str; + int start_from=first_output; + + stream_output=-1; + if(output) + { + *output=(int)wtx.vout.size(); + } + while( (stream_output < 0) && (start_from<(int)wtx.vout.size()) ) + { + stream_output=-1; + for (int j = start_from; j < (int)wtx.vout.size(); ++j) + { + if(stream_output < 0) + { + if(given_conditions) + { + for(int c=0;c<(int)(*given_conditions).size();c++) + { + (*given_conditions)[c].m_TmpMatch=false; + } + } + keys.clear(); + const CScript& script1 = wtx.vout[j].scriptPubKey; + CScript::const_iterator pc1 = script1.begin(); + + mc_gState->m_TmpScript->Clear(); + mc_gState->m_TmpScript->SetScript((unsigned char*)(&pc1[0]),(size_t)(script1.end()-pc1),MC_SCR_TYPE_SCRIPTPUBKEY); + + if(mc_gState->m_TmpScript->IsOpReturnScript()) + { + if(mc_gState->m_TmpScript->GetNumElements()) // 2 OP_DROPs + OP_RETURN - item key + { + mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(&format,&chunk_hashes,&chunk_count,&total_chunk_size); +// chunk_hashes=NULL; +// mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(&format); + + unsigned char short_txid[MC_AST_SHORT_TXID_SIZE]; + mc_gState->m_TmpScript->SetElement(0); + + if(mc_gState->m_TmpScript->GetEntity(short_txid) == 0) + { + if(memcmp(short_txid,stream_id,MC_AST_SHORT_TXID_SIZE) == 0) + { + stream_output=j; + for(int e=1;em_TmpScript->GetNumElements()-1;e++) + { + mc_gState->m_TmpScript->SetElement(e); + // Should be spkk + if(mc_gState->m_TmpScript->GetItemKey(item_key,&item_key_size)) // Item key + { + return entry; + } + item_key[item_key_size]=0; + + if(given_conditions) + { + for(int c=0;c<(int)(*given_conditions).size();c++) + { + if((*given_conditions)[c].m_Type == MC_QCT_KEY) + { + if(strcmp((char*)item_key,(*given_conditions)[c].m_Value.c_str()) == 0) + { + (*given_conditions)[c].m_TmpMatch=true; + } + } + } + } + + keys.push_back(string(item_key,item_key+item_key_size)); + } + + if(given_conditions) + { + for(int c=0;c<(int)(*given_conditions).size();c++) + { + if((*given_conditions)[c].m_Type == MC_QCT_KEY) + { + if(!(*given_conditions)[c].m_TmpMatch) + { + stream_output=-1; + } + } + } + } + + const unsigned char *elem; + +// elem = mc_gState->m_TmpScript->GetData(mc_gState->m_TmpScript->GetNumElements()-1,&elem_size); + retrieve_status = GetFormattedData(mc_gState->m_TmpScript,&elem,&out_size,chunk_hashes,chunk_count,total_chunk_size); +// item_value=OpReturnEntry(elem,elem_size,wtx.GetHash(),j); + format_item_value=OpReturnFormatEntry(elem,out_size,wtx.GetHash(),j,format,&format_text_str,retrieve_status); + } + } + } + } + } + } + + if(stream_output < 0) + { + return entry; + } + if(output) + { + *output=stream_output; + } + + publishers.clear(); + publishers_set.clear(); + for (int i = 0; i < (int)wtx.vin.size(); ++i) + { + int op_addr_offset,op_addr_size,is_redeem_script,sighash_type; + + const CScript& script2 = wtx.vin[i].scriptSig; + CScript::const_iterator pc2 = script2.begin(); + + ptr=mc_ExtractAddressFromInputScript((unsigned char*)(&pc2[0]),(int)(script2.end()-pc2),&op_addr_offset,&op_addr_size,&is_redeem_script,&sighash_type,0); + if(ptr) + { + if( (sighash_type == SIGHASH_ALL) || ( (sighash_type == SIGHASH_SINGLE) && (i == stream_output) ) ) + { + uint160 publisher_hash=Hash160(ptr+op_addr_offset,ptr+op_addr_offset+op_addr_size); + if(publishers_set.count(publisher_hash) == 0) + { + publishers_set.insert(publisher_hash); + string publisher_str; + if(is_redeem_script) + { + publisher_str=CBitcoinAddress((CScriptID)publisher_hash).ToString(); + } + else + { + publisher_str=CBitcoinAddress((CKeyID)publisher_hash).ToString(); + } + + if(given_conditions) + { + for(int c=0;c<(int)(*given_conditions).size();c++) + { + if((*given_conditions)[c].m_Type == MC_QCT_PUBLISHER) + { + if(strcmp(publisher_str.c_str(),(*given_conditions)[c].m_Value.c_str()) == 0) + { + (*given_conditions)[c].m_TmpMatch=true; + } + } + } + } + + publishers.push_back(publisher_str); + } + } + } + } + + if(given_conditions) + { + for(int c=0;c<(int)(*given_conditions).size();c++) + { + if((*given_conditions)[c].m_Type == MC_QCT_PUBLISHER) + { + if(!(*given_conditions)[c].m_TmpMatch) + { + stream_output=-1; + } + } + } + } + + if(stream_output < 0) + { + start_from++; + } + } + + if(stream_output < 0) + { + return entry; + } + + + entry.push_back(Pair("publishers", publishers)); + entry.push_back(Pair("keys", keys)); + if(mc_gState->m_Compatibility & MC_VCM_1_0) + { + entry.push_back(Pair("key", keys[0])); + } + entry.push_back(Pair("offchain", (retrieve_status & MC_OST_STORAGE_MASK) == MC_OST_OFF_CHAIN)); + if( ( retrieve_status & MC_OST_CONTROL_NO_DATA ) == 0) + { + entry.push_back(Pair("available", AvailableFromStatus(retrieve_status))); + if(retrieve_status & MC_OST_ERROR_MASK) + { + string error_str; + int errorCode; + error_str=OffChainError(retrieve_status,&errorCode); + entry.push_back(Pair("error", error_str)); + } + } + entry.push_back(Pair("data", format_item_value)); + + if(verbose) + { + WalletTxToJSON(wtx, entry, true, stream_output); + } + else + { + MinimalWalletTxToJSON(wtx, entry); + } + + return entry; +} + Object TxOutEntry(const CTxOut& TxOutIn,int vout,const CTxIn& TxIn,uint256 hash,mc_Buffer *amounts,mc_Script *lpScript) { From 32c6bc0875e5f741705c778406fd3ec9d83b214c Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 5 Aug 2018 12:43:14 +0300 Subject: [PATCH 149/157] liststreamqueryitems --- src/rpc/rpcstreams.cpp | 247 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 227 insertions(+), 20 deletions(-) diff --git a/src/rpc/rpcstreams.cpp b/src/rpc/rpcstreams.cpp index 3fa0350a..79448d0e 100644 --- a/src/rpc/rpcstreams.cpp +++ b/src/rpc/rpcstreams.cpp @@ -12,7 +12,8 @@ #define MC_QPR_MAX_UNCHECKED_TX_LIST_SIZE 1048576 #define MC_QPR_MAX_MERGED_TX_LIST_SIZE 1024 -#define MC_QPR_MAX_DIRTY_TX_LIST_SIZE 256 +#define MC_QPR_MAX_DIRTY_TX_LIST_SIZE 5000 +#define MC_QPR_MAX_CLEAN_TX_LIST_SIZE 5000 #define MC_QPR_MAX_SECONDARY_TX_LIST_SIZE 1048576 #define MC_QPR_TX_CHECK_COST 100 #define MC_QPR_MAX_TX_PER_BLOCK 4 @@ -2125,23 +2126,23 @@ int GetAndQueryDirtyList(vector& conditions, mc_EntityDetails switch(conditions[i].m_Type) { case MC_QCT_KEY: - entStat.m_Entity.m_EntityType=MC_TET_STREAM_KEY; + entStat.m_Entity.m_EntityType |= MC_TET_STREAM_KEY; getSubKeyEntityFromKey(conditions[i].m_Value,entStat,&vConditionEntities[i]); break; case MC_QCT_PUBLISHER: - entStat.m_Entity.m_EntityType=MC_TET_STREAM_PUBLISHER; - getSubKeyEntityFromKey(conditions[i].m_Value,entStat,&vConditionEntities[i]); + entStat.m_Entity.m_EntityType |= MC_TET_STREAM_PUBLISHER; + getSubKeyEntityFromPublisher(conditions[i].m_Value,entStat,&vConditionEntities[i]); break; } } else { - entStat.m_Entity.m_EntityType=MC_TET_STREAM; + entStat.m_Entity.m_EntityType |= MC_TET_STREAM; memcpy(&vConditionEntities[i],&entStat.m_Entity,sizeof(mc_TxEntity)); } if(vConditionEntities[i].m_EntityType) { - vConditionListSizes[i]=pwalletTxsMain->GetListSize(&vConditionEntities[i],entStat.m_Generation,NULL); + vConditionListSizes[i]=pwalletTxsMain->GetListSize(&vConditionEntities[i],entStat.m_Generation,NULL); if(vConditionListSizes[i]>max_size) { max_size=vConditionListSizes[i]; @@ -2156,7 +2157,6 @@ int GetAndQueryDirtyList(vector& conditions, mc_EntityDetails { int min_size=max_size+1; int min_condition=conditions_count; - conditions_count=false; for(i=0;i<=conditions_count;i++) { if(vConditionMerged[i] == 0) @@ -2172,11 +2172,11 @@ int GetAndQueryDirtyList(vector& conditions, mc_EntityDetails if(min_condition MC_QPR_MAX_UNCHECKED_TX_LIST_SIZE) { - throw JSONRPCError(RPC_NOT_SUBSCRIBED, "Not subscribed to this stream"); + throw JSONRPCError(RPC_NOT_SUPPORTED, "This query may take too much time"); } pwalletTxsMain->GetList(&vConditionEntities[min_condition],entStat.m_Generation,1,min_size,entity_rows); conditions_used++; @@ -2186,6 +2186,7 @@ int GetAndQueryDirtyList(vector& conditions, mc_EntityDetails { mc_TxEntityRow *lpEntTx; lpEntTx=(mc_TxEntityRow*)entity_rows->GetRow(row); + lpEntTx->m_TempPos=0; if( (lpEntTx->m_Flags & MC_TFL_IS_EXTENSION) == 0 ) { clean_count++; @@ -2204,9 +2205,13 @@ int GetAndQueryDirtyList(vector& conditions, mc_EntityDetails merge_lists=false; } } + else + { + merge_lists=false; + } } - last_state=-2; + last_state=2; clean_count=0; dirty_count=0; out_row=0; @@ -2216,21 +2221,25 @@ int GetAndQueryDirtyList(vector& conditions, mc_EntityDetails lpEntTx=(mc_TxEntityRow*)entity_rows->GetRow(row); if(lpEntTx->m_Flags & MC_TFL_IS_EXTENSION) { - lpEntTx->m_Generation=last_state; + lpEntTx->m_TempPos=last_state; + if(lpEntTx->m_TempPos == 2) + { + dirty_count++; + } } else { - switch(lpEntTx->m_Generation) + switch(lpEntTx->m_TempPos) { - case -1: + case 1: break; - case -2: + case 2: dirty_count++; break; default: if(conditions_used < conditions_count) { - lpEntTx->m_Generation=-2; + lpEntTx->m_TempPos=2; dirty_count++; } else @@ -2239,15 +2248,15 @@ int GetAndQueryDirtyList(vector& conditions, mc_EntityDetails } break; } - last_state=lpEntTx->m_Generation; + last_state=lpEntTx->m_TempPos; } - if(lpEntTx->m_Generation != -1) + if(lpEntTx->m_TempPos != 1) { if(out_row < row) { memcpy(entity_rows->GetRow(out_row),lpEntTx,entity_rows->m_Size); - out_row++; } + out_row++; } } @@ -2256,15 +2265,213 @@ int GetAndQueryDirtyList(vector& conditions, mc_EntityDetails return dirty_count; } +void FillContitionsList(vector& conditions, Value param) +{ + bool key_found=false; + bool publisher_found=false; + bool field_parsed; + + if(param.type() != obj_type) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid query, should be object "); + } + + BOOST_FOREACH(const Pair& d, param.get_obj()) + { + field_parsed=false; + if(d.name_ == "key") + { + if(key_found) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Only one of the key fields can appear in the object"); + } + if(d.value_.type()==str_type) + { + conditions.push_back(mc_QueryCondition(MC_QCT_KEY,d.value_.get_str())); + } + else + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid key, should be string"); + } + field_parsed=true; + key_found=true; + } + + if(d.name_ == "keys") + { + if( mc_gState->m_Features->MultipleStreamKeys() == 0 ) + { + throw JSONRPCError(RPC_NOT_SUPPORTED, "Multiple keys are not supported by this protocol version"); + } + if(key_found) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Only one of the key fields can appear in the object"); + } + if(d.value_.type() == array_type) + { + for(int i=0;i<(int)d.value_.get_array().size();i++) + { + if(d.value_.get_array()[i].type()==str_type) + { + conditions.push_back(mc_QueryCondition(MC_QCT_KEY,d.value_.get_array()[i].get_str())); + } + else + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid key, should be string"); + } + } + } + else + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid keys, should be array"); + } + field_parsed=true; + key_found=true; + } + + if(d.name_ == "publisher") + { + if(publisher_found) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Only one of the publisher fields can appear in the object"); + } + if(d.value_.type()==str_type) + { + conditions.push_back(mc_QueryCondition(MC_QCT_PUBLISHER,d.value_.get_str())); + } + else + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid publisher, should be string"); + } + field_parsed=true; + publisher_found=true; + } + + if(d.name_ == "publishers") + { + if(publisher_found) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Only one of the publisher fields can appear in the object"); + } + if(d.value_.type() == array_type) + { + for(int i=0;i<(int)d.value_.get_array().size();i++) + { + if(d.value_.get_array()[i].type()==str_type) + { + conditions.push_back(mc_QueryCondition(MC_QCT_PUBLISHER,d.value_.get_array()[i].get_str())); + } + else + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid publisher, should be string"); + } + } + } + else + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid publishers, should be array"); + } + field_parsed=true; + publisher_found=true; + } + + if(!field_parsed) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid field: %s",d.name_.c_str())); + } + + } +} + Value liststreamqueryitems(const Array& params, bool fHelp) { - if (fHelp || params.size() < 2 || params.size() > 6) + if (fHelp || params.size() < 2 || params.size() > 3) throw runtime_error("Help message not found\n"); if((mc_gState->m_WalletMode & MC_WMD_TXS) == 0) { throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported with this wallet version. For full streams functionality, run \"multichaind -walletdbversion=2 -rescan\" "); } + + vector conditions; + vector * lpConditions; - return Value::null; + mc_EntityDetails stream_entity; + parseStreamIdentifier(params[0],&stream_entity); + + bool verbose=false; + int dirty_count; + + if (params.size() > 2) + { + verbose=paramtobool(params[2]); + } + + FillContitionsList(conditions,params[1]); + + if(conditions.size() == 0) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid query, cannot be empty"); + } + + mc_Buffer *entity_rows=mc_gState->m_TmpBuffers->m_RpcEntityRows; + entity_rows->Clear(); + + dirty_count=GetAndQueryDirtyList(conditions,&stream_entity,false,entity_rows); + + if(dirty_count > MC_QPR_MAX_DIRTY_TX_LIST_SIZE) + { + throw JSONRPCError(RPC_NOT_SUPPORTED, "This query may take too much time"); + } + + if(entity_rows->GetCount() > MC_QPR_MAX_CLEAN_TX_LIST_SIZE) + { + throw JSONRPCError(RPC_NOT_SUPPORTED, "Resulting list is too large"); + } + + Array retArray; + int last_output; + uint256 last_hash=0; + for(int i=0;iGetCount();i++) + { + mc_TxEntityRow *lpEntTx; + lpEntTx=(mc_TxEntityRow*)entity_rows->GetRow(i); + lpConditions=NULL; + if(lpEntTx->m_TempPos != 1) + { + if(lpEntTx->m_TempPos == 2) + { + lpConditions=&conditions; + } + uint256 hash; + int first_output=mc_GetHashAndFirstOutput(lpEntTx,&hash); + if(last_hash == hash) + { + if(first_output <= last_output) + { + first_output=-1; + } + } + else + { + last_output=-1; + } + last_hash=hash; + if(first_output >= 0) + { + const CWalletTx& wtx=pwalletTxsMain->GetWalletTx(hash,NULL,NULL); + Object entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,verbose,lpConditions,&last_output); + if(entry.size()) + { + retArray.push_back(entry); + } + else + { + last_output=-1; + } + } + } + } + + return retArray; } From 36c76df99ac43c9319a6e6b3684136b59fdbc1a9 Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 5 Aug 2018 13:04:34 +0300 Subject: [PATCH 150/157] liststreamqueryitems help --- src/rpc/rpchelp.cpp | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/rpc/rpchelp.cpp b/src/rpc/rpchelp.cpp index e803bb4f..e9c3f823 100644 --- a/src/rpc/rpchelp.cpp +++ b/src/rpc/rpchelp.cpp @@ -4118,7 +4118,31 @@ void mc_InitRPCHelpMap18() )); - mapHelpStrings.insert(std::make_pair("AAAAAAA", + mapHelpStrings.insert(std::make_pair("liststreamqueryitems", + "liststreamqueryitems \"stream-identifier\" query ( verbose )\n" + "\nReturns stream items for specific query.\n" + "\nArguments:\n" + "1. \"stream-identifier\" (string, required) Stream identifier - one of the following: stream txid, stream reference, stream name.\n" + "2. query (object, required) Query\n" + " {\n" + " \"key\" : \"key\" (string, optional, default: \"\") Item key\n" + " or\n" + " \"keys\" : keys (array, optional) Item keys, array of strings\n" + " or\n" + " \"publisher\" : \"publisher\" (string, optional, default: \"\") Publisher\n" + " or\n" + " \"publishers\" : publishers (array, optional) Publishers, array of strings\n" + " }\n" + "3. verbose (boolean, optional, default=false) If true, returns information about item transaction \n" + "\nResult:\n" + "\"stream-items\" (array) List of stream items for specific query.\n" + "\nExamples:\n" + + HelpExampleCli("liststreamqueryitems", "\"test-stream\" \"{\\\"keys\\\":[\\\"key01\\\",\"key02\"]}\"") + + HelpExampleCli("liststreamqueryitems", "\"test-stream\" \"{\\\"keys\\\":[\\\"key01\\\",\"key02\"],\\\"publisher\\\":\\\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XZ\\\"}\" true ") + + HelpExampleRpc("liststreamqueryitems", "\"test-stream\", \"{\\\"keys\\\":[\\\"key01\\\",\"key02\"],\\\"publisher\\\":\\\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XZ\\\"}\", false") + )); + + mapHelpStrings.insert(std::make_pair("AAAAAAA", "" )); From 7096905aa65a504de14d5edf532f192c0bf65ef1 Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 5 Aug 2018 13:25:45 +0300 Subject: [PATCH 151/157] Multiple txs in liststreamtxitems --- src/rpc/rpcclient.cpp | 2 ++ src/rpc/rpchelp.cpp | 6 ++++-- src/rpc/rpcstreams.cpp | 37 ++++++++++++++++++++++++------------- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/src/rpc/rpcclient.cpp b/src/rpc/rpcclient.cpp index a5512c9b..b46e6148 100644 --- a/src/rpc/rpcclient.cpp +++ b/src/rpc/rpcclient.cpp @@ -178,6 +178,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "listassettransactions", 4 }, { "getassettransaction", 2 }, { "getstreamitem", 2 }, + { "liststreamtxitems", 1 }, { "liststreamtxitems", 2 }, { "liststreamitems", 1 }, { "liststreamitems", 2 }, @@ -341,6 +342,7 @@ static const CRPCConvertParamMayBeString vRPCConvertParamsMayBeString[] = { "unsubscribe", 0 }, { "liststreamkeys", 1 }, { "liststreampublishers", 1 }, + { "liststreamtxitems", 1 }, { "listassets", 0 }, { "liststreams", 0 }, { "listupgrades", 0 }, diff --git a/src/rpc/rpchelp.cpp b/src/rpc/rpchelp.cpp index e9c3f823..3fd477cd 100644 --- a/src/rpc/rpchelp.cpp +++ b/src/rpc/rpchelp.cpp @@ -3709,11 +3709,13 @@ void mc_InitRPCHelpMap16() )); mapHelpStrings.insert(std::make_pair("liststreamtxitems", - "liststreamtxitems \"stream-identifier\" \"txid\" ( verbose )\n" + "liststreamtxitems \"stream-identifier\" txids ( verbose )\n" "\nReturns stream items.\n" "\nArguments:\n" "1. \"stream-identifier\" (string, required) Stream identifier - one of the following: stream txid, stream reference, stream name.\n" - "2. \"txid\" (string, required) The transaction id\n" + "2. \"txids\" (string, required) Transaction IDs, comma delimited\n" + " or\n" + "2. txids (array, required) Array of transaction IDs\n" "3. verbose (boolean, optional, default=false) If true, returns information about item transaction \n" "\nResult:\n" "\"stream-items\" (array) Array of stream items.\n" diff --git a/src/rpc/rpcstreams.cpp b/src/rpc/rpcstreams.cpp index 79448d0e..30884720 100644 --- a/src/rpc/rpcstreams.cpp +++ b/src/rpc/rpcstreams.cpp @@ -981,9 +981,6 @@ Value liststreamtxitems(const Array& params, bool fHelp) throw JSONRPCError(RPC_NOT_SUBSCRIBED, "Not subscribed to this stream"); } - - uint256 hash = ParseHashV(params[1], "parameter 2"); - bool verbose=false; if (params.size() > 2) @@ -991,21 +988,35 @@ Value liststreamtxitems(const Array& params, bool fHelp) verbose=paramtobool(params[2]); } - const CWalletTx& wtx=pwalletTxsMain->GetWalletTx(hash,NULL,NULL); - Array output_array; - int first_output=0; - int stream_output; - while(first_output < (int)wtx.vout.size()) + + vector inputStrings; + + inputStrings=ParseStringList(params[1]); + + for(int j=0;j<(int)inputStrings.size();j++) { - Object entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,verbose,NULL,&stream_output); + uint256 hash = ParseHashV(inputStrings[j], "txid"); - if(stream_output < (int)wtx.vout.size()) + const CWalletTx& wtx=pwalletTxsMain->GetWalletTx(hash,NULL,NULL); + + int first_output=0; + int stream_output; + while(first_output < (int)wtx.vout.size()) { - output_array.push_back(entry); + Object entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,verbose,NULL,&stream_output); + + if(stream_output < (int)wtx.vout.size()) + { + output_array.push_back(entry); + } + first_output=stream_output+1; } - first_output=stream_output+1; - } + } + +// uint256 hash = ParseHashV(params[1], "parameter 2"); + + return output_array; } From 28f7960f51954aa5145175b6e20706e49929cc11 Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 5 Aug 2018 13:34:14 +0300 Subject: [PATCH 152/157] Version 2.0 alpha 4 --- src/version/version.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version/version.cpp b/src/version/version.cpp index 8f8f2d9c..d1c8e5d2 100644 --- a/src/version/version.cpp +++ b/src/version/version.cpp @@ -13,7 +13,7 @@ int mc_State::VersionInfo(int version) return custom_version; } - int this_build=20000103; + int this_build=20000104; int this_protocol=20003; if(version < 0) From 5579db0b7089d644de70efe1040f7d22e3202c7f Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 6 Aug 2018 10:11:43 +0300 Subject: [PATCH 153/157] maxqueryscanitems runtime parameter --- src/chainparams/globals.h | 1 + src/core/init.cpp | 1 + src/core/main.h | 1 + src/rpc/rpcstreams.cpp | 12 +++++++----- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/chainparams/globals.h b/src/chainparams/globals.h index b0cdc6a9..0e55f380 100644 --- a/src/chainparams/globals.h +++ b/src/chainparams/globals.h @@ -23,6 +23,7 @@ int64_t MAX_MONEY = 21000000 * COIN; unsigned int MAX_SCRIPT_ELEMENT_SIZE=520; // script.h int MIN_BLOCKS_BETWEEN_UPGRADES = 100; int MAX_OP_RETURN_SHOWN=16384; +int MAX_STREAM_QUERY_ITEMS=5000; int MAX_FORMATTED_DATA_DEPTH=100; unsigned int MAX_OP_RETURN_OP_DROP_COUNT=100000000; uint32_t JSON_NO_DOUBLE_FORMATTING=0; diff --git a/src/core/init.cpp b/src/core/init.cpp index baf43e92..9b1e3610 100644 --- a/src/core/init.cpp +++ b/src/core/init.cpp @@ -505,6 +505,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += " -hideknownopdrops= " + strprintf(_("Remove recognized MultiChain OP_DROP metadata from the responses to JSON_RPC calls (default: %u)"), 0) + "\n"; strUsage += " -maxshowndata= " + strprintf(_("The maximum number of bytes to show in the data field of API responses. (default: %u)"), MAX_OP_RETURN_SHOWN) + "\n"; strUsage += " " + _("Pieces of data larger than this will be returned as an object with txid, vout and size fields, for use with the gettxoutdata command.") + "\n"; + strUsage += " -maxqueryscanitems= " + strprintf(_("The maximum number of txs to be decoded in JSON_RPC calls. (default: %u)"), MAX_STREAM_QUERY_ITEMS) + "\n"; strUsage += " -v1apicompatible= " + strprintf(_("JSON_RPC calls responses compatible with MultiChain 1.0 (default: %u)"), 0) + "\n"; // strUsage += " -apidecimaldigits= " + _("maximal number of decimal digits in API output (default: auto)") + "\n"; diff --git a/src/core/main.h b/src/core/main.h index 4095809c..2444f3f7 100644 --- a/src/core/main.h +++ b/src/core/main.h @@ -72,6 +72,7 @@ static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 50000; static const unsigned int DEFAULT_MAX_SUCCESSORS_FROM_ONE_NODE = 10; /* MCHN END */ extern int MAX_OP_RETURN_SHOWN; +extern int MAX_STREAM_QUERY_ITEMS; extern int MAX_FORMATTED_DATA_DEPTH; extern int MIN_BLOCKS_BETWEEN_UPGRADES; extern unsigned int OFFCHAIN_MSG_PADDING; diff --git a/src/rpc/rpcstreams.cpp b/src/rpc/rpcstreams.cpp index 30884720..9d1fbbcd 100644 --- a/src/rpc/rpcstreams.cpp +++ b/src/rpc/rpcstreams.cpp @@ -2411,7 +2411,7 @@ Value liststreamqueryitems(const Array& params, bool fHelp) parseStreamIdentifier(params[0],&stream_entity); bool verbose=false; - int dirty_count; + int dirty_count,max_count; if (params.size() > 2) { @@ -2429,13 +2429,15 @@ Value liststreamqueryitems(const Array& params, bool fHelp) entity_rows->Clear(); dirty_count=GetAndQueryDirtyList(conditions,&stream_entity,false,entity_rows); - - if(dirty_count > MC_QPR_MAX_DIRTY_TX_LIST_SIZE) + max_count=GetArg("-maxqueryscanitems",MAX_STREAM_QUERY_ITEMS); + if(dirty_count > max_count) { - throw JSONRPCError(RPC_NOT_SUPPORTED, "This query may take too much time"); + throw JSONRPCError(RPC_NOT_SUPPORTED, + strprintf("This query requires decoding %d items, which is above the maxqueryscanitems limit of %d.", + dirty_count,max_count)); } - if(entity_rows->GetCount() > MC_QPR_MAX_CLEAN_TX_LIST_SIZE) + if(entity_rows->GetCount() > max_count) { throw JSONRPCError(RPC_NOT_SUPPORTED, "Resulting list is too large"); } From 4c1fe2833cf14424545cebad7a0bc1d7d4f46dd2 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 6 Aug 2018 11:39:19 +0300 Subject: [PATCH 154/157] Fixed missing creator list in liststreams --- src/rpc/rpcutils.cpp | 49 +++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index d16262ce..a3b2e5a8 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -666,31 +666,42 @@ Object StreamEntry(const unsigned char *txid,uint32_t output_level) string param_value((char*)ptr+value_offset,(char*)ptr+value_offset+value_size); fields.push_back(Pair(param_name, param_value)); } - else - { - if(ptr[offset+1] == MC_ENT_SPRM_ISSUER) - { - if(value_size == 24) - { - unsigned char tptr[4]; - memcpy(tptr,ptr+value_offset+sizeof(uint160),4); - if(mc_GetLE(tptr,4) & MC_PFL_IS_SCRIPTHASH) - { - openers.push_back(CBitcoinAddress(*(CScriptID*)(ptr+value_offset)).ToString()); - } - else - { - openers.push_back(CBitcoinAddress(*(CKeyID*)(ptr+value_offset)).ToString()); - } - } - } - } } offset=new_offset; } vfields=fields; } + offset=0; + while(offset>=0) + { + new_offset=entity.NextParam(offset,&value_offset,&value_size); + if(value_offset > 0) + { + if(ptr[offset] == 0) + { + if(ptr[offset+1] == MC_ENT_SPRM_ISSUER) + { + if(value_size == 24) + { + unsigned char tptr[4]; + memcpy(tptr,ptr+value_offset+sizeof(uint160),4); + if(mc_GetLE(tptr,4) & MC_PFL_IS_SCRIPTHASH) + { + openers.push_back(CBitcoinAddress(*(CScriptID*)(ptr+value_offset)).ToString()); + } + else + { + openers.push_back(CBitcoinAddress(*(CKeyID*)(ptr+value_offset)).ToString()); + } + } + } + } + } + offset=new_offset; + } + + entry.push_back(Pair("details",vfields)); } From d9dbe66ce35cbd87ba5c191dbcbccd2037a407d2 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 6 Aug 2018 14:58:15 +0300 Subject: [PATCH 155/157] maxqueryscanitems in setruntimeparam --- src/core/init.cpp | 6 +++--- src/rpc/rpchelp.cpp | 3 ++- src/rpc/rpcmisc.cpp | 2 ++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/core/init.cpp b/src/core/init.cpp index 9b1e3610..07ea282a 100644 --- a/src/core/init.cpp +++ b/src/core/init.cpp @@ -502,11 +502,11 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += " -flushsourcechunks= " + _("Flush offchain items created by this node to disk immediately when created, default 1") + "\n"; strUsage += "\n" + _("MultiChain API response parameters") + "\n"; - strUsage += " -hideknownopdrops= " + strprintf(_("Remove recognized MultiChain OP_DROP metadata from the responses to JSON_RPC calls (default: %u)"), 0) + "\n"; + strUsage += " -hideknownopdrops= " + strprintf(_("Remove recognized MultiChain OP_DROP metadata from the responses to JSON-RPC calls (default: %u)"), 0) + "\n"; strUsage += " -maxshowndata= " + strprintf(_("The maximum number of bytes to show in the data field of API responses. (default: %u)"), MAX_OP_RETURN_SHOWN) + "\n"; strUsage += " " + _("Pieces of data larger than this will be returned as an object with txid, vout and size fields, for use with the gettxoutdata command.") + "\n"; - strUsage += " -maxqueryscanitems= " + strprintf(_("The maximum number of txs to be decoded in JSON_RPC calls. (default: %u)"), MAX_STREAM_QUERY_ITEMS) + "\n"; - strUsage += " -v1apicompatible= " + strprintf(_("JSON_RPC calls responses compatible with MultiChain 1.0 (default: %u)"), 0) + "\n"; + strUsage += " -maxqueryscanitems= " + strprintf(_("The maximum number of txs to be decoded during JSON-RPC querying commands. (default: %u)"), MAX_STREAM_QUERY_ITEMS) + "\n"; + strUsage += " -v1apicompatible= " + strprintf(_("JSON-RPC calls responses compatible with MultiChain 1.0 (default: %u)"), 0) + "\n"; // strUsage += " -apidecimaldigits= " + _("maximal number of decimal digits in API output (default: auto)") + "\n"; strUsage += "\n" + _("Wallet optimization options:") + "\n"; diff --git a/src/rpc/rpchelp.cpp b/src/rpc/rpchelp.cpp index 3fd477cd..de09acda 100644 --- a/src/rpc/rpchelp.cpp +++ b/src/rpc/rpchelp.cpp @@ -3550,6 +3550,7 @@ void mc_InitRPCHelpMap15() " miningturnover,\n" " lockadminminerounds,\n" " maxshowndata, \n" + " maxqueryscanitems, \n" " bantx,\n" " lockblock,\n" " autosubscribe,\n" @@ -4130,7 +4131,7 @@ void mc_InitRPCHelpMap18() " \"key\" : \"key\" (string, optional, default: \"\") Item key\n" " or\n" " \"keys\" : keys (array, optional) Item keys, array of strings\n" - " or\n" + " and/or\n" " \"publisher\" : \"publisher\" (string, optional, default: \"\") Publisher\n" " or\n" " \"publishers\" : publishers (array, optional) Publishers, array of strings\n" diff --git a/src/rpc/rpcmisc.cpp b/src/rpc/rpcmisc.cpp index 8f11ae37..358cd151 100644 --- a/src/rpc/rpcmisc.cpp +++ b/src/rpc/rpcmisc.cpp @@ -225,6 +225,7 @@ Value getruntimeparams(const json_spirit::Array& params, bool fHelp) obj.push_back(Pair("lockblock",GetArg("-lockblock",""))); obj.push_back(Pair("hideknownopdrops",GetBoolArg("-hideknownopdrops",false))); obj.push_back(Pair("maxshowndata",GetArg("-maxshowndata",MAX_OP_RETURN_SHOWN))); + obj.push_back(Pair("maxqueryscanitems",GetArg("-maxqueryscanitems",MAX_STREAM_QUERY_ITEMS))); obj.push_back(Pair("v1apicompatible",GetBoolArg("-v1apicompatible",false))); obj.push_back(Pair("miningrequirespeers",Params().MiningRequiresPeers())); obj.push_back(Pair("mineemptyrounds",Params().MineEmptyRounds())); @@ -360,6 +361,7 @@ Value setruntimeparam(const json_spirit::Array& params, bool fHelp) } if( (param_name == "lockadminminerounds") || (param_name == "maxshowndata") || + (param_name == "maxqueryscanitems") || (param_name == "dropmessagestest") ) { if( (params[1].type() == int_type) || (params[1].type() == str_type) ) From ff46bdb120ebf0c305afad5e473394ca74b8a955 Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 8 Aug 2018 10:04:31 +0300 Subject: [PATCH 156/157] Fixed possible crash when inserting new chunk --- src/wallet/chunkcollector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wallet/chunkcollector.cpp b/src/wallet/chunkcollector.cpp index 5063e4aa..f1b72ed1 100644 --- a/src/wallet/chunkcollector.cpp +++ b/src/wallet/chunkcollector.cpp @@ -578,7 +578,7 @@ int mc_ChunkCollector::InsertChunkInternal( m_TotalChunkCount++; m_TotalChunkSize+=chunk_size; - if(m_MemPoolNext->GetCount() >= 2*m_MaxMemPoolSize) + if(m_MemPool->GetCount() >= 2*m_MaxMemPoolSize) { CommitInternal(0); } From de34537a5e9fec999ce5d10a514c0c8c7d7fb123 Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 8 Aug 2018 22:23:48 +0300 Subject: [PATCH 157/157] Fixed memory leak on our tx count --- src/wallet/wallet.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index e63473f4..d344607d 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -3136,7 +3136,7 @@ bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, stri } // Track how many getdata requests our transaction gets - mapRequestCount[wtxNew.GetHash()] = 0; + // mapRequestCount[wtxNew.GetHash()] = 0; // Broadcast