Skip to content
This repository has been archived by the owner on Nov 15, 2023. It is now read-only.

move paras inherent filtering to runtime #4028

Merged
merged 112 commits into from
Nov 16, 2021
Merged
Changes from 1 commit
Commits
Show all changes
112 commits
Select commit Hold shift + click to select a range
87b6cde
move things around, add filter methods
drahnr Oct 6, 2021
bf1f98b
validator keys, modify availability bitfields according to disputes
drahnr Oct 8, 2021
7c2711d
simplify, keep the filter -> sanitize generic for both usecases
drahnr Oct 13, 2021
702bceb
minor
drahnr Oct 13, 2021
de29623
assure tests still work, reduce changeset
drahnr Oct 13, 2021
98271ae
integration
drahnr Oct 13, 2021
f7003e8
start entropy passing
drahnr Oct 13, 2021
eb27dbb
fixins
drahnr Oct 14, 2021
0e5d2ff
compile, 1 failing test
drahnr Oct 14, 2021
ceaefae
filter with coverage
drahnr Oct 15, 2021
2f8385f
fixins
drahnr Oct 15, 2021
c169ae0
Update runtime/parachains/src/paras_inherent.rs
drahnr Oct 18, 2021
623a283
slip of the pen
drahnr Oct 18, 2021
9e1cb25
improve test cases
drahnr Oct 18, 2021
e0f4b9e
misc
drahnr Oct 18, 2021
9ec43fa
fix
drahnr Oct 18, 2021
8bd9af6
fixins
drahnr Oct 18, 2021
2c23251
test avoid extra into() calls in assert_noop!
drahnr Oct 18, 2021
b815172
chores
drahnr Oct 18, 2021
903382b
ff
drahnr Oct 18, 2021
75b48eb
test fixup superfluous into call
drahnr Oct 18, 2021
1c2b4de
chore: pfmt
drahnr Oct 18, 2021
c38771f
improve apply_block_weight_limit to try to maximize the number of suf…
drahnr Oct 18, 2021
2908356
new code treats the lack of backed candidates as ok
drahnr Oct 18, 2021
82617f2
Use vrf based entropy
drahnr Oct 18, 2021
1b236a5
fixup vrf random
drahnr Oct 19, 2021
56d2752
add warn
drahnr Oct 19, 2021
e11c985
slip of the pen
drahnr Oct 19, 2021
4e13c1b
fixup
drahnr Oct 19, 2021
7d00562
assure ordering
drahnr Oct 19, 2021
4e06e8d
rethink apply_weights
drahnr Oct 19, 2021
383711d
mock
drahnr Oct 19, 2021
bcd70eb
use a closure as predicate check
drahnr Oct 20, 2021
84cbec3
extract and use DisputedBitfield
drahnr Oct 20, 2021
f347b3f
chore: simplify
drahnr Oct 20, 2021
0df5cf4
remove stray dbg
drahnr Oct 20, 2021
376d158
chore: fmt
drahnr Oct 20, 2021
2d2f8a8
address feedback
drahnr Oct 20, 2021
a1a567c
fix test, halfway there
drahnr Oct 20, 2021
64a759a
stage1
drahnr Oct 21, 2021
98fb6d4
dbg stuff
drahnr Oct 21, 2021
2085069
make group selection align
drahnr Oct 21, 2021
2795af0
fix session index
drahnr Oct 22, 2021
3271fd3
fix wrongly returned candidates
drahnr Oct 22, 2021
4375b83
cleanup
drahnr Oct 22, 2021
cbed703
chore fmt
drahnr Oct 22, 2021
88a7ee1
fix ensure check
drahnr Oct 24, 2021
a7332e0
make good case test work
drahnr Oct 24, 2021
849af66
more tests for bitfields
drahnr Oct 24, 2021
bfa1f1a
create sanitize_backed_candidates
drahnr Oct 29, 2021
c2826a7
fixup tests
drahnr Oct 29, 2021
d2f03f4
update guide
drahnr Oct 31, 2021
209e267
add check referenced in the guide
drahnr Oct 31, 2021
03e77ad
improve weights code
drahnr Oct 31, 2021
3b89d79
fmt
drahnr Oct 31, 2021
20249dd
fixins
drahnr Oct 31, 2021
57e8daa
Update roadmap/implementers-guide/src/runtime/inclusion.md
drahnr Nov 1, 2021
352872b
compiling + address review
drahnr Nov 1, 2021
c592b48
add comments
drahnr Nov 1, 2021
7dd4773
fix weight calc
drahnr Nov 1, 2021
6cb7b8a
Merge remote-tracking branch 'origin/master' into bernhard-inherent-f…
drahnr Nov 3, 2021
ca447a8
address review comments and test failure
drahnr Nov 3, 2021
7b75f7a
fix
drahnr Nov 3, 2021
d524b34
fix: condition
drahnr Nov 5, 2021
0e637ae
Fix random_sel function
Lldenaurois Nov 5, 2021
8113944
Fix overlength block check
Lldenaurois Nov 5, 2021
d290479
Merge remote-tracking branch 'parity/master' into bernhard-inherent-f…
Lldenaurois Nov 6, 2021
9671728
Zeke + Ladi commit for disputes filtering + integration test builder …
Lldenaurois Nov 6, 2021
a8b4771
Add benchmarks for code upgrades
Lldenaurois Nov 6, 2021
70feb6e
Code upgrade bench; Feature gate TestWeightInfo
emostov Nov 6, 2021
f861c2c
Try and make CI happier
emostov Nov 6, 2021
35407fc
Feature gate enter test to not(benchmarks)
emostov Nov 8, 2021
6c0c078
Make sure no unused imports/fn
emostov Nov 8, 2021
0e88948
refactor, re-use, the beginning
drahnr Nov 5, 2021
aecdd96
Fix issue with frame benchmarking dep compilation
emostov Nov 8, 2021
ef35d97
More precise feature gating for some derives
emostov Nov 8, 2021
4536741
integrate piece-wise
drahnr Nov 8, 2021
5b1f1c0
foo
drahnr Nov 8, 2021
fa0de2e
fixins
drahnr Nov 8, 2021
55a7f71
chore fmt
drahnr Nov 8, 2021
dac3b8e
fixins
drahnr Nov 8, 2021
61db8a7
rename const generic
drahnr Nov 8, 2021
80174ec
Merge branch 'bernhard-inherent-filtering2' into bernhard-inherent-fi…
emostov Nov 8, 2021
7f1293b
Update runtime/parachains/src/paras_inherent.rs
drahnr Nov 10, 2021
60a9be5
Fix compilation
bkchr Nov 11, 2021
e36e12a
limit to test
drahnr Nov 11, 2021
a2dcd20
remove unused spam slots
drahnr Nov 11, 2021
9a9eb22
spellcheck
drahnr Nov 11, 2021
f1ef3e1
remove a tick, fix a typo
drahnr Nov 11, 2021
c9e5872
Add Code upgrade weights
Lldenaurois Nov 11, 2021
a7a835c
Merge remote-tracking branch 'parity/master' into bernhard-inherent-f…
Lldenaurois Nov 11, 2021
8f1ad09
comment improvements + >=
drahnr Nov 12, 2021
8ac5e56
remove another tick
drahnr Nov 12, 2021
ee89736
Update runtime/parachains/src/paras_inherent/benchmarking.rs
drahnr Nov 12, 2021
ca5cc54
saturating fixins + some spaces
drahnr Nov 12, 2021
c18e141
fix
drahnr Nov 13, 2021
ba34d06
benchmarking - preliminary results
drahnr Nov 14, 2021
f71e233
Add training wheels
Lldenaurois Nov 12, 2021
8a42171
Refactor some early exit logic for enter
emostov Nov 15, 2021
ac0c325
Gracefully handle filtering bitfields & candidates (#4280)
emostov Nov 15, 2021
e566544
Refactor inherent data weight limiting logic (#4287)
emostov Nov 15, 2021
62cda42
Apply suggestions from code review
emostov Nov 15, 2021
933cba7
Update runtime/parachains/src/builder.rs
drahnr Nov 15, 2021
2c85807
Update runtime/parachains/src/builder.rs
emostov Nov 15, 2021
168ac98
Update runtime/parachains/src/paras_inherent.rs
emostov Nov 16, 2021
2a0869b
final pass
drahnr Nov 16, 2021
811f168
Run cargo +nightly-2021-10-29 fmt
emostov Nov 16, 2021
e12bc31
Update implementors guide with `sanitize_*` & `enter` (#4294)
emostov Nov 16, 2021
a52ffb7
Make spell check happier
emostov Nov 16, 2021
53fcce5
Merge branch 'zeke-bernhard-inherent-filtering-go-green' into bernhar…
emostov Nov 16, 2021
9e7eed5
Make wasm runtimes compile with benchmarks enabled (#4303)
emostov Nov 16, 2021
3d760b6
Update weight files for runtime
emostov Nov 16, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fixins
drahnr committed Oct 18, 2021

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
commit 8bd9af6e34c31caaacd6e864181ac3ae9d2a986f
1 change: 1 addition & 0 deletions runtime/parachains/src/disputes.rs
Original file line number Diff line number Diff line change
@@ -738,6 +738,7 @@ impl<T: Config> Pallet<T> {
Ok(fresh)
}

/// Removes all duplicate disputes.
fn filter_multi_dispute_data(statement_sets: &mut MultiDisputeStatementSet) {
let config = <configuration::Pallet<T>>::config();

64 changes: 44 additions & 20 deletions runtime/parachains/src/paras_inherent.rs
Original file line number Diff line number Diff line change
@@ -59,7 +59,7 @@ const INCLUSION_INHERENT_CLAIMED_WEIGHT: Weight = 1_000_000_000;
// we assume that 75% of an paras inherent's weight is used processing backed candidates
const MINIMAL_INCLUSION_INHERENT_WEIGHT: Weight = INCLUSION_INHERENT_CLAIMED_WEIGHT / 4;

/// A bitfield concering concluded disputes for candidates
/// A bitfield concerning concluded disputes for candidates
/// associated to the core index equivalent to the bit position.
#[derive(Default, PartialEq, Eq, Clone, Encode, Decode, RuntimeDebug, TypeInfo)]
pub(crate) struct DisputedBitfield(pub(crate) BitVec<bitvec::order::Lsb0, u8>);
@@ -134,6 +134,8 @@ pub mod pallet {
const INHERENT_IDENTIFIER: InherentIdentifier = PARACHAINS_INHERENT_IDENTIFIER;

fn create_inherent(data: &InherentData) -> Option<Self::Call> {
let parent_hash = <frame_system::Pallet<T>>::parent_hash();

let ParachainsInherentData::<T::Header> {
bitfields,
backed_candidates,
@@ -146,11 +148,21 @@ pub mod pallet {
Err(_) => {
log::warn!(target: LOG_TARGET, "ParachainsInherentData failed to decode");

return None
let parent_header = unimplemented!();
ParachainsInherentData {
// this is ok, since there are no backed candidates
// to pick from in case of overweight.
entropy: SeedEntropy::default(),

parent_header,

bitfields: Default::default(),
backed_candidates: Default::default(),
disputes: Default::default(),
}
},
};

let parent_hash = <frame_system::Pallet<T>>::parent_hash();
let current_session = <shared::Pallet<T>>::session_index();

// filter out any unneeded dispute statements
@@ -192,6 +204,8 @@ pub mod pallet {
)
.ok()?;

// XXX @Lldenaurois
// FIXME these weights are garbage
let remaining_weight = MINIMAL_INCLUSION_INHERENT_WEIGHT;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This seems off. We rather want something like a 'maximum allowed weight' whereas I think this is a 'minimum possible eeight'. Thoughts?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is one of the mentioned values that need to be replaced by sane values from @Lldenaurois PR. I'll annotate those values/fns with // XXX @Lldenaurois for easier discovery on merge.

let (_backed_candidates_weight, backed_candidates, bitfields) = apply_weight_limit::<T, _>(
expected_bits,
@@ -429,7 +443,8 @@ macro_rules! ensure2 {

/// Calculate the weight of a single backed candidate.
fn backed_candidate_weight<T: Config>(backed_candidate: &BackedCandidate<<T>::Hash>) -> Weight {
// FIXME
// XXX @Lldenaurois
// FIXME these weights are garbage
const CODE_UPGRADE_WEIGHT: Weight = 10_000 as Weight;
const DISPUTE_PER_STATEMENT_WEIGHT: Weight = 1_000 as Weight;

@@ -443,6 +458,8 @@ fn backed_candidate_weight<T: Config>(backed_candidate: &BackedCandidate<<T>::Ha

/// Calculate the weight of a individual bitfield.
fn bitfield_weight<T: Config>(bitfield: &AvailabilityBitfield) -> Weight {
// XXX @Lldenaurois
// FIXME these weights are garbage
7_000 as Weight
drahnr marked this conversation as resolved.
Show resolved Hide resolved
}

@@ -454,7 +471,7 @@ fn bitfield_weight<T: Config>(bitfield: &AvailabilityBitfield) -> Weight {
/// Since there is the relation of `backed candidate <-> occupied core <-> bitfield`
/// this is used to pick the candidate, but also include all relevant
/// bitfields.
fn apply_weight_limit<T: Config, F: Fn(CoreIndex) -> Option<ParaId>>(
fn apply_weight_limit<T: Config + inclusion::Config, F: Fn(CoreIndex) -> Option<ParaId>>(
expected_bits: usize,
mut candidates: Vec<BackedCandidate<<T>::Hash>>,
mut bitfields: UncheckedSignedAvailabilityBitfields,
@@ -466,7 +483,7 @@ fn apply_weight_limit<T: Config, F: Fn(CoreIndex) -> Option<ParaId>>(
.iter()
.map(|backed_candidate| backed_candidate_weight::<T>(backed_candidate))
.sum();
total += bitfields.iter().map(|bitfield| bitfield_weight::<T>(bitfield.unchecked_payload())).sum();
total += bitfields.iter().map(|bitfield| bitfield_weight::<T>(bitfield.unchecked_payload())).sum::<Weight>();

if max_weight < total {
return (total, candidates, bitfields)
@@ -489,10 +506,10 @@ fn apply_weight_limit<T: Config, F: Fn(CoreIndex) -> Option<ParaId>>(
let mut candidates_core_index: BTreeMap<CandidateHash, CoreIndex> = (0..expected_bits)
.map(|bit_index| core_lookup(CoreIndex::from(bit_index as u32)))
.filter_map(|opt_para_id| opt_para_id)
.map(|para_id| {
.filter_map(|para_id| {
PendingAvailability::<T>::get(&para_id)
})
.map(|cpa: CandidatePendingAvailability<T,<T>::BlockNumber>| {
.map(|cpa: CandidatePendingAvailability<<T>::Hash,<T>::BlockNumber>| {
(cpa.candidate_hash(), cpa.core_occupied())
})
.collect();
@@ -513,13 +530,13 @@ fn apply_weight_limit<T: Config, F: Fn(CoreIndex) -> Option<ParaId>>(
for (i, bitfield) in bitfields.iter().enumerate() {
// lookup the core index that is responsible for the candidate
if let Some(core_index) = candidates_core_index.get(&picked_candidate.hash()) {
if bitfield.unchecked_payload().0[core_index.0 as _] {
if bitfield.unchecked_payload().0[core_index.0 as usize] {
// avoid duplicate accounting if it was already included before
if bitfields_to_include_coverage[i] {
// mark the `i`-th bit
covered_bitfields.set(i, true);
// account for the added weight of the bitfield
bitfields_weight += bitfield_weight(bitfield.unchecked_payload());
bitfields_weight += bitfield_weight::<T>(bitfield.unchecked_payload());
}
}
}
@@ -560,6 +577,11 @@ fn apply_weight_limit<T: Config, F: Fn(CoreIndex) -> Option<ParaId>>(
/// While this function technically returns a set of unchecked bitfields,
/// they were actually checked and filtered to allow using it in both
/// cases, as `filtering` and `checking` stage.
///
/// `EARLY_RETURN` determines the behavior.
/// `false` assures that all inputs are filtered, and invalid ones are filtered out.
/// It also skips signature verification.
/// `true` returns an `Err(_)` on the first check failing.
pub(crate) fn sanitize_bitfields<T: Config + crate::inclusion::Config, const EARLY_RETURN: bool>(
unchecked_bitfields: UncheckedSignedAvailabilityBitfields,
disputed_bits: DisputedBitfield,
@@ -606,17 +628,19 @@ pub(crate) fn sanitize_bitfields<T: Config + crate::inclusion::Config, const EAR

let validator_public = &validators[validator_index.0 as usize];

let signed_bitfield = if let Ok(signed_bitfield) =
unchecked_bitfield.try_into_checked(&signing_context, validator_public)
{
signed_bitfield
} else if EARLY_RETURN {
fail!(crate::inclusion::pallet::Error::<T>::InvalidBitfieldSignature);
// only check the signatures when returning early
if EARLY_RETURN {
let signed_bitfield = if let Ok(signed_bitfield) =
unchecked_bitfield.try_into_checked(&signing_context, validator_public)
{
signed_bitfield
} else {
fail!(crate::inclusion::pallet::Error::<T>::InvalidBitfieldSignature);
};
bitfields.push(signed_bitfield.into_unchecked());
} else {
continue
};

bitfields.push(signed_bitfield.into_unchecked());
bitfields.push(unchecked_bitfield);
}

last_index = Some(validator_index);
}