Spurious MustUseOneshotSender panic on zebrad shutdown

[teor2345]

Version

zebrad 1.0.0-alpha.0

Commit a418284 "Create the global span immediately after activating tracing" from PR #1568

Platform

Linux ... 5.4.83 #1-NixOS SMP Fri Dec 11 12:23:33 UTC 2020 x86_64 GNU/Linux

Description

zebrad can panic with a spurious MustUseOneshotSender panic on shutdown.

I tried this:

Shutting down zebrad using Control-C (SIGINT)

I expected to see this happen:

zebrad exits without panicking

Instead, this happened:

zebrad panics with a spurious MustUseOneshotSender error, even though the sender invariant is irrelevant during shutdown.

Possible Solutions

Stop enforcing and relying on the sender invariant during shutdown:

• Set a flag during shutdown, and use it to ignore any MustUseOneshotSender errors
• As soon as the flag is set, Clients should stop polling the background Receivers tied to their Senders

Commands

zebrad start from a debug build, using the default config

Related Issues

We should re-test the fix for this issue after #1351, because it will change the interrupt handler so it works even when zebrad is busy.

#1576 is a similar panic in the checkpointer during shutdown.

Logs

Jan 11 17:19:08.139  INFO {zebrad="827cb238" net=Mainnet}:peer{addr=161.53.201.181:8233}:msg_as_req{msg=getheaders}:inbound:state: zebra_state::service: responding to peer GetBlocks or GetHeaders final_height=Height(0) response_len=0 chain_tip_height=Height(0) stop_height=None intersection_height=Some(Height(0))
Jan 11 17:19:08.156  INFO {zebrad="827cb238" net=Mainnet}:peer{addr=93.51.53.177:8233}:msg_as_req{msg=getheaders}:inbound:state: zebra_state::service: responding to peer GetBlocks or GetHeaders final_height=Height(0) response_len=0 chain_tip_height=Height(0) stop_height=None intersection_height=Some(Height(0))
^CJan 11 17:19:08.177  INFO {zebrad="827cb238" net=Mainnet}:sig{kind=SignalKind(2) name="SIGINT"}: zebrad::signal: received SIGINT, starting shutdown
The application panicked (crashed).
Message:  unused oneshot sender: oneshot must be used or canceled: MustUseOneshotSender { tx: Some(Sender { inner: Inner { complete: false, data: Lock { locked: false, data: UnsafeCell }, rx_task: Lock { locked: false, data: UnsafeCell }, tx_task: Lock { locked: false, data: UnsafeCell } } }) }
Location: zebra-network/src/peer/client.rs:183

  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ SPANTRACE ━━━━━━━━━━━━━━━━━━━━━━━━━━Jan 11 17:19:08.177  INFO {zebrad="827cb238" net=Mainnet}:peer{addr=185.86.76.62:8233}:msg_as_req{msg=getheaders}:inbound:state: zebra_state::service: responding to peer GetBlocks or GetHeaders final_height=Height(0) response_len=0 chain_tip_height=Height(0) stop_height=None intersection_height=Some(Height(0))
━━━━━━━━━
  
   0: zebra_network::peer::client::drop
      at zebra-network/src/peer/client.rs:180

[rex4539]

Seeing the same.

^CJan 16 17:10:12.203 INFO {zebrad="8a7b0232" net="Main"}:sig{kind=SignalKind(2) name="SIGINT"}: zebrad::signal: received SIGINT, starting shutdown The application panicked (crashed). Message: unused oneshot sender: oneshot must be used or canceled: MustUseOneshotSender { tx: Some(Sender { inner: Inner { complete: false, data: Lock { locked: false, data: UnsafeCell }, rx_task: Lock { locked: false, data: UnsafeCell }, tx_task: Lock { locked: false, data: UnsafeCell } } }) } Location: zebra-network/src/peer/client.rs:183

Metadata:
version: 1.0.0-alpha.0
git commit: 8a7b023
Zcash network: Mainnet

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ SPANTRACE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━

0: zebra_network::peer::client::drop
at zebra-network/src/peer/client.rs:180

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ BACKTRACE ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
1: backtrace::backtrace::libunwind::trace::h56b5b89d46ea18ee
at /Users/rex/.cargo/registry/src/jackfan.us.kg-1ecc6299db9ec823/backtrace-0.3.54/src/backtrace/libunwind.rs:90
88 │ #[inline(always)]
89 │ pub unsafe fn trace(mut cb: &mut dyn FnMut(&super::Frame) -> bool) {
90 > uw::_Unwind_Backtrace(trace_fn, &mut cb as *mut _ as *mut _);
91 │
92 │ extern "C" fn trace_fn(
2: backtrace::backtrace::trace_unsynchronized::h59c70a87a59809c5
at /Users/rex/.cargo/registry/src/jackfan.us.kg-1ecc6299db9ec823/backtrace-0.3.54/src/backtrace/mod.rs:66
64 │ /// See information on trace for caveats on cb panicking.
65 │ pub unsafe fn trace_unsynchronized<F: FnMut(&Frame) -> bool>(mut cb: F) {
66 > trace_imp(&mut cb)
67 │ }
68 │
3: backtrace::backtrace::trace::h9e0f4b8e68f89b43
at /Users/rex/.cargo/registry/src/jackfan.us.kg-1ecc6299db9ec823/backtrace-0.3.54/src/backtrace/mod.rs:53
51 │ pub fn trace<F: FnMut(&Frame) -> bool>(cb: F) {
52 │ let _guard = crate::lock::lock();
53 > unsafe { trace_unsynchronized(cb) }
54 │ }
55 │
4: backtrace::capture::Backtrace::create::h8d054d598eb8a93e
at /Users/rex/.cargo/registry/src/jackfan.us.kg-1ecc6299db9ec823/backtrace-0.3.54/src/capture.rs:176
174 │ let mut frames = Vec::new();
175 │ let mut actual_start_index = None;
176 > trace(|frame| {
177 │ frames.push(BacktraceFrame {
178 │ frame: Frame::Raw(frame.clone()),
5: backtrace::capture::Backtrace::new::h89e80605e5dd4af5
at /Users/rex/.cargo/registry/src/jackfan.us.kg-1ecc6299db9ec823/backtrace-0.3.54/src/capture.rs:140
138 │ #[inline(never)] // want to make sure there's a frame here to remove
139 │ pub fn new() -> Backtrace {
140 > let mut bt = Self::create(Self::new as usize);
141 │ bt.resolve();
142 │ bt
6: color_eyre::config::PanicHook::panic_report::h907c19ae3792a475
at /Users/rex/.cargo/registry/src/jackfan.us.kg-1ecc6299db9ec823/color-eyre-0.5.10/src/config.rs:982
980 │
981 │ let backtrace = if capture_bt {
982 > Some(backtrace::Backtrace::new())
983 │ } else {
984 │ None
7: <zebrad::application::ZebradApp as abscissa_core::application::Application>::register_components::{{closure}}::hf128d32c8f32dd29
at /Users/rex/zebra/zebrad/src/application.rs:206
204 │
205 │ std::panic::set_hook(Box::new(move |panic_info| {
206 > let panic_report = panic_hook.panic_report(panic_info);
207 │ eprintln!("{}", panic_report);
208 │
8: std::panicking::rust_panic_with_hook::he74e9eac8174d6dd
at /rustc/a62a76047ea24aad7639f14eb3ce0e620b77bdb7/library/std/src/panicking.rs:595
9: std::panicking::begin_panic_handler::{{closure}}::h122c1e0e85c150a9
at /rustc/a62a76047ea24aad7639f14eb3ce0e620b77bdb7/library/std/src/panicking.rs:497
10: std::sys_common::backtrace::__rust_end_short_backtrace::he103784f07f740ac
at /rustc/a62a76047ea24aad7639f14eb3ce0e620b77bdb7/library/std/src/sys_common/backtrace.rs:141
11: rust_begin_unwind
at /rustc/a62a76047ea24aad7639f14eb3ce0e620b77bdb7/library/std/src/panicking.rs:493
12: std::panicking::begin_panic_fmt::h9385c4006b676237
at /rustc/a62a76047ea24aad7639f14eb3ce0e620b77bdb7/library/std/src/panicking.rs:435
13: <zebra_network::peer::client::MustUseOneshotSender as core::ops::drop::Drop>::drop::h97d490ca3a0d130b
at /Users/rex/zebra/zebra-network/src/peer/client.rs:183
181 │ fn drop(&mut self) {
182 │ // is_canceled() will not panic, because we check is_none() first
183 > assert!(
184 │ self.tx.is_none() || self.is_canceled(),
185 │ "unused oneshot sender: oneshot must be used or canceled: {:?}",
14: core::ptr::drop_in_place<zebra_network::peer::client::MustUseOneshotSender<core::result::Result<zebra_network::protocol::internal::response::Response,zebra_network::peer::error::SharedPeerError>>>::ha9607cb686389006
at /Users/rex/.rustup/toolchains/nightly-x86_64-apple-darwin/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:179
177 │ #[lang = "drop_in_place"]
178 │ #[allow(unconditional_recursion)]
179 > pub unsafe fn drop_in_place<T: ?Sized>(to_drop: *mut T) {
180 │ // Code here does not matter - this is replaced by the
181 │ // real drop glue by the compiler.
15: core::ptr::drop_in_place<zebra_network::peer::connection::Connection<tower::load_shed::LoadShed<tower::buffer::service::Bufferzebrad::components::inbound::Inbound,zebra_network::protocol::internal::request::Request>,futures_util::sink::with::With<futures_util::stream::stream::split::SplitSink<tokio_util::codec::framed::Framedtokio::net::tcp::stream::TcpStream,zebra_network::protocol::external::codec::Codec,zebra_network::protocol::external::message::Message>,zebra_network::protocol::external::message::Message,zebra_network::protocol::external::message::Message,futures_util::future::ready::Ready<core::result::Result<zebra_network::protocol::external::message::Message,zebra_chain::serialization::error::SerializationError>>,<zebra_network::peer::handshake::Handshake<tower::load_shed::LoadShed<tower::buffer::service::Bufferzebrad::components::inbound::Inbound,zebra_network::protocol::internal::request::Request>> as tower_service::Service<(tokio::net::tcp::stream::TcpStream,std::net::addr::SocketAddr)>>::call::{{closure}}::{{closure}}>>::run<core::pin::Pin<alloc::boxed::Box<dyn futures_core::stream::Stream+Item = core::result::Result<zebra_network::protocol::external::message::Message,zebra_chain::serialization::error::SerializationError>+core::marker::Send>>>::{{closure}}>::hd43552bfe385642f
at /Users/rex/.rustup/toolchains/nightly-x86_64-apple-darwin/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:351
349 │ /// assert_eq!([2, 3, 0, 1], array);
350 │ /// }
351 > /// ```
352 │ ///
353 │ /// Swapping two overlapping regions:
16: core::ptr::drop_in_place<core::future::from_generator::GenFuture<zebra_network::peer::connection::Connection<tower::load_shed::LoadShed<tower::buffer::service::Bufferzebrad::components::inbound::Inbound,zebra_network::protocol::internal::request::Request>,futures_util::sink::with::With<futures_util::stream::stream::split::SplitSink<tokio_util::codec::framed::Framedtokio::net::tcp::stream::TcpStream,zebra_network::protocol::external::codec::Codec,zebra_network::protocol::external::message::Message>,zebra_network::protocol::external::message::Message,zebra_network::protocol::external::message::Message,futures_util::future::ready::Ready<core::result::Result<zebra_network::protocol::external::message::Message,zebra_chain::serialization::error::SerializationError>>,<zebra_network::peer::handshake::Handshake<tower::load_shed::LoadShed<tower::buffer::service::Bufferzebrad::components::inbound::Inbound,zebra_network::protocol::internal::request::Request>> as tower_service::Service<(tokio::net::tcp::stream::TcpStream,std::net::addr::SocketAddr)>>::call::{{closure}}::{{closure}}>>::run<core::pin::Pin<alloc::boxed::Box<dyn futures_core::stream::Stream+Item = core::result::Result<zebra_network::protocol::external::message::Message,zebra_chain::serialization::error::SerializationError>+core::marker::Send>>>::{{closure}}>>::h7dcc754286155b5c
at /Users/rex/.rustup/toolchains/nightly-x86_64-apple-darwin/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:179
177 │ #[lang = "drop_in_place"]
178 │ #[allow(unconditional_recursion)]
179 > pub unsafe fn drop_in_place<T: ?Sized>(to_drop: *mut T) {
180 │ // Code here does not matter - this is replaced by the
181 │ // real drop glue by the compiler.
17: core::ptr::drop_in_place<tracing_futures::Instrumented<core::future::from_generator::GenFuture<zebra_network::peer::connection::Connection<tower::load_shed::LoadShed<tower::buffer::service::Bufferzebrad::components::inbound::Inbound,zebra_network::protocol::internal::request::Request>,futures_util::sink::with::With<futures_util::stream::stream::split::SplitSink<tokio_util::codec::framed::Framedtokio::net::tcp::stream::TcpStream,zebra_network::protocol::external::codec::Codec,zebra_network::protocol::external::message::Message>,zebra_network::protocol::external::message::Message,zebra_network::protocol::external::message::Message,futures_util::future::ready::Ready<core::result::Result<zebra_network::protocol::external::message::Message,zebra_chain::serialization::error::SerializationError>>,<zebra_network::peer::handshake::Handshake<tower::load_shed::LoadShed<tower::buffer::service::Bufferzebrad::components::inbound::Inbound,zebra_network::protocol::internal::request::Request>> as tower_service::Service<(tokio::net::tcp::stream::TcpStream,std::net::addr::SocketAddr)>>::call::{{closure}}::{{closure}}>>::run<core::pin::Pin<alloc::boxed::Box<dyn futures_core::stream::Stream+Item = core::result::Result<zebra_network::protocol::external::message::Message,zebra_chain::serialization::error::SerializationError>+core::marker::Send>>>::{{closure}}>>>::h3a0a7fe8157ca015
at /Users/rex/.rustup/toolchains/nightly-x86_64-apple-darwin/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:179
177 │ #[lang = "drop_in_place"]
178 │ #[allow(unconditional_recursion)]
179 > pub unsafe fn drop_in_place<T: ?Sized>(to_drop: *mut T) {
180 │ // Code here does not matter - this is replaced by the
181 │ // real drop glue by the compiler.
18: core::ptr::drop_in_place<alloc::boxed::Box<dyn core::future::future::Future+Output = ()+core::marker::Send>>::ha58d68489ae00a95
at /Users/rex/.rustup/toolchains/nightly-x86_64-apple-darwin/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:179
177 │ #[lang = "drop_in_place"]
178 │ #[allow(unconditional_recursion)]
179 > pub unsafe fn drop_in_place<T: ?Sized>(to_drop: *mut T) {
180 │ // Code here does not matter - this is replaced by the
181 │ // real drop glue by the compiler.
19: core::ptr::drop_in_place<core::pin::Pin<alloc::boxed::Box<dyn core::future::future::Future+Output = ()+core::marker::Send>>>::h5befb420e05c353c
at /Users/rex/.rustup/toolchains/nightly-x86_64-apple-darwin/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:179
177 │ #[lang = "drop_in_place"]
178 │ #[allow(unconditional_recursion)]
179 > pub unsafe fn drop_in_place<T: ?Sized>(to_drop: *mut T) {
180 │ // Code here does not matter - this is replaced by the
181 │ // real drop glue by the compiler.
20: core::ptr::drop_in_place<tracing::instrument::Instrumented<core::pin::Pin<alloc::boxed::Box<dyn core::future::future::Future+Output = ()+core::marker::Send>>>>::h38b61524d27c19b4
at /Users/rex/.rustup/toolchains/nightly-x86_64-apple-darwin/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:179
177 │ #[lang = "drop_in_place"]
178 │ #[allow(unconditional_recursion)]
179 > pub unsafe fn drop_in_place<T: ?Sized>(to_drop: *mut T) {
180 │ // Code here does not matter - this is replaced by the
181 │ // real drop glue by the compiler.
21: core::ptr::drop_in_place<tokio::runtime::task::core::Stage<tracing::instrument::Instrumented<core::pin::Pin<alloc::boxed::Box<dyn core::future::future::Future+Output = ()+core::marker::Send>>>>>::hdfc9d4ed70157de8
at /Users/rex/.rustup/toolchains/nightly-x86_64-apple-darwin/lib/rustlib/src/rust/library/core/src/ptr/mod.rs:179
177 │ #[lang = "drop_in_place"]
178 │ #[allow(unconditional_recursion)]
179 > pub unsafe fn drop_in_place<T: ?Sized>(to_drop: *mut T) {
180 │ // Code here does not matter - this is replaced by the
181 │ // real drop glue by the compiler.