Plugin: shutdown notification

doc/PLUGINS.md

@@ -791,6 +791,16 @@ here, with the peer's signatures attached.
 }
 ```
 
+### `shutdown`
+
+Called when lightningd is shutting down, or this plugin has been
+shutdown by the plugin stop command.  The plugin has 30 seconds to
+exit itself, otherwise it's killed.
+
+Because lightningd can crash or be killed, a plugin cannot rely on
+this function always called.
+
+
 ## Hooks
 
 Hooks allow a plugin to define custom behavior for `lightningd`

lightningd/bitcoind.h

@@ -32,6 +32,9 @@ struct bitcoind {
 	/* Ignore results, we're shutting down. */
 	bool shutdown;
 
+	/* Timer if we're waiting for it to warm up. */
+	struct oneshot *checkchain_timer;
+
 	struct list_head pending_getfilteredblock;
 
 	/* Map each method to a plugin, so we can have multiple plugins

lightningd/chaintopology.c

@@ -46,10 +46,10 @@ static void maybe_completed_init(struct chain_topology *topo)
 
 static void next_topology_timer(struct chain_topology *topo)
 {
-	/* This takes care of its own lifetime. */
-	notleak(new_reltimer(topo->timers, topo,
-			     time_from_sec(topo->poll_seconds),
-			     try_extend_tip, topo));
+	assert(!topo->extend_timer);
+	topo->extend_timer = new_reltimer(topo->ld->timers, topo,
+					  time_from_sec(topo->poll_seconds),
+					  try_extend_tip, topo);
 }
 
 static bool we_broadcast(const struct chain_topology *topo,
@@ -437,6 +437,7 @@ static void update_feerates(struct bitcoind *bitcoind,
 
 static void start_fee_estimate(struct chain_topology *topo)
 {
+	topo->updatefee_timer = NULL;
 	/* Once per new block head, update fee estimates. */
 	bitcoind_estimate_fees(topo->bitcoind, NUM_FEERATES, update_feerates,
 			       topo);
@@ -582,10 +583,10 @@ AUTODATA(json_command, &parse_feerate_command);
 
 static void next_updatefee_timer(struct chain_topology *topo)
 {
-	/* This takes care of its own lifetime. */
-	notleak(new_reltimer(topo->timers, topo,
-			     time_from_sec(topo->poll_seconds),
-			     start_fee_estimate, topo));
+	assert(!topo->updatefee_timer);
+	topo->updatefee_timer = new_reltimer(topo->ld->timers, topo,
+					     time_from_sec(topo->poll_seconds),
+					     start_fee_estimate, topo);
 }
 
 struct sync_waiter {
@@ -935,6 +936,7 @@ static void get_new_block(struct bitcoind *bitcoind,
 
 static void try_extend_tip(struct chain_topology *topo)
 {
+	topo->extend_timer = NULL;
 	bitcoind_getrawblockbyheight(topo->bitcoind, topo->tip->height + 1,
 				     get_new_block, topo);
 }
@@ -1144,24 +1146,25 @@ check_chain(struct bitcoind *bitcoind, const char *chain,
 		return;
 	}
 
-	notleak(new_reltimer(bitcoind->ld->timers, bitcoind,
-			     /* Be 4x more aggressive in this case. */
-			     time_divide(time_from_sec(bitcoind->ld->topology
-						       ->poll_seconds), 4),
-			     retry_check_chain, bitcoind->ld->topology));
-	}
+	assert(!bitcoind->checkchain_timer);
+	bitcoind->checkchain_timer
+		= new_reltimer(bitcoind->ld->timers, bitcoind,
+			       /* Be 4x more aggressive in this case. */
+			       time_divide(time_from_sec(bitcoind->ld->topology
+							 ->poll_seconds), 4),
+			       retry_check_chain, bitcoind->ld->topology);
+}
 
 static void retry_check_chain(struct chain_topology *topo)
 {
+	topo->bitcoind->checkchain_timer = NULL;
 	bitcoind_getchaininfo(topo->bitcoind, false, check_chain, topo);
 }
 
 void setup_topology(struct chain_topology *topo,
-		    struct timers *timers,
 		    u32 min_blockheight, u32 max_blockheight)
 {
 	memset(&topo->feerate, 0, sizeof(topo->feerate));
-	topo->timers = timers;
 
 	topo->min_blockheight = min_blockheight;
 	topo->max_blockheight = max_blockheight;
@@ -1173,6 +1176,7 @@ void setup_topology(struct chain_topology *topo,
 	log_debug(topo->ld->log, "All Bitcoin plugin commands registered");
 
 	/* Sanity checks, then topology initialization. */
+	topo->bitcoind->checkchain_timer = NULL;
 	bitcoind_getchaininfo(topo->bitcoind, true, check_chain, topo);
 
 	tal_add_destructor(topo, destroy_chain_topology);
@@ -1187,3 +1191,12 @@ void begin_topology(struct chain_topology *topo)
 {
 	try_extend_tip(topo);
 }
+
+void stop_topology(struct chain_topology *topo)
+{
+	/* Remove timers while we're cleaning up plugins. */
+	tal_free(topo->bitcoind->checkchain_timer);
+	tal_free(topo->extend_timer);
+	tal_free(topo->updatefee_timer);
+}
+

lightningd/chaintopology.h

@@ -113,8 +113,8 @@ struct chain_topology {
 	/* The bitcoind. */
 	struct bitcoind *bitcoind;
 
-	/* Our timer list. */
-	struct timers *timers;
+	/* Timers we're running. */
+	struct oneshot *extend_timer, *updatefee_timer;
 
 	/* Bitcoin transactions we're broadcasting */
 	struct list_head outgoing_txs;
@@ -193,11 +193,13 @@ void broadcast_tx_ahf(struct chain_topology *topo,
 				     const char *err));
 
 struct chain_topology *new_topology(struct lightningd *ld, struct log *log);
-void setup_topology(struct chain_topology *topology, struct timers *timers,
+void setup_topology(struct chain_topology *topology,
 		    u32 min_blockheight, u32 max_blockheight);
 
 void begin_topology(struct chain_topology *topo);
 
+void stop_topology(struct chain_topology *topo);
+
 struct txlocator *locate_tx(const void *ctx, const struct chain_topology *topo, const struct bitcoin_txid *txid);
 
 static inline bool topology_synced(const struct chain_topology *topo)

lightningd/lightningd.c

@@ -1035,8 +1035,7 @@ int main(int argc, char *argv[])
 
 	/*~ Initialize block topology.  This does its own io_loop to
 	 * talk to bitcoind, so does its own db transactions. */
-	setup_topology(ld->topology, ld->timers,
-		       min_blockheight, max_blockheight);
+	setup_topology(ld->topology, min_blockheight, max_blockheight);
 
 	db_begin_transaction(ld->wallet->db);
 
@@ -1142,12 +1141,15 @@ int main(int argc, char *argv[])
 		stop_response = tal_steal(NULL, ld->stop_response);
 	}
 
+	/* Stop topology callbacks. */
+	stop_topology(ld->topology);
+
 	/* We're not going to collect our children. */
 	remove_sigchild_handler();
-	shutdown_subdaemons(ld);
 
-	/* Remove plugins. */
-	plugins_free(ld->plugins);
+	/* Tell plugins we're shutting down. */
+	shutdown_plugins(ld);
+	shutdown_subdaemons(ld);
 
 	/* Clean up the JSON-RPC. This needs to happen in a DB transaction since
 	 * it might actually be touching the DB in some destructors, e.g.,

lightningd/notification.c

@@ -554,3 +554,14 @@ void notify_channel_open_failed(struct lightningd *ld,
 	jsonrpc_notification_end(n);
 	plugins_notify(ld->plugins, take(n));
 }
+
+REGISTER_NOTIFICATION(plugin_shutdown, NULL);
+
+bool notify_plugin_shutdown(struct lightningd *ld, struct plugin *p)
+{
+	struct jsonrpc_notification *n =
+		jsonrpc_notification_start(NULL, "shutdown");
+
+	jsonrpc_notification_end(n);
+	return plugin_single_notify(p, take(n));
+}

lightningd/notification.h

@@ -104,4 +104,7 @@ void notify_openchannel_peer_sigs(struct lightningd *ld,
 
 void notify_channel_open_failed(struct lightningd *ld,
                                 const struct channel_id *cid);
+
+/* Tell this plugin to shutdown: returns true if it was subscribed. */
+bool notify_plugin_shutdown(struct lightningd *ld, struct plugin *p);
 #endif /* LIGHTNING_LIGHTNINGD_NOTIFICATION_H */

lightningd/plugin.c

@@ -89,22 +89,6 @@ struct plugins *plugins_new(const tal_t *ctx, struct log_book *log_book,
 	return p;
 }
 
-void plugins_free(struct plugins *plugins)
-{
-	struct plugin *p;
-
-	plugins->shutdown = true;
-
-	/* Plugins are usually the unit of allocation, and they are internally
-	 * consistent, so let's free each plugin first. */
-	while (!list_empty(&plugins->plugins)) {
-		p = list_pop(&plugins->plugins, struct plugin, list);
-		tal_free(p);
-	}
-
-	tal_free(plugins);
-}
-
 /* Check that all the plugin's subscriptions are actually for known
  * notification topics. Emit a warning if that's not the case, but
  * don't kill the plugin. */
@@ -205,8 +189,12 @@ static void destroy_plugin(struct plugin *p)
 
 	/* If we are shutting down, do not continue to checking if
 	 * the dying plugin is important.  */
-	if (p->plugins->shutdown)
+	if (p->plugins->shutdown) {
+		/* But return if this was the last plugin! */
+		if (list_empty(&p->plugins->plugins))
+			io_break(p->plugins);
 		return;
+	}
 
 	/* Now check if the dying plugin is important.  */
 	if (p->important) {
@@ -1958,21 +1946,36 @@ static bool plugin_subscriptions_contains(struct plugin *plugin,
 	return false;
 }
 
+bool plugin_single_notify(struct plugin *p,
+			  const struct jsonrpc_notification *n TAKES)
+{
+	bool interested;
+	if (plugin_subscriptions_contains(p, n->method)) {
+		plugin_send(p, json_stream_dup(p, n->stream, p->log));
+		interested = true;
+	} else
+		interested = false;
+
+	if (taken(n))
+		tal_free(n);
+
+	return interested;
+}
+
 void plugins_notify(struct plugins *plugins,
 		    const struct jsonrpc_notification *n TAKES)
 {
 	struct plugin *p;
 
+	if (taken(n))
+		tal_steal(tmpctx, n);
+
 	/* If we're shutting down, ld->plugins will be NULL */
 	if (plugins) {
 		list_for_each(&plugins->plugins, p, list) {
-			if (plugin_subscriptions_contains(p, n->method))
-				plugin_send(p, json_stream_dup(p, n->stream,
-							       p->log));
+			plugin_single_notify(p, n);
 		}
 	}
-	if (taken(n))
-		tal_free(n);
 }
 
 static void destroy_request(struct jsonrpc_request *req,
@@ -2067,3 +2070,49 @@ bool was_plugin_destroyed(struct plugin_destroyed *pd)
 	tal_free(pd);
 	return true;
 }
+
+static void plugin_shutdown_timeout(struct lightningd *ld)
+{
+	io_break(ld->plugins);
+}
+
+void shutdown_plugins(struct lightningd *ld)
+{
+	struct plugin *p, *next;
+
+	/* This makes sure we don't complain about important plugins
+	 * vanishing! */
+	ld->plugins->shutdown = true;
+
+	/* Tell them all to shutdown; if they care. */
+	list_for_each_safe(&ld->plugins->plugins, p, next, list) {
+		/* Kill immediately, deletes self from list. */
+		if (!notify_plugin_shutdown(ld, p))
+			tal_free(p);
+	}
+
+	/* If anyone was interested in shutdown, give them time. */
+	if (!list_empty(&ld->plugins->plugins)) {
+		struct oneshot *t;
+
+		/* 30 seconds should do it. */
+		t = new_reltimer(ld->timers, ld,
+				 time_from_sec(30),
+				 plugin_shutdown_timeout, ld);
+
+		io_loop_with_timers(ld);
+		tal_free(t);
+
+		/* Report and free remaining plugins. */
+		while (!list_empty(&ld->plugins->plugins)) {
+			p = list_pop(&ld->plugins->plugins, struct plugin, list);
+			log_debug(ld->log,
+				  "%s: failed to shutdown, killing.",
+				  p->shortname);
+			tal_free(p);
+		}
+	}
+
+	/* NULL stops notifications trying to access plugins. */
+	ld->plugins = tal_free(ld->plugins);
+}

lightningd/plugin.h

@@ -189,26 +189,6 @@ void plugins_add_default_dir(struct plugins *plugins);
  */
 void plugins_init(struct plugins *plugins);
 
-/**
- * Free all resources that are held by plugins in the correct order.
- *
- * This function ensures that the resources dangling off of the plugins struct
- * are freed in the correct order. This is necessary since the struct manages
- * two orthogonal sets of resources:
- *
- *  - Plugins
- *  - Hook calls and notifications
- *
- * The tal hierarchy is organized in a plugin centric way, i.e., the plugins
- * may exit in an arbitrary order and they'll unregister pointers in the other
- * resources. However this will fail if `tal_free` decides to free one of the
- * non-plugin resources (technically a sibling in the allocation tree) before
- * the plugins we will get a use-after-free. This function fixes this by
- * freeing in the correct order without adding additional child-relationships
- * in the allocation structure and without adding destructors.
- */
-void plugins_free(struct plugins *plugins);
-
 /**
  * Register a plugin for initialization and execution.
  *
@@ -271,6 +251,11 @@ bool plugins_send_getmanifest(struct plugins *plugins);
 void plugin_kill(struct plugin *plugin, enum log_level loglevel,
 		 const char *fmt, ...);
 
+/**
+ * Tell all the plugins we're shutting down, and free them.
+ */
+void shutdown_plugins(struct lightningd *ld);
+
 /**
  * Returns the plugin which registers the command with name {cmd_name}
  */
@@ -351,6 +336,17 @@ char *add_plugin_dir(struct plugins *plugins, const char *dir,
  */
 void clear_plugins(struct plugins *plugins);
 
+/**
+ * Send notification to this single plugin, if interested.
+ *
+ * Returns true if it was subscribed to the notification.
+ */
+bool plugin_single_notify(struct plugin *p,
+			  const struct jsonrpc_notification *n TAKES);
+
+/**
+ * Send notification to all interested plugins.
+ */
 void plugins_notify(struct plugins *plugins,
 		    const struct jsonrpc_notification *n TAKES);