DB: Store failcode of local forward into DB

Store failcode of local forward into DB, and thus listforwards will show
the failcode for local failed forward.
We'd be limited to differentiating malformed HTLC failures (if it's the
next hop failing) or an opaque failire if any later node fails, so here
we just store failcode for local forwards.

CHANGELOG.md

@@ -18,6 +18,7 @@ Note: You should always set `allow-deprecated-apis=false` to test for
 changes.
 
 - JSON API: `newaddr` output field `address`: use `bech32` or `p2sh-segwit` instead.
+- JSON API: `listforwards` has a `failreason` field.
 
 ### Removed
 

lightningd/peer_htlcs.c

@@ -140,11 +140,26 @@ static void fail_out_htlc(struct htlc_out *hout, const char *localfail)
 {
 	htlc_out_check(hout, __func__);
 	assert(hout->failcode || hout->failuremsg);
+	struct lightningd *ld = hout->key.channel->peer->ld;
+
 	if (hout->am_origin) {
 		payment_failed(hout->key.channel->peer->ld, hout, localfail);
 	} else if (hout->in) {
 		fail_in_htlc(hout->in, hout->failcode, hout->failuremsg,
 			     hout->key.channel->scid);
+
+		if(find_htlc_out(&ld->htlcs_out, hout->key.channel, hout->key.id))
+			wallet_forwarded_payment_add(ld->wallet,
+						 hout->in, hout,
+						 FORWARD_FAILED,
+						 hout->failcode);
+		else
+			/* if we haven't added htlc_out into ld->htlcs_out, just set NULL
+			 * for hout field in forward failure */
+			wallet_forwarded_payment_add(ld->wallet,
+						 hout->in, NULL,
+						 FORWARD_FAILED,
+						 hout->failcode);
 	}
 }
 
@@ -406,9 +421,18 @@ static void rcvd_htlc_reply(struct subd *subd, const u8 *msg, const int *fds UNU
 						  (int)tal_count(failurestr),
 						  (const char *)failurestr);
 			payment_failed(ld, hout, localfail);
-		} else if (hout->in)
+		} else if (hout->in) {
 			local_fail_htlc(hout->in, failure_code,
-					hout->key.channel->scid);
+					 hout->key.channel->scid);
+
+			/* here we haven't call connect_htlc_out(),
+			 * so set htlc field with NULL */
+			wallet_forwarded_payment_add(ld->wallet,
+					 hout->in, NULL,
+					 FORWARD_FAILED,
+					 failure_code);
+		}
+
 		/* Prevent hout from being failed twice. */
 		tal_del_destructor(hout, destroy_hout_subd_died);
 		tal_free(hout);
@@ -500,10 +524,15 @@ static void forward_htlc(struct htlc_in *hin,
 	struct amount_msat fee;
 	struct lightningd *ld = hin->key.channel->peer->ld;
 	struct channel *next = active_channel_by_id(ld, next_hop, NULL);
+	struct htlc_out *hout = NULL;
 
 	/* Unknown peer, or peer not ready. */
 	if (!next || !next->scid) {
 		local_fail_htlc(hin, WIRE_UNKNOWN_NEXT_PEER, NULL);
+		wallet_forwarded_payment_add(hin->key.channel->peer->ld->wallet,
+					 hin, NULL,
+					 FORWARD_FAILED,
+					 hin->failcode);
 		return;
 	}
 
@@ -567,14 +596,27 @@ static void forward_htlc(struct htlc_in *hin,
 		goto fail;
 	}
 
+	hout = tal(tmpctx, struct htlc_out);
 	failcode = send_htlc_out(next, amt_to_forward,
 				 outgoing_cltv_value, &hin->payment_hash,
-				 next_onion, hin, NULL);
+				 next_onion, hin, &hout);
+
 	if (!failcode)
 		return;
 
+	/* In fact, we didn't get the new htlc_out in these 2 cases */
+	if (failcode == WIRE_UNKNOWN_NEXT_PEER ||
+		failcode == WIRE_TEMPORARY_CHANNEL_FAILURE) {
+		tal_free(hout);
+		hout = NULL;
+	}
+
 fail:
 	local_fail_htlc(hin, failcode, next->scid);
+	wallet_forwarded_payment_add(ld->wallet,
+				 hin, hout,
+				 FORWARD_FAILED,
+				 hin->failcode);
 }
 
 /* Temporary information, while we resolve the next hop */
@@ -602,6 +644,10 @@ static void channel_resolve_reply(struct subd *gossip, const u8 *msg,
 
 	if (!peer_id) {
 		local_fail_htlc(gr->hin, WIRE_UNKNOWN_NEXT_PEER, NULL);
+		wallet_forwarded_payment_add(gr->hin->key.channel->peer->ld->wallet,
+					 gr->hin, NULL,
+					 FORWARD_FAILED,
+					 gr->hin->failcode);
 		tal_free(gr);
 		return;
 	}
@@ -749,7 +795,7 @@ static void fulfill_our_htlc_out(struct channel *channel, struct htlc_out *hout,
 	else if (hout->in) {
 		fulfill_htlc(hout->in, preimage);
 		wallet_forwarded_payment_add(ld->wallet, hout->in, hout,
-					     FORWARD_SETTLED);
+					 FORWARD_SETTLED, 0);
 	}
 }
 
@@ -842,7 +888,11 @@ static bool peer_failed_our_htlc(struct channel *channel,
 	htlc_out_check(hout, __func__);
 
 	if (hout->in)
-		wallet_forwarded_payment_add(ld->wallet, hout->in, hout, FORWARD_FAILED);
+		wallet_forwarded_payment_add(ld->wallet,
+					 hout->in,
+					 hout,
+					 FORWARD_FAILED,
+					 hout->failcode);
 
 	return true;
 }
@@ -877,9 +927,14 @@ void onchain_failed_our_htlc(const struct channel *channel,
 					  why);
 		payment_failed(ld, hout, localfail);
 		tal_free(localfail);
-	} else if (hout->in)
+	} else if (hout->in) {
 		local_fail_htlc(hout->in, WIRE_PERMANENT_CHANNEL_FAILURE,
-				hout->key.channel->scid);
+					 hout->key.channel->scid);
+		wallet_forwarded_payment_add(hout->key.channel->peer->ld->wallet,
+					 hout->in, hout,
+					 FORWARD_FAILED,
+					 hout->failcode);
+	}
 }
 
 static void remove_htlc_in(struct channel *channel, struct htlc_in *hin)
@@ -1001,7 +1056,7 @@ static bool update_out_htlc(struct channel *channel,
 
 		if (hout->in)
 			wallet_forwarded_payment_add(ld->wallet, hout->in, hout,
-						     FORWARD_OFFERED);
+						     FORWARD_OFFERED, 0);
 
 		/* For our own HTLCs, we commit payment to db lazily */
 		if (hout->origin_htlc_id == 0)
@@ -1428,6 +1483,10 @@ void peer_got_revoke(struct channel *channel, const u8 *msg)
 
 		hin = find_htlc_in(&ld->htlcs_in, channel, changed[i].id);
 		local_fail_htlc(hin, failcodes[i], NULL);
+		wallet_forwarded_payment_add(ld->wallet,
+					 hin, NULL,
+					 FORWARD_FAILED,
+					 hin->failcode);
 	}
 	wallet_channel_save(ld->wallet, channel);
 }
@@ -1902,6 +1961,32 @@ static void listforwardings_add_forwardings(struct json_stream *response, struct
 				     cur->fee,
 				     "fee", "fee_msat");
 		json_add_string(response, "status", forward_status_name(cur->status));
+
+		if(cur->failcode != 0) {
+			assert(cur->status == FORWARD_FAILED);
+			if(amount_msat_eq(cur->msat_out, AMOUNT_MSAT(0)) &&
+						cur->channel_out.u64 == 0)
+				json_add_string(response,
+						"failreason",
+						tal_fmt(cur,
+						"(Failed with no out_htlc)"
+						" failcode 0x%04x (%s)",
+						cur->failcode,
+						onion_type_name(cur->failcode)));
+			else
+				json_add_string(response,
+						"failreason",
+						tal_fmt(cur, "failcode 0x%04x (%s)",
+						cur->failcode,
+						onion_type_name(cur->failcode)));
+		} else if (cur->status == FORWARD_FAILED) {
+			json_add_string(response,
+					"failreason",
+					"Not local failure("
+					"malformed HTLC failure"
+					" OR later node failed)");
+		}
+
 		json_object_end(response);
 	}
 	json_array_end(response);

wallet/db.c

@@ -360,6 +360,7 @@ static struct migration dbmigrations[] = {
 	  ", in_msatoshi INTEGER"
 	  ", out_msatoshi INTEGER"
 	  ", state INTEGER"
+	  ", failcode INTEGER"
 	  ", UNIQUE(in_htlc_id, out_htlc_id)"
 	  ");", NULL },
 	/* Add a direction for failed payments. */

wallet/wallet.c

@@ -2507,7 +2507,8 @@ struct channeltx *wallet_channeltxs_get(struct wallet *w, const tal_t *ctx,
 
 void wallet_forwarded_payment_add(struct wallet *w, const struct htlc_in *in,
 				  const struct htlc_out *out,
-				  enum forward_status state)
+				  enum forward_status state,
+				  enum onion_type failcode)
 {
 	sqlite3_stmt *stmt;
 	stmt = db_prepare(
@@ -2519,14 +2520,45 @@ void wallet_forwarded_payment_add(struct wallet *w, const struct htlc_in *in,
 		", out_channel_scid"
 		", in_msatoshi"
 		", out_msatoshi"
-		", state) VALUES (?, ?, ?, ?, ?, ?, ?);");
+		", state"
+		", failcode) VALUES (?, ?, ?, ?, ?, ?, ?, ?);");
 	sqlite3_bind_int64(stmt, 1, in->dbid);
-	sqlite3_bind_int64(stmt, 2, out->dbid);
+
+	if(out) {
+		sqlite3_bind_int64(stmt, 2, out->dbid);
+	} else {
+		assert(failcode != 0);
+		assert(state == FORWARD_FAILED);
+		sqlite3_bind_null(stmt, 2);
+	}
+
 	sqlite3_bind_int64(stmt, 3, in->key.channel->scid->u64);
-	sqlite3_bind_int64(stmt, 4, out->key.channel->scid->u64);
+
+	if(out) {
+		sqlite3_bind_int64(stmt, 4, out->key.channel->scid->u64);
+	} else {
+		assert(failcode != 0);
+		assert(state == FORWARD_FAILED);
+		sqlite3_bind_null(stmt, 4);
+	}
+
 	sqlite3_bind_amount_msat(stmt, 5, in->msat);
-	sqlite3_bind_amount_msat(stmt, 6, out->msat);
+
+	if(out) {
+		sqlite3_bind_amount_msat(stmt, 6, out->msat);
+	} else {
+		assert(failcode != 0);
+		assert(state == FORWARD_FAILED);
+		sqlite3_bind_null(stmt, 6);
+	}
+
 	sqlite3_bind_int(stmt, 7, wallet_forward_status_in_db(state));
+	if(failcode != 0) {
+		assert(state == FORWARD_FAILED);
+		sqlite3_bind_int(stmt, 8, (int)failcode);
+	} else {
+		sqlite3_bind_null(stmt, 8);
+	}
 	db_exec_prepared(w->db, stmt);
 }
 
@@ -2562,6 +2594,7 @@ const struct forwarding *wallet_forwarded_payments_get(struct wallet *w,
 	stmt = db_prepare(w->db,
 			  "SELECT"
 			  "  f.state"
+			  ", f.failcode"
 			  ", in_msatoshi"
 			  ", out_msatoshi"
 			  ", hin.payment_hash as payment_hash"
@@ -2574,8 +2607,24 @@ const struct forwarding *wallet_forwarded_payments_get(struct wallet *w,
 		tal_resize(&results, count+1);
 		struct forwarding *cur = &results[count];
 		cur->status = sqlite3_column_int(stmt, 0);
-		cur->msat_in = sqlite3_column_amount_msat(stmt, 1);
-		cur->msat_out = sqlite3_column_amount_msat(stmt, 2);
+
+		if (sqlite3_column_type(stmt, 1) != SQLITE_NULL) {
+			assert(cur->status == FORWARD_FAILED);
+			cur->failcode = sqlite3_column_int(stmt, 1);
+		} else {
+			cur->failcode = 0;
+		}
+
+		cur->msat_in = sqlite3_column_amount_msat(stmt, 2);
+
+		if (sqlite3_column_type(stmt, 3) != SQLITE_NULL) {
+			cur->msat_out = sqlite3_column_amount_msat(stmt, 3);
+		} else {
+			assert(cur->status == FORWARD_FAILED);
+			assert(cur->failcode != 0);
+			cur->msat_out = AMOUNT_MSAT(0);
+		}
+
 		if (!amount_msat_sub(&cur->fee, cur->msat_in, cur->msat_out)) {
 			log_broken(w->log, "Forwarded in %s less than out %s!",
 				   type_to_string(tmpctx, struct amount_msat,
@@ -2585,15 +2634,24 @@ const struct forwarding *wallet_forwarded_payments_get(struct wallet *w,
 			cur->fee = AMOUNT_MSAT(0);
 		}
 
-		if (sqlite3_column_type(stmt, 3) != SQLITE_NULL) {
+		if (sqlite3_column_type(stmt, 4) != SQLITE_NULL) {
 			cur->payment_hash = tal(ctx, struct sha256_double);
-			sqlite3_column_sha256_double(stmt, 3, cur->payment_hash);
+			sqlite3_column_sha256_double(stmt, 4, cur->payment_hash);
 		} else {
+			assert(cur->status == FORWARD_FAILED);
+			assert(cur->failcode != 0);
 			cur->payment_hash = NULL;
 		}
 
-		cur->channel_in.u64 = sqlite3_column_int64(stmt, 4);
-		cur->channel_out.u64 = sqlite3_column_int64(stmt, 5);
+		cur->channel_in.u64 = sqlite3_column_int64(stmt, 5);
+
+		if (sqlite3_column_type(stmt, 6) != SQLITE_NULL) {
+			cur->channel_out.u64 = sqlite3_column_int64(stmt, 6);
+		} else {
+			assert(cur->status == FORWARD_FAILED);
+			assert(cur->failcode != 0);
+			cur->channel_out.u64 = 0;
+		}
 	}
 
 	db_stmt_done(stmt);

wallet/wallet.h

@@ -17,6 +17,7 @@
 #include <lightningd/log.h>
 #include <onchaind/onchain_wire.h>
 #include <wally_bip32.h>
+#include <wire/gen_onion_wire.h>
 
 enum onion_type;
 struct amount_msat;
@@ -165,6 +166,7 @@ struct forwarding {
 	struct amount_msat msat_in, msat_out, fee;
 	struct sha256_double *payment_hash;
 	enum forward_status status;
+	enum onion_type failcode;
 };
 
 /* A database backed shachain struct. The datastructure is
@@ -1029,7 +1031,8 @@ struct channeltx *wallet_channeltxs_get(struct wallet *w, const tal_t *ctx,
  */
 void wallet_forwarded_payment_add(struct wallet *w, const struct htlc_in *in,
 				  const struct htlc_out *out,
-				  enum forward_status state);
+				  enum forward_status state,
+				  enum onion_type failcode);
 
 /**
  * Retrieve summary of successful forwarded payments' fees