From 6eceb67cb5cc90f53ad7735a8a866cfc245f78e5 Mon Sep 17 00:00:00 2001 From: Nalin Bhardwaj Date: Thu, 27 May 2021 03:25:50 +0530 Subject: [PATCH 1/4] wallet: sort listsendpays by ID Changelog-Changed: JSON: `listsendpays` output is now ordered by `id`. --- wallet/db_postgres_sqlgen.c | 6 +-- wallet/db_sqlite3_sqlgen.c | 6 +-- wallet/statements_gettextgen.po | 96 ++++++++++++++++----------------- wallet/wallet.c | 3 +- 4 files changed, 56 insertions(+), 55 deletions(-) diff --git a/wallet/db_postgres_sqlgen.c b/wallet/db_postgres_sqlgen.c index c178b7b5e008..9525c7814b4f 100644 --- a/wallet/db_postgres_sqlgen.c +++ b/wallet/db_postgres_sqlgen.c @@ -1593,8 +1593,8 @@ struct db_query db_postgres_queries[] = { .readonly = false, }, { - .name = "SELECT id, status, destination, msatoshi, payment_hash, timestamp, payment_preimage, path_secrets, route_nodes, route_channels, msatoshi_sent, description, bolt11, failonionreply, total_msat, partid, local_offer_id FROM payments WHERE payment_hash = ?;", - .query = "SELECT id, status, destination, msatoshi, payment_hash, timestamp, payment_preimage, path_secrets, route_nodes, route_channels, msatoshi_sent, description, bolt11, failonionreply, total_msat, partid, local_offer_id FROM payments WHERE payment_hash = $1;", + .name = "SELECT id, status, destination, msatoshi, payment_hash, timestamp, payment_preimage, path_secrets, route_nodes, route_channels, msatoshi_sent, description, bolt11, failonionreply, total_msat, partid, local_offer_id FROM payments WHERE payment_hash = ? ORDER BY id;", + .query = "SELECT id, status, destination, msatoshi, payment_hash, timestamp, payment_preimage, path_secrets, route_nodes, route_channels, msatoshi_sent, description, bolt11, failonionreply, total_msat, partid, local_offer_id FROM payments WHERE payment_hash = $1 ORDER BY id;", .placeholders = 1, .readonly = true, }, @@ -1906,4 +1906,4 @@ struct db_query db_postgres_queries[] = { #endif /* LIGHTNINGD_WALLET_GEN_DB_POSTGRES */ -// SHA256STAMP:3ab5cc5d3610b26e4df12de155aa85990ac825c0e75a1adf410e67e2ca173327 +// SHA256STAMP:435d8c98449934c86167d11929b515312babce55bae5487dc3cdc201cb4ba0fe diff --git a/wallet/db_sqlite3_sqlgen.c b/wallet/db_sqlite3_sqlgen.c index 85e3c8e3344f..f6121dad17e7 100644 --- a/wallet/db_sqlite3_sqlgen.c +++ b/wallet/db_sqlite3_sqlgen.c @@ -1593,8 +1593,8 @@ struct db_query db_sqlite3_queries[] = { .readonly = false, }, { - .name = "SELECT id, status, destination, msatoshi, payment_hash, timestamp, payment_preimage, path_secrets, route_nodes, route_channels, msatoshi_sent, description, bolt11, failonionreply, total_msat, partid, local_offer_id FROM payments WHERE payment_hash = ?;", - .query = "SELECT id, status, destination, msatoshi, payment_hash, timestamp, payment_preimage, path_secrets, route_nodes, route_channels, msatoshi_sent, description, bolt11, failonionreply, total_msat, partid, local_offer_id FROM payments WHERE payment_hash = ?;", + .name = "SELECT id, status, destination, msatoshi, payment_hash, timestamp, payment_preimage, path_secrets, route_nodes, route_channels, msatoshi_sent, description, bolt11, failonionreply, total_msat, partid, local_offer_id FROM payments WHERE payment_hash = ? ORDER BY id;", + .query = "SELECT id, status, destination, msatoshi, payment_hash, timestamp, payment_preimage, path_secrets, route_nodes, route_channels, msatoshi_sent, description, bolt11, failonionreply, total_msat, partid, local_offer_id FROM payments WHERE payment_hash = ? ORDER BY id;", .placeholders = 1, .readonly = true, }, @@ -1906,4 +1906,4 @@ struct db_query db_sqlite3_queries[] = { #endif /* LIGHTNINGD_WALLET_GEN_DB_SQLITE3 */ -// SHA256STAMP:3ab5cc5d3610b26e4df12de155aa85990ac825c0e75a1adf410e67e2ca173327 +// SHA256STAMP:435d8c98449934c86167d11929b515312babce55bae5487dc3cdc201cb4ba0fe diff --git a/wallet/statements_gettextgen.po b/wallet/statements_gettextgen.po index 574487c12eba..afd5883f41c4 100644 --- a/wallet/statements_gettextgen.po +++ b/wallet/statements_gettextgen.po @@ -1055,190 +1055,190 @@ msgid "UPDATE payments SET failonionreply=? , faildestperm=? , failind msgstr "" #: wallet/wallet.c:3160 -msgid "SELECT id, status, destination, msatoshi, payment_hash, timestamp, payment_preimage, path_secrets, route_nodes, route_channels, msatoshi_sent, description, bolt11, failonionreply, total_msat, partid, local_offer_id FROM payments WHERE payment_hash = ?;" +msgid "SELECT id, status, destination, msatoshi, payment_hash, timestamp, payment_preimage, path_secrets, route_nodes, route_channels, msatoshi_sent, description, bolt11, failonionreply, total_msat, partid, local_offer_id FROM payments WHERE payment_hash = ? ORDER BY id;" msgstr "" -#: wallet/wallet.c:3182 +#: wallet/wallet.c:3183 msgid "SELECT id, status, destination, msatoshi, payment_hash, timestamp, payment_preimage, path_secrets, route_nodes, route_channels, msatoshi_sent, description, bolt11, failonionreply, total_msat, partid, local_offer_id FROM payments ORDER BY id;" msgstr "" -#: wallet/wallet.c:3233 +#: wallet/wallet.c:3234 msgid "SELECT id, status, destination, msatoshi, payment_hash, timestamp, payment_preimage, path_secrets, route_nodes, route_channels, msatoshi_sent, description, bolt11, failonionreply, total_msat, partid, local_offer_id FROM payments WHERE local_offer_id = ?;" msgstr "" -#: wallet/wallet.c:3278 +#: wallet/wallet.c:3279 msgid "DELETE FROM htlc_sigs WHERE channelid = ?" msgstr "" -#: wallet/wallet.c:3285 +#: wallet/wallet.c:3286 msgid "INSERT INTO htlc_sigs (channelid, signature) VALUES (?, ?)" msgstr "" -#: wallet/wallet.c:3297 +#: wallet/wallet.c:3298 msgid "SELECT blobval FROM vars WHERE name='genesis_hash'" msgstr "" -#: wallet/wallet.c:3321 +#: wallet/wallet.c:3322 msgid "INSERT INTO vars (name, blobval) VALUES ('genesis_hash', ?);" msgstr "" -#: wallet/wallet.c:3339 +#: wallet/wallet.c:3340 msgid "SELECT txid, outnum FROM utxoset WHERE spendheight < ?" msgstr "" -#: wallet/wallet.c:3351 +#: wallet/wallet.c:3352 msgid "DELETE FROM utxoset WHERE spendheight < ?" msgstr "" -#: wallet/wallet.c:3359 wallet/wallet.c:3473 +#: wallet/wallet.c:3360 wallet/wallet.c:3474 msgid "INSERT INTO blocks (height, hash, prev_hash) VALUES (?, ?, ?);" msgstr "" -#: wallet/wallet.c:3378 +#: wallet/wallet.c:3379 msgid "DELETE FROM blocks WHERE hash = ?" msgstr "" -#: wallet/wallet.c:3384 +#: wallet/wallet.c:3385 msgid "SELECT * FROM blocks WHERE height >= ?;" msgstr "" -#: wallet/wallet.c:3393 +#: wallet/wallet.c:3394 msgid "DELETE FROM blocks WHERE height > ?" msgstr "" -#: wallet/wallet.c:3405 +#: wallet/wallet.c:3406 msgid "UPDATE outputs SET spend_height = ?, status = ? WHERE prev_out_tx = ? AND prev_out_index = ?" msgstr "" -#: wallet/wallet.c:3423 +#: wallet/wallet.c:3424 msgid "UPDATE utxoset SET spendheight = ? WHERE txid = ? AND outnum = ?" msgstr "" -#: wallet/wallet.c:3446 wallet/wallet.c:3484 +#: wallet/wallet.c:3447 wallet/wallet.c:3485 msgid "INSERT INTO utxoset ( txid, outnum, blockheight, spendheight, txindex, scriptpubkey, satoshis) VALUES(?, ?, ?, ?, ?, ?, ?);" msgstr "" -#: wallet/wallet.c:3510 +#: wallet/wallet.c:3511 msgid "SELECT height FROM blocks WHERE height = ?" msgstr "" -#: wallet/wallet.c:3523 +#: wallet/wallet.c:3524 msgid "SELECT txid, spendheight, scriptpubkey, satoshis FROM utxoset WHERE blockheight = ? AND txindex = ? AND outnum = ? AND spendheight IS NULL" msgstr "" -#: wallet/wallet.c:3565 +#: wallet/wallet.c:3566 msgid "SELECT blockheight, txindex, outnum FROM utxoset WHERE spendheight = ?" msgstr "" -#: wallet/wallet.c:3596 wallet/wallet.c:3756 +#: wallet/wallet.c:3597 wallet/wallet.c:3757 msgid "SELECT blockheight FROM transactions WHERE id=?" msgstr "" -#: wallet/wallet.c:3606 +#: wallet/wallet.c:3607 msgid "INSERT INTO transactions ( id, blockheight, txindex, rawtx) VALUES (?, ?, ?, ?);" msgstr "" -#: wallet/wallet.c:3627 +#: wallet/wallet.c:3628 msgid "UPDATE transactions SET blockheight = ?, txindex = ? WHERE id = ?" msgstr "" -#: wallet/wallet.c:3644 +#: wallet/wallet.c:3645 msgid "INSERT INTO transaction_annotations (txid, idx, location, type, channel) VALUES (?, ?, ?, ?, ?) ON CONFLICT(txid,idx) DO NOTHING;" msgstr "" -#: wallet/wallet.c:3676 +#: wallet/wallet.c:3677 msgid "SELECT type, channel_id FROM transactions WHERE id=?" msgstr "" -#: wallet/wallet.c:3692 +#: wallet/wallet.c:3693 msgid "UPDATE transactions SET type = ?, channel_id = ? WHERE id = ?" msgstr "" -#: wallet/wallet.c:3711 +#: wallet/wallet.c:3712 msgid "SELECT type FROM transactions WHERE id=?" msgstr "" -#: wallet/wallet.c:3734 +#: wallet/wallet.c:3735 msgid "SELECT rawtx FROM transactions WHERE id=?" msgstr "" -#: wallet/wallet.c:3780 +#: wallet/wallet.c:3781 msgid "SELECT blockheight, txindex FROM transactions WHERE id=?" msgstr "" -#: wallet/wallet.c:3808 +#: wallet/wallet.c:3809 msgid "SELECT id FROM transactions WHERE blockheight=?" msgstr "" -#: wallet/wallet.c:3827 +#: wallet/wallet.c:3828 msgid "INSERT INTO channeltxs ( channel_id, type, transaction_id, input_num, blockheight) VALUES (?, ?, ?, ?, ?);" msgstr "" -#: wallet/wallet.c:3851 +#: wallet/wallet.c:3852 msgid "SELECT DISTINCT(channel_id) FROM channeltxs WHERE type = ?;" msgstr "" -#: wallet/wallet.c:3872 +#: wallet/wallet.c:3873 msgid "SELECT c.type, c.blockheight, t.rawtx, c.input_num, c.blockheight - t.blockheight + 1 AS depth, t.id as txid FROM channeltxs c JOIN transactions t ON t.id = c.transaction_id WHERE c.channel_id = ? ORDER BY c.id ASC;" msgstr "" -#: wallet/wallet.c:3917 +#: wallet/wallet.c:3918 msgid "UPDATE forwarded_payments SET in_msatoshi=?, out_msatoshi=?, state=?, resolved_time=?, failcode=? WHERE in_htlc_id=?" msgstr "" -#: wallet/wallet.c:3975 +#: wallet/wallet.c:3976 msgid "INSERT INTO forwarded_payments ( in_htlc_id, out_htlc_id, in_channel_scid, out_channel_scid, in_msatoshi, out_msatoshi, state, received_time, resolved_time, failcode) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);" msgstr "" -#: wallet/wallet.c:4034 +#: wallet/wallet.c:4035 msgid "SELECT CAST(COALESCE(SUM(in_msatoshi - out_msatoshi), 0) AS BIGINT)FROM forwarded_payments WHERE state = ?;" msgstr "" -#: wallet/wallet.c:4083 +#: wallet/wallet.c:4084 msgid "SELECT f.state, in_msatoshi, out_msatoshi, hin.payment_hash as payment_hash, in_channel_scid, out_channel_scid, f.received_time, f.resolved_time, f.failcode FROM forwarded_payments f LEFT JOIN channel_htlcs hin ON (f.in_htlc_id = hin.id) WHERE (1 = ? OR f.state = ?) AND (1 = ? OR f.in_channel_scid = ?) AND (1 = ? OR f.out_channel_scid = ?)" msgstr "" -#: wallet/wallet.c:4205 +#: wallet/wallet.c:4206 msgid "SELECT t.id, t.rawtx, t.blockheight, t.txindex, t.type as txtype, c2.short_channel_id as txchan, a.location, a.idx as ann_idx, a.type as annotation_type, c.short_channel_id FROM transactions t LEFT JOIN transaction_annotations a ON (a.txid = t.id) LEFT JOIN channels c ON (a.channel = c.id) LEFT JOIN channels c2 ON (t.channel_id = c2.id) ORDER BY t.blockheight, t.txindex ASC" msgstr "" -#: wallet/wallet.c:4299 +#: wallet/wallet.c:4300 msgid "INSERT INTO penalty_bases ( channel_id, commitnum, txid, outnum, amount) VALUES (?, ?, ?, ?, ?);" msgstr "" -#: wallet/wallet.c:4324 +#: wallet/wallet.c:4325 msgid "SELECT commitnum, txid, outnum, amount FROM penalty_bases WHERE channel_id = ?" msgstr "" -#: wallet/wallet.c:4348 +#: wallet/wallet.c:4349 msgid "DELETE FROM penalty_bases WHERE channel_id = ? AND commitnum = ?" msgstr "" -#: wallet/wallet.c:4366 +#: wallet/wallet.c:4367 msgid "SELECT 1 FROM offers WHERE offer_id = ?;" msgstr "" -#: wallet/wallet.c:4379 +#: wallet/wallet.c:4380 msgid "INSERT INTO offers ( offer_id, bolt12, label, status) VALUES (?, ?, ?, ?);" msgstr "" -#: wallet/wallet.c:4406 +#: wallet/wallet.c:4407 msgid "SELECT bolt12, label, status FROM offers WHERE offer_id = ?;" msgstr "" -#: wallet/wallet.c:4434 +#: wallet/wallet.c:4435 msgid "SELECT offer_id FROM offers;" msgstr "" -#: wallet/wallet.c:4460 +#: wallet/wallet.c:4461 msgid "UPDATE offers SET status=? WHERE offer_id = ?;" msgstr "" -#: wallet/wallet.c:4471 +#: wallet/wallet.c:4472 msgid "UPDATE invoices SET state=? WHERE state=? AND local_offer_id = ?;" msgstr "" -#: wallet/wallet.c:4499 +#: wallet/wallet.c:4500 msgid "SELECT status FROM offers WHERE offer_id = ?;" msgstr "" @@ -1257,4 +1257,4 @@ msgstr "" #: wallet/test/run-wallet.c:1647 msgid "INSERT INTO channels (id) VALUES (1);" msgstr "" -# SHA256STAMP:5d466ed1c95975f0155bdb67e995c14b7927634e810d319c1ba616327c37d80a +# SHA256STAMP:fdfbb1278ba9e09884c9205e54fc16bb9e66a0362a5aaad11a7efd42746d8e72 diff --git a/wallet/wallet.c b/wallet/wallet.c index 7f53e7d35cfd..70a895e6bf0f 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -3176,7 +3176,8 @@ wallet_payment_list(const tal_t *ctx, ", partid" ", local_offer_id" " FROM payments" - " WHERE payment_hash = ?;")); + " WHERE payment_hash = ?" + " ORDER BY id;")); db_bind_sha256(stmt, 0, payment_hash); } else { stmt = db_prepare_v2(wallet->db, SQL("SELECT" From 1df0adb442ed190eef424480fa49aa2274e57211 Mon Sep 17 00:00:00 2001 From: Nalin Bhardwaj Date: Thu, 27 May 2021 03:26:58 +0530 Subject: [PATCH 2/4] plugins/pay: sort output payments in listpays Changelog-Changed: `listpays` output is now ordered by the `created_at` timestamp. --- plugins/pay.c | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/plugins/pay.c b/plugins/pay.c index c3007dc64715..b08c55c64213 100644 --- a/plugins/pay.c +++ b/plugins/pay.c @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -1701,6 +1702,17 @@ static bool pay_mpp_eq(const struct pay_mpp *pm, const struct sha256 *payment_ha return memcmp(pm->payment_hash, payment_hash, sizeof(struct sha256)) == 0; } +static int cmp_pay_mpp(const struct pay_mpp *a, + const struct pay_mpp *b, + void *unused UNUSED) +{ + if (a->timestamp < b->timestamp) + return -1; + if (a->timestamp == b->timestamp) + return 0; + return 1; +} + HTABLE_DEFINE_TYPE(struct pay_mpp, pay_mpp_key, pay_mpp_hash, pay_mpp_eq, pay_map); @@ -1796,6 +1808,7 @@ static struct command_result *listsendpays_done(struct command *cmd, struct pay_map pay_map; struct pay_map_iter it; struct pay_mpp *pm; + struct pay_mpp *pays; pay_map_init(&pay_map); @@ -1864,17 +1877,26 @@ static struct command_result *listsendpays_done(struct command *cmd, } } - /* Now we've collapsed them, provide summary. */ - ret = jsonrpc_stream_success(cmd); - json_array_start(ret, "pays"); + pays = tal_arr(NULL, struct pay_mpp, pay_map_count(&pay_map)); + i = 0; for (pm = pay_map_first(&pay_map, &it); pm; pm = pay_map_next(&pay_map, &it)) { - add_new_entry(ret, buf, pm); + pays[i++] = *pm; } pay_map_clear(&pay_map); + asort(pays, tal_count(pays), cmp_pay_mpp, NULL); + + /* Now we've collapsed and sorted them, provide summary. */ + ret = jsonrpc_stream_success(cmd); + json_array_start(ret, "pays"); + + for (i = 0; i < tal_count(pays); i++) + add_new_entry(ret, buf, &pays[i]); + tal_free(pays); + json_array_end(ret); return command_finished(cmd, ret); } From d737fb5f1ec859fbc70ce686966cfc1c69f19042 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 31 May 2021 12:07:30 +0930 Subject: [PATCH 3/4] pytest: test pay ordering. Signed-off-by: Rusty Russell --- tests/test_pay.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_pay.py b/tests/test_pay.py index 2223c92feca0..5f75e0f0e02a 100644 --- a/tests/test_pay.py +++ b/tests/test_pay.py @@ -3498,6 +3498,20 @@ def test_listpays_ongoing_attempt(node_factory, bitcoind, executor): l1.rpc.listpays() +def test_listsendpays_and_listpays_order(node_factory): + """listsendpays should be in increasing id order, listpays in created_at""" + l1, l2 = node_factory.line_graph(2) + for i in range(5): + inv = l2.rpc.invoice(1000 - i, "test {}".format(i), "test")['bolt11'] + l1.rpc.pay(inv) + + ids = [p['id'] for p in l1.rpc.listsendpays()['payments']] + assert ids == sorted(ids) + + created_at = [p['created_at'] for p in l1.rpc.listpays()['pays']] + assert created_at == sorted(created_at) + + @pytest.mark.developer("needs use_shadow") def test_mpp_waitblockheight_routehint_conflict(node_factory, bitcoind, executor): ''' From 0d622272ceeea4218111093218c4881da54e1f92 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 31 May 2021 12:09:27 +0930 Subject: [PATCH 4/4] doc: document ordering of listpays/listsendpays. Signed-off-by: Rusty Russell --- doc/lightning-listpays.7 | 5 ++++- doc/lightning-listpays.7.md | 2 ++ doc/lightning-listsendpays.7 | 4 ++-- doc/lightning-listsendpays.7.md | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/doc/lightning-listpays.7 b/doc/lightning-listpays.7 index 0d2c828a1f2d..672918f16167 100644 --- a/doc/lightning-listpays.7 +++ b/doc/lightning-listpays.7 @@ -59,6 +59,9 @@ If \fBstatus\fR is "failed": \fBerroronion\fR (hex, optional): the error onion returned on failure, if any\. .RE + +The returned array is ordered by increasing \fBcreated_at\fR fields\. + .SH AUTHOR Rusty Russell \fI is mainly responsible\. @@ -71,4 +74,4 @@ Rusty Russell \fI is mainly responsible\. Main web site: \fIhttps://github.com/ElementsProject/lightning\fR -\" SHA256STAMP:e27d57394bef9bdaf9b99ae0d9050c9044c194ab66f6c94c43b532a86e1a0031 +\" SHA256STAMP:4037a4dd9746b5dfc91ef2f5b4fa9000a334689157d9ac86dc4dba5c82628cfe diff --git a/doc/lightning-listpays.7.md b/doc/lightning-listpays.7.md index 0ade89123bb0..aab53b6c0b1d 100644 --- a/doc/lightning-listpays.7.md +++ b/doc/lightning-listpays.7.md @@ -37,6 +37,8 @@ If **status** is "failed": - **erroronion** (hex, optional): the error onion returned on failure, if any. [comment]: # (GENERATE-FROM-SCHEMA-END) +The returned array is ordered by increasing **created_at** fields. + AUTHOR ------ diff --git a/doc/lightning-listsendpays.7 b/doc/lightning-listsendpays.7 index 5cb045fffb72..8f594c82cf26 100644 --- a/doc/lightning-listsendpays.7 +++ b/doc/lightning-listsendpays.7 @@ -18,7 +18,7 @@ command per \fIpay\fR, so this command should be used with caution\. .SH RETURN VALUE -On success, an array of objects is returned\. Each object contains: +On success, an array of objects is returned, ordered by increasing \fIid\fR\. Each object contains: \fIid\fR @@ -75,4 +75,4 @@ responsible\. Main web site: \fIhttps://github.com/ElementsProject/lightning\fR -\" SHA256STAMP:693da7a03235656092cc349b6b77335f71c3a1427d88f1cc8007ea7dd64a8e18 +\" SHA256STAMP:c6b36205c8067723bb9edc20f3645754faa9c9d26568d6c899721b43a1e99812 diff --git a/doc/lightning-listsendpays.7.md b/doc/lightning-listsendpays.7.md index 22a5e0e9952f..7752b7e73591 100644 --- a/doc/lightning-listsendpays.7.md +++ b/doc/lightning-listsendpays.7.md @@ -20,7 +20,7 @@ command per *pay*, so this command should be used with caution. RETURN VALUE ------------ -On success, an array of objects is returned. Each object contains: +On success, an array of objects is returned, ordered by increasing *id*. Each object contains: *id* unique internal value assigned at creation