diff --git a/protocols/horizon/main.go b/protocols/horizon/main.go index 4a486da4d7..ed4d0e550e 100644 --- a/protocols/horizon/main.go +++ b/protocols/horizon/main.go @@ -166,8 +166,12 @@ type AssetStat struct { } `json:"_links"` base.Asset - PT string `json:"paging_token"` - Amount string `json:"amount"` + PT string `json:"paging_token"` + Accounts AssetStatAccounts `json:"accounts"` + // Action needed in release: horizon-v3.0.0: deprecated field + Amount string `json:"amount"` + Balances AssetStatBalances `json:"balances"` + // Action needed in release: horizon-v3.0.0: deprecated field NumAccounts int32 `json:"num_accounts"` Flags AccountFlags `json:"flags"` } @@ -177,6 +181,20 @@ func (res AssetStat) PagingToken() string { return res.PT } +// AssetStatBalances represents the summarized balances for a single Asset +type AssetStatBalances struct { + Authorized string `json:"authorized"` + AuthorizedToMaintainLiabilities string `json:"authorized_to_maintain_liabilities"` + Unauthorized string `json:"unauthorized"` +} + +// AssetStatAccounts represents the summarized acount numbers for a single Asset +type AssetStatAccounts struct { + Authorized int32 `json:"authorized"` + AuthorizedToMaintainLiabilities int32 `json:"authorized_to_maintain_liabilities"` + Unauthorized int32 `json:"unauthorized"` +} + // Balance represents an account's holdings for a single currency type type Balance struct { Balance string `json:"balance"` diff --git a/services/horizon/CHANGELOG.md b/services/horizon/CHANGELOG.md index d990ca7c34..11d2dba972 100644 --- a/services/horizon/CHANGELOG.md +++ b/services/horizon/CHANGELOG.md @@ -7,6 +7,8 @@ file. This project adheres to [Semantic Versioning](http://semver.org/). * Add an endpoint which determines if Horizon is healthy enough to receive traffic ([3435](https://github.com/stellar/go/pull/3435)). * Sanitize route regular expressions for Prometheus metrics ([3459](https://github.com/stellar/go/pull/3459)). +* Add asset stat summaries per trust-line flag category ([3454](https://github.com/stellar/go/pull/3454)). + - The `amount`, and `num_accounts` fields in `/assets` endpoint are deprecated. Fields will be removed in Horizon 3.0. You can find the same data under `balances.authorized`, and `accounts.authorized`, respectively. ## v2.0.0 diff --git a/services/horizon/internal/actions/asset_test.go b/services/horizon/internal/actions/asset_test.go index eec4d45e71..7663bd4e29 100644 --- a/services/horizon/internal/actions/asset_test.go +++ b/services/horizon/internal/actions/asset_test.go @@ -5,6 +5,8 @@ import ( "strings" "testing" + "github.com/stretchr/testify/assert" + "github.com/stellar/go/protocols/horizon" "github.com/stellar/go/protocols/horizon/base" "github.com/stellar/go/services/horizon/internal/db2/history" @@ -123,10 +125,30 @@ func TestAssetStats(t *testing.T) { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, AssetIssuer: issuer.AccountID, AssetCode: "USD", + Accounts: history.ExpAssetStatAccounts{ + Authorized: 2, + AuthorizedToMaintainLiabilities: 3, + Unauthorized: 4, + }, + Balances: history.ExpAssetStatBalances{ + Authorized: "1", + AuthorizedToMaintainLiabilities: "2", + Unauthorized: "3", + }, Amount: "1", NumAccounts: 2, } usdAssetStatResponse := horizon.AssetStat{ + Accounts: horizon.AssetStatAccounts{ + Authorized: usdAssetStat.Accounts.Authorized, + AuthorizedToMaintainLiabilities: usdAssetStat.Accounts.AuthorizedToMaintainLiabilities, + Unauthorized: usdAssetStat.Accounts.Unauthorized, + }, + Balances: horizon.AssetStatBalances{ + Authorized: "0.0000001", + AuthorizedToMaintainLiabilities: "0.0000002", + Unauthorized: "0.0000003", + }, Amount: "0.0000001", NumAccounts: usdAssetStat.NumAccounts, Asset: base.Asset{ @@ -142,10 +164,30 @@ func TestAssetStats(t *testing.T) { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, AssetIssuer: issuer.AccountID, AssetCode: "ETHER", + Accounts: history.ExpAssetStatAccounts{ + Authorized: 1, + AuthorizedToMaintainLiabilities: 2, + Unauthorized: 3, + }, + Balances: history.ExpAssetStatBalances{ + Authorized: "23", + AuthorizedToMaintainLiabilities: "46", + Unauthorized: "92", + }, Amount: "23", NumAccounts: 1, } etherAssetStatResponse := horizon.AssetStat{ + Accounts: horizon.AssetStatAccounts{ + Authorized: etherAssetStat.Accounts.Authorized, + AuthorizedToMaintainLiabilities: etherAssetStat.Accounts.AuthorizedToMaintainLiabilities, + Unauthorized: etherAssetStat.Accounts.Unauthorized, + }, + Balances: horizon.AssetStatBalances{ + Authorized: "0.0000023", + AuthorizedToMaintainLiabilities: "0.0000046", + Unauthorized: "0.0000092", + }, Amount: "0.0000023", NumAccounts: etherAssetStat.NumAccounts, Asset: base.Asset{ @@ -161,10 +203,30 @@ func TestAssetStats(t *testing.T) { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, AssetIssuer: otherIssuer.AccountID, AssetCode: "USD", + Accounts: history.ExpAssetStatAccounts{ + Authorized: 2, + AuthorizedToMaintainLiabilities: 3, + Unauthorized: 4, + }, + Balances: history.ExpAssetStatBalances{ + Authorized: "1", + AuthorizedToMaintainLiabilities: "2", + Unauthorized: "3", + }, Amount: "1", NumAccounts: 2, } otherUSDAssetStatResponse := horizon.AssetStat{ + Accounts: horizon.AssetStatAccounts{ + Authorized: otherUSDAssetStat.Accounts.Authorized, + AuthorizedToMaintainLiabilities: otherUSDAssetStat.Accounts.AuthorizedToMaintainLiabilities, + Unauthorized: otherUSDAssetStat.Accounts.Unauthorized, + }, + Balances: horizon.AssetStatBalances{ + Authorized: "0.0000001", + AuthorizedToMaintainLiabilities: "0.0000002", + Unauthorized: "0.0000003", + }, Amount: "0.0000001", NumAccounts: otherUSDAssetStat.NumAccounts, Asset: base.Asset{ @@ -182,10 +244,30 @@ func TestAssetStats(t *testing.T) { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, AssetIssuer: otherIssuer.AccountID, AssetCode: "EUR", + Accounts: history.ExpAssetStatAccounts{ + Authorized: 3, + AuthorizedToMaintainLiabilities: 4, + Unauthorized: 5, + }, + Balances: history.ExpAssetStatBalances{ + Authorized: "111", + AuthorizedToMaintainLiabilities: "222", + Unauthorized: "333", + }, Amount: "111", NumAccounts: 3, } eurAssetStatResponse := horizon.AssetStat{ + Accounts: horizon.AssetStatAccounts{ + Authorized: eurAssetStat.Accounts.Authorized, + AuthorizedToMaintainLiabilities: eurAssetStat.Accounts.AuthorizedToMaintainLiabilities, + Unauthorized: eurAssetStat.Accounts.Unauthorized, + }, + Balances: horizon.AssetStatBalances{ + Authorized: "0.0000111", + AuthorizedToMaintainLiabilities: "0.0000222", + Unauthorized: "0.0000333", + }, Amount: "0.0000111", NumAccounts: eurAssetStat.NumAccounts, Asset: base.Asset{ @@ -311,23 +393,12 @@ func TestAssetStats(t *testing.T) { t.Run(testCase.name, func(t *testing.T) { r := makeRequest(t, testCase.queryParams, map[string]string{}, q.Session) results, err := handler.GetResourcePage(httptest.NewRecorder(), r) - if err != nil { - t.Fatalf("unexpected error %v", err) - } - - if len(results) != len(testCase.expected) { - t.Fatalf( - "expectes results to have length %v but got %v", - len(results), - len(testCase.expected), - ) - } + assert.NoError(t, err) + assert.Len(t, results, len(testCase.expected)) for i, item := range results { assetStat := item.(horizon.AssetStat) - if assetStat != testCase.expected[i] { - t.Fatalf("expected %v but got %v", testCase.expected[i], assetStat) - } + assert.Equal(t, testCase.expected[i], assetStat) } }) } @@ -344,6 +415,16 @@ func TestAssetStatsIssuerDoesNotExist(t *testing.T) { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, AssetIssuer: "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", AssetCode: "USD", + Accounts: history.ExpAssetStatAccounts{ + Authorized: 2, + AuthorizedToMaintainLiabilities: 3, + Unauthorized: 4, + }, + Balances: history.ExpAssetStatBalances{ + Authorized: "1", + AuthorizedToMaintainLiabilities: "2", + Unauthorized: "3", + }, Amount: "1", NumAccounts: 2, } @@ -356,6 +437,16 @@ func TestAssetStatsIssuerDoesNotExist(t *testing.T) { tt.Assert.NoError(err) expectedAssetStatResponse := horizon.AssetStat{ + Accounts: horizon.AssetStatAccounts{ + Authorized: 2, + AuthorizedToMaintainLiabilities: 3, + Unauthorized: 4, + }, + Balances: horizon.AssetStatBalances{ + Authorized: "0.0000001", + AuthorizedToMaintainLiabilities: "0.0000002", + Unauthorized: "0.0000003", + }, Amount: "0.0000001", NumAccounts: usdAssetStat.NumAccounts, Asset: base.Asset{ diff --git a/services/horizon/internal/db2/history/asset_stats.go b/services/horizon/internal/db2/history/asset_stats.go index ec914ede7a..81174b676a 100644 --- a/services/horizon/internal/db2/history/asset_stats.go +++ b/services/horizon/internal/db2/history/asset_stats.go @@ -16,6 +16,8 @@ func assetStatToMap(assetStat ExpAssetStat) map[string]interface{} { "asset_type": assetStat.AssetType, "asset_code": assetStat.AssetCode, "asset_issuer": assetStat.AssetIssuer, + "accounts": assetStat.Accounts, + "balances": assetStat.Balances, "amount": assetStat.Amount, "num_accounts": assetStat.NumAccounts, } diff --git a/services/horizon/internal/db2/history/asset_stats_test.go b/services/horizon/internal/db2/history/asset_stats_test.go index 0238f821a1..63d478a583 100644 --- a/services/horizon/internal/db2/history/asset_stats_test.go +++ b/services/horizon/internal/db2/history/asset_stats_test.go @@ -21,6 +21,16 @@ func TestInsertAssetStats(t *testing.T) { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, AssetIssuer: "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", AssetCode: "USD", + Accounts: ExpAssetStatAccounts{ + Authorized: 2, + AuthorizedToMaintainLiabilities: 3, + Unauthorized: 4, + }, + Balances: ExpAssetStatBalances{ + Authorized: "1", + AuthorizedToMaintainLiabilities: "2", + Unauthorized: "3", + }, Amount: "1", NumAccounts: 2, }, @@ -28,6 +38,16 @@ func TestInsertAssetStats(t *testing.T) { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum12, AssetIssuer: "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", AssetCode: "ETHER", + Accounts: ExpAssetStatAccounts{ + Authorized: 1, + AuthorizedToMaintainLiabilities: 3, + Unauthorized: 4, + }, + Balances: ExpAssetStatBalances{ + Authorized: "23", + AuthorizedToMaintainLiabilities: "2", + Unauthorized: "3", + }, Amount: "23", NumAccounts: 1, }, @@ -52,6 +72,16 @@ func TestInsertAssetStat(t *testing.T) { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, AssetIssuer: "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", AssetCode: "USD", + Accounts: ExpAssetStatAccounts{ + Authorized: 2, + AuthorizedToMaintainLiabilities: 3, + Unauthorized: 4, + }, + Balances: ExpAssetStatBalances{ + Authorized: "1", + AuthorizedToMaintainLiabilities: "2", + Unauthorized: "3", + }, Amount: "1", NumAccounts: 2, }, @@ -59,6 +89,16 @@ func TestInsertAssetStat(t *testing.T) { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum12, AssetIssuer: "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", AssetCode: "ETHER", + Accounts: ExpAssetStatAccounts{ + Authorized: 1, + AuthorizedToMaintainLiabilities: 3, + Unauthorized: 4, + }, + Balances: ExpAssetStatBalances{ + Authorized: "23", + AuthorizedToMaintainLiabilities: "2", + Unauthorized: "3", + }, Amount: "23", NumAccounts: 1, }, @@ -85,6 +125,16 @@ func TestInsertAssetStatAlreadyExistsError(t *testing.T) { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, AssetIssuer: "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", AssetCode: "USD", + Accounts: ExpAssetStatAccounts{ + Authorized: 2, + AuthorizedToMaintainLiabilities: 3, + Unauthorized: 4, + }, + Balances: ExpAssetStatBalances{ + Authorized: "1", + AuthorizedToMaintainLiabilities: "2", + Unauthorized: "3", + }, Amount: "1", NumAccounts: 2, } @@ -120,6 +170,16 @@ func TestUpdateAssetStatDoesNotExistsError(t *testing.T) { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, AssetIssuer: "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", AssetCode: "USD", + Accounts: ExpAssetStatAccounts{ + Authorized: 2, + AuthorizedToMaintainLiabilities: 3, + Unauthorized: 4, + }, + Balances: ExpAssetStatBalances{ + Authorized: "1", + AuthorizedToMaintainLiabilities: "2", + Unauthorized: "3", + }, Amount: "1", NumAccounts: 2, } @@ -143,6 +203,16 @@ func TestUpdateStat(t *testing.T) { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, AssetIssuer: "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", AssetCode: "USD", + Accounts: ExpAssetStatAccounts{ + Authorized: 2, + AuthorizedToMaintainLiabilities: 3, + Unauthorized: 4, + }, + Balances: ExpAssetStatBalances{ + Authorized: "1", + AuthorizedToMaintainLiabilities: "2", + Unauthorized: "3", + }, Amount: "1", NumAccounts: 2, } @@ -177,6 +247,16 @@ func TestGetAssetStatDoesNotExist(t *testing.T) { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, AssetIssuer: "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", AssetCode: "USD", + Accounts: ExpAssetStatAccounts{ + Authorized: 2, + AuthorizedToMaintainLiabilities: 3, + Unauthorized: 4, + }, + Balances: ExpAssetStatBalances{ + Authorized: "1", + AuthorizedToMaintainLiabilities: "2", + Unauthorized: "3", + }, Amount: "1", NumAccounts: 2, } @@ -196,6 +276,16 @@ func TestRemoveAssetStat(t *testing.T) { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, AssetIssuer: "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", AssetCode: "USD", + Accounts: ExpAssetStatAccounts{ + Authorized: 2, + AuthorizedToMaintainLiabilities: 3, + Unauthorized: 4, + }, + Balances: ExpAssetStatBalances{ + Authorized: "1", + AuthorizedToMaintainLiabilities: "2", + Unauthorized: "3", + }, Amount: "1", NumAccounts: 2, } @@ -320,6 +410,16 @@ func TestGetAssetStatsFiltersAndCursor(t *testing.T) { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, AssetIssuer: "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", AssetCode: "USD", + Accounts: ExpAssetStatAccounts{ + Authorized: 2, + AuthorizedToMaintainLiabilities: 3, + Unauthorized: 4, + }, + Balances: ExpAssetStatBalances{ + Authorized: "1", + AuthorizedToMaintainLiabilities: "2", + Unauthorized: "3", + }, Amount: "1", NumAccounts: 2, } @@ -327,6 +427,16 @@ func TestGetAssetStatsFiltersAndCursor(t *testing.T) { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum12, AssetIssuer: "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", AssetCode: "ETHER", + Accounts: ExpAssetStatAccounts{ + Authorized: 1, + AuthorizedToMaintainLiabilities: 3, + Unauthorized: 4, + }, + Balances: ExpAssetStatBalances{ + Authorized: "23", + AuthorizedToMaintainLiabilities: "2", + Unauthorized: "3", + }, Amount: "23", NumAccounts: 1, } @@ -334,6 +444,16 @@ func TestGetAssetStatsFiltersAndCursor(t *testing.T) { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, AssetIssuer: "GA5WBPYA5Y4WAEHXWR2UKO2UO4BUGHUQ74EUPKON2QHV4WRHOIRNKKH2", AssetCode: "USD", + Accounts: ExpAssetStatAccounts{ + Authorized: 2, + AuthorizedToMaintainLiabilities: 3, + Unauthorized: 4, + }, + Balances: ExpAssetStatBalances{ + Authorized: "1", + AuthorizedToMaintainLiabilities: "2", + Unauthorized: "3", + }, Amount: "1", NumAccounts: 2, } @@ -341,6 +461,16 @@ func TestGetAssetStatsFiltersAndCursor(t *testing.T) { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, AssetIssuer: "GA5WBPYA5Y4WAEHXWR2UKO2UO4BUGHUQ74EUPKON2QHV4WRHOIRNKKH2", AssetCode: "EUR", + Accounts: ExpAssetStatAccounts{ + Authorized: 3, + AuthorizedToMaintainLiabilities: 2, + Unauthorized: 4, + }, + Balances: ExpAssetStatBalances{ + Authorized: "111", + AuthorizedToMaintainLiabilities: "2", + Unauthorized: "3", + }, Amount: "111", NumAccounts: 3, } diff --git a/services/horizon/internal/db2/history/main.go b/services/horizon/internal/db2/history/main.go index 1cbba043c6..f940387ed8 100644 --- a/services/horizon/internal/db2/history/main.go +++ b/services/horizon/internal/db2/history/main.go @@ -4,6 +4,8 @@ package history import ( "database/sql" + "database/sql/driver" + "encoding/json" "fmt" "sync" "time" @@ -321,11 +323,13 @@ type Asset struct { // ExpAssetStat is a row in the exp_asset_stats table representing the stats per Asset type ExpAssetStat struct { - AssetType xdr.AssetType `db:"asset_type"` - AssetCode string `db:"asset_code"` - AssetIssuer string `db:"asset_issuer"` - Amount string `db:"amount"` - NumAccounts int32 `db:"num_accounts"` + AssetType xdr.AssetType `db:"asset_type"` + AssetCode string `db:"asset_code"` + AssetIssuer string `db:"asset_issuer"` + Accounts ExpAssetStatAccounts `db:"accounts"` + Balances ExpAssetStatBalances `db:"balances"` + Amount string `db:"amount"` + NumAccounts int32 `db:"num_accounts"` } // PagingToken returns a cursor for this asset stat @@ -338,6 +342,58 @@ func (e ExpAssetStat) PagingToken() string { ) } +// ExpAssetStatAccounts represents the summarized acount numbers for a single Asset +type ExpAssetStatAccounts struct { + Authorized int32 `json:"authorized"` + AuthorizedToMaintainLiabilities int32 `json:"authorized_to_maintain_liabilities"` + Unauthorized int32 `json:"unauthorized"` +} + +func (e ExpAssetStatAccounts) Value() (driver.Value, error) { + return json.Marshal(e) +} + +func (e *ExpAssetStatAccounts) Scan(src interface{}) error { + source, ok := src.([]byte) + if !ok { + return errors.New("Type assertion .([]byte) failed.") + } + + return json.Unmarshal(source, &e) +} + +func (a ExpAssetStatAccounts) Add(b ExpAssetStatAccounts) ExpAssetStatAccounts { + return ExpAssetStatAccounts{ + Authorized: a.Authorized + b.Authorized, + AuthorizedToMaintainLiabilities: a.AuthorizedToMaintainLiabilities + b.AuthorizedToMaintainLiabilities, + Unauthorized: a.Unauthorized + b.Unauthorized, + } +} + +func (a ExpAssetStatAccounts) IsZero() bool { + return a.Authorized == 0 && a.AuthorizedToMaintainLiabilities == 0 && a.Unauthorized == 0 +} + +// ExpAssetStatBalances represents the summarized balances for a single Asset +type ExpAssetStatBalances struct { + Authorized string `json:"authorized"` + AuthorizedToMaintainLiabilities string `json:"authorized_to_maintain_liabilities"` + Unauthorized string `json:"unauthorized"` +} + +func (e ExpAssetStatBalances) Value() (driver.Value, error) { + return json.Marshal(e) +} + +func (e *ExpAssetStatBalances) Scan(src interface{}) error { + source, ok := src.([]byte) + if !ok { + return errors.New("Type assertion .([]byte) failed.") + } + + return json.Unmarshal(source, &e) +} + // QAssetStats defines exp_asset_stats related queries. type QAssetStats interface { InsertAssetStats(stats []ExpAssetStat, batchSize int) error diff --git a/services/horizon/internal/db2/schema/bindata.go b/services/horizon/internal/db2/schema/bindata.go index 8cbeb8a342..178933884c 100644 --- a/services/horizon/internal/db2/schema/bindata.go +++ b/services/horizon/internal/db2/schema/bindata.go @@ -38,6 +38,7 @@ // migrations/41_add_sponsor_to_state_tables.sql (800B) // migrations/42_add_num_sponsored_and_num_sponsoring_to_accounts.sql (276B) // migrations/43_add_claimable_balances_flags.sql (145B) +// migrations/44_asset_stat_accounts_and_balances.sql (439B) // migrations/4_add_protocol_version.sql (188B) // migrations/5_create_trades_table.sql (1.1kB) // migrations/6_create_assets_table.sql (366B) @@ -873,6 +874,26 @@ func migrations43_add_claimable_balances_flagsSql() (*asset, error) { return a, nil } +var _migrations44_asset_stat_accounts_and_balancesSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x8c\x90\x4f\x4f\xc2\x30\x18\xc6\xef\xef\xa7\x78\x6e\x48\x2c\x9f\x60\xf1\x30\xec\x2e\xa6\x6e\x84\x75\xe7\xe6\x5d\x69\x74\x84\xb5\x84\xb6\xd1\xf8\xe9\x8d\x44\x70\x60\xa2\x5c\xfb\xfc\xf9\xf5\x79\x17\x0b\xdc\x8f\xc3\xcb\x81\x93\x43\xb7\x27\x2a\x95\xae\xd6\xd0\xe5\x52\x55\x70\xef\x7b\xc3\x31\xba\x64\x62\xe2\x14\xa9\x94\x12\x8f\x8d\xea\x9e\x6b\xb0\xb5\x21\xfb\x14\xf1\xd4\x36\xf5\x52\x4c\xa5\x9e\x77\xec\xad\xfb\x96\x0a\xea\x56\xb2\xd4\xbf\xcb\x80\xb6\xd2\x04\xe0\xa7\xeb\x01\xdb\x18\x7c\x6f\xfa\x3c\xec\x36\x26\xf4\x5b\x67\xd3\xdd\x8c\x73\x7a\x0d\x87\xe1\xc3\x6d\x66\x02\x3e\x8f\xe6\xe4\x9f\x8b\x63\xfc\xcc\xbb\x21\xce\xe3\x57\x72\x5e\xfc\xb3\xf3\xa8\x5d\x2f\x6d\x2b\x8d\xba\xd1\xa8\x3b\xa5\xc4\xa5\xe7\xfc\x85\xa9\xa7\x20\x9a\x1e\x57\x86\x37\xff\x37\x56\xae\x9b\xd5\x35\x55\x5c\xbc\x9e\x38\x05\x7d\x06\x00\x00\xff\xff\x9e\x44\x07\x8e\xb7\x01\x00\x00") + +func migrations44_asset_stat_accounts_and_balancesSqlBytes() ([]byte, error) { + return bindataRead( + _migrations44_asset_stat_accounts_and_balancesSql, + "migrations/44_asset_stat_accounts_and_balances.sql", + ) +} + +func migrations44_asset_stat_accounts_and_balancesSql() (*asset, error) { + bytes, err := migrations44_asset_stat_accounts_and_balancesSqlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "migrations/44_asset_stat_accounts_and_balances.sql", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xb0, 0x3b, 0xce, 0x82, 0x13, 0x91, 0xc0, 0xc2, 0x93, 0xb1, 0x17, 0xbc, 0x57, 0x34, 0xdd, 0x98, 0xc8, 0x36, 0xee, 0x29, 0xd, 0x1e, 0x69, 0xb, 0x3f, 0x31, 0xbc, 0x41, 0xd4, 0x7d, 0xa5, 0xfd}} + return a, nil +} + var _migrations4_add_protocol_versionSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x84\xcd\xb1\x0a\xc2\x30\x10\x06\xe0\x3d\x4f\xf1\xef\x52\x70\xef\x14\x4d\x9d\xce\x44\x4a\x32\x38\x15\xd1\xa3\x06\x6a\xae\x5c\x82\xe2\xdb\xbb\xba\x88\x4f\xf0\x75\x1d\x36\x8f\x3c\xeb\xa5\x31\xd2\x6a\x2c\xc5\x61\x44\xb4\x3b\x1a\x10\x3c\x9d\x71\xcf\xb5\x89\xbe\xa7\x85\x6f\x33\x6b\x85\x01\xac\x73\xd8\x07\x4a\x47\x8f\x55\xa5\xc9\x55\x96\xe9\xc9\x5a\xb3\x14\xe4\xd2\x78\x66\x85\x1b\x0e\x36\x51\xc4\x16\x3e\x44\xf8\x44\xd4\x1b\xf3\x6d\x39\x79\x95\xff\x9a\x1b\xc3\xe9\x97\xd5\x9b\x4f\x00\x00\x00\xff\xff\x83\xbb\x30\x2e\xbc\x00\x00\x00") func migrations4_add_protocol_versionSqlBytes() ([]byte, error) { @@ -1142,6 +1163,7 @@ var _bindata = map[string]func() (*asset, error){ "migrations/41_add_sponsor_to_state_tables.sql": migrations41_add_sponsor_to_state_tablesSql, "migrations/42_add_num_sponsored_and_num_sponsoring_to_accounts.sql": migrations42_add_num_sponsored_and_num_sponsoring_to_accountsSql, "migrations/43_add_claimable_balances_flags.sql": migrations43_add_claimable_balances_flagsSql, + "migrations/44_asset_stat_accounts_and_balances.sql": migrations44_asset_stat_accounts_and_balancesSql, "migrations/4_add_protocol_version.sql": migrations4_add_protocol_versionSql, "migrations/5_create_trades_table.sql": migrations5_create_trades_tableSql, "migrations/6_create_assets_table.sql": migrations6_create_assets_tableSql, @@ -1231,6 +1253,7 @@ var _bintree = &bintree{nil, map[string]*bintree{ "41_add_sponsor_to_state_tables.sql": &bintree{migrations41_add_sponsor_to_state_tablesSql, map[string]*bintree{}}, "42_add_num_sponsored_and_num_sponsoring_to_accounts.sql": &bintree{migrations42_add_num_sponsored_and_num_sponsoring_to_accountsSql, map[string]*bintree{}}, "43_add_claimable_balances_flags.sql": &bintree{migrations43_add_claimable_balances_flagsSql, map[string]*bintree{}}, + "44_asset_stat_accounts_and_balances.sql": &bintree{migrations44_asset_stat_accounts_and_balancesSql, map[string]*bintree{}}, "4_add_protocol_version.sql": &bintree{migrations4_add_protocol_versionSql, map[string]*bintree{}}, "5_create_trades_table.sql": &bintree{migrations5_create_trades_tableSql, map[string]*bintree{}}, "6_create_assets_table.sql": &bintree{migrations6_create_assets_tableSql, map[string]*bintree{}}, diff --git a/services/horizon/internal/db2/schema/migrations/44_asset_stat_accounts_and_balances.sql b/services/horizon/internal/db2/schema/migrations/44_asset_stat_accounts_and_balances.sql new file mode 100644 index 0000000000..93beade74b --- /dev/null +++ b/services/horizon/internal/db2/schema/migrations/44_asset_stat_accounts_and_balances.sql @@ -0,0 +1,19 @@ +-- +migrate Up + +ALTER TABLE exp_asset_stats +ADD COLUMN accounts JSONB, +ADD COLUMN balances JSONB; +UPDATE exp_asset_stats + SET + accounts = jsonb_build_object('authorized', num_accounts), + balances = jsonb_build_object('authorized', amount); + +ALTER TABLE exp_asset_stats +ALTER COLUMN accounts SET NOT NULL, +ALTER COLUMN balances SET NOT NULL; + +-- +migrate Down + +ALTER TABLE exp_asset_stats +DROP COLUMN accounts, +DROP COLUMN balances; diff --git a/services/horizon/internal/docs/reference/endpoints/assets-all.md b/services/horizon/internal/docs/reference/endpoints/assets-all.md index 4d665033e5..9c6463d744 100644 --- a/services/horizon/internal/docs/reference/endpoints/assets-all.md +++ b/services/horizon/internal/docs/reference/endpoints/assets-all.md @@ -8,9 +8,6 @@ replacement: https://developers.stellar.org/api/resources/assets/ This endpoint represents all [assets](../resources/asset.md). It will give you all the assets in the system along with various statistics about each. -### Notes -- The attribute `num_accounts` includes authorized trust lines only. - ## Request ``` @@ -81,8 +78,16 @@ If called normally this endpoint responds with a [page](../resources/page.md) of "asset_code": "BANANA", "asset_issuer": "GDSBCQO34HWPGUGQSP3QBFEXVTSR2PW46UIGTHVWGWJGQKH3AFNHXHXN", "paging_token": "BANANA_GDSBCQO34HWPGUGQSP3QBFEXVTSR2PW46UIGTHVWGWJGQKH3AFNHXHXN_credit_alphanum4", - "amount": "10000.0000000", - "num_accounts": 2126, + "accounts": { + "authorized": 2126, + "authorized_to_maintain_liabilities": 32, + "unauthorized": 5 + }, + "balances": { + "authorized": "10000.0000000", + "authorized_to_maintain_liabilities": "3000.0000000", + "unauthorized": "4000.0000000" + }, "flags": { "auth_required": true, "auth_revocable": false @@ -98,8 +103,16 @@ If called normally this endpoint responds with a [page](../resources/page.md) of "asset_code": "BTC", "asset_issuer": "GBAUUA74H4XOQYRSOW2RZUA4QL5PB37U3JS5NE3RTB2ELJVMIF5RLMAG", "paging_token": "BTC_GBAUUA74H4XOQYRSOW2RZUA4QL5PB37U3JS5NE3RTB2ELJVMIF5RLMAG_credit_alphanum4", - "amount": "5000.0000000", - "num_accounts": 32, + "accounts": { + "authorized": 32, + "authorized_to_maintain_liabilities": 124, + "unauthorized": 6 + }, + "balances": { + "authorized": "5000.0000000", + "authorized_to_maintain_liabilities": "8000.0000000", + "unauthorized": "2000.0000000" + }, "flags": { "auth_required": false, "auth_revocable": false @@ -115,8 +128,16 @@ If called normally this endpoint responds with a [page](../resources/page.md) of "asset_code": "USD", "asset_issuer": "GBAUUA74H4XOQYRSOW2RZUA4QL5PB37U3JS5NE3RTB2ELJVMIF5RLMAG", "paging_token": "USD_GBAUUA74H4XOQYRSOW2RZUA4QL5PB37U3JS5NE3RTB2ELJVMIF5RLMAG_credit_alphanum4", - "amount": "1000000000.0000000", - "num_accounts": 91547871, + "accounts": { + "authorized": 91547871, + "authorized_to_maintain_liabilities": 45773935, + "unauthorized": 22886967 + }, + "balances": { + "authorized": "1000000000.0000000", + "authorized_to_maintain_liabilities": "500000000.0000000", + "unauthorized": "250000000.0000000" + }, "flags": { "auth_required": false, "auth_revocable": false diff --git a/services/horizon/internal/docs/reference/resources/asset.md b/services/horizon/internal/docs/reference/resources/asset.md index 989f4be889..35bec0d292 100644 --- a/services/horizon/internal/docs/reference/resources/asset.md +++ b/services/horizon/internal/docs/reference/resources/asset.md @@ -16,8 +16,8 @@ To learn more about the concept of assets in the Stellar network, take a look at | asset_type | string | The type of this asset: "credit_alphanum4", or "credit_alphanum12". | | asset_code | string | The code of this asset. | | asset_issuer | string | The issuer of this asset. | -| amount | number | The number of units of credit issued. | -| num_accounts | number | The number of accounts that: 1) trust this asset and 2) where if the asset has the auth_required flag then the account is authorized to hold the asset. | +| accounts | object | The number of accounts holding this asset, summarized by each state of the trust line flags. | +| balances | object | The number of units of credit issued, summarized by each state of the trust line flags. | | flags | object | The flags denote the enabling/disabling of certain asset issuer privileges. | | paging_token | string | A [paging token](./page.md) suitable for use as the `cursor` parameter to transaction collection resources. | @@ -46,8 +46,16 @@ To learn more about the concept of assets in the Stellar network, take a look at "asset_code": "USD", "asset_issuer": "GBAUUA74H4XOQYRSOW2RZUA4QL5PB37U3JS5NE3RTB2ELJVMIF5RLMAG", "paging_token": "USD_GBAUUA74H4XOQYRSOW2RZUA4QL5PB37U3JS5NE3RTB2ELJVMIF5RLMAG_credit_alphanum4", - "amount": "100.0000000", - "num_accounts": 91547871, + "accounts": { + "authorized": 91547871, + "authorized_to_maintain_liabilities": 45773935, + "unauthorized": 22886967 + }, + "balances": { + "authorized": "100.0000000", + "authorized_to_maintain_liabilities": "50.0000000", + "unauthorized": "25.0000000" + }, "flags": { "auth_required": false, "auth_revocable": false diff --git a/services/horizon/internal/ingest/main.go b/services/horizon/internal/ingest/main.go index 14690ffc24..c9bed61ced 100644 --- a/services/horizon/internal/ingest/main.go +++ b/services/horizon/internal/ingest/main.go @@ -49,7 +49,8 @@ const ( // - 11: Protocol 14: CAP-23 and CAP-33. // - 12: Trigger state rebuild due to `absTime` -> `abs_time` rename // in ClaimableBalances predicates. - CurrentVersion = 12 + // - 13: Trigger state rebuild to include more than just authorized assets. + CurrentVersion = 13 // MaxDBConnections is the size of the postgres connection pool dedicated to Horizon ingestion: // * Ledger ingestion, diff --git a/services/horizon/internal/ingest/processors/asset_stats_processor.go b/services/horizon/internal/ingest/processors/asset_stats_processor.go index a4f5cab5d0..72989fd133 100644 --- a/services/horizon/internal/ingest/processors/asset_stats_processor.go +++ b/services/horizon/internal/ingest/processors/asset_stats_processor.go @@ -2,7 +2,6 @@ package processors import ( "database/sql" - "math/big" "github.com/stellar/go/ingest" "github.com/stellar/go/services/horizon/internal/db2/history" @@ -109,8 +108,10 @@ func (p *AssetStatsProcessor) Commit() error { assetStatsDeltas := p.assetStatSet.All() for _, delta := range assetStatsDeltas { var rowsAffected int64 + var stat history.ExpAssetStat + var err error - stat, err := p.assetStatsQ.GetAssetStat( + stat, err = p.assetStatsQ.GetAssetStat( delta.AssetType, delta.AssetCode, delta.AssetIssuer, @@ -121,39 +122,53 @@ func (p *AssetStatsProcessor) Commit() error { } if assetStatNotFound { - // Insert - if delta.NumAccounts < 0 { + // Safety checks + if delta.Accounts.Authorized < 0 { + return ingest.NewStateError(errors.Errorf( + "Authorized accounts negative but DB entry does not exist for asset: %s %s %s", + delta.AssetType, + delta.AssetCode, + delta.AssetIssuer, + )) + } else if delta.Accounts.AuthorizedToMaintainLiabilities < 0 { + return ingest.NewStateError(errors.Errorf( + "AuthorizedToMaintainLiabilities accounts negative but DB entry does not exist for asset: %s %s %s", + delta.AssetType, + delta.AssetCode, + delta.AssetIssuer, + )) + } else if delta.Accounts.Unauthorized < 0 { return ingest.NewStateError(errors.Errorf( - "NumAccounts negative but DB entry does not exist for asset: %s %s %s", + "Unauthorized accounts negative but DB entry does not exist for asset: %s %s %s", delta.AssetType, delta.AssetCode, delta.AssetIssuer, )) } + // Insert var errInsert error rowsAffected, errInsert = p.assetStatsQ.InsertAssetStat(delta) if errInsert != nil { return errors.Wrap(errInsert, "could not insert asset stat") } } else { - statBalance, ok := new(big.Int).SetString(stat.Amount, 10) - if !ok { - return errors.New("Error parsing: " + stat.Amount) + var statBalances assetStatBalances + if err = statBalances.Parse(&stat.Balances); err != nil { + return errors.Wrap(err, "Error parsing balances") } - deltaBalance, ok := new(big.Int).SetString(delta.Amount, 10) - if !ok { - return errors.New("Error parsing: " + stat.Amount) + var deltaBalances assetStatBalances + if err = deltaBalances.Parse(&delta.Balances); err != nil { + return errors.Wrap(err, "Error parsing balances") } - // statBalance = statBalance + deltaBalance - statBalance.Add(statBalance, deltaBalance) - statAccounts := stat.NumAccounts + delta.NumAccounts + statBalances = statBalances.Add(deltaBalances) + statAccounts := stat.Accounts.Add(delta.Accounts) - if statAccounts == 0 { + if statAccounts.IsZero() { // Remove stats - if statBalance.Cmp(big.NewInt(0)) != 0 { + if !statBalances.IsZero() { return ingest.NewStateError(errors.Errorf( "Removing asset stat by final amount non-zero for: %s %s %s", delta.AssetType, @@ -175,8 +190,10 @@ func (p *AssetStatsProcessor) Commit() error { AssetType: delta.AssetType, AssetCode: delta.AssetCode, AssetIssuer: delta.AssetIssuer, - Amount: statBalance.String(), - NumAccounts: statAccounts, + Accounts: statAccounts, + Balances: statBalances.ConvertToHistoryObject(), + Amount: statBalances.Authorized.String(), + NumAccounts: statAccounts.Authorized, }) if err != nil { return errors.Wrap(err, "could not update asset stat") @@ -202,50 +219,26 @@ func (p *AssetStatsProcessor) adjustAssetStat( preTrustline *xdr.TrustLineEntry, postTrustline *xdr.TrustLineEntry, ) error { - var deltaBalance xdr.Int64 - var deltaAccounts int32 - var trustline xdr.TrustLineEntry + deltaAccounts := map[xdr.Uint32]int32{} + deltaBalances := map[xdr.Uint32]int64{} + + if preTrustline == nil && postTrustline == nil { + return ingest.NewStateError(errors.New("both pre and post trustlines cannot be nil")) + } - if preTrustline != nil && postTrustline == nil { + var trustline xdr.TrustLineEntry + if preTrustline != nil { trustline = *preTrustline - // removing a trustline - if xdr.TrustLineFlags(preTrustline.Flags).IsAuthorized() { - deltaAccounts = -1 - deltaBalance = -preTrustline.Balance - } - } else if preTrustline == nil && postTrustline != nil { - trustline = *postTrustline - // adding a trustline - if xdr.TrustLineFlags(postTrustline.Flags).IsAuthorized() { - deltaAccounts = 1 - deltaBalance = postTrustline.Balance - } - } else if preTrustline != nil && postTrustline != nil { + deltaAccounts[preTrustline.Flags] -= 1 + deltaBalances[preTrustline.Flags] -= int64(preTrustline.Balance) + } + if postTrustline != nil { trustline = *postTrustline - // updating a trustline - if xdr.TrustLineFlags(preTrustline.Flags).IsAuthorized() && - xdr.TrustLineFlags(postTrustline.Flags).IsAuthorized() { - // trustline remains authorized - deltaAccounts = 0 - deltaBalance = postTrustline.Balance - preTrustline.Balance - } else if xdr.TrustLineFlags(preTrustline.Flags).IsAuthorized() && - !xdr.TrustLineFlags(postTrustline.Flags).IsAuthorized() { - // trustline was authorized and became unauthorized - deltaAccounts = -1 - deltaBalance = -preTrustline.Balance - } else if !xdr.TrustLineFlags(preTrustline.Flags).IsAuthorized() && - xdr.TrustLineFlags(postTrustline.Flags).IsAuthorized() { - // trustline was unauthorized and became authorized - deltaAccounts = 1 - deltaBalance = postTrustline.Balance - } - // else, trustline was unauthorized and remains unauthorized - // so there is no change to accounts or balances - } else { - return ingest.NewStateError(errors.New("both pre and post trustlines cannot be nil")) + deltaAccounts[postTrustline.Flags] += 1 + deltaBalances[postTrustline.Flags] += int64(postTrustline.Balance) } - err := p.assetStatSet.AddDelta(trustline.Asset, int64(deltaBalance), deltaAccounts) + err := p.assetStatSet.AddDelta(trustline.Asset, deltaBalances, deltaAccounts) if err != nil { return errors.Wrap(err, "error running AssetStatSet.AddDelta") } diff --git a/services/horizon/internal/ingest/processors/asset_stats_processor_test.go b/services/horizon/internal/ingest/processors/asset_stats_processor_test.go index 439005d8e2..cf7fb320c9 100644 --- a/services/horizon/internal/ingest/processors/asset_stats_processor_test.go +++ b/services/horizon/internal/ingest/processors/asset_stats_processor_test.go @@ -58,6 +58,12 @@ func (s *AssetStatsProcessorTestSuiteState) TestCreateTrustLine() { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, AssetIssuer: trustLineIssuer.Address(), AssetCode: "EUR", + Accounts: history.ExpAssetStatAccounts{Authorized: 1}, + Balances: history.ExpAssetStatBalances{ + Authorized: "0", + AuthorizedToMaintainLiabilities: "0", + Unauthorized: "0", + }, Amount: "0", NumAccounts: 1, }, @@ -83,8 +89,21 @@ func (s *AssetStatsProcessorTestSuiteState) TestCreateTrustLineUnauthorized() { }) s.Assert().NoError(err) - s.mockQ. - On("InsertAssetStats", []history.ExpAssetStat{}, maxBatchSize).Return(nil).Once() + s.mockQ.On("InsertAssetStats", []history.ExpAssetStat{ + { + AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, + AssetIssuer: trustLineIssuer.Address(), + AssetCode: "EUR", + Accounts: history.ExpAssetStatAccounts{Unauthorized: 1}, + Balances: history.ExpAssetStatBalances{ + Authorized: "0", + AuthorizedToMaintainLiabilities: "0", + Unauthorized: "0", + }, + Amount: "0", + NumAccounts: 0, + }, + }, maxBatchSize).Return(nil).Once() } func TestAssetStatsProcessorTestSuiteLedger(t *testing.T) { @@ -223,10 +242,39 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestInsertTrustLine() { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, AssetIssuer: trustLineIssuer.Address(), AssetCode: "EUR", + Accounts: history.ExpAssetStatAccounts{ + Authorized: 1, + }, + Balances: history.ExpAssetStatBalances{ + Authorized: "10", + AuthorizedToMaintainLiabilities: "0", + Unauthorized: "0", + }, Amount: "10", NumAccounts: 1, }).Return(int64(1), nil).Once() + s.mockQ.On("GetAssetStat", + xdr.AssetTypeAssetTypeCreditAlphanum4, + "USD", + trustLineIssuer.Address(), + ).Return(history.ExpAssetStat{}, sql.ErrNoRows).Once() + s.mockQ.On("InsertAssetStat", history.ExpAssetStat{ + AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, + AssetIssuer: trustLineIssuer.Address(), + AssetCode: "USD", + Accounts: history.ExpAssetStatAccounts{ + Unauthorized: 1, + }, + Balances: history.ExpAssetStatBalances{ + Authorized: "0", + AuthorizedToMaintainLiabilities: "0", + Unauthorized: "10", + }, + Amount: "0", + NumAccounts: 0, + }).Return(int64(1), nil).Once() + s.Assert().NoError(s.processor.Commit()) } @@ -273,6 +321,12 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestUpdateTrustLine() { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, AssetIssuer: trustLineIssuer.Address(), AssetCode: "EUR", + Accounts: history.ExpAssetStatAccounts{Authorized: 1}, + Balances: history.ExpAssetStatBalances{ + Authorized: "100", + AuthorizedToMaintainLiabilities: "0", + Unauthorized: "0", + }, Amount: "100", NumAccounts: 1, }, nil).Once() @@ -280,6 +334,12 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestUpdateTrustLine() { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, AssetIssuer: trustLineIssuer.Address(), AssetCode: "EUR", + Accounts: history.ExpAssetStatAccounts{Authorized: 1}, + Balances: history.ExpAssetStatBalances{ + Authorized: "110", + AuthorizedToMaintainLiabilities: "0", + Unauthorized: "0", + }, Amount: "110", NumAccounts: 1, }).Return(int64(1), nil).Once() @@ -290,44 +350,60 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestUpdateTrustLine() { func (s *AssetStatsProcessorTestSuiteLedger) TestUpdateTrustLineAuthorization() { lastModifiedLedgerSeq := xdr.Uint32(1234) - trustLine := xdr.TrustLineEntry{ + // EUR trustline: 100 unauthorized -> 10 authorized + eurTrustLine := xdr.TrustLineEntry{ AccountId: xdr.MustAddress("GAOQJGUAB7NI7K7I62ORBXMN3J4SSWQUQ7FOEPSDJ322W2HMCNWPHXFB"), Asset: xdr.MustNewCreditAsset("EUR", trustLineIssuer.Address()), Balance: 100, } - updatedTrustLine := xdr.TrustLineEntry{ + eurUpdatedTrustLine := xdr.TrustLineEntry{ AccountId: xdr.MustAddress("GAOQJGUAB7NI7K7I62ORBXMN3J4SSWQUQ7FOEPSDJ322W2HMCNWPHXFB"), Asset: xdr.MustNewCreditAsset("EUR", trustLineIssuer.Address()), Balance: 10, Flags: xdr.Uint32(xdr.TrustLineFlagsAuthorizedFlag), } - otherTrustLine := xdr.TrustLineEntry{ + // USD trustline: 100 authorized -> 10 unauthorized + usdTrustLine := xdr.TrustLineEntry{ AccountId: xdr.MustAddress("GAOQJGUAB7NI7K7I62ORBXMN3J4SSWQUQ7FOEPSDJ322W2HMCNWPHXFB"), Asset: xdr.MustNewCreditAsset("USD", trustLineIssuer.Address()), Balance: 100, Flags: xdr.Uint32(xdr.TrustLineFlagsAuthorizedFlag), } - otherUpdatedTrustLine := xdr.TrustLineEntry{ + usdUpdatedTrustLine := xdr.TrustLineEntry{ AccountId: xdr.MustAddress("GAOQJGUAB7NI7K7I62ORBXMN3J4SSWQUQ7FOEPSDJ322W2HMCNWPHXFB"), Asset: xdr.MustNewCreditAsset("USD", trustLineIssuer.Address()), Balance: 10, } + // ETH trustline: 100 authorized -> 10 authorized_to_maintain_liabilities + ethTrustLine := xdr.TrustLineEntry{ + AccountId: xdr.MustAddress("GAOQJGUAB7NI7K7I62ORBXMN3J4SSWQUQ7FOEPSDJ322W2HMCNWPHXFB"), + Asset: xdr.MustNewCreditAsset("ETH", trustLineIssuer.Address()), + Balance: 100, + Flags: xdr.Uint32(xdr.TrustLineFlagsAuthorizedFlag), + } + ethUpdatedTrustLine := xdr.TrustLineEntry{ + AccountId: xdr.MustAddress("GAOQJGUAB7NI7K7I62ORBXMN3J4SSWQUQ7FOEPSDJ322W2HMCNWPHXFB"), + Asset: xdr.MustNewCreditAsset("ETH", trustLineIssuer.Address()), + Balance: 10, + Flags: xdr.Uint32(xdr.TrustLineFlagsAuthorizedToMaintainLiabilitiesFlag), + } + err := s.processor.ProcessChange(ingest.Change{ Type: xdr.LedgerEntryTypeTrustline, Pre: &xdr.LedgerEntry{ LastModifiedLedgerSeq: lastModifiedLedgerSeq - 1, Data: xdr.LedgerEntryData{ Type: xdr.LedgerEntryTypeTrustline, - TrustLine: &trustLine, + TrustLine: &eurTrustLine, }, }, Post: &xdr.LedgerEntry{ LastModifiedLedgerSeq: lastModifiedLedgerSeq, Data: xdr.LedgerEntryData{ Type: xdr.LedgerEntryTypeTrustline, - TrustLine: &updatedTrustLine, + TrustLine: &eurUpdatedTrustLine, }, }, }) @@ -339,14 +415,33 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestUpdateTrustLineAuthorization() LastModifiedLedgerSeq: lastModifiedLedgerSeq - 1, Data: xdr.LedgerEntryData{ Type: xdr.LedgerEntryTypeTrustline, - TrustLine: &otherTrustLine, + TrustLine: &usdTrustLine, }, }, Post: &xdr.LedgerEntry{ LastModifiedLedgerSeq: lastModifiedLedgerSeq, Data: xdr.LedgerEntryData{ Type: xdr.LedgerEntryTypeTrustline, - TrustLine: &otherUpdatedTrustLine, + TrustLine: &usdUpdatedTrustLine, + }, + }, + }) + s.Assert().NoError(err) + + err = s.processor.ProcessChange(ingest.Change{ + Type: xdr.LedgerEntryTypeTrustline, + Pre: &xdr.LedgerEntry{ + LastModifiedLedgerSeq: lastModifiedLedgerSeq - 1, + Data: xdr.LedgerEntryData{ + Type: xdr.LedgerEntryTypeTrustline, + TrustLine: ðTrustLine, + }, + }, + Post: &xdr.LedgerEntry{ + LastModifiedLedgerSeq: lastModifiedLedgerSeq, + Data: xdr.LedgerEntryData{ + Type: xdr.LedgerEntryTypeTrustline, + TrustLine: ðUpdatedTrustLine, }, }, }) @@ -356,11 +451,33 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestUpdateTrustLineAuthorization() xdr.AssetTypeAssetTypeCreditAlphanum4, "EUR", trustLineIssuer.Address(), - ).Return(history.ExpAssetStat{}, sql.ErrNoRows).Once() - s.mockQ.On("InsertAssetStat", history.ExpAssetStat{ + ).Return(history.ExpAssetStat{ AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, AssetIssuer: trustLineIssuer.Address(), AssetCode: "EUR", + Accounts: history.ExpAssetStatAccounts{ + Unauthorized: 1, + }, + Balances: history.ExpAssetStatBalances{ + Authorized: "0", + AuthorizedToMaintainLiabilities: "0", + Unauthorized: "100", + }, + Amount: "0", + NumAccounts: 0, + }, nil).Once() + s.mockQ.On("UpdateAssetStat", history.ExpAssetStat{ + AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, + AssetIssuer: trustLineIssuer.Address(), + AssetCode: "EUR", + Accounts: history.ExpAssetStatAccounts{ + Authorized: 1, + }, + Balances: history.ExpAssetStatBalances{ + Authorized: "10", + AuthorizedToMaintainLiabilities: "0", + Unauthorized: "0", + }, Amount: "10", NumAccounts: 1, }).Return(int64(1), nil).Once() @@ -373,18 +490,78 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestUpdateTrustLineAuthorization() AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, AssetIssuer: trustLineIssuer.Address(), AssetCode: "USD", + Accounts: history.ExpAssetStatAccounts{ + Authorized: 1, + }, + Balances: history.ExpAssetStatBalances{ + Authorized: "100", + AuthorizedToMaintainLiabilities: "0", + Unauthorized: "0", + }, Amount: "100", NumAccounts: 1, }, nil).Once() - s.mockQ.On("RemoveAssetStat", + s.mockQ.On("UpdateAssetStat", history.ExpAssetStat{ + AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, + AssetIssuer: trustLineIssuer.Address(), + AssetCode: "USD", + Accounts: history.ExpAssetStatAccounts{ + Unauthorized: 1, + }, + Balances: history.ExpAssetStatBalances{ + Authorized: "0", + AuthorizedToMaintainLiabilities: "0", + Unauthorized: "10", + }, + Amount: "0", + NumAccounts: 0, + }).Return(int64(1), nil).Once() + + s.mockQ.On("GetAssetStat", xdr.AssetTypeAssetTypeCreditAlphanum4, - "USD", + "ETH", trustLineIssuer.Address(), - ).Return(int64(1), nil).Once() + ).Return(history.ExpAssetStat{ + AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, + AssetIssuer: trustLineIssuer.Address(), + AssetCode: "ETH", + Accounts: history.ExpAssetStatAccounts{ + Authorized: 1, + }, + Balances: history.ExpAssetStatBalances{ + Authorized: "100", + AuthorizedToMaintainLiabilities: "0", + Unauthorized: "0", + }, + Amount: "100", + NumAccounts: 1, + }, nil).Once() + s.mockQ.On("UpdateAssetStat", history.ExpAssetStat{ + AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, + AssetIssuer: trustLineIssuer.Address(), + AssetCode: "ETH", + Accounts: history.ExpAssetStatAccounts{ + AuthorizedToMaintainLiabilities: 1, + }, + Balances: history.ExpAssetStatBalances{ + Authorized: "0", + AuthorizedToMaintainLiabilities: "10", + Unauthorized: "0", + }, + Amount: "0", + NumAccounts: 0, + }).Return(int64(1), nil).Once() + s.Assert().NoError(s.processor.Commit()) } func (s *AssetStatsProcessorTestSuiteLedger) TestRemoveTrustLine() { + authorizedTrustLine := xdr.TrustLineEntry{ + AccountId: xdr.MustAddress("GAOQJGUAB7NI7K7I62ORBXMN3J4SSWQUQ7FOEPSDJ322W2HMCNWPHXFB"), + Asset: xdr.MustNewCreditAsset("EUR", trustLineIssuer.Address()), + Balance: 0, + Flags: xdr.Uint32(xdr.TrustLineFlagsAuthorizedFlag), + } unauthorizedTrustLine := xdr.TrustLineEntry{ AccountId: xdr.MustAddress("GAOQJGUAB7NI7K7I62ORBXMN3J4SSWQUQ7FOEPSDJ322W2HMCNWPHXFB"), Asset: xdr.MustNewCreditAsset("USD", trustLineIssuer.Address()), @@ -395,13 +572,8 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestRemoveTrustLine() { Type: xdr.LedgerEntryTypeTrustline, Pre: &xdr.LedgerEntry{ Data: xdr.LedgerEntryData{ - Type: xdr.LedgerEntryTypeTrustline, - TrustLine: &xdr.TrustLineEntry{ - AccountId: xdr.MustAddress("GAOQJGUAB7NI7K7I62ORBXMN3J4SSWQUQ7FOEPSDJ322W2HMCNWPHXFB"), - Asset: xdr.MustNewCreditAsset("EUR", trustLineIssuer.Address()), - Balance: 0, - Flags: xdr.Uint32(xdr.TrustLineFlagsAuthorizedFlag), - }, + Type: xdr.LedgerEntryTypeTrustline, + TrustLine: &authorizedTrustLine, }, }, Post: nil, @@ -428,6 +600,14 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestRemoveTrustLine() { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, AssetIssuer: trustLineIssuer.Address(), AssetCode: "EUR", + Accounts: history.ExpAssetStatAccounts{ + Authorized: 1, + }, + Balances: history.ExpAssetStatBalances{ + Authorized: "0", + AuthorizedToMaintainLiabilities: "0", + Unauthorized: "0", + }, Amount: "0", NumAccounts: 1, }, nil).Once() @@ -436,6 +616,32 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestRemoveTrustLine() { "EUR", trustLineIssuer.Address(), ).Return(int64(1), nil).Once() + + s.mockQ.On("GetAssetStat", + xdr.AssetTypeAssetTypeCreditAlphanum4, + "USD", + trustLineIssuer.Address(), + ).Return(history.ExpAssetStat{ + AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, + AssetIssuer: trustLineIssuer.Address(), + AssetCode: "USD", + Accounts: history.ExpAssetStatAccounts{ + Unauthorized: 1, + }, + Balances: history.ExpAssetStatBalances{ + Authorized: "0", + AuthorizedToMaintainLiabilities: "0", + Unauthorized: "0", + }, + Amount: "0", + NumAccounts: 0, + }, nil).Once() + s.mockQ.On("RemoveAssetStat", + xdr.AssetTypeAssetTypeCreditAlphanum4, + "USD", + trustLineIssuer.Address(), + ).Return(int64(1), nil).Once() + s.Assert().NoError(s.processor.Commit()) } @@ -496,6 +702,14 @@ func (s *AssetStatsProcessorTestSuiteLedger) TestProcessUpgradeChange() { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, AssetIssuer: trustLineIssuer.Address(), AssetCode: "EUR", + Accounts: history.ExpAssetStatAccounts{ + Authorized: 1, + }, + Balances: history.ExpAssetStatBalances{ + Authorized: "10", + AuthorizedToMaintainLiabilities: "0", + Unauthorized: "0", + }, Amount: "10", NumAccounts: 1, }).Return(int64(1), nil).Once() diff --git a/services/horizon/internal/ingest/processors/asset_stats_set.go b/services/horizon/internal/ingest/processors/asset_stats_set.go index 97026ca1c9..d8c7858d9c 100644 --- a/services/horizon/internal/ingest/processors/asset_stats_set.go +++ b/services/horizon/internal/ingest/processors/asset_stats_set.go @@ -13,27 +13,104 @@ type assetStatKey struct { assetCode string assetIssuer string } + type assetStatValue struct { - amount *big.Int - numAccounts int32 + assetStatKey + balances assetStatBalances + accounts history.ExpAssetStatAccounts +} + +type assetStatBalances struct { + Authorized *big.Int + AuthorizedToMaintainLiabilities *big.Int + Unauthorized *big.Int +} + +func (a *assetStatBalances) Parse(b *history.ExpAssetStatBalances) error { + authorized, ok := new(big.Int).SetString(b.Authorized, 10) + if !ok { + return errors.New("Error parsing: " + b.Authorized) + } + a.Authorized = authorized + + authorizedToMaintainLiabilities, ok := new(big.Int).SetString(b.AuthorizedToMaintainLiabilities, 10) + if !ok { + return errors.New("Error parsing: " + b.AuthorizedToMaintainLiabilities) + } + a.AuthorizedToMaintainLiabilities = authorizedToMaintainLiabilities + + unauthorized, ok := new(big.Int).SetString(b.Unauthorized, 10) + if !ok { + return errors.New("Error parsing: " + b.Unauthorized) + } + a.Unauthorized = unauthorized + + return nil +} + +func (a assetStatBalances) Add(b assetStatBalances) assetStatBalances { + return assetStatBalances{ + Authorized: big.NewInt(0).Add(a.Authorized, b.Authorized), + AuthorizedToMaintainLiabilities: big.NewInt(0).Add(a.AuthorizedToMaintainLiabilities, b.AuthorizedToMaintainLiabilities), + Unauthorized: big.NewInt(0).Add(a.Unauthorized, b.Unauthorized), + } +} + +func (a assetStatBalances) IsZero() bool { + return a.Authorized.Cmp(big.NewInt(0)) == 0 && a.AuthorizedToMaintainLiabilities.Cmp(big.NewInt(0)) == 0 && a.Unauthorized.Cmp(big.NewInt(0)) == 0 +} + +func (a assetStatBalances) ConvertToHistoryObject() history.ExpAssetStatBalances { + return history.ExpAssetStatBalances{ + Authorized: a.Authorized.String(), + AuthorizedToMaintainLiabilities: a.AuthorizedToMaintainLiabilities.String(), + Unauthorized: a.Unauthorized.String(), + } +} + +func (value assetStatValue) ConvertToHistoryObject() history.ExpAssetStat { + balances := value.balances.ConvertToHistoryObject() + return history.ExpAssetStat{ + AssetType: value.assetType, + AssetCode: value.assetCode, + AssetIssuer: value.assetIssuer, + Accounts: value.accounts, + Balances: balances, + Amount: balances.Authorized, + NumAccounts: value.accounts.Authorized, + } } // AssetStatSet represents a collection of asset stats type AssetStatSet map[assetStatKey]*assetStatValue -// Add updates the set with a trustline entry from a history archive snapshot -// if the trustline is authorized. +// Add updates the set with a trustline entry from a history archive snapshot. func (s AssetStatSet) Add(trustLine xdr.TrustLineEntry) error { - if !xdr.TrustLineFlags(trustLine.Flags).IsAuthorized() { - return nil - } - - return s.AddDelta(trustLine.Asset, int64(trustLine.Balance), 1) + flags := trustLine.Flags + return s.AddDelta( + trustLine.Asset, + map[xdr.Uint32]int64{flags: int64(trustLine.Balance)}, + map[xdr.Uint32]int32{flags: 1}, + ) } -// AddDelta adds a delta balance and delta accounts to a given asset. -func (s AssetStatSet) AddDelta(asset xdr.Asset, deltaBalance int64, deltaAccounts int32) error { - if deltaBalance == 0 && deltaAccounts == 0 { +// AddDelta adds a delta balance and delta accounts to a given asset trustline. +func (s AssetStatSet) AddDelta(asset xdr.Asset, deltaBalances map[xdr.Uint32]int64, deltaAccounts map[xdr.Uint32]int32) error { + accountsEmpty := true + for _, v := range deltaAccounts { + if v != 0 { + accountsEmpty = false + break + } + } + balancesEmpty := true + for _, v := range deltaBalances { + if v != 0 { + balancesEmpty = false + break + } + } + if accountsEmpty && balancesEmpty { return nil } @@ -44,22 +121,50 @@ func (s AssetStatSet) AddDelta(asset xdr.Asset, deltaBalance int64, deltaAccount current, ok := s[key] if !ok { - s[key] = &assetStatValue{ - amount: big.NewInt(int64(deltaBalance)), - numAccounts: deltaAccounts, + current = &assetStatValue{assetStatKey: key, balances: assetStatBalances{ + Authorized: big.NewInt(0), + AuthorizedToMaintainLiabilities: big.NewInt(0), + Unauthorized: big.NewInt(0), + }} + s[key] = current + } + + for k, v := range deltaAccounts { + flags := xdr.TrustLineFlags(k) + if flags.IsAuthorized() { + current.accounts.Authorized += v + } else if flags.IsAuthorizedToMaintainLiabilitiesFlag() { + current.accounts.AuthorizedToMaintainLiabilities += v + } else { + current.accounts.Unauthorized += v } - } else { - current.amount.Add(current.amount, big.NewInt(int64(deltaBalance))) - current.numAccounts += deltaAccounts - // Note: it's possible that after operations above: - // numAccounts != 0 && amount == 0 (ex. two accounts send some of their assets to third account) - // OR - // numAccounts == 0 && amount != 0 (ex. issuer issued an asset) - if current.numAccounts == 0 && current.amount.Cmp(big.NewInt(0)) == 0 { - delete(s, key) + } + + for k, v := range deltaBalances { + flags := xdr.TrustLineFlags(k) + bigV := big.NewInt(v) + if flags.IsAuthorized() { + current.balances.Authorized.Add(current.balances.Authorized, bigV) + } else if flags.IsAuthorizedToMaintainLiabilitiesFlag() { + current.balances.AuthorizedToMaintainLiabilities.Add(current.balances.AuthorizedToMaintainLiabilities, bigV) + } else { + current.balances.Unauthorized.Add(current.balances.Unauthorized, bigV) } } + // Note: it's possible that after operations above: + // numAccounts != 0 && amount == 0 (ex. two accounts send some of their assets to third account) + // OR + // numAccounts == 0 && amount != 0 (ex. issuer issued an asset) + if current.balances.Authorized.Cmp(big.NewInt(0)) == 0 && + current.balances.AuthorizedToMaintainLiabilities.Cmp(big.NewInt(0)) == 0 && + current.balances.Unauthorized.Cmp(big.NewInt(0)) == 0 && + current.accounts.Authorized == 0 && + current.accounts.AuthorizedToMaintainLiabilities == 0 && + current.accounts.Unauthorized == 0 { + delete(s, key) + } + return nil } @@ -73,26 +178,14 @@ func (s AssetStatSet) Remove(assetType xdr.AssetType, assetCode string, assetIss delete(s, key) - return history.ExpAssetStat{ - AssetType: key.assetType, - AssetCode: key.assetCode, - AssetIssuer: key.assetIssuer, - Amount: value.amount.String(), - NumAccounts: value.numAccounts, - }, true + return value.ConvertToHistoryObject(), true } // All returns a list of all `history.ExpAssetStat` contained within the set func (s AssetStatSet) All() []history.ExpAssetStat { assetStats := make([]history.ExpAssetStat, 0, len(s)) - for key, value := range s { - assetStats = append(assetStats, history.ExpAssetStat{ - AssetType: key.assetType, - AssetCode: key.assetCode, - AssetIssuer: key.assetIssuer, - Amount: value.amount.String(), - NumAccounts: value.numAccounts, - }) + for _, value := range s { + assetStats = append(assetStats, value.ConvertToHistoryObject()) } return assetStats } diff --git a/services/horizon/internal/ingest/processors/asset_stats_set_test.go b/services/horizon/internal/ingest/processors/asset_stats_set_test.go index ef842ec898..211b6b9152 100644 --- a/services/horizon/internal/ingest/processors/asset_stats_set_test.go +++ b/services/horizon/internal/ingest/processors/asset_stats_set_test.go @@ -40,21 +40,6 @@ func assertAllEquals(t *testing.T, set AssetStatSet, expected []history.ExpAsset } } -func TestAssetStatSetIgnoresUnauthorizedTrustlines(t *testing.T) { - set := AssetStatSet{} - err := set.Add(xdr.TrustLineEntry{ - AccountId: xdr.MustAddress("GAOQJGUAB7NI7K7I62ORBXMN3J4SSWQUQ7FOEPSDJ322W2HMCNWPHXFB"), - Asset: xdr.MustNewCreditAsset("EUR", trustLineIssuer.Address()), - Balance: 1, - }) - if err != nil { - t.Fatalf("unexpected error %v", err) - } - if all := set.All(); len(all) != 0 { - t.Fatalf("expected empty list but got %v", all) - } -} - func TestAddAndRemoveAssetStats(t *testing.T) { set := AssetStatSet{} eur := "EUR" @@ -62,6 +47,14 @@ func TestAddAndRemoveAssetStats(t *testing.T) { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, AssetCode: eur, AssetIssuer: trustLineIssuer.Address(), + Accounts: history.ExpAssetStatAccounts{ + Authorized: 1, + }, + Balances: history.ExpAssetStatBalances{ + Authorized: "1", + AuthorizedToMaintainLiabilities: "0", + Unauthorized: "0", + }, Amount: "1", NumAccounts: 1, } @@ -87,7 +80,9 @@ func TestAddAndRemoveAssetStats(t *testing.T) { t.Fatalf("unexpected error %v", err) } + eurAssetStat.Balances.Authorized = "25" eurAssetStat.Amount = "25" + eurAssetStat.Accounts.Authorized++ eurAssetStat.NumAccounts++ assertAllEquals(t, set, []history.ExpAssetStat{eurAssetStat}) @@ -113,19 +108,58 @@ func TestAddAndRemoveAssetStats(t *testing.T) { t.Fatalf("unexpected error %v", err) } + // Add an authorized_to_maintain_liabilities trust line + err = set.Add(xdr.TrustLineEntry{ + AccountId: xdr.MustAddress("GAOQJGUAB7NI7K7I62ORBXMN3J4SSWQUQ7FOEPSDJ322W2HMCNWPHXFB"), + Asset: xdr.MustNewCreditAsset(ether, trustLineIssuer.Address()), + Balance: 4, + Flags: xdr.Uint32(xdr.TrustLineFlagsAuthorizedToMaintainLiabilitiesFlag), + }) + if err != nil { + t.Fatalf("unexpected error %v", err) + } + + // Add an unauthorized trust line + err = set.Add(xdr.TrustLineEntry{ + AccountId: xdr.MustAddress("GAOQJGUAB7NI7K7I62ORBXMN3J4SSWQUQ7FOEPSDJ322W2HMCNWPHXFB"), + Asset: xdr.MustNewCreditAsset(ether, trustLineIssuer.Address()), + Balance: 5, + }) + if err != nil { + t.Fatalf("unexpected error %v", err) + } + expected := []history.ExpAssetStat{ - history.ExpAssetStat{ + { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum12, AssetCode: ether, AssetIssuer: trustLineIssuer.Address(), + Accounts: history.ExpAssetStatAccounts{ + Authorized: 1, + AuthorizedToMaintainLiabilities: 1, + Unauthorized: 1, + }, + Balances: history.ExpAssetStatBalances{ + Authorized: "3", + AuthorizedToMaintainLiabilities: "4", + Unauthorized: "5", + }, Amount: "3", NumAccounts: 1, }, eurAssetStat, - history.ExpAssetStat{ + { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, AssetCode: usd, AssetIssuer: trustLineIssuer.Address(), + Accounts: history.ExpAssetStatAccounts{ + Authorized: 1, + }, + Balances: history.ExpAssetStatBalances{ + Authorized: "10", + AuthorizedToMaintainLiabilities: "0", + Unauthorized: "0", + }, Amount: "10", NumAccounts: 1, }, @@ -166,6 +200,14 @@ func TestOverflowAssetStatSet(t *testing.T) { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, AssetCode: eur, AssetIssuer: trustLineIssuer.Address(), + Accounts: history.ExpAssetStatAccounts{ + Authorized: 1, + }, + Balances: history.ExpAssetStatBalances{ + Authorized: "9223372036854775807", + AuthorizedToMaintainLiabilities: "0", + Unauthorized: "0", + }, Amount: "9223372036854775807", NumAccounts: 1, } @@ -191,6 +233,14 @@ func TestOverflowAssetStatSet(t *testing.T) { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, AssetCode: eur, AssetIssuer: trustLineIssuer.Address(), + Accounts: history.ExpAssetStatAccounts{ + Authorized: 2, + }, + Balances: history.ExpAssetStatBalances{ + Authorized: "18446744073709551614", + AuthorizedToMaintainLiabilities: "0", + Unauthorized: "0", + }, Amount: "18446744073709551614", NumAccounts: 2, } diff --git a/services/horizon/internal/ingest/verify.go b/services/horizon/internal/ingest/verify.go index 6f9c1f47c6..d6e1d2d0cd 100644 --- a/services/horizon/internal/ingest/verify.go +++ b/services/horizon/internal/ingest/verify.go @@ -27,7 +27,7 @@ const assetStatsBatchSize = 500 // check them. // There is a test that checks it, to fix it: update the actual `verifyState` // method instead of just updating this value! -const stateVerifierExpectedIngestionVersion = 12 +const stateVerifierExpectedIngestionVersion = 13 // verifyState is called as a go routine from pipeline post hook every 64 // ledgers. It checks if the state is correct. If another go routine is already diff --git a/services/horizon/internal/ingest/verify_range_state_test.go b/services/horizon/internal/ingest/verify_range_state_test.go index 2c0240b9b9..b522267f55 100644 --- a/services/horizon/internal/ingest/verify_range_state_test.go +++ b/services/horizon/internal/ingest/verify_range_state_test.go @@ -5,6 +5,7 @@ package ingest import ( "context" "database/sql" + "fmt" "io" "testing" @@ -15,6 +16,7 @@ import ( "github.com/stellar/go/historyarchive" "github.com/stellar/go/ingest" "github.com/stellar/go/ingest/ledgerbackend" + "github.com/stellar/go/keypair" "github.com/stellar/go/services/horizon/internal/db2" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/services/horizon/internal/ingest/processors" @@ -433,3 +435,47 @@ func (s *VerifyRangeStateTestSuite) TestSuccessWithVerify() { ) clonedQ.AssertExpectations(s.T()) } + +func (s *VerifyRangeStateTestSuite) TestVerifyFailsWhenAssetStatsMismatch() { + set := processors.AssetStatSet{} + + trustLineIssuer := xdr.MustAddress("GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H") + set.Add(xdr.TrustLineEntry{ + AccountId: xdr.MustAddress(keypair.MustRandom().Address()), + Balance: 123, + Asset: xdr.MustNewCreditAsset("EUR", trustLineIssuer.Address()), + Flags: xdr.Uint32(xdr.TrustLineFlagsAuthorizedToMaintainLiabilitiesFlag), + }) + + stat := history.ExpAssetStat{ + AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, + AssetCode: "EUR", + AssetIssuer: trustLineIssuer.Address(), + Accounts: history.ExpAssetStatAccounts{ + Unauthorized: 1, + }, + Balances: history.ExpAssetStatBalances{ + Authorized: "0", + AuthorizedToMaintainLiabilities: "0", + Unauthorized: "123", + }, + Amount: "0", + NumAccounts: 0, + } + + s.historyQ.MockQAssetStats.On("GetAssetStats", "", "", db2.PageQuery{ + Order: "asc", + Limit: assetStatsBatchSize, + }).Return([]history.ExpAssetStat{stat}, nil).Once() + s.historyQ.MockQAssetStats.On("GetAssetStats", "", "", db2.PageQuery{ + Cursor: stat.PagingToken(), + Order: "asc", + Limit: assetStatsBatchSize, + }).Return([]history.ExpAssetStat{}, nil).Once() + + err := checkAssetStats(set, s.historyQ) + s.Assert().EqualError(err, fmt.Sprintf("db asset stat with code EUR issuer %s does not match asset stat from HAS", trustLineIssuer.Address())) + + // Satisfy the mock + s.historyQ.Rollback() +} diff --git a/services/horizon/internal/resourceadapter/asset_stat.go b/services/horizon/internal/resourceadapter/asset_stat.go index ffc9bf2e84..7b0735a71f 100644 --- a/services/horizon/internal/resourceadapter/asset_stat.go +++ b/services/horizon/internal/resourceadapter/asset_stat.go @@ -23,10 +23,15 @@ func PopulateAssetStat( res.Asset.Type = xdr.AssetTypeToString[row.AssetType] res.Asset.Code = row.AssetCode res.Asset.Issuer = row.AssetIssuer + res.Accounts = protocol.AssetStatAccounts(row.Accounts) res.Amount, err = amount.IntStringToAmount(row.Amount) if err != nil { return errors.Wrap(err, "Invalid amount in PopulateAssetStat") } + err = populateAssetStatBalances(&res.Balances, row.Balances) + if err != nil { + return err + } res.NumAccounts = row.NumAccounts flags := int8(issuer.Flags) res.Flags = protocol.AccountFlags{ @@ -45,3 +50,22 @@ func PopulateAssetStat( res.Links.Toml = hal.NewLink(toml) return } + +func populateAssetStatBalances(res *protocol.AssetStatBalances, row history.ExpAssetStatBalances) (err error) { + res.Authorized, err = amount.IntStringToAmount(row.Authorized) + if err != nil { + return errors.Wrapf(err, "Invalid amount in PopulateAssetStatBalances: %q", row.Authorized) + } + + res.AuthorizedToMaintainLiabilities, err = amount.IntStringToAmount(row.AuthorizedToMaintainLiabilities) + if err != nil { + return errors.Wrapf(err, "Invalid amount in PopulateAssetStatBalances: %q", row.AuthorizedToMaintainLiabilities) + } + + res.Unauthorized, err = amount.IntStringToAmount(row.Unauthorized) + if err != nil { + return errors.Wrapf(err, "Invalid amount in PopulateAssetStatBalances: %q", row.Unauthorized) + } + + return nil +} diff --git a/services/horizon/internal/resourceadapter/asset_stat_test.go b/services/horizon/internal/resourceadapter/asset_stat_test.go index 59ba0f852e..2acd29c157 100644 --- a/services/horizon/internal/resourceadapter/asset_stat_test.go +++ b/services/horizon/internal/resourceadapter/asset_stat_test.go @@ -16,6 +16,16 @@ func TestPopulateExpAssetStat(t *testing.T) { AssetType: xdr.AssetTypeAssetTypeCreditAlphanum4, AssetCode: "XIM", AssetIssuer: "GBZ35ZJRIKJGYH5PBKLKOZ5L6EXCNTO7BKIL7DAVVDFQ2ODJEEHHJXIM", + Accounts: history.ExpAssetStatAccounts{ + Authorized: 429, + AuthorizedToMaintainLiabilities: 214, + Unauthorized: 107, + }, + Balances: history.ExpAssetStatBalances{ + Authorized: "100000000000000000000", + AuthorizedToMaintainLiabilities: "50000000000000000000", + Unauthorized: "2500000000000000000", + }, Amount: "100000000000000000000", // 10T NumAccounts: 429, } @@ -32,6 +42,12 @@ func TestPopulateExpAssetStat(t *testing.T) { assert.Equal(t, "credit_alphanum4", res.Type) assert.Equal(t, "XIM", res.Code) assert.Equal(t, "GBZ35ZJRIKJGYH5PBKLKOZ5L6EXCNTO7BKIL7DAVVDFQ2ODJEEHHJXIM", res.Issuer) + assert.Equal(t, int32(429), res.Accounts.Authorized) + assert.Equal(t, int32(214), res.Accounts.AuthorizedToMaintainLiabilities) + assert.Equal(t, int32(107), res.Accounts.Unauthorized) + assert.Equal(t, "10000000000000.0000000", res.Balances.Authorized) + assert.Equal(t, "5000000000000.0000000", res.Balances.AuthorizedToMaintainLiabilities) + assert.Equal(t, "250000000000.0000000", res.Balances.Unauthorized) assert.Equal(t, "10000000000000.0000000", res.Amount) assert.Equal(t, int32(429), res.NumAccounts) assert.Equal(t, horizon.AccountFlags{}, res.Flags)