From 76f6f32c583e960381fea76c778030839c065ef7 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 11 Jun 2024 14:48:47 -0400 Subject: [PATCH 01/29] feat: indexer base types --- indexer/base/README.md | 5 + indexer/base/column.go | 149 +++++++++++++++++ indexer/base/entity.go | 40 +++++ indexer/base/go.mod | 6 + indexer/base/go.sum | 0 indexer/base/kind.go | 304 ++++++++++++++++++++++++++++++++++ indexer/base/listener.go | 114 +++++++++++++ indexer/base/module_schema.go | 8 + indexer/base/table.go | 20 +++ 9 files changed, 646 insertions(+) create mode 100644 indexer/base/README.md create mode 100644 indexer/base/column.go create mode 100644 indexer/base/entity.go create mode 100644 indexer/base/go.mod create mode 100644 indexer/base/go.sum create mode 100644 indexer/base/kind.go create mode 100644 indexer/base/listener.go create mode 100644 indexer/base/module_schema.go create mode 100644 indexer/base/table.go diff --git a/indexer/base/README.md b/indexer/base/README.md new file mode 100644 index 000000000000..0b96a27dc63c --- /dev/null +++ b/indexer/base/README.md @@ -0,0 +1,5 @@ +# Indexer Base + +The indexer base module is designed to provide a stable, zero-dependency base layer for the built-in indexer functionality. Packages that integrate with the indexer should feel free to depend on this package without fear of any external dependencies being pulled in. + +The basic types for specifying index sources, targets and decoders are provided here along with a basic engine that ties these together. A package wishing to be an indexing source could accept an instance of `Engine` directly to be compatible with indexing. A package wishing to be a decoder can use the `Entity` and `Table` types. A package defining an indexing target should implement the `Indexer` interface. \ No newline at end of file diff --git a/indexer/base/column.go b/indexer/base/column.go new file mode 100644 index 000000000000..14fa4e55b322 --- /dev/null +++ b/indexer/base/column.go @@ -0,0 +1,149 @@ +package indexerbase + +import "fmt" + +// Column represents a column in a table schema. +type Column struct { + // Name is the name of the column. + Name string + + // Kind is the basic type of the column. + Kind Kind + + // Nullable indicates whether null values are accepted for the column. + Nullable bool + + // AddressPrefix is the address prefix of the column's kind, currently only used for Bech32AddressKind. + AddressPrefix string + + // EnumDefinition is the definition of the enum type and is only valid when Kind is EnumKind. + EnumDefinition EnumDefinition +} + +// EnumDefinition represents the definition of an enum type. +type EnumDefinition struct { + // Name is the name of the enum type. + Name string + + // Values is a list of distinct values that are part of the enum type. + Values []string +} + +// Validate validates the column. +func (c Column) Validate() error { + // non-empty name + if c.Name == "" { + return fmt.Errorf("column name cannot be empty") + } + + // valid kind + if err := c.Kind.Validate(); err != nil { + return fmt.Errorf("invalid column type for %q: %w", c.Name, err) + } + + // address prefix only valid with Bech32AddressKind + if c.Kind == Bech32AddressKind && c.AddressPrefix == "" { + return fmt.Errorf("missing address prefix for column %q", c.Name) + } else if c.Kind != Bech32AddressKind && c.AddressPrefix != "" { + return fmt.Errorf("address prefix is only valid for column %q with type Bech32AddressKind", c.Name) + } + + // enum definition only valid with EnumKind + if c.Kind == EnumKind { + if err := c.EnumDefinition.Validate(); err != nil { + return fmt.Errorf("invalid enum definition for column %q: %w", c.Name, err) + } + } else if c.Kind != EnumKind && c.EnumDefinition.Name != "" && c.EnumDefinition.Values != nil { + return fmt.Errorf("enum definition is only valid for column %q with type EnumKind", c.Name) + } + + return nil +} + +// Validate validates the enum definition. +func (e EnumDefinition) Validate() error { + if e.Name == "" { + return fmt.Errorf("enum definition name cannot be empty") + } + if len(e.Values) == 0 { + return fmt.Errorf("enum definition values cannot be empty") + } + seen := make(map[string]bool, len(e.Values)) + for i, v := range e.Values { + if v == "" { + return fmt.Errorf("enum definition value at index %d cannot be empty for enum %s", i, e.Name) + } + if seen[v] { + return fmt.Errorf("duplicate enum definition value %q for enum %s", v, e.Name) + } + seen[v] = true + } + return nil +} + +// ValidateValue validates that the value conforms to the column's kind and nullability. +func (c Column) ValidateValue(value any) error { + if value == nil { + if !c.Nullable { + return fmt.Errorf("column %q cannot be null", c.Name) + } + return nil + } + return c.Kind.ValidateValue(value) +} + +// ValidateKey validates that the value conforms to the set of columns as a Key in an EntityUpdate. +// See EntityUpdate.Key for documentation on the requirements of such values. +func ValidateKey(cols []Column, value any) error { + if len(cols) == 0 { + return nil + } + + if len(cols) == 1 { + return cols[0].ValidateValue(value) + } + + values, ok := value.([]any) + if !ok { + return fmt.Errorf("expected slice of values for key columns, got %T", value) + } + + if len(cols) != len(values) { + return fmt.Errorf("expected %d key columns, got %d values", len(cols), len(value.([]any))) + } + for i, col := range cols { + if err := col.ValidateValue(values[i]); err != nil { + return fmt.Errorf("invalid value for key column %q: %w", col.Name, err) + } + } + return nil +} + +// ValidateValue validates that the value conforms to the set of columns as a Value in an EntityUpdate. +// See EntityUpdate.Value for documentation on the requirements of such values. +func ValidateValue(cols []Column, value any) error { + valueUpdates, ok := value.(ValueUpdates) + if ok { + colMap := map[string]Column{} + for _, col := range cols { + colMap[col.Name] = col + } + var errs []error + valueUpdates.Iterate(func(colName string, value any) bool { + col, ok := colMap[colName] + if !ok { + errs = append(errs, fmt.Errorf("unknown column %q in value updates", colName)) + } + if err := col.ValidateValue(value); err != nil { + errs = append(errs, fmt.Errorf("invalid value for column %q: %w", colName, err)) + } + return true + }) + if len(errs) > 0 { + return fmt.Errorf("validation errors: %v", errs) + } + return nil + } else { + return ValidateKey(cols, value) + } +} diff --git a/indexer/base/entity.go b/indexer/base/entity.go new file mode 100644 index 000000000000..95f016037fd8 --- /dev/null +++ b/indexer/base/entity.go @@ -0,0 +1,40 @@ +package indexerbase + +// EntityUpdate represents an update operation on an entity in the schema. +type EntityUpdate struct { + // TableName is the name of the table that the entity belongs to in the schema. + TableName string + + // Key returns the value of the primary key of the entity and must conform to these constraints with respect + // that the schema that is defined for the entity: + // - if key represents a single column, then the value must be valid for the first column in that + // column list. For instance, if there is one column in the key of type String, then the value must be of + // type string + // - if key represents multiple columns, then the value must be a slice of values where each value is valid + // for the corresponding column in the column list. For instance, if there are two columns in the key of + // type String, String, then the value must be a slice of two strings. + // If the key has no columns, meaning that this is a singleton entity, then this value is ignored and can be nil. + Key any + + // Value returns the non-primary key columns of the entity and can either conform to the same constraints + // as EntityUpdate.Key or it may be and instance of ValueUpdates. ValueUpdates can be used as a performance + // optimization to avoid copying the values of the entity into the update and/or to omit unchanged columns. + // If this is a delete operation, then this value is ignored and can be nil. + Value any + + // Delete is a flag that indicates whether this update is a delete operation. If true, then the Value field + // is ignored and can be nil. + Delete bool +} + +// ValueUpdates is an interface that represents the value columns of an entity update. Columns that +// were not updated may be excluded from the update. Consumers should be aware that implementations +// may not filter out columns that were unchanged. However, if a column is omitted from the update +// it should be considered unchanged. +type ValueUpdates interface { + + // Iterate iterates over the columns and values in the entity update. The function should return + // true to continue iteration or false to stop iteration. Each column value should conform + // to the requirements of that column's type in the schema. + Iterate(func(col string, value any) bool) +} diff --git a/indexer/base/go.mod b/indexer/base/go.mod new file mode 100644 index 000000000000..c369648761e8 --- /dev/null +++ b/indexer/base/go.mod @@ -0,0 +1,6 @@ +module cosmossdk.io/indexer/base + +// NOTE: this go.mod should have zero dependencies and remain on an older version of Go +// to be compatible with legacy codebases. + +go 1.19 diff --git a/indexer/base/go.sum b/indexer/base/go.sum new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/indexer/base/kind.go b/indexer/base/kind.go new file mode 100644 index 000000000000..d1873944fb1b --- /dev/null +++ b/indexer/base/kind.go @@ -0,0 +1,304 @@ +package indexerbase + +import ( + "encoding/json" + "fmt" + "time" +) + +// Kind represents the basic type of a column in the table schema. +// Each kind defines the types of go values which should be accepted +// by listeners and generated by decoders when providing entity updates. +type Kind int + +const ( + // InvalidKind indicates that an invalid type. + InvalidKind Kind = iota + + // StringKind is a string type and values of this type must be of the go type string + // or implement fmt.Stringer(). + StringKind + + // BytesKind is a bytes type and values of this type must be of the go type []byte. + BytesKind + + // Int8Kind is an int8 type and values of this type must be of the go type int8. + Int8Kind + + // Uint8Kind is a uint8 type and values of this type must be of the go type uint8. + Uint8Kind + + // Int16Kind is an int16 type and values of this type must be of the go type int16. + Int16Kind + + // Uint16Kind is a uint16 type and values of this type must be of the go type uint16. + Uint16Kind + + // Int32Kind is an int32 type and values of this type must be of the go type int32. + Int32Kind + + // Uint32Kind is a uint32 type and values of this type must be of the go type uint32. + Uint32Kind + + // Int64Kind is an int64 type and values of this type must be of the go type int64. + Int64Kind + + // Uint64Kind is a uint64 type and values of this type must be of the go type uint64. + Uint64Kind + + // IntegerKind represents an arbitrary precision integer number. Values of this type must + // be of the go type string or a type that implements fmt.Stringer with the resulted string + // formatted as an integer number. + IntegerKind + + // DecimalKind represents an arbitrary precision decimal or integer number. Values of this type + // must be of the go type string or a type that implements fmt.Stringer with the resulting string + // formatted as decimal numbers with an optional fractional part. Exponential E-notation + // is supported but NaN and Infinity are not. + DecimalKind + + // BoolKind is a boolean type and values of this type must be of the go type bool. + BoolKind + + // TimeKind is a time type and values of this type must be of the go type time.Time. + TimeKind + + // DurationKind is a duration type and values of this type must be of the go type time.Duration. + DurationKind + + // Float32Kind is a float32 type and values of this type must be of the go type float32. + Float32Kind + + // Float64Kind is a float64 type and values of this type must be of the go type float64. + Float64Kind + + // Bech32AddressKind is a bech32 address type and values of this type must be of the go type string or []byte + // or a type which implements fmt.Stringer. Columns of this type are expected to set the AddressPrefix field + // in the column definition to the bech32 address prefix. + Bech32AddressKind + + // EnumKind is an enum type and values of this type must be of the go type string or implement fmt.Stringer. + // Columns of this type are expected to set the EnumDefinition field in the column definition to the enum + // definition. + EnumKind + + // JSONKind is a JSON type and values of this type can either be of go type json.RawMessage + // or any type that can be marshaled to JSON using json.Marshal. + JSONKind +) + +// Validate returns an error if the kind is invalid. +func (t Kind) Validate() error { + if t <= InvalidKind { + return fmt.Errorf("unknown type: %d", t) + } + if t > JSONKind { + return fmt.Errorf("invalid type: %d", t) + } + return nil +} + +// ValidateValue returns an error if the value does not the type go type specified by the kind. +// Some columns may accept nil values, however, this method does not have any notion of +// nullability. It only checks that the value is of the correct type. +func (t Kind) ValidateValue(value any) error { + switch t { + case StringKind: + _, ok := value.(string) + _, ok2 := value.(fmt.Stringer) + if !ok && !ok2 { + return fmt.Errorf("expected string or type that implements fmt.Stringer, got %T", value) + } + case BytesKind: + _, ok := value.([]byte) + if !ok { + return fmt.Errorf("expected []byte, got %T", value) + } + case Int8Kind: + _, ok := value.(int8) + if !ok { + return fmt.Errorf("expected int8, got %T", value) + } + case Uint8Kind: + _, ok := value.(uint8) + if !ok { + return fmt.Errorf("expected uint8, got %T", value) + } + case Int16Kind: + _, ok := value.(int16) + if !ok { + return fmt.Errorf("expected int16, got %T", value) + } + case Uint16Kind: + _, ok := value.(uint16) + if !ok { + return fmt.Errorf("expected uint16, got %T", value) + } + case Int32Kind: + _, ok := value.(int32) + if !ok { + return fmt.Errorf("expected int32, got %T", value) + } + case Uint32Kind: + _, ok := value.(uint32) + if !ok { + return fmt.Errorf("expected uint32, got %T", value) + } + case Int64Kind: + _, ok := value.(int64) + if !ok { + return fmt.Errorf("expected int64, got %T", value) + } + case Uint64Kind: + _, ok := value.(uint64) + if !ok { + return fmt.Errorf("expected uint64, got %T", value) + } + case IntegerKind: + _, ok := value.(string) + _, ok2 := value.(fmt.Stringer) + if !ok && !ok2 { + return fmt.Errorf("expected string or type that implements fmt.Stringer, got %T", value) + } + case DecimalKind: + _, ok := value.(string) + _, ok2 := value.(fmt.Stringer) + if !ok && !ok2 { + return fmt.Errorf("expected string or type that implements fmt.Stringer, got %T", value) + } + case BoolKind: + _, ok := value.(bool) + if !ok { + return fmt.Errorf("expected bool, got %T", value) + } + case TimeKind: + _, ok := value.(time.Time) + if !ok { + return fmt.Errorf("expected time.Time, got %T", value) + } + case DurationKind: + _, ok := value.(time.Duration) + if !ok { + return fmt.Errorf("expected time.Duration, got %T", value) + } + case Float32Kind: + _, ok := value.(float32) + if !ok { + return fmt.Errorf("expected float32, got %T", value) + } + case Float64Kind: + _, ok := value.(float64) + if !ok { + return fmt.Errorf("expected float64, got %T", value) + } + case Bech32AddressKind: + _, ok := value.(string) + _, ok2 := value.([]byte) + _, ok3 := value.(fmt.Stringer) + if !ok && !ok2 && !ok3 { + return fmt.Errorf("expected string or []byte, got %T", value) + } + case EnumKind: + _, ok := value.(string) + _, ok2 := value.(fmt.Stringer) + if !ok && !ok2 { + return fmt.Errorf("expected string or type that implements fmt.Stringer, got %T", value) + } + case JSONKind: + return nil + default: + return fmt.Errorf("invalid type: %d", t) + } + return nil +} + +// String returns a string representation of the kind. +func (t Kind) String() string { + switch t { + case StringKind: + return "string" + case BytesKind: + return "bytes" + case Int8Kind: + return "int8" + case Uint8Kind: + return "uint8" + case Int16Kind: + return "int16" + case Uint16Kind: + return "uint16" + case Int32Kind: + return "int32" + case Uint32Kind: + return "uint32" + case Int64Kind: + return "int64" + case Uint64Kind: + return "uint64" + case DecimalKind: + return "decimal" + case IntegerKind: + return "integer" + case BoolKind: + return "bool" + case TimeKind: + return "time" + case DurationKind: + return "duration" + case Float32Kind: + return "float32" + case Float64Kind: + return "float64" + case Bech32AddressKind: + return "bech32address" + case EnumKind: + return "enum" + case JSONKind: + return "json" + default: + return "" + } +} + +// KindForGoValue finds the simplest kind that can represent the given go value. It will not, however, +// return kinds such as IntegerKind, DecimalKind, Bech32AddressKind, or EnumKind which all can be +// represented as strings. It will return InvalidKind if the value is not a simple type. +func KindForGoValue(value any) Kind { + switch value.(type) { + case string, fmt.Stringer: + return StringKind + case []byte: + return BytesKind + case int8: + return Int8Kind + case uint8: + return Uint8Kind + case int16: + return Int16Kind + case uint16: + return Uint16Kind + case int32: + return Int32Kind + case uint32: + return Uint32Kind + case int64: + return Int64Kind + case uint64: + return Uint64Kind + case float32: + return Float32Kind + case float64: + return Float64Kind + case bool: + return BoolKind + case time.Time: + return TimeKind + case time.Duration: + return DurationKind + case json.RawMessage: + return JSONKind + default: + } + + return InvalidKind +} diff --git a/indexer/base/listener.go b/indexer/base/listener.go new file mode 100644 index 000000000000..cccd6f08e153 --- /dev/null +++ b/indexer/base/listener.go @@ -0,0 +1,114 @@ +package indexerbase + +import ( + "encoding/json" +) + +// Listener is an interface that defines methods for listening to both raw and logical blockchain data. +// It is valid for any of the methods to be nil, in which case the listener will not be called for that event. +// Listeners should understand the guarantees that are provided by the source they are listening to and +// understand which methods will or will not be called. For instance, most blockchains will not do logical +// decoding of data out of the box, so the EnsureLogicalSetup and OnEntityUpdate methods will not be called. +// These methods will only be called when listening logical decoding is setup. +type Listener struct { + // StartBlock is called at the beginning of processing a block. + StartBlock func(uint64) error + + // OnBlockHeader is called when a block header is received. + OnBlockHeader func(BlockHeaderData) error + + // OnTx is called when a transaction is received. + OnTx func(TxData) error + + // OnEvent is called when an event is received. + OnEvent func(EventData) error + + // OnKVPair is called when a key-value has been written to the store for a given module. + OnKVPair func(module string, key, value []byte, delete bool) error + + // Commit is called when state is commited, usually at the end of a block. Any + // indexers should commit their data when this is called and return an error if + // they are unable to commit. + Commit func() error + + // EnsureLogicalSetup should be called whenever the blockchain process starts OR whenever + // logical decoding of a module is initiated. An indexer listening to this event + // should ensure that they have performed whatever initialization steps (such as database + // migrations) required to receive OnEntityUpdate events for the given module. If the + // schema is incompatible with the existing schema, the listener should return an error. + // If the listener is persisting state for the module, it should return the last block + // that was saved for the module so that the framework can determine whether it is safe + // to resume indexing from the current height or whether there is a gap (usually an error). + // If the listener does not persist any state for the module, it should return 0 for lastBlock + // and nil for error. + // If the listener has initialized properly and would like to persist state for the module, + // but does not have any persisted state yet, it should return -1 for lastBlock and nil for error. + // In this case, the framework will perform a "catch-up sync" calling OnEntityUpdate for every + // entity already in the module followed by CommitCatchupSync before processing new block data. + EnsureLogicalSetup func(module string, schema ModuleSchema) (lastBlock int64, err error) + + // OnEntityUpdate is called whenever an entity is updated in the module. This is only called + // when logical data is available. It should be assumed that the same data in raw form + // is also passed to OnKVPair. + OnEntityUpdate func(module string, update EntityUpdate) error + + // CommitCatchupSync is called after all existing entities for a module have been passed to + // OnEntityUpdate during a catch-up sync which has been initiated by return -1 for lastBlock + // in EnsureLogicalSetup. The listener should commit all the data that has been received at + // this point and also save the block number as the last block that has been processed so + // that processing of regular block data can resume from this point in the future. + CommitCatchupSync func(module string, block uint64) error +} + +// BlockHeaderData represents the raw block header data that is passed to a listener. +type BlockHeaderData struct { + // Height is the height of the block. + Height uint64 + + // Bytes is the raw byte representation of the block header. + Bytes ToBytes + + // JSON is the JSON representation of the block header. It should generally be a JSON object. + JSON ToJSON +} + +// TxData represents the raw transaction data that is passed to a listener. +type TxData struct { + // TxIndex is the index of the transaction in the block. + TxIndex int32 + + // Bytes is the raw byte representation of the transaction. + Bytes ToBytes + + // JSON is the JSON representation of the transaction. It should generally be a JSON object. + JSON ToJSON +} + +// EventData represents event data that is passed to a listener. +type EventData struct { + // TxIndex is the index of the transaction in the block to which this event is associated. + // It should be set to a negative number if the event is not associated with a transaction. + // Canonically -1 should be used to represent begin block processing and -2 should be used to + // represent end block processing. + TxIndex int32 + + // MsgIndex is the index of the message in the transaction to which this event is associated. + // If TxIndex is negative, this index could correspond to the index of the message in + // begin or end block processing if such indexes exist, or it can be set to zero. + MsgIndex uint32 + + // EventIndex is the index of the event in the message to which this event is associated. + EventIndex uint32 + + // Type is the type of the event. + Type string + + // Data is the JSON representation of the event data. It should generally be a JSON object. + Data ToJSON +} + +// ToBytes is a function that lazily returns the raw byte representation of data. +type ToBytes = func() ([]byte, error) + +// ToJSON is a function that lazily returns the JSON representation of data. +type ToJSON = func() (json.RawMessage, error) diff --git a/indexer/base/module_schema.go b/indexer/base/module_schema.go new file mode 100644 index 000000000000..4e8b81c2be3c --- /dev/null +++ b/indexer/base/module_schema.go @@ -0,0 +1,8 @@ +package indexerbase + +// ModuleSchema represents the logical schema of a module for purposes of indexing and querying. +type ModuleSchema struct { + + // Tables is a list of tables that are part of the schema for the module. + Tables []Table +} diff --git a/indexer/base/table.go b/indexer/base/table.go new file mode 100644 index 000000000000..2d076f6d3eb8 --- /dev/null +++ b/indexer/base/table.go @@ -0,0 +1,20 @@ +package indexerbase + +// Table represents a table in the schema of a module. +type Table struct { + // Name is the name of the table. + Name string + + // KeyColumns is a list of columns that make up the primary key of the table. + KeyColumns []Column + + // ValueColumns is a list of columns that are not part of the primary key of the table. + ValueColumns []Column + + // RetainDeletions is a flag that indicates whether the indexer should retain + // deleted rows in the database and flag them as deleted rather than actually + // deleting the row. For many types of data in state, the data is deleted even + // though it is still valid in order to save space. Indexers will want to have + // the option of retaining such data and distinguishing from other "true" deletions. + RetainDeletions bool +} From 63aeb85ecec56fd4cda2f21852ab5e9fde0b6770 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 11 Jun 2024 15:11:07 -0400 Subject: [PATCH 02/29] WIP on tests --- indexer/base/column.go | 4 +- indexer/base/column_test.go | 7 + indexer/base/kind.go | 19 ++- indexer/base/kind_test.go | 290 ++++++++++++++++++++++++++++++++++++ 4 files changed, 312 insertions(+), 8 deletions(-) create mode 100644 indexer/base/column_test.go create mode 100644 indexer/base/kind_test.go diff --git a/indexer/base/column.go b/indexer/base/column.go index 14fa4e55b322..30b75d161c82 100644 --- a/indexer/base/column.go +++ b/indexer/base/column.go @@ -82,6 +82,8 @@ func (e EnumDefinition) Validate() error { } // ValidateValue validates that the value conforms to the column's kind and nullability. +// It currently does not do any validation that IntegerKind, DecimalKind, Bech32AddressKind, or EnumKind +// values are valid for their respective types behind conforming to the correct go type. func (c Column) ValidateValue(value any) error { if value == nil { if !c.Nullable { @@ -89,7 +91,7 @@ func (c Column) ValidateValue(value any) error { } return nil } - return c.Kind.ValidateValue(value) + return c.Kind.ValidateValueType(value) } // ValidateKey validates that the value conforms to the set of columns as a Key in an EntityUpdate. diff --git a/indexer/base/column_test.go b/indexer/base/column_test.go new file mode 100644 index 000000000000..b646247b058c --- /dev/null +++ b/indexer/base/column_test.go @@ -0,0 +1,7 @@ +package indexerbase + +import "testing" + +func TestColumnValidate(t *testing.T) { + +} diff --git a/indexer/base/kind.go b/indexer/base/kind.go index d1873944fb1b..0c86ab90f4c1 100644 --- a/indexer/base/kind.go +++ b/indexer/base/kind.go @@ -47,7 +47,7 @@ const ( Uint64Kind // IntegerKind represents an arbitrary precision integer number. Values of this type must - // be of the go type string or a type that implements fmt.Stringer with the resulted string + // be of the go type int64, string or a type that implements fmt.Stringer with the resulted string // formatted as an integer number. IntegerKind @@ -98,10 +98,12 @@ func (t Kind) Validate() error { return nil } -// ValidateValue returns an error if the value does not the type go type specified by the kind. +// ValidateValueType returns an error if the value does not the type go type specified by the kind. // Some columns may accept nil values, however, this method does not have any notion of // nullability. It only checks that the value is of the correct type. -func (t Kind) ValidateValue(value any) error { +// It also doesn't perform any validation that IntegerKind, DecimalKind, Bech32AddressKind, or EnumKind +// values are valid for their respective types. +func (t Kind) ValidateValueType(value any) error { switch t { case StringKind: _, ok := value.(string) @@ -157,7 +159,8 @@ func (t Kind) ValidateValue(value any) error { case IntegerKind: _, ok := value.(string) _, ok2 := value.(fmt.Stringer) - if !ok && !ok2 { + _, ok3 := value.(int64) + if !ok && !ok2 && !ok3 { return fmt.Errorf("expected string or type that implements fmt.Stringer, got %T", value) } case DecimalKind: @@ -262,7 +265,10 @@ func (t Kind) String() string { // KindForGoValue finds the simplest kind that can represent the given go value. It will not, however, // return kinds such as IntegerKind, DecimalKind, Bech32AddressKind, or EnumKind which all can be -// represented as strings. It will return InvalidKind if the value is not a simple type. +// represented as strings. Generally all values which do not have a more specific type will +// return JSONKind because the framework cannot decide at this point whether the value +// can or cannot be marshaled to JSON. This method should generally only be used as a fallback +// when the kind of a column is not specified more specifically. func KindForGoValue(value any) Kind { switch value.(type) { case string, fmt.Stringer: @@ -298,7 +304,6 @@ func KindForGoValue(value any) Kind { case json.RawMessage: return JSONKind default: + return JSONKind } - - return InvalidKind } diff --git a/indexer/base/kind_test.go b/indexer/base/kind_test.go new file mode 100644 index 000000000000..18075535b067 --- /dev/null +++ b/indexer/base/kind_test.go @@ -0,0 +1,290 @@ +package indexerbase + +import ( + "strings" + "testing" + "time" +) + +func TestKind_Validate(t *testing.T) { + validKinds := []Kind{ + StringKind, + BytesKind, + Int8Kind, + Uint8Kind, + Int16Kind, + Uint16Kind, + Int32Kind, + Uint32Kind, + Int64Kind, + Uint64Kind, + IntegerKind, + DecimalKind, + BoolKind, + EnumKind, + Bech32AddressKind, + } + + for _, kind := range validKinds { + if err := kind.Validate(); err != nil { + t.Errorf("expected valid kind %s to pass validation, got: %v", kind, err) + } + } + + invalidKinds := []Kind{ + Kind(-1), + InvalidKind, + Kind(100), + } + + for _, kind := range invalidKinds { + if err := kind.Validate(); err == nil { + t.Errorf("expected invalid kind %s to fail validation, got: %v", kind, err) + } + } +} + +func TestKind_ValidateValue(t *testing.T) { + tests := []struct { + kind Kind + value any + valid bool + }{ + { + kind: StringKind, + value: "hello", + valid: true, + }, + { + kind: StringKind, + value: &strings.Builder{}, + valid: true, + }, + { + kind: StringKind, + value: []byte("hello"), + valid: false, + }, + { + kind: BytesKind, + value: []byte("hello"), + valid: true, + }, + { + kind: BytesKind, + value: "hello", + valid: false, + }, + { + kind: Int8Kind, + value: int8(1), + valid: true, + }, + { + kind: Int8Kind, + value: int16(1), + valid: false, + }, + { + kind: Uint8Kind, + value: uint8(1), + valid: true, + }, + { + kind: Uint8Kind, + value: uint16(1), + valid: false, + }, + { + kind: Int16Kind, + value: int16(1), + valid: true, + }, + { + kind: Int16Kind, + value: int32(1), + valid: false, + }, + { + kind: Uint16Kind, + value: uint16(1), + valid: true, + }, + { + kind: Uint16Kind, + value: uint32(1), + valid: false, + }, + { + kind: Int32Kind, + value: int32(1), + valid: true, + }, + { + kind: Int32Kind, + value: int64(1), + valid: false, + }, + { + kind: Uint32Kind, + value: uint32(1), + valid: true, + }, + { + kind: Uint32Kind, + value: uint64(1), + valid: false, + }, + { + kind: Int64Kind, + value: int64(1), + valid: true, + }, + { + kind: Int64Kind, + value: int32(1), + valid: false, + }, + { + kind: Uint64Kind, + value: uint64(1), + valid: true, + }, + { + kind: Uint64Kind, + value: uint32(1), + valid: false, + }, + { + kind: IntegerKind, + value: "1", + valid: true, + }, + //{ + // kind: IntegerKind, + // value: (&strings.Builder{}).WriteString("1"), + // valid: true, + //}, + { + kind: IntegerKind, + value: int32(1), + valid: false, + }, + { + kind: IntegerKind, + value: int64(1), + valid: true, + }, + { + kind: DecimalKind, + value: "1.0", + valid: true, + }, + { + kind: DecimalKind, + value: "1", + valid: true, + }, + { + kind: DecimalKind, + value: "1.1e4", + valid: true, + }, + //{ + // kind: DecimalKind, + // value: (&strings.Builder{}).WriteString("1.0"), + // valid: true, + //}, + { + kind: DecimalKind, + value: int32(1), + valid: false, + }, + { + kind: Bech32AddressKind, + value: "cosmos1hsk6jryyqjfhp5g7c0nh4n6dd45ygctnxglp5h", + valid: true, + }, + //{ + // kind: Bech32AddressKind, + // value: (&strings.Builder{}).WriteString("cosmos1hsk6jryyqjfhp5g7c0nh4n6dd45ygctnxglp5h"), + // valid: true, + //}, + { + kind: Bech32AddressKind, + value: 1, + valid: false, + }, + { + kind: BoolKind, + value: true, + valid: true, + }, + { + kind: BoolKind, + value: false, + valid: true, + }, + { + kind: BoolKind, + value: 1, + valid: false, + }, + { + kind: EnumKind, + value: "hello", + valid: true, + }, + //{ + // kind: EnumKind, + // value: (&strings.Builder{}).WriteString("hello"), + // valid: true, + //}, + { + kind: EnumKind, + value: 1, + valid: false, + }, + { + kind: TimeKind, + value: time.Now(), + valid: true, + }, + { + kind: TimeKind, + value: "hello", + valid: false, + }, + { + kind: DurationKind, + value: time.Second, + valid: true, + }, + { + kind: DurationKind, + value: "hello", + valid: false, + }, + { + kind: Float32Kind, + value: float32(1.0), + valid: true, + }, + { + kind: Float32Kind, + value: float64(1.0), + valid: false, + }, + // TODO float64, json + } + + for i, tt := range tests { + err := tt.kind.ValidateValueType(tt.value) + if tt.valid && err != nil { + t.Errorf("test %d: expected valid value %v for kind %s to pass validation, got: %v", i, tt.value, tt.kind, err) + } + if !tt.valid && err == nil { + t.Errorf("test %d: expected invalid value %v for kind %s to fail validation, got: %v", i, tt.value, tt.kind, err) + } + } +} From 28ed78b03229666b589c08fe8113007fc29cb1e8 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 11 Jun 2024 17:47:23 -0400 Subject: [PATCH 03/29] feat(indexer/base): add Manager and DecodeableModule --- indexer/base/catch_up.go | 8 ++ indexer/base/decoder.go | 92 +++++++++++++++ indexer/base/listener.go | 2 +- indexer/base/log.go | 20 ++++ indexer/base/manager.go | 234 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 355 insertions(+), 1 deletion(-) create mode 100644 indexer/base/catch_up.go create mode 100644 indexer/base/decoder.go create mode 100644 indexer/base/log.go create mode 100644 indexer/base/manager.go diff --git a/indexer/base/catch_up.go b/indexer/base/catch_up.go new file mode 100644 index 000000000000..432e644183a9 --- /dev/null +++ b/indexer/base/catch_up.go @@ -0,0 +1,8 @@ +package indexerbase + +// CatchUpSource is an interface that allows indexers to start indexing modules with pre-existing state. +type CatchUpSource interface { + + // IterateAllKVPairs iterates over all key-value pairs for a given module. + IterateAllKVPairs(moduleName string, fn func(key, value []byte) error) error +} diff --git a/indexer/base/decoder.go b/indexer/base/decoder.go new file mode 100644 index 000000000000..8ce21bd5a5bf --- /dev/null +++ b/indexer/base/decoder.go @@ -0,0 +1,92 @@ +package indexerbase + +import "sort" + +type DecoderResolver interface { + // Iterate iterates over all module decoders which should be initialized at startup. + Iterate(func(string, ModuleDecoder) error) error + + // LookupDecoder allows for resolving decoders dynamically. For instance, some module-like + // things may come into existence dynamically (like x/accounts or EVM or WASM contracts). + // The first time the manager sees one of these appearing in KV-store writes, it will + // lookup a decoder for it and cache it for future use. This check will only happen the first + // time a module is seen. In order to start decoding an existing module, the indexing manager + // needs to be restarted, usually with a node restart and indexer catch-up needs to be run. + LookupDecoder(moduleName string) (decoder ModuleDecoder, found bool, err error) +} + +// DecodableModule is an interface that modules can implement to provide a ModuleDecoder. +// Usually these modules would also implement appmodule.AppModule, but that is not included +// to leave this package free of any dependencies. +type DecodableModule interface { + + // ModuleDecoder returns a ModuleDecoder for the module. + ModuleDecoder() (ModuleDecoder, error) +} + +// ModuleDecoder is a struct that contains the schema and a KVDecoder for a module. +type ModuleDecoder struct { + // Schema is the schema for the module. + Schema ModuleSchema + + // KVDecoder is a function that decodes a key-value pair into an EntityUpdate. + // If modules pass logical updates directly to the engine and don't require logical decoding of raw bytes, + // then this function should be nil. + KVDecoder KVDecoder +} + +// KVDecoder is a function that decodes a key-value pair into an EntityUpdate. +// If the KV-pair doesn't represent an entity update, the function should return false +// as the second return value. Error should only be non-nil when the decoder expected +// to parse a valid update and was unable to. +type KVDecoder = func(key, value []byte) (EntityUpdate, bool, error) + +type appModuleDecoderResolver[ModuleT any] struct { + moduleSet map[string]ModuleT +} + +// NewAppModuleDecoderResolver returns DecoderResolver that will discover modules implementing +// DecodeableModule in the provided module set. +func NewAppModuleDecoderResolver[ModuleT any](moduleSet map[string]ModuleT) DecoderResolver { + return &appModuleDecoderResolver[ModuleT]{ + moduleSet: moduleSet, + } +} + +func (a appModuleDecoderResolver[ModuleT]) Iterate(f func(string, ModuleDecoder) error) error { + keys := make([]string, 0, len(a.moduleSet)) + for k := range a.moduleSet { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + module := a.moduleSet[k] + dm, ok := any(module).(DecodableModule) + if ok { + decoder, err := dm.ModuleDecoder() + if err != nil { + return err + } + err = f(k, decoder) + if err != nil { + return err + } + } + } + return nil +} + +func (a appModuleDecoderResolver[ModuleT]) LookupDecoder(moduleName string) (ModuleDecoder, bool, error) { + mod, ok := a.moduleSet[moduleName] + if !ok { + return ModuleDecoder{}, false, nil + } + + dm, ok := any(mod).(DecodableModule) + if !ok { + return ModuleDecoder{}, false, nil + } + + decoder, err := dm.ModuleDecoder() + return decoder, true, err +} diff --git a/indexer/base/listener.go b/indexer/base/listener.go index cccd6f08e153..3bdd1db272f2 100644 --- a/indexer/base/listener.go +++ b/indexer/base/listener.go @@ -24,7 +24,7 @@ type Listener struct { OnEvent func(EventData) error // OnKVPair is called when a key-value has been written to the store for a given module. - OnKVPair func(module string, key, value []byte, delete bool) error + OnKVPair func(moduleName string, key, value []byte, delete bool) error // Commit is called when state is commited, usually at the end of a block. Any // indexers should commit their data when this is called and return an error if diff --git a/indexer/base/log.go b/indexer/base/log.go new file mode 100644 index 000000000000..eba6f37b5059 --- /dev/null +++ b/indexer/base/log.go @@ -0,0 +1,20 @@ +package indexerbase + +// Logger is the expected interface of loggers used in the indexer. +type Logger interface { + // Info takes a message and a set of key/value pairs and logs with level INFO. + // The key of the tuple must be a string. + Info(msg string, keyVals ...any) + + // Warn takes a message and a set of key/value pairs and logs with level WARN. + // The key of the tuple must be a string. + Warn(msg string, keyVals ...any) + + // Error takes a message and a set of key/value pairs and logs with level ERR. + // The key of the tuple must be a string. + Error(msg string, keyVals ...any) + + // Debug takes a message and a set of key/value pairs and logs with level DEBUG. + // The key of the tuple must be a string. + Debug(msg string, keyVals ...any) +} diff --git a/indexer/base/manager.go b/indexer/base/manager.go new file mode 100644 index 000000000000..c68210c9aa5c --- /dev/null +++ b/indexer/base/manager.go @@ -0,0 +1,234 @@ +package indexerbase + +type Manager struct { + logger Logger + decoderResolver DecoderResolver + decoders map[string]KVDecoder + listeners []Listener + needLogicalDecoding bool + listener Listener +} + +// ManagerOptions are the options for creating a new Manager. +type ManagerOptions struct { + // DecoderResolver is the resolver for module decoders. It is required. + DecoderResolver DecoderResolver + + // Listeners are the listeners that will be called when the manager receives events. + Listeners []Listener + + // CatchUpSource is the source that will be used do initial indexing of modules with pre-existing + // state. It is optional, but if it is not provided, indexing can only be starting when a node + // is synced from genesis. + CatchUpSource CatchUpSource + + // Logger is the logger that will be used by the manager. It is optional. + Logger Logger +} + +// NewManager creates a new Manager with the provided options. +func NewManager(opts ManagerOptions) (*Manager, error) { + if opts.Logger != nil { + opts.Logger.Info("Initializing indexer manager") + } + + mgr := &Manager{ + logger: opts.Logger, + decoderResolver: opts.DecoderResolver, + decoders: map[string]KVDecoder{}, + listeners: opts.Listeners, + needLogicalDecoding: false, + listener: Listener{}, + } + + return mgr, nil +} + +// Listener returns that listener that should be passed directly to the blockchain for managing +// all indexing. +func (p *Manager) init() error { + // check each subscribed listener to see if we actually need to register the listener + + for _, listener := range p.listeners { + if listener.StartBlock != nil { + p.listener.StartBlock = p.startBlock + break + } + } + + for _, listener := range p.listeners { + if listener.OnBlockHeader != nil { + p.listener.OnBlockHeader = p.onBlockHeader + break + } + } + + for _, listener := range p.listeners { + if listener.OnTx != nil { + p.listener.OnTx = p.onTx + break + } + } + + for _, listener := range p.listeners { + if listener.OnEvent != nil { + p.listener.OnEvent = p.onEvent + break + } + } + + for _, listener := range p.listeners { + if listener.Commit != nil { + p.listener.Commit = p.commit + break + } + } + + for _, listener := range p.listeners { + if listener.OnEntityUpdate != nil { + p.needLogicalDecoding = true + p.listener.OnKVPair = p.onKVPair + break + } + } + + if p.listener.OnKVPair == nil { + for _, listener := range p.listeners { + if listener.OnKVPair != nil { + p.listener.OnKVPair = p.onKVPair + break + } + } + } + + if p.needLogicalDecoding { + err := p.decoderResolver.Iterate(func(moduleName string, module ModuleDecoder) error { + p.decoders[moduleName] = module.KVDecoder + // TODO + return nil + }) + if err != nil { + return err + } + } + + return nil +} + +func (p *Manager) startBlock(height uint64) error { + if p.logger != nil { + p.logger.Debug("start block", "height", height) + } + + for _, listener := range p.listeners { + if listener.StartBlock == nil { + continue + } + if err := listener.StartBlock(height); err != nil { + return err + } + } + return nil +} + +func (p *Manager) onBlockHeader(data BlockHeaderData) error { + if p.logger != nil { + p.logger.Debug("block header", "height", data.Height) + } + + for _, listener := range p.listeners { + if listener.OnBlockHeader == nil { + continue + } + if err := listener.OnBlockHeader(data); err != nil { + return err + } + } + return nil +} + +func (p *Manager) onTx(data TxData) error { + for _, listener := range p.listeners { + if listener.OnTx == nil { + continue + } + if err := listener.OnTx(data); err != nil { + return err + } + } + return nil +} + +func (p *Manager) onEvent(data EventData) error { + for _, listener := range p.listeners { + if err := listener.OnEvent(data); err != nil { + return err + } + } + return nil +} + +func (p *Manager) commit() error { + if p.logger != nil { + p.logger.Debug("commit") + } + + for _, listener := range p.listeners { + if err := listener.Commit(); err != nil { + return err + } + } + return nil +} + +func (p *Manager) onKVPair(storeKey string, key, value []byte, delete bool) error { + if p.logger != nil { + p.logger.Debug("kv pair", "storeKey", storeKey, "delete", delete) + } + + for _, listener := range p.listeners { + if listener.OnKVPair == nil { + continue + } + if err := listener.OnKVPair(storeKey, key, value, delete); err != nil { + return err + } + } + + if !p.needLogicalDecoding { + return nil + } + + decoder, ok := p.decoders[storeKey] + if !ok { + return nil + } + + update, handled, err := decoder(key, value) + if err != nil { + return err + } + if !handled { + p.logger.Info("not decoded", "storeKey", storeKey, "tableName", update.TableName) + return nil + } + + p.logger.Info("decoded", + "storeKey", storeKey, + "tableName", update.TableName, + "key", update.Key, + "values", update.Value, + "delete", update.Delete, + ) + + for _, indexer := range p.listeners { + if indexer.OnEntityUpdate == nil { + continue + } + if err := indexer.OnEntityUpdate(storeKey, update); err != nil { + return err + } + } + + return nil +} From 21b787a2a18a9135edbac767fd343aba42b1aeee Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 11 Jun 2024 18:00:34 -0400 Subject: [PATCH 04/29] WIP --- indexer/base/manager.go | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/indexer/base/manager.go b/indexer/base/manager.go index c68210c9aa5c..89f09a11aa70 100644 --- a/indexer/base/manager.go +++ b/indexer/base/manager.go @@ -181,16 +181,16 @@ func (p *Manager) commit() error { return nil } -func (p *Manager) onKVPair(storeKey string, key, value []byte, delete bool) error { +func (p *Manager) onKVPair(moduleName string, key, value []byte, delete bool) error { if p.logger != nil { - p.logger.Debug("kv pair", "storeKey", storeKey, "delete", delete) + p.logger.Debug("kv pair", "moduleName", moduleName, "delete", delete) } for _, listener := range p.listeners { if listener.OnKVPair == nil { continue } - if err := listener.OnKVPair(storeKey, key, value, delete); err != nil { + if err := listener.OnKVPair(moduleName, key, value, delete); err != nil { return err } } @@ -199,8 +199,22 @@ func (p *Manager) onKVPair(storeKey string, key, value []byte, delete bool) erro return nil } - decoder, ok := p.decoders[storeKey] + decoder, ok := p.decoders[moduleName] if !ok { + // check for decoder when first seeing a module + md, found, err := p.decoderResolver.LookupDecoder(moduleName) + if err != nil { + return err + } + if found { + p.decoders[moduleName] = md.KVDecoder + decoder = md.KVDecoder + } else { + p.decoders[moduleName] = nil + } + } + + if decoder == nil { return nil } @@ -209,12 +223,12 @@ func (p *Manager) onKVPair(storeKey string, key, value []byte, delete bool) erro return err } if !handled { - p.logger.Info("not decoded", "storeKey", storeKey, "tableName", update.TableName) + p.logger.Info("not decoded", "moduleName", moduleName, "tableName", update.TableName) return nil } p.logger.Info("decoded", - "storeKey", storeKey, + "moduleName", moduleName, "tableName", update.TableName, "key", update.Key, "values", update.Value, @@ -225,7 +239,7 @@ func (p *Manager) onKVPair(storeKey string, key, value []byte, delete bool) erro if indexer.OnEntityUpdate == nil { continue } - if err := indexer.OnEntityUpdate(storeKey, update); err != nil { + if err := indexer.OnEntityUpdate(moduleName, update); err != nil { return err } } From b2e57cdbf5a0121af4a951c0e339d62db4d93c6c Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Wed, 12 Jun 2024 12:12:25 -0400 Subject: [PATCH 05/29] update listener --- indexer/base/listener.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/indexer/base/listener.go b/indexer/base/listener.go index 3bdd1db272f2..b6957c7b76eb 100644 --- a/indexer/base/listener.go +++ b/indexer/base/listener.go @@ -11,6 +11,12 @@ import ( // decoding of data out of the box, so the EnsureLogicalSetup and OnEntityUpdate methods will not be called. // These methods will only be called when listening logical decoding is setup. type Listener struct { + // Initialize is called when the listener is initialized before any other methods are called. + // The lastBlock return value should be the last block height the listener persisted if it is + // persisting block data, 0 if it is not interested in persisting block data, or -1 if it is + // persisting block data but has not persisted any data yet. + Initialize func(InitializationData) (lastBlock int64, err error) + // StartBlock is called at the beginning of processing a block. StartBlock func(uint64) error @@ -58,6 +64,25 @@ type Listener struct { // this point and also save the block number as the last block that has been processed so // that processing of regular block data can resume from this point in the future. CommitCatchupSync func(module string, block uint64) error + + // SubscribedModules is a map of modules that the listener is interested in receiving events for in OnKVPair and + // logical decoding listeners (if these are registered). If this is left nil but listeners are registered, + // it is assumed that the listener is interested in all modules. + SubscribedModules map[string]bool +} + +// InitializationData represents initialization data that is passed to a listener. +type InitializationData struct { + + // HasEventAlignedWrites indicates that the blockchain data source will emit KV-pair events + // in an order aligned with transaction, message and event callbacks. If this is true + // then indexers can assume that KV-pair data is associated with these specific transactions, messages + // and events. This may be useful for indexers which store a log of all operations (such as immutable + // or version controlled databases) so that the history log can include fine grain correlation between + // state updates and transactions, messages and events. If this value is false, then indexers should + // assume that KV-pair data occurs out of order with respect to transaction, message and event callbacks - + // the only safe assumption being that KV-pair data is associated with the block in which it was emitted. + HasEventAlignedWrites bool } // BlockHeaderData represents the raw block header data that is passed to a listener. From 23138518b487ef3cd7795a1eaac5676c27ec6ca2 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Wed, 12 Jun 2024 12:16:15 -0400 Subject: [PATCH 06/29] docs --- indexer/base/decoder.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/indexer/base/decoder.go b/indexer/base/decoder.go index 8ce21bd5a5bf..42264d70c61d 100644 --- a/indexer/base/decoder.go +++ b/indexer/base/decoder.go @@ -9,9 +9,10 @@ type DecoderResolver interface { // LookupDecoder allows for resolving decoders dynamically. For instance, some module-like // things may come into existence dynamically (like x/accounts or EVM or WASM contracts). // The first time the manager sees one of these appearing in KV-store writes, it will - // lookup a decoder for it and cache it for future use. This check will only happen the first - // time a module is seen. In order to start decoding an existing module, the indexing manager - // needs to be restarted, usually with a node restart and indexer catch-up needs to be run. + // lookup a decoder for it and cache it for future use. The manager will also perform + // a catch-up sync before passing any new writes to ensure that all historical state has + // been synced if there is any This check will only happen the first time a module is seen + // by the manager in a given process (a process restart will cause this check to happen again). LookupDecoder(moduleName string) (decoder ModuleDecoder, found bool, err error) } From 6466ac5d25296bd23ab2d649a99cf6b5aec2e9e0 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Mon, 17 Jun 2024 17:58:08 -0400 Subject: [PATCH 07/29] updates --- indexer/base/column.go | 151 -------------- indexer/base/column_test.go | 7 - indexer/base/decoder.go | 46 ++-- indexer/base/entity.go | 40 ---- indexer/base/kind_test.go | 290 -------------------------- indexer/base/listener.go | 31 ++- indexer/base/manager.go | 149 +++++++++---- indexer/base/process.go | 86 ++++++++ indexer/base/{catch_up.go => sync.go} | 4 +- indexer/base/table.go | 20 -- 10 files changed, 243 insertions(+), 581 deletions(-) delete mode 100644 indexer/base/column.go delete mode 100644 indexer/base/column_test.go delete mode 100644 indexer/base/entity.go delete mode 100644 indexer/base/kind_test.go create mode 100644 indexer/base/process.go rename indexer/base/{catch_up.go => sync.go} (56%) delete mode 100644 indexer/base/table.go diff --git a/indexer/base/column.go b/indexer/base/column.go deleted file mode 100644 index 30b75d161c82..000000000000 --- a/indexer/base/column.go +++ /dev/null @@ -1,151 +0,0 @@ -package indexerbase - -import "fmt" - -// Column represents a column in a table schema. -type Column struct { - // Name is the name of the column. - Name string - - // Kind is the basic type of the column. - Kind Kind - - // Nullable indicates whether null values are accepted for the column. - Nullable bool - - // AddressPrefix is the address prefix of the column's kind, currently only used for Bech32AddressKind. - AddressPrefix string - - // EnumDefinition is the definition of the enum type and is only valid when Kind is EnumKind. - EnumDefinition EnumDefinition -} - -// EnumDefinition represents the definition of an enum type. -type EnumDefinition struct { - // Name is the name of the enum type. - Name string - - // Values is a list of distinct values that are part of the enum type. - Values []string -} - -// Validate validates the column. -func (c Column) Validate() error { - // non-empty name - if c.Name == "" { - return fmt.Errorf("column name cannot be empty") - } - - // valid kind - if err := c.Kind.Validate(); err != nil { - return fmt.Errorf("invalid column type for %q: %w", c.Name, err) - } - - // address prefix only valid with Bech32AddressKind - if c.Kind == Bech32AddressKind && c.AddressPrefix == "" { - return fmt.Errorf("missing address prefix for column %q", c.Name) - } else if c.Kind != Bech32AddressKind && c.AddressPrefix != "" { - return fmt.Errorf("address prefix is only valid for column %q with type Bech32AddressKind", c.Name) - } - - // enum definition only valid with EnumKind - if c.Kind == EnumKind { - if err := c.EnumDefinition.Validate(); err != nil { - return fmt.Errorf("invalid enum definition for column %q: %w", c.Name, err) - } - } else if c.Kind != EnumKind && c.EnumDefinition.Name != "" && c.EnumDefinition.Values != nil { - return fmt.Errorf("enum definition is only valid for column %q with type EnumKind", c.Name) - } - - return nil -} - -// Validate validates the enum definition. -func (e EnumDefinition) Validate() error { - if e.Name == "" { - return fmt.Errorf("enum definition name cannot be empty") - } - if len(e.Values) == 0 { - return fmt.Errorf("enum definition values cannot be empty") - } - seen := make(map[string]bool, len(e.Values)) - for i, v := range e.Values { - if v == "" { - return fmt.Errorf("enum definition value at index %d cannot be empty for enum %s", i, e.Name) - } - if seen[v] { - return fmt.Errorf("duplicate enum definition value %q for enum %s", v, e.Name) - } - seen[v] = true - } - return nil -} - -// ValidateValue validates that the value conforms to the column's kind and nullability. -// It currently does not do any validation that IntegerKind, DecimalKind, Bech32AddressKind, or EnumKind -// values are valid for their respective types behind conforming to the correct go type. -func (c Column) ValidateValue(value any) error { - if value == nil { - if !c.Nullable { - return fmt.Errorf("column %q cannot be null", c.Name) - } - return nil - } - return c.Kind.ValidateValueType(value) -} - -// ValidateKey validates that the value conforms to the set of columns as a Key in an EntityUpdate. -// See EntityUpdate.Key for documentation on the requirements of such values. -func ValidateKey(cols []Column, value any) error { - if len(cols) == 0 { - return nil - } - - if len(cols) == 1 { - return cols[0].ValidateValue(value) - } - - values, ok := value.([]any) - if !ok { - return fmt.Errorf("expected slice of values for key columns, got %T", value) - } - - if len(cols) != len(values) { - return fmt.Errorf("expected %d key columns, got %d values", len(cols), len(value.([]any))) - } - for i, col := range cols { - if err := col.ValidateValue(values[i]); err != nil { - return fmt.Errorf("invalid value for key column %q: %w", col.Name, err) - } - } - return nil -} - -// ValidateValue validates that the value conforms to the set of columns as a Value in an EntityUpdate. -// See EntityUpdate.Value for documentation on the requirements of such values. -func ValidateValue(cols []Column, value any) error { - valueUpdates, ok := value.(ValueUpdates) - if ok { - colMap := map[string]Column{} - for _, col := range cols { - colMap[col.Name] = col - } - var errs []error - valueUpdates.Iterate(func(colName string, value any) bool { - col, ok := colMap[colName] - if !ok { - errs = append(errs, fmt.Errorf("unknown column %q in value updates", colName)) - } - if err := col.ValidateValue(value); err != nil { - errs = append(errs, fmt.Errorf("invalid value for column %q: %w", colName, err)) - } - return true - }) - if len(errs) > 0 { - return fmt.Errorf("validation errors: %v", errs) - } - return nil - } else { - return ValidateKey(cols, value) - } -} diff --git a/indexer/base/column_test.go b/indexer/base/column_test.go deleted file mode 100644 index b646247b058c..000000000000 --- a/indexer/base/column_test.go +++ /dev/null @@ -1,7 +0,0 @@ -package indexerbase - -import "testing" - -func TestColumnValidate(t *testing.T) { - -} diff --git a/indexer/base/decoder.go b/indexer/base/decoder.go index 434903ac1c98..75eb3abe811c 100644 --- a/indexer/base/decoder.go +++ b/indexer/base/decoder.go @@ -2,20 +2,6 @@ package indexerbase import "sort" -type DecoderResolver interface { - // Iterate iterates over all module decoders which should be initialized at startup. - Iterate(func(string, ModuleDecoder) error) error - - // LookupDecoder allows for resolving decoders dynamically. For instance, some module-like - // things may come into existence dynamically (like x/accounts or EVM or WASM contracts). - // The first time the manager sees one of these appearing in KV-store writes, it will - // lookup a decoder for it and cache it for future use. The manager will also perform - // a catch-up sync before passing any new writes to ensure that all historical state has - // been synced if there is any This check will only happen the first time a module is seen - // by the manager in a given process (a process restart will cause this check to happen again). - LookupDecoder(moduleName string) (decoder ModuleDecoder, found bool, err error) -} - // DecodableModule is an interface that modules can implement to provide a ModuleDecoder. // Usually these modules would also implement appmodule.AppModule, but that is not included type DecodableModule interface { @@ -39,21 +25,35 @@ type ModuleDecoder struct { // If the KV-pair doesn't represent an object update, the function should return false // as the second return value. Error should only be non-nil when the decoder expected // to parse a valid update and was unable to. -type KVDecoder = func(key, value []byte) (ObjectUpdate, bool, error) +type KVDecoder = func(key, value []byte, deleted bool) (ObjectUpdate, bool, error) -type appModuleDecoderResolver[ModuleT any] struct { - moduleSet map[string]ModuleT +type DecoderResolver interface { + // Iterate iterates over all module decoders which should be initialized at startup. + Iterate(func(string, ModuleDecoder) error) error + + // LookupDecoder allows for resolving decoders dynamically. For instance, some module-like + // things may come into existence dynamically (like x/accounts or EVM or WASM contracts). + // The first time the manager sees one of these appearing in KV-store writes, it will + // lookup a decoder for it and cache it for future use. The manager will also perform + // a catch-up sync before passing any new writes to ensure that all historical state has + // been synced if there is any This check will only happen the first time a module is seen + // by the manager in a given process (a process restart will cause this check to happen again). + LookupDecoder(moduleName string) (decoder ModuleDecoder, found bool, err error) } -// NewAppModuleDecoderResolver returns DecoderResolver that will discover modules implementing +type moduleSetDecoderResolver struct { + moduleSet map[string]interface{} +} + +// ModuleSetDecoderResolver returns DecoderResolver that will discover modules implementing // DecodeableModule in the provided module set. -func NewAppModuleDecoderResolver[ModuleT any](moduleSet map[string]ModuleT) DecoderResolver { - return &appModuleDecoderResolver[ModuleT]{ +func ModuleSetDecoderResolver(moduleSet map[string]interface{}) DecoderResolver { + return &moduleSetDecoderResolver{ moduleSet: moduleSet, } } -func (a appModuleDecoderResolver[ModuleT]) Iterate(f func(string, ModuleDecoder) error) error { +func (a moduleSetDecoderResolver) Iterate(f func(string, ModuleDecoder) error) error { keys := make([]string, 0, len(a.moduleSet)) for k := range a.moduleSet { keys = append(keys, k) @@ -76,7 +76,7 @@ func (a appModuleDecoderResolver[ModuleT]) Iterate(f func(string, ModuleDecoder) return nil } -func (a appModuleDecoderResolver[ModuleT]) LookupDecoder(moduleName string) (ModuleDecoder, bool, error) { +func (a moduleSetDecoderResolver) LookupDecoder(moduleName string) (ModuleDecoder, bool, error) { mod, ok := a.moduleSet[moduleName] if !ok { return ModuleDecoder{}, false, nil @@ -89,4 +89,4 @@ func (a appModuleDecoderResolver[ModuleT]) LookupDecoder(moduleName string) (Mod decoder, err := dm.ModuleDecoder() return decoder, true, err -} \ No newline at end of file +} diff --git a/indexer/base/entity.go b/indexer/base/entity.go deleted file mode 100644 index 95f016037fd8..000000000000 --- a/indexer/base/entity.go +++ /dev/null @@ -1,40 +0,0 @@ -package indexerbase - -// EntityUpdate represents an update operation on an entity in the schema. -type EntityUpdate struct { - // TableName is the name of the table that the entity belongs to in the schema. - TableName string - - // Key returns the value of the primary key of the entity and must conform to these constraints with respect - // that the schema that is defined for the entity: - // - if key represents a single column, then the value must be valid for the first column in that - // column list. For instance, if there is one column in the key of type String, then the value must be of - // type string - // - if key represents multiple columns, then the value must be a slice of values where each value is valid - // for the corresponding column in the column list. For instance, if there are two columns in the key of - // type String, String, then the value must be a slice of two strings. - // If the key has no columns, meaning that this is a singleton entity, then this value is ignored and can be nil. - Key any - - // Value returns the non-primary key columns of the entity and can either conform to the same constraints - // as EntityUpdate.Key or it may be and instance of ValueUpdates. ValueUpdates can be used as a performance - // optimization to avoid copying the values of the entity into the update and/or to omit unchanged columns. - // If this is a delete operation, then this value is ignored and can be nil. - Value any - - // Delete is a flag that indicates whether this update is a delete operation. If true, then the Value field - // is ignored and can be nil. - Delete bool -} - -// ValueUpdates is an interface that represents the value columns of an entity update. Columns that -// were not updated may be excluded from the update. Consumers should be aware that implementations -// may not filter out columns that were unchanged. However, if a column is omitted from the update -// it should be considered unchanged. -type ValueUpdates interface { - - // Iterate iterates over the columns and values in the entity update. The function should return - // true to continue iteration or false to stop iteration. Each column value should conform - // to the requirements of that column's type in the schema. - Iterate(func(col string, value any) bool) -} diff --git a/indexer/base/kind_test.go b/indexer/base/kind_test.go deleted file mode 100644 index 18075535b067..000000000000 --- a/indexer/base/kind_test.go +++ /dev/null @@ -1,290 +0,0 @@ -package indexerbase - -import ( - "strings" - "testing" - "time" -) - -func TestKind_Validate(t *testing.T) { - validKinds := []Kind{ - StringKind, - BytesKind, - Int8Kind, - Uint8Kind, - Int16Kind, - Uint16Kind, - Int32Kind, - Uint32Kind, - Int64Kind, - Uint64Kind, - IntegerKind, - DecimalKind, - BoolKind, - EnumKind, - Bech32AddressKind, - } - - for _, kind := range validKinds { - if err := kind.Validate(); err != nil { - t.Errorf("expected valid kind %s to pass validation, got: %v", kind, err) - } - } - - invalidKinds := []Kind{ - Kind(-1), - InvalidKind, - Kind(100), - } - - for _, kind := range invalidKinds { - if err := kind.Validate(); err == nil { - t.Errorf("expected invalid kind %s to fail validation, got: %v", kind, err) - } - } -} - -func TestKind_ValidateValue(t *testing.T) { - tests := []struct { - kind Kind - value any - valid bool - }{ - { - kind: StringKind, - value: "hello", - valid: true, - }, - { - kind: StringKind, - value: &strings.Builder{}, - valid: true, - }, - { - kind: StringKind, - value: []byte("hello"), - valid: false, - }, - { - kind: BytesKind, - value: []byte("hello"), - valid: true, - }, - { - kind: BytesKind, - value: "hello", - valid: false, - }, - { - kind: Int8Kind, - value: int8(1), - valid: true, - }, - { - kind: Int8Kind, - value: int16(1), - valid: false, - }, - { - kind: Uint8Kind, - value: uint8(1), - valid: true, - }, - { - kind: Uint8Kind, - value: uint16(1), - valid: false, - }, - { - kind: Int16Kind, - value: int16(1), - valid: true, - }, - { - kind: Int16Kind, - value: int32(1), - valid: false, - }, - { - kind: Uint16Kind, - value: uint16(1), - valid: true, - }, - { - kind: Uint16Kind, - value: uint32(1), - valid: false, - }, - { - kind: Int32Kind, - value: int32(1), - valid: true, - }, - { - kind: Int32Kind, - value: int64(1), - valid: false, - }, - { - kind: Uint32Kind, - value: uint32(1), - valid: true, - }, - { - kind: Uint32Kind, - value: uint64(1), - valid: false, - }, - { - kind: Int64Kind, - value: int64(1), - valid: true, - }, - { - kind: Int64Kind, - value: int32(1), - valid: false, - }, - { - kind: Uint64Kind, - value: uint64(1), - valid: true, - }, - { - kind: Uint64Kind, - value: uint32(1), - valid: false, - }, - { - kind: IntegerKind, - value: "1", - valid: true, - }, - //{ - // kind: IntegerKind, - // value: (&strings.Builder{}).WriteString("1"), - // valid: true, - //}, - { - kind: IntegerKind, - value: int32(1), - valid: false, - }, - { - kind: IntegerKind, - value: int64(1), - valid: true, - }, - { - kind: DecimalKind, - value: "1.0", - valid: true, - }, - { - kind: DecimalKind, - value: "1", - valid: true, - }, - { - kind: DecimalKind, - value: "1.1e4", - valid: true, - }, - //{ - // kind: DecimalKind, - // value: (&strings.Builder{}).WriteString("1.0"), - // valid: true, - //}, - { - kind: DecimalKind, - value: int32(1), - valid: false, - }, - { - kind: Bech32AddressKind, - value: "cosmos1hsk6jryyqjfhp5g7c0nh4n6dd45ygctnxglp5h", - valid: true, - }, - //{ - // kind: Bech32AddressKind, - // value: (&strings.Builder{}).WriteString("cosmos1hsk6jryyqjfhp5g7c0nh4n6dd45ygctnxglp5h"), - // valid: true, - //}, - { - kind: Bech32AddressKind, - value: 1, - valid: false, - }, - { - kind: BoolKind, - value: true, - valid: true, - }, - { - kind: BoolKind, - value: false, - valid: true, - }, - { - kind: BoolKind, - value: 1, - valid: false, - }, - { - kind: EnumKind, - value: "hello", - valid: true, - }, - //{ - // kind: EnumKind, - // value: (&strings.Builder{}).WriteString("hello"), - // valid: true, - //}, - { - kind: EnumKind, - value: 1, - valid: false, - }, - { - kind: TimeKind, - value: time.Now(), - valid: true, - }, - { - kind: TimeKind, - value: "hello", - valid: false, - }, - { - kind: DurationKind, - value: time.Second, - valid: true, - }, - { - kind: DurationKind, - value: "hello", - valid: false, - }, - { - kind: Float32Kind, - value: float32(1.0), - valid: true, - }, - { - kind: Float32Kind, - value: float64(1.0), - valid: false, - }, - // TODO float64, json - } - - for i, tt := range tests { - err := tt.kind.ValidateValueType(tt.value) - if tt.valid && err != nil { - t.Errorf("test %d: expected valid value %v for kind %s to pass validation, got: %v", i, tt.value, tt.kind, err) - } - if !tt.valid && err == nil { - t.Errorf("test %d: expected invalid value %v for kind %s to fail validation, got: %v", i, tt.value, tt.kind, err) - } - } -} diff --git a/indexer/base/listener.go b/indexer/base/listener.go index f0accb620824..2f20c611887f 100644 --- a/indexer/base/listener.go +++ b/indexer/base/listener.go @@ -15,7 +15,8 @@ type Listener struct { // The lastBlockPersisted return value should be the last block height the listener persisted if it is // persisting block data, 0 if it is not interested in persisting block data, or -1 if it is // persisting block data but has not persisted any data yet. This check allows the indexer - // framework to ensure that the listener has not missed blocks. + // framework to ensure that the listener has not missed blocks. Data sources MUST call + // initialize before any other method is called, otherwise, no data will be processed. Initialize func(InitializationData) (lastBlockPersisted int64, err error) // StartBlock is called at the beginning of processing a block. @@ -31,11 +32,12 @@ type Listener struct { OnEvent func(EventData) error // OnKVPair is called when a key-value has been written to the store for a given module. - OnKVPair func(moduleName string, key, value []byte, delete bool) error + OnKVPair func(KVPairData) error // Commit is called when state is committed, usually at the end of a block. Any // indexers should commit their data when this is called and return an error if - // they are unable to commit. + // they are unable to commit. Data sources MUST call Commit when data is committed, + // otherwise it should be assumed that indexers have not persisted their state. Commit func() error // InitializeModuleSchema should be called whenever the blockchain process starts OR whenever @@ -49,7 +51,7 @@ type Listener struct { // OnObjectUpdate is called whenever an object is updated in a module's state. This is only called // when logical data is available. It should be assumed that the same data in raw form // is also passed to OnKVPair. - OnObjectUpdate func(module string, update ObjectUpdate) error + OnObjectUpdate func(ObjectUpdateData) error } // InitializationData represents initialization data that is passed to a listener. @@ -117,3 +119,24 @@ type ToBytes = func() ([]byte, error) // ToJSON is a function that lazily returns the JSON representation of data. type ToJSON = func() (json.RawMessage, error) + +// KVPairData represents key-value pair data that is passed to a listener. +type KVPairData struct { + // ModuleName is the name of the module that the key-value pair belongs to. + ModuleName string + + // Key is the key of the key-value pair. + Key []byte + + // Value is the value of the key-value pair. It should be ignored when Delete is true. + Value []byte + + // Delete is a flag that indicates that the key-value pair was deleted. If it is false, + // then it is assumed that this has been a set operation. + Delete bool +} + +type ObjectUpdateData struct { + ModuleName string + Update ObjectUpdate +} diff --git a/indexer/base/manager.go b/indexer/base/manager.go index 89f09a11aa70..254b96b38355 100644 --- a/indexer/base/manager.go +++ b/indexer/base/manager.go @@ -4,9 +4,10 @@ type Manager struct { logger Logger decoderResolver DecoderResolver decoders map[string]KVDecoder - listeners []Listener needLogicalDecoding bool listener Listener + listenerProcesses []*listenerProcess + done chan struct{} } // ManagerOptions are the options for creating a new Manager. @@ -17,13 +18,17 @@ type ManagerOptions struct { // Listeners are the listeners that will be called when the manager receives events. Listeners []Listener - // CatchUpSource is the source that will be used do initial indexing of modules with pre-existing + // SyncSource is the source that will be used do initial indexing of modules with pre-existing // state. It is optional, but if it is not provided, indexing can only be starting when a node // is synced from genesis. - CatchUpSource CatchUpSource + SyncSource SyncSource // Logger is the logger that will be used by the manager. It is optional. Logger Logger + + BufferSize int + + Done chan struct{} } // NewManager creates a new Manager with the provided options. @@ -32,13 +37,23 @@ func NewManager(opts ManagerOptions) (*Manager, error) { opts.Logger.Info("Initializing indexer manager") } + done := opts.Done + if done == nil { + done = make(chan struct{}) + } + mgr := &Manager{ logger: opts.Logger, decoderResolver: opts.DecoderResolver, decoders: map[string]KVDecoder{}, - listeners: opts.Listeners, needLogicalDecoding: false, listener: Listener{}, + done: done, + } + + err := mgr.init(opts.Listeners) + if err != nil { + return nil, err } return mgr, nil @@ -46,46 +61,46 @@ func NewManager(opts ManagerOptions) (*Manager, error) { // Listener returns that listener that should be passed directly to the blockchain for managing // all indexing. -func (p *Manager) init() error { +func (p *Manager) init(listeners []Listener) error { // check each subscribed listener to see if we actually need to register the listener - for _, listener := range p.listeners { + for _, listener := range listeners { if listener.StartBlock != nil { p.listener.StartBlock = p.startBlock break } } - for _, listener := range p.listeners { + for _, listener := range listeners { if listener.OnBlockHeader != nil { p.listener.OnBlockHeader = p.onBlockHeader break } } - for _, listener := range p.listeners { + for _, listener := range listeners { if listener.OnTx != nil { p.listener.OnTx = p.onTx break } } - for _, listener := range p.listeners { + for _, listener := range listeners { if listener.OnEvent != nil { p.listener.OnEvent = p.onEvent break } } - for _, listener := range p.listeners { + for _, listener := range listeners { if listener.Commit != nil { p.listener.Commit = p.commit break } } - for _, listener := range p.listeners { - if listener.OnEntityUpdate != nil { + for _, listener := range listeners { + if listener.OnObjectUpdate != nil { p.needLogicalDecoding = true p.listener.OnKVPair = p.onKVPair break @@ -93,7 +108,7 @@ func (p *Manager) init() error { } if p.listener.OnKVPair == nil { - for _, listener := range p.listeners { + for _, listener := range listeners { if listener.OnKVPair != nil { p.listener.OnKVPair = p.onKVPair break @@ -112,6 +127,21 @@ func (p *Manager) init() error { } } + // initialize go routines for each listener + for _, listener := range listeners { + proc := &listenerProcess{ + listener: listener, + packetChan: make(chan packet), + commitDoneChan: make(chan error), + } + p.listenerProcesses = append(p.listenerProcesses, proc) + + // TODO initialize + // TODO initialize module schema + + go proc.run() + } + return nil } @@ -120,12 +150,13 @@ func (p *Manager) startBlock(height uint64) error { p.logger.Debug("start block", "height", height) } - for _, listener := range p.listeners { - if listener.StartBlock == nil { + for _, proc := range p.listenerProcesses { + if proc.listener.StartBlock != nil { continue } - if err := listener.StartBlock(height); err != nil { - return err + proc.packetChan <- packet{ + packetType: packetTypeStartBlock, + data: height, } } return nil @@ -136,35 +167,50 @@ func (p *Manager) onBlockHeader(data BlockHeaderData) error { p.logger.Debug("block header", "height", data.Height) } - for _, listener := range p.listeners { - if listener.OnBlockHeader == nil { + for _, proc := range p.listenerProcesses { + if proc.listener.OnBlockHeader == nil { continue } - if err := listener.OnBlockHeader(data); err != nil { - return err + proc.packetChan <- packet{ + packetType: packetTypeOnBlockHeader, + data: data, } } return nil } func (p *Manager) onTx(data TxData) error { - for _, listener := range p.listeners { - if listener.OnTx == nil { + if p.logger != nil { + p.logger.Debug("tx", "txIndex", data.TxIndex) + } + + for _, proc := range p.listenerProcesses { + if proc.listener.OnTx == nil { continue } - if err := listener.OnTx(data); err != nil { - return err + proc.packetChan <- packet{ + packetType: packetTypeOnTx, + data: data, } } return nil } func (p *Manager) onEvent(data EventData) error { - for _, listener := range p.listeners { - if err := listener.OnEvent(data); err != nil { - return err + if p.logger != nil { + p.logger.Debug("event", "txIndex", data.TxIndex, "msgIndex", data.MsgIndex, "eventIndex", data.EventIndex) + } + + for _, proc := range p.listenerProcesses { + if proc.listener.OnEvent == nil { + continue + } + proc.packetChan <- packet{ + packetType: packetTypeOnEvent, + data: data, } } + return nil } @@ -173,25 +219,39 @@ func (p *Manager) commit() error { p.logger.Debug("commit") } - for _, listener := range p.listeners { - if err := listener.Commit(); err != nil { + for _, proc := range p.listenerProcesses { + if proc.listener.Commit == nil { + continue + } + proc.packetChan <- packet{ + packetType: packetTypeCommit, + } + } + + // wait for all listeners to finish committing + for _, proc := range p.listenerProcesses { + err := <-proc.commitDoneChan + if err != nil { return err } } + return nil } -func (p *Manager) onKVPair(moduleName string, key, value []byte, delete bool) error { +func (p *Manager) onKVPair(data KVPairData) error { + moduleName := data.ModuleName if p.logger != nil { - p.logger.Debug("kv pair", "moduleName", moduleName, "delete", delete) + p.logger.Debug("kv pair received", "moduleName", moduleName) } - for _, listener := range p.listeners { - if listener.OnKVPair == nil { + for _, proc := range p.listenerProcesses { + if proc.listener.OnKVPair == nil { continue } - if err := listener.OnKVPair(moduleName, key, value, delete); err != nil { - return err + proc.packetChan <- packet{ + packetType: packetTypeOnKVPair, + data: data, } } @@ -218,29 +278,30 @@ func (p *Manager) onKVPair(moduleName string, key, value []byte, delete bool) er return nil } - update, handled, err := decoder(key, value) + update, handled, err := decoder(data.Key, data.Value, data.Delete) if err != nil { return err } if !handled { - p.logger.Info("not decoded", "moduleName", moduleName, "tableName", update.TableName) + p.logger.Debug("not decoded", "moduleName", moduleName, "objectType", update.TypeName) return nil } - p.logger.Info("decoded", + p.logger.Debug("decoded", "moduleName", moduleName, - "tableName", update.TableName, + "objectType", update.TypeName, "key", update.Key, "values", update.Value, "delete", update.Delete, ) - for _, indexer := range p.listeners { - if indexer.OnEntityUpdate == nil { + for _, proc := range p.listenerProcesses { + if proc.listener.OnObjectUpdate == nil { continue } - if err := indexer.OnEntityUpdate(moduleName, update); err != nil { - return err + proc.packetChan <- packet{ + packetType: packetTypeOnObjectUpdate, + data: update, } } diff --git a/indexer/base/process.go b/indexer/base/process.go new file mode 100644 index 000000000000..ff12343fa1fb --- /dev/null +++ b/indexer/base/process.go @@ -0,0 +1,86 @@ +package indexerbase + +import "fmt" + +type packetType int + +const ( + packetTypeStartBlock = iota + packetTypeOnBlockHeader + packetTypeOnTx + packetTypeOnEvent + packetTypeOnKVPair + packetTypeOnObjectUpdate + packetTypeCommit +) + +type packet struct { + packetType packetType + data interface{} +} + +type listenerProcess struct { + listener Listener + packetChan chan packet + err error + commitDoneChan chan error + cancel chan struct{} +} + +func (l *listenerProcess) run() { + for { + select { + case packet := <-l.packetChan: + if l.processPacket(packet) { + return // stop processing packets + } + case <-l.cancel: + return + } + } +} + +func (l *listenerProcess) processPacket(p packet) bool { + if l.err != nil { + if p.packetType == packetTypeCommit { + l.commitDoneChan <- l.err + return true + } + return false + } + + switch p.packetType { + case packetTypeStartBlock: + if l.listener.StartBlock != nil { + l.err = l.listener.StartBlock(p.data.(uint64)) + } + case packetTypeOnBlockHeader: + if l.listener.OnBlockHeader != nil { + l.err = l.listener.OnBlockHeader(p.data.(BlockHeaderData)) + } + case packetTypeOnTx: + if l.listener.OnTx != nil { + l.err = l.listener.OnTx(p.data.(TxData)) + } + case packetTypeOnEvent: + if l.listener.OnEvent != nil { + l.err = l.listener.OnEvent(p.data.(EventData)) + } + case packetTypeOnKVPair: + if l.listener.OnKVPair != nil { + l.err = l.listener.OnKVPair(p.data.(KVPairData)) + } + case packetTypeOnObjectUpdate: + if l.listener.OnObjectUpdate != nil { + l.err = l.listener.OnObjectUpdate(p.data.(ObjectUpdateData)) + } + case packetTypeCommit: + if l.listener.Commit != nil { + l.err = l.listener.Commit() + } + l.commitDoneChan <- l.err + default: + l.err = fmt.Errorf("unknown packet type: %d", p.packetType) + } + return false +} diff --git a/indexer/base/catch_up.go b/indexer/base/sync.go similarity index 56% rename from indexer/base/catch_up.go rename to indexer/base/sync.go index 432e644183a9..1a9618d34538 100644 --- a/indexer/base/catch_up.go +++ b/indexer/base/sync.go @@ -1,7 +1,7 @@ package indexerbase -// CatchUpSource is an interface that allows indexers to start indexing modules with pre-existing state. -type CatchUpSource interface { +// SyncSource is an interface that allows indexers to start indexing modules with pre-existing state. +type SyncSource interface { // IterateAllKVPairs iterates over all key-value pairs for a given module. IterateAllKVPairs(moduleName string, fn func(key, value []byte) error) error diff --git a/indexer/base/table.go b/indexer/base/table.go deleted file mode 100644 index 2d076f6d3eb8..000000000000 --- a/indexer/base/table.go +++ /dev/null @@ -1,20 +0,0 @@ -package indexerbase - -// Table represents a table in the schema of a module. -type Table struct { - // Name is the name of the table. - Name string - - // KeyColumns is a list of columns that make up the primary key of the table. - KeyColumns []Column - - // ValueColumns is a list of columns that are not part of the primary key of the table. - ValueColumns []Column - - // RetainDeletions is a flag that indicates whether the indexer should retain - // deleted rows in the database and flag them as deleted rather than actually - // deleting the row. For many types of data in state, the data is deleted even - // though it is still valid in order to save space. Indexers will want to have - // the option of retaining such data and distinguishing from other "true" deletions. - RetainDeletions bool -} From c87f02076544a750bd35672631a6846788699122 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Mon, 17 Jun 2024 19:30:28 -0400 Subject: [PATCH 08/29] updates --- indexer/base/README.md | 6 +- indexer/base/manager.go | 110 ++++++++++++++++++++++++++++++----- indexer/base/manager_test.go | 1 + 3 files changed, 101 insertions(+), 16 deletions(-) create mode 100644 indexer/base/manager_test.go diff --git a/indexer/base/README.md b/indexer/base/README.md index 356b99fcf3c3..dc03d818c094 100644 --- a/indexer/base/README.md +++ b/indexer/base/README.md @@ -11,9 +11,9 @@ The basic types for specifying index sources, targets and decoders are provided ```mermaid sequenceDiagram actor Source - participant Indexer - Source ->> Indexer: Initialize + participant Indexer Source -->> Indexer: InitializeModuleSchema + Source ->> Indexer: Initialize loop Block Source ->> Indexer: StartBlock Source ->> Indexer: OnBlockHeader @@ -25,7 +25,7 @@ sequenceDiagram end ``` -`Initialize` must be called before any other method and should only be invoked once. `InitializeModuleSchema` should be called at most once for every module with logical data. +`InitializeModuleSchema` should be called at most once for every module with logical data and all calls to should happen even before `Initialize` is called. After that `Initialize` MUST be called before any other method and should only be invoked once. Sources will generally only call `InitializeModuleSchema` and `OnObjectUpdate` if they have native logical decoding capabilities. Usually, the indexer framework will provide this functionality based on `OnKVPair` data and `IndexableModule` implementations. diff --git a/indexer/base/manager.go b/indexer/base/manager.go index 254b96b38355..37a8dbc6cb6f 100644 --- a/indexer/base/manager.go +++ b/indexer/base/manager.go @@ -1,5 +1,7 @@ package indexerbase +import "fmt" + type Manager struct { logger Logger decoderResolver DecoderResolver @@ -7,6 +9,7 @@ type Manager struct { needLogicalDecoding bool listener Listener listenerProcesses []*listenerProcess + initialized bool done chan struct{} } @@ -51,7 +54,7 @@ func NewManager(opts ManagerOptions) (*Manager, error) { done: done, } - err := mgr.init(opts.Listeners) + err := mgr.setup(opts.Listeners) if err != nil { return nil, err } @@ -61,9 +64,11 @@ func NewManager(opts ManagerOptions) (*Manager, error) { // Listener returns that listener that should be passed directly to the blockchain for managing // all indexing. -func (p *Manager) init(listeners []Listener) error { +func (p *Manager) setup(listeners []Listener) error { // check each subscribed listener to see if we actually need to register the listener + p.listener.Initialize = p.initialize + for _, listener := range listeners { if listener.StartBlock != nil { p.listener.StartBlock = p.startBlock @@ -117,14 +122,8 @@ func (p *Manager) init(listeners []Listener) error { } if p.needLogicalDecoding { - err := p.decoderResolver.Iterate(func(moduleName string, module ModuleDecoder) error { - p.decoders[moduleName] = module.KVDecoder - // TODO - return nil - }) - if err != nil { - return err - } + p.listener.InitializeModuleSchema = p.initializeModuleSchema + p.listener.OnObjectUpdate = p.onObjectUpdate } // initialize go routines for each listener @@ -135,14 +134,63 @@ func (p *Manager) init(listeners []Listener) error { commitDoneChan: make(chan error), } p.listenerProcesses = append(p.listenerProcesses, proc) + } + + return nil +} + +func (p *Manager) initialize(data InitializationData) (lastBlockPersisted int64, err error) { + if p.logger != nil { + p.logger.Debug("initialize") + } + + // setup logical decoding + if p.needLogicalDecoding { + err = p.decoderResolver.Iterate(func(moduleName string, module ModuleDecoder) error { + // if the schema was already initialized by the data source by InitializeModuleSchema, + // then this is an error + if _, ok := p.decoders[moduleName]; ok { + return fmt.Errorf("module schema for %s already initialized", moduleName) + } + + p.decoders[moduleName] = module.KVDecoder - // TODO initialize - // TODO initialize module schema + for _, proc := range p.listenerProcesses { + if proc.listener.InitializeModuleSchema == nil { + continue + } + err := proc.listener.InitializeModuleSchema(moduleName, module.Schema) + if err != nil { + return err + } + } + + return nil + }) + if err != nil { + return + } + } + // call initialize + for _, proc := range p.listenerProcesses { + if proc.listener.Initialize == nil { + continue + } + lastBlockPersisted, err = proc.listener.Initialize(data) + if err != nil { + return + } + } + + // start go routines + for _, proc := range p.listenerProcesses { go proc.run() } - return nil + p.initialized = true + + return } func (p *Manager) startBlock(height uint64) error { @@ -307,3 +355,39 @@ func (p *Manager) onKVPair(data KVPairData) error { return nil } + +func (p *Manager) initializeModuleSchema(module string, schema ModuleSchema) error { + if p.initialized { + return fmt.Errorf("cannot initialize module schema after initialization") + } + + for _, proc := range p.listenerProcesses { + // set the decoder for the module to so that we know that it is already initialized, + // but that also we are not handling decoding - in this case the data source + // should be doing the decoding and passing it to the manager in OnObjectUpdate + p.decoders[module] = nil + + if proc.listener.InitializeModuleSchema == nil { + continue + } + err := proc.listener.InitializeModuleSchema(module, schema) + if err != nil { + return err + } + } + + return nil +} + +func (p *Manager) onObjectUpdate(data ObjectUpdateData) error { + for _, proc := range p.listenerProcesses { + if proc.listener.OnObjectUpdate == nil { + continue + } + proc.packetChan <- packet{ + packetType: packetTypeOnObjectUpdate, + data: data, + } + } + return nil +} diff --git a/indexer/base/manager_test.go b/indexer/base/manager_test.go new file mode 100644 index 000000000000..ef34355a0709 --- /dev/null +++ b/indexer/base/manager_test.go @@ -0,0 +1 @@ +package indexerbase From 4c793d7a447a926a513fa196152b834f5fb0ee29 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Mon, 17 Jun 2024 20:19:46 -0400 Subject: [PATCH 09/29] docs --- indexer/base/README.md | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/indexer/base/README.md b/indexer/base/README.md index dc03d818c094..21b5dd2fd2a5 100644 --- a/indexer/base/README.md +++ b/indexer/base/README.md @@ -11,17 +11,27 @@ The basic types for specifying index sources, targets and decoders are provided ```mermaid sequenceDiagram actor Source + actor Manager participant Indexer - Source -->> Indexer: InitializeModuleSchema - Source ->> Indexer: Initialize + Source -->> Manager: InitializeModuleSchema + Manager ->> Indexer: InitializeModuleSchema + Source ->> Manager: Initialize + Manager ->> Indexer: Initialize loop Block - Source ->> Indexer: StartBlock - Source ->> Indexer: OnBlockHeader - Source -->> Indexer: OnTx - Source -->> Indexer: OnEvent - Source -->> Indexer: OnKVPair - Source -->> Indexer: OnObjectUpdate - Source ->> Indexer: Commit + Source ->> Manager: StartBlock + Manager ->> Indexer: StartBlock + Source -->> Manager: OnBlockHeader + Manager -->> Indexer: OnBlockHeader + Source -->> Manager: OnTx + Manager -->> Indexer: OnTx + Source -->> Manager: OnEvent + Manager -->> Indexer: OnEvent + Source -->> Manager: OnKVPair + Manager -->> Indexer: OnKVPair + Source -->> Manager: OnObjectUpdate + Manager -->> Indexer: OnObjectUpdate + Source ->> Manager: Commit + Manager ->> Indexer: Commit end ``` From 1088827651ab4758829501b91410c26d1261f2cf Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Wed, 3 Jul 2024 13:47:33 +0200 Subject: [PATCH 10/29] feat(schema/appdata)!: refactoring and packet support --- schema/appdata/data.go | 94 ++++++++++++++++++++++++++++++ schema/appdata/listener.go | 113 ++++++------------------------------- schema/appdata/packet.go | 61 ++++++++++++++++++++ schema/decoder.go | 25 ++++++-- 4 files changed, 191 insertions(+), 102 deletions(-) create mode 100644 schema/appdata/data.go create mode 100644 schema/appdata/packet.go diff --git a/schema/appdata/data.go b/schema/appdata/data.go new file mode 100644 index 000000000000..10c58a42fa20 --- /dev/null +++ b/schema/appdata/data.go @@ -0,0 +1,94 @@ +package appdata + +import ( + "encoding/json" + + "cosmossdk.io/schema" +) + +// ModuleInitializationData represents data for related to module initialization, in particular +// the module's schema. +type ModuleInitializationData struct { + // ModuleName is the name of the module. + ModuleName string + + // Schema is the schema of the module. + Schema schema.ModuleSchema +} + +// StartBlockData represents the data that is passed to a listener when a block is started. +type StartBlockData struct { + // Height is the height of the block. + Height uint64 + + // Bytes is the raw byte representation of the block header. It may be nil if the source does not provide it. + HeaderBytes ToBytes + + // JSON is the JSON representation of the block header. It should generally be a JSON object. + // It may be nil if the source does not provide it. + HeaderJSON ToJSON +} + +// TxData represents the raw transaction data that is passed to a listener. +type TxData struct { + // TxIndex is the index of the transaction in the block. + TxIndex int32 + + // Bytes is the raw byte representation of the transaction. + Bytes ToBytes + + // JSON is the JSON representation of the transaction. It should generally be a JSON object. + JSON ToJSON +} + +// EventData represents event data that is passed to a listener. +type EventData struct { + // TxIndex is the index of the transaction in the block to which this event is associated. + // It should be set to a negative number if the event is not associated with a transaction. + // Canonically -1 should be used to represent begin block processing and -2 should be used to + // represent end block processing. + TxIndex int32 + + // MsgIndex is the index of the message in the transaction to which this event is associated. + // If TxIndex is negative, this index could correspond to the index of the message in + // begin or end block processing if such indexes exist, or it can be set to zero. + MsgIndex uint32 + + // EventIndex is the index of the event in the message to which this event is associated. + EventIndex uint32 + + // Type is the type of the event. + Type string + + // Data is the JSON representation of the event data. It should generally be a JSON object. + Data ToJSON +} + +// ToBytes is a function that lazily returns the raw byte representation of data. +type ToBytes = func() ([]byte, error) + +// ToJSON is a function that lazily returns the JSON representation of data. +type ToJSON = func() (json.RawMessage, error) + +type KVPairData struct { + Updates []ModuleKVPairUpdate +} + +type ModuleKVPairUpdate struct { + // ModuleName is the name of the module that the key-value pair belongs to. + ModuleName string + + Update schema.KVPairUpdate +} + +// ObjectUpdateData represents object update data that is passed to a listener. +type ObjectUpdateData struct { + // ModuleName is the name of the module that the update corresponds to. + ModuleName string + + // Updates are the object updates. + Updates []schema.ObjectUpdate +} + +// CommitData represents commit data +type CommitData struct{} diff --git a/schema/appdata/listener.go b/schema/appdata/listener.go index e0868ae0c474..d4786cb02564 100644 --- a/schema/appdata/listener.go +++ b/schema/appdata/listener.go @@ -1,30 +1,22 @@ package appdata -import ( - "encoding/json" - - "cosmossdk.io/schema" -) - // Listener is an interface that defines methods for listening to both raw and logical blockchain data. // It is valid for any of the methods to be nil, in which case the listener will not be called for that event. // Listeners should understand the guarantees that are provided by the source they are listening to and // understand which methods will or will not be called. For instance, most blockchains will not do logical -// decoding of data out of the box, so the InitializeModuleSchema and OnObjectUpdate methods will not be called. +// decoding of data out of the box, so the InitializeModuleData and OnObjectUpdate methods will not be called. // These methods will only be called when listening logical decoding is setup. type Listener struct { - // Initialize is called when the listener is initialized before any other methods are called. - // The lastBlockPersisted return value should be the last block height the listener persisted if it is - // persisting block data, 0 if it is not interested in persisting block data, or -1 if it is - // persisting block data but has not persisted any data yet. This check allows the indexer - // framework to ensure that the listener has not missed blocks. - Initialize func(InitializationData) (lastBlockPersisted int64, err error) + // InitializeModuleData should be called whenever the blockchain process starts OR whenever + // logical decoding of a module is initiated. An indexer listening to this event + // should ensure that they have performed whatever initialization steps (such as database + // migrations) required to receive OnObjectUpdate events for the given module. If the + // indexer's schema is incompatible with the module's on-chain schema, the listener should return + // an error. Module names must conform to the NameFormat regular expression. + InitializeModuleData func(ModuleInitializationData) error // StartBlock is called at the beginning of processing a block. - StartBlock func(uint64) error - - // OnBlockHeader is called when a block header is received. - OnBlockHeader func(BlockHeaderData) error + StartBlock func(StartBlockData) error // OnTx is called when a transaction is received. OnTx func(TxData) error @@ -34,89 +26,16 @@ type Listener struct { // OnKVPair is called when a key-value has been written to the store for a given module. // Module names must conform to the NameFormat regular expression. - OnKVPair func(moduleName string, key, value []byte, delete bool) error - - // Commit is called when state is committed, usually at the end of a block. Any - // indexers should commit their data when this is called and return an error if - // they are unable to commit. - Commit func() error - - // InitializeModuleSchema should be called whenever the blockchain process starts OR whenever - // logical decoding of a module is initiated. An indexer listening to this event - // should ensure that they have performed whatever initialization steps (such as database - // migrations) required to receive OnObjectUpdate events for the given module. If the - // indexer's schema is incompatible with the module's on-chain schema, the listener should return - // an error. Module names must conform to the NameFormat regular expression. - InitializeModuleSchema func(moduleName string, moduleSchema schema.ModuleSchema) error + OnKVPair func(updates KVPairData) error // OnObjectUpdate is called whenever an object is updated in a module's state. This is only called // when logical data is available. It should be assumed that the same data in raw form // is also passed to OnKVPair. Module names must conform to the NameFormat regular expression. - OnObjectUpdate func(moduleName string, update schema.ObjectUpdate) error -} + OnObjectUpdate func(ObjectUpdateData) error -// InitializationData represents initialization data that is passed to a listener. -type InitializationData struct { - // HasEventAlignedWrites indicates that the blockchain data source will emit KV-pair events - // in an order aligned with transaction, message and event callbacks. If this is true - // then indexers can assume that KV-pair data is associated with these specific transactions, messages - // and events. This may be useful for indexers which store a log of all operations (such as immutable - // or version controlled databases) so that the history log can include fine grain correlation between - // state updates and transactions, messages and events. If this value is false, then indexers should - // assume that KV-pair data occurs out of order with respect to transaction, message and event callbacks - - // the only safe assumption being that KV-pair data is associated with the block in which it was emitted. - HasEventAlignedWrites bool -} - -// BlockHeaderData represents the raw block header data that is passed to a listener. -type BlockHeaderData struct { - // Height is the height of the block. - Height uint64 - - // Bytes is the raw byte representation of the block header. - Bytes ToBytes - - // JSON is the JSON representation of the block header. It should generally be a JSON object. - JSON ToJSON -} - -// TxData represents the raw transaction data that is passed to a listener. -type TxData struct { - // TxIndex is the index of the transaction in the block. - TxIndex int32 - - // Bytes is the raw byte representation of the transaction. - Bytes ToBytes - - // JSON is the JSON representation of the transaction. It should generally be a JSON object. - JSON ToJSON -} - -// EventData represents event data that is passed to a listener. -type EventData struct { - // TxIndex is the index of the transaction in the block to which this event is associated. - // It should be set to a negative number if the event is not associated with a transaction. - // Canonically -1 should be used to represent begin block processing and -2 should be used to - // represent end block processing. - TxIndex int32 - - // MsgIndex is the index of the message in the transaction to which this event is associated. - // If TxIndex is negative, this index could correspond to the index of the message in - // begin or end block processing if such indexes exist, or it can be set to zero. - MsgIndex uint32 - - // EventIndex is the index of the event in the message to which this event is associated. - EventIndex uint32 - - // Type is the type of the event. - Type string - - // Data is the JSON representation of the event data. It should generally be a JSON object. - Data ToJSON + // Commit is called when state is committed, usually at the end of a block. Any + // indexers should commit their data when this is called and return an error if + // they are unable to commit. Data sources MUST call Commit when data is committed, + // otherwise it should be assumed that indexers have not persisted their state. + Commit func(CommitData) error } - -// ToBytes is a function that lazily returns the raw byte representation of data. -type ToBytes = func() ([]byte, error) - -// ToJSON is a function that lazily returns the JSON representation of data. -type ToJSON = func() (json.RawMessage, error) diff --git a/schema/appdata/packet.go b/schema/appdata/packet.go new file mode 100644 index 000000000000..e5fe6be966b7 --- /dev/null +++ b/schema/appdata/packet.go @@ -0,0 +1,61 @@ +package appdata + +// Packet is the interface that all listener data structures implement so that this data can be "packetized" +// and processed in a stream, possibly asynchronously. +type Packet interface { + apply(*Listener) error +} + +// SendPacket sends a packet to a listener invoking the appropriate callback for this packet if one is registered. +func (l Listener) SendPacket(p Packet) error { + return p.apply(&l) +} + +func (m ModuleInitializationData) apply(l *Listener) error { + if l.InitializeModuleData == nil { + return nil + } + return l.InitializeModuleData(m) +} + +func (b StartBlockData) apply(l *Listener) error { + if l.StartBlock == nil { + return nil + } + return l.StartBlock(b) +} + +func (t TxData) apply(l *Listener) error { + if l.OnTx == nil { + return nil + } + return l.OnTx(t) +} + +func (e EventData) apply(l *Listener) error { + if l.OnEvent == nil { + return nil + } + return l.OnEvent(e) +} + +func (k KVPairData) apply(l *Listener) error { + if l.OnKVPair == nil { + return nil + } + return l.OnKVPair(k) +} + +func (o ObjectUpdateData) apply(l *Listener) error { + if l.OnObjectUpdate == nil { + return nil + } + return l.OnObjectUpdate(o) +} + +func (c CommitData) apply(l *Listener) error { + if l.Commit == nil { + return nil + } + return l.Commit(c) +} diff --git a/schema/decoder.go b/schema/decoder.go index 1d228d1c06ae..86aedec9f9dc 100644 --- a/schema/decoder.go +++ b/schema/decoder.go @@ -18,8 +18,23 @@ type ModuleCodec struct { KVDecoder KVDecoder } -// KVDecoder is a function that decodes a key-value pair into an ObjectUpdate. -// If the KV-pair doesn't represent an object update, the function should return false -// as the second return value. Error should only be non-nil when the decoder expected -// to parse a valid update and was unable to. -type KVDecoder = func(key, value []byte) (ObjectUpdate, bool, error) +// KVDecoder is a function that decodes a key-value pair into one or more ObjectUpdate's. +// If the KV-pair doesn't represent object updates, the function should return nil as the first +// and no error. The error result should only be non-nil when the decoder expected +// to parse a valid update and was unable to. In the case of an error, the decoder may return +// a non-nil value for the first return value, which can indicate which parts of the update +// were decodable to aid debugging. +type KVDecoder = func(KVPairUpdate) ([]ObjectUpdate, error) + +// KVPairUpdate represents a key-value pair set or delete. +type KVPairUpdate struct { + // Key is the key of the key-value pair. + Key []byte + + // Value is the value of the key-value pair. It should be ignored when Delete is true. + Value []byte + + // Delete is a flag that indicates that the key-value pair was deleted. If it is false, + // then it is assumed that this has been a set operation. + Delete bool +} From 812c6ee4178933697055996dec685df887fed08a Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Wed, 3 Jul 2024 13:53:44 +0200 Subject: [PATCH 11/29] docs --- schema/appdata/data.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/schema/appdata/data.go b/schema/appdata/data.go index 10c58a42fa20..7e02fbc5db8f 100644 --- a/schema/appdata/data.go +++ b/schema/appdata/data.go @@ -70,14 +70,17 @@ type ToBytes = func() ([]byte, error) // ToJSON is a function that lazily returns the JSON representation of data. type ToJSON = func() (json.RawMessage, error) +// KVPairData represents a batch of key-value pair data that is passed to a listener. type KVPairData struct { Updates []ModuleKVPairUpdate } +// ModuleKVPairUpdate represents a key-value pair update for a specific module. type ModuleKVPairUpdate struct { // ModuleName is the name of the module that the key-value pair belongs to. ModuleName string + // Update is the key-value pair update. Update schema.KVPairUpdate } @@ -90,5 +93,5 @@ type ObjectUpdateData struct { Updates []schema.ObjectUpdate } -// CommitData represents commit data +// CommitData represents commit data. It is empty for now, but fields could be added later. type CommitData struct{} From bca3bb0eaf2e5510d1ee52b679f040d2165b3cd3 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Wed, 3 Jul 2024 22:41:15 +0200 Subject: [PATCH 12/29] feat(schema/appdata): async listener mux'ing --- schema/appdata/async.go | 128 +++++++++++++++++++++++++++++++ schema/appdata/async_test.go | 1 + schema/appdata/mux.go | 121 +++++++++++++++++++++++++++++ schema/appdata/mux_test.go | 142 +++++++++++++++++++++++++++++++++++ 4 files changed, 392 insertions(+) create mode 100644 schema/appdata/async.go create mode 100644 schema/appdata/async_test.go create mode 100644 schema/appdata/mux.go create mode 100644 schema/appdata/mux_test.go diff --git a/schema/appdata/async.go b/schema/appdata/async.go new file mode 100644 index 000000000000..ae98f8b7c04e --- /dev/null +++ b/schema/appdata/async.go @@ -0,0 +1,128 @@ +package appdata + +// AsyncListenerMux returns a listener that forwards received events to all the provided listeners asynchronously +// with each listener processing in a separate go routine. All callbacks in the returned listener will return nil +// except for Commit which will return an error or nil once all listeners have processed the commit. The doneChan +// is used to signal that the listeners should stop listening and return. bufferSize is the size of the buffer for the +// channels used to send events to the listeners. +func AsyncListenerMux(listeners []Listener, bufferSize int, doneChan <-chan struct{}) Listener { + asyncListeners := make([]Listener, len(listeners)) + commitChans := make([]chan error, len(listeners)) + for i, l := range listeners { + commitChan := make(chan error) + commitChans[i] = commitChan + asyncListeners[i] = AsyncListener(l, bufferSize, commitChan, doneChan) + } + mux := ListenerMux(asyncListeners...) + muxCommit := mux.Commit + mux.Commit = func(data CommitData) error { + err := muxCommit(data) + if err != nil { + return err + } + + for _, commitChan := range commitChans { + err := <-commitChan + if err != nil { + return err + } + } + return nil + } + + return mux +} + +// AsyncListener returns a listener that forwards received events to the provided listener listening in asynchronously +// in a separate go routine. The listener that is returned will return nil for all methods including Commit and +// an error or nil will only be returned in commitChan once the sender has sent commit and the receiving listener has +// processed it. Thus commitChan can be used as a synchronization and error checking mechanism. The go routine +// that is being used for listening will exit when doneChan is closed and no more events will be received by the listener. +// bufferSize is the size of the buffer for the channel that is used to send events to the listener. +// Instead of using AsyncListener directly, it is recommended to use AsyncListenerMux which does coordination directly +// via its Commit callback. +func AsyncListener(listener Listener, bufferSize int, commitChan chan<- error, doneChan <-chan struct{}) Listener { + packetChan := make(chan Packet, bufferSize) + res := Listener{} + + go func() { + var err error + for { + select { + case packet := <-packetChan: + if err != nil { + // if we have an error, don't process any more packets + // and return the error and finish when it's time to commit + if _, ok := packet.(CommitData); ok { + commitChan <- err + return + } + } else { + // process the packet + err = listener.SendPacket(packet) + // if it's a commit + if _, ok := packet.(CommitData); ok { + commitChan <- err + if err != nil { + return + } + } + } + + case <-doneChan: + return + } + } + }() + + if listener.InitializeModuleData != nil { + res.InitializeModuleData = func(data ModuleInitializationData) error { + packetChan <- data + return nil + } + } + + if listener.StartBlock != nil { + res.StartBlock = func(data StartBlockData) error { + packetChan <- data + return nil + } + } + + if listener.OnTx != nil { + res.OnTx = func(data TxData) error { + packetChan <- data + return nil + } + } + + if listener.OnEvent != nil { + res.OnEvent = func(data EventData) error { + packetChan <- data + return nil + } + } + + if listener.OnKVPair != nil { + res.OnKVPair = func(data KVPairData) error { + packetChan <- data + return nil + } + } + + if listener.OnObjectUpdate != nil { + res.OnObjectUpdate = func(data ObjectUpdateData) error { + packetChan <- data + return nil + } + } + + if listener.Commit != nil { + res.Commit = func(data CommitData) error { + packetChan <- data + return nil + } + } + + return res +} diff --git a/schema/appdata/async_test.go b/schema/appdata/async_test.go new file mode 100644 index 000000000000..5d8a93960585 --- /dev/null +++ b/schema/appdata/async_test.go @@ -0,0 +1 @@ +package appdata diff --git a/schema/appdata/mux.go b/schema/appdata/mux.go new file mode 100644 index 000000000000..cc7b662b9128 --- /dev/null +++ b/schema/appdata/mux.go @@ -0,0 +1,121 @@ +package appdata + +// ListenerMux returns a listener that forwards received events to all the provided listeners in order. +// A callback is only registered if a non-nil callback is present in at least one of the listeners. +func ListenerMux(listeners ...Listener) Listener { + mux := Listener{} + + for _, l := range listeners { + if l.InitializeModuleData != nil { + mux.InitializeModuleData = func(data ModuleInitializationData) error { + for _, l := range listeners { + if l.InitializeModuleData != nil { + if err := l.InitializeModuleData(data); err != nil { + return err + } + } + } + return nil + } + break + } + } + + for _, l := range listeners { + if l.StartBlock != nil { + mux.StartBlock = func(data StartBlockData) error { + for _, l := range listeners { + if l.StartBlock != nil { + if err := l.StartBlock(data); err != nil { + return err + } + } + } + return nil + } + break + } + } + + for _, l := range listeners { + if l.OnTx != nil { + mux.OnTx = func(data TxData) error { + for _, l := range listeners { + if l.OnTx != nil { + if err := l.OnTx(data); err != nil { + return err + } + } + } + return nil + } + break + } + } + + for _, listener := range listeners { + if listener.OnEvent != nil { + mux.OnEvent = func(data EventData) error { + for _, l := range listeners { + if l.OnEvent != nil { + if err := l.OnEvent(data); err != nil { + return err + } + } + } + return nil + } + break + } + } + + for _, listener := range listeners { + if listener.OnKVPair != nil { + mux.OnKVPair = func(data KVPairData) error { + for _, l := range listeners { + if l.OnKVPair != nil { + if err := l.OnKVPair(data); err != nil { + return err + } + } + } + return nil + } + break + } + } + + for _, listener := range listeners { + if listener.OnObjectUpdate != nil { + mux.OnObjectUpdate = func(data ObjectUpdateData) error { + for _, l := range listeners { + if l.OnObjectUpdate != nil { + if err := l.OnObjectUpdate(data); err != nil { + return err + } + } + } + return nil + } + break + } + } + + for _, listener := range listeners { + if listener.Commit != nil { + mux.Commit = func(data CommitData) error { + for _, l := range listeners { + if l.Commit != nil { + if err := l.Commit(data); err != nil { + return err + } + } + } + return nil + } + break + } + } + + return mux +} diff --git a/schema/appdata/mux_test.go b/schema/appdata/mux_test.go new file mode 100644 index 000000000000..01238629e304 --- /dev/null +++ b/schema/appdata/mux_test.go @@ -0,0 +1,142 @@ +package appdata + +import "testing" + +func TestListenerMux(t *testing.T) { + t.Run("empty", func(t *testing.T) { + res := ListenerMux(Listener{}, Listener{}) + if res.InitializeModuleData != nil { + t.Error("expected nil") + } + if res.StartBlock != nil { + t.Error("expected nil") + } + if res.OnTx != nil { + t.Error("expected nil") + } + if res.OnEvent != nil { + t.Error("expected nil") + } + if res.OnKVPair != nil { + t.Error("expected nil") + } + if res.OnObjectUpdate != nil { + t.Error("expected nil") + } + if res.Commit != nil { + t.Error("expected nil") + } + }) + + t.Run("all called once", func(t *testing.T) { + var calls []string + res := ListenerMux(Listener{ + InitializeModuleData: func(ModuleInitializationData) error { + calls = append(calls, "InitializeModuleData 1") + return nil + }, + StartBlock: func(StartBlockData) error { + calls = append(calls, "StartBlock 1") + return nil + }, + OnTx: func(TxData) error { + calls = append(calls, "OnTx 1") + return nil + }, + OnEvent: func(EventData) error { + calls = append(calls, "OnEvent 1") + return nil + }, + OnKVPair: func(KVPairData) error { + calls = append(calls, "OnKVPair 1") + return nil + }, + OnObjectUpdate: func(ObjectUpdateData) error { + calls = append(calls, "OnObjectUpdate 1") + return nil + }, + Commit: func(CommitData) error { + calls = append(calls, "Commit 1") + return nil + }, + }, Listener{ + InitializeModuleData: func(ModuleInitializationData) error { + calls = append(calls, "InitializeModuleData 2") + return nil + }, + StartBlock: func(StartBlockData) error { + calls = append(calls, "StartBlock 2") + return nil + }, + OnTx: func(TxData) error { + calls = append(calls, "OnTx 2") + return nil + }, + OnEvent: func(EventData) error { + calls = append(calls, "OnEvent 2") + return nil + }, + OnKVPair: func(KVPairData) error { + calls = append(calls, "OnKVPair 2") + return nil + }, + OnObjectUpdate: func(ObjectUpdateData) error { + calls = append(calls, "OnObjectUpdate 2") + return nil + }, + Commit: func(CommitData) error { + calls = append(calls, "Commit 2") + return nil + }, + }) + + if err := res.InitializeModuleData(ModuleInitializationData{}); err != nil { + t.Error(err) + } + if err := res.StartBlock(StartBlockData{}); err != nil { + t.Error(err) + } + if err := res.OnTx(TxData{}); err != nil { + t.Error(err) + } + if err := res.OnEvent(EventData{}); err != nil { + t.Error(err) + } + if err := res.OnKVPair(KVPairData{}); err != nil { + t.Error(err) + } + if err := res.OnObjectUpdate(ObjectUpdateData{}); err != nil { + t.Error(err) + } + if err := res.Commit(CommitData{}); err != nil { + t.Error(err) + } + + expected := []string{ + "InitializeModuleData 1", + "InitializeModuleData 2", + "StartBlock 1", + "StartBlock 2", + "OnTx 1", + "OnTx 2", + "OnEvent 1", + "OnEvent 2", + "OnKVPair 1", + "OnKVPair 2", + "OnObjectUpdate 1", + "OnObjectUpdate 2", + "Commit 1", + "Commit 2", + } + + if len(calls) != len(expected) { + t.Fatalf("expected %d calls, got %d", len(expected), len(calls)) + } + + for i := range calls { + if calls[i] != expected[i] { + t.Errorf("expected %q, got %q", expected[i], calls[i]) + } + } + }) +} From 26734fb615a7a5998da27feb5a4e3e1aa4efe8fe Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Wed, 3 Jul 2024 23:00:33 +0200 Subject: [PATCH 13/29] tests --- schema/appdata/async_test.go | 75 ++++++++++++++++ schema/appdata/mux_test.go | 169 ++++++++++++++++------------------- 2 files changed, 153 insertions(+), 91 deletions(-) diff --git a/schema/appdata/async_test.go b/schema/appdata/async_test.go index 5d8a93960585..1fb52a38233e 100644 --- a/schema/appdata/async_test.go +++ b/schema/appdata/async_test.go @@ -1 +1,76 @@ package appdata + +import ( + "fmt" + "testing" +) + +func TestAsyncListenerMux(t *testing.T) { + +} + +func TestAsyncListener(t *testing.T) { + t.Run("call done", func(t *testing.T) { + commitChan := make(chan error) + doneChan := make(chan struct{}) + var calls []string + listener := callCollector(1, func(name string, i int, _ Packet) { + calls = append(calls, fmt.Sprintf("%s %d", name, i)) + }) + res := AsyncListener(listener, 16, commitChan, doneChan) + + callAllCallbacksOnces(t, res) + + err := <-commitChan + if err != nil { + t.Fatalf("expected nil, got %v", err) + } + + checkExpectedCallOrder(t, calls, []string{ + "InitializeModuleData 1", + "StartBlock 1", + "OnTx 1", + "OnEvent 1", + "OnKVPair 1", + "OnObjectUpdate 1", + "Commit 1", + }) + + calls = nil + + doneChan <- struct{}{} + + callAllCallbacksOnces(t, res) + + checkExpectedCallOrder(t, calls, nil) + }) + + t.Run("error", func(t *testing.T) { + commitChan := make(chan error) + doneChan := make(chan struct{}) + var calls []string + listener := callCollector(1, func(name string, i int, _ Packet) { + calls = append(calls, fmt.Sprintf("%s %d", name, i)) + }) + + listener.OnKVPair = func(updates KVPairData) error { + return fmt.Errorf("error") + } + + res := AsyncListener(listener, 16, commitChan, doneChan) + + callAllCallbacksOnces(t, res) + + err := <-commitChan + if err == nil || err.Error() != "error" { + t.Fatalf("expected error, got %v", err) + } + + checkExpectedCallOrder(t, calls, []string{ + "InitializeModuleData 1", + "StartBlock 1", + "OnTx 1", + "OnEvent 1", + }) + }) +} diff --git a/schema/appdata/mux_test.go b/schema/appdata/mux_test.go index 01238629e304..20ed9644d85d 100644 --- a/schema/appdata/mux_test.go +++ b/schema/appdata/mux_test.go @@ -1,6 +1,8 @@ package appdata -import "testing" +import ( + "testing" +) func TestListenerMux(t *testing.T) { t.Run("empty", func(t *testing.T) { @@ -30,89 +32,15 @@ func TestListenerMux(t *testing.T) { t.Run("all called once", func(t *testing.T) { var calls []string - res := ListenerMux(Listener{ - InitializeModuleData: func(ModuleInitializationData) error { - calls = append(calls, "InitializeModuleData 1") - return nil - }, - StartBlock: func(StartBlockData) error { - calls = append(calls, "StartBlock 1") - return nil - }, - OnTx: func(TxData) error { - calls = append(calls, "OnTx 1") - return nil - }, - OnEvent: func(EventData) error { - calls = append(calls, "OnEvent 1") - return nil - }, - OnKVPair: func(KVPairData) error { - calls = append(calls, "OnKVPair 1") - return nil - }, - OnObjectUpdate: func(ObjectUpdateData) error { - calls = append(calls, "OnObjectUpdate 1") - return nil - }, - Commit: func(CommitData) error { - calls = append(calls, "Commit 1") - return nil - }, - }, Listener{ - InitializeModuleData: func(ModuleInitializationData) error { - calls = append(calls, "InitializeModuleData 2") - return nil - }, - StartBlock: func(StartBlockData) error { - calls = append(calls, "StartBlock 2") - return nil - }, - OnTx: func(TxData) error { - calls = append(calls, "OnTx 2") - return nil - }, - OnEvent: func(EventData) error { - calls = append(calls, "OnEvent 2") - return nil - }, - OnKVPair: func(KVPairData) error { - calls = append(calls, "OnKVPair 2") - return nil - }, - OnObjectUpdate: func(ObjectUpdateData) error { - calls = append(calls, "OnObjectUpdate 2") - return nil - }, - Commit: func(CommitData) error { - calls = append(calls, "Commit 2") - return nil - }, - }) - - if err := res.InitializeModuleData(ModuleInitializationData{}); err != nil { - t.Error(err) - } - if err := res.StartBlock(StartBlockData{}); err != nil { - t.Error(err) - } - if err := res.OnTx(TxData{}); err != nil { - t.Error(err) - } - if err := res.OnEvent(EventData{}); err != nil { - t.Error(err) - } - if err := res.OnKVPair(KVPairData{}); err != nil { - t.Error(err) - } - if err := res.OnObjectUpdate(ObjectUpdateData{}); err != nil { - t.Error(err) - } - if err := res.Commit(CommitData{}); err != nil { - t.Error(err) + onCall := func(name string, i int, _ Packet) { + calls = append(calls, name) } - expected := []string{ + res := ListenerMux(callCollector(1, onCall), callCollector(2, onCall)) + + callAllCallbacksOnces(t, res) + + checkExpectedCallOrder(t, calls, []string{ "InitializeModuleData 1", "InitializeModuleData 2", "StartBlock 1", @@ -127,16 +55,75 @@ func TestListenerMux(t *testing.T) { "OnObjectUpdate 2", "Commit 1", "Commit 2", - } + }) + }) +} - if len(calls) != len(expected) { - t.Fatalf("expected %d calls, got %d", len(expected), len(calls)) - } +func callAllCallbacksOnces(t *testing.T, listener Listener) { + if err := listener.InitializeModuleData(ModuleInitializationData{}); err != nil { + t.Error(err) + } + if err := listener.StartBlock(StartBlockData{}); err != nil { + t.Error(err) + } + if err := listener.OnTx(TxData{}); err != nil { + t.Error(err) + } + if err := listener.OnEvent(EventData{}); err != nil { + t.Error(err) + } + if err := listener.OnKVPair(KVPairData{}); err != nil { + t.Error(err) + } + if err := listener.OnObjectUpdate(ObjectUpdateData{}); err != nil { + t.Error(err) + } + if err := listener.Commit(CommitData{}); err != nil { + t.Error(err) + } +} + +func callCollector(i int, onCall func(string, int, Packet)) Listener { + return Listener{ + InitializeModuleData: func(ModuleInitializationData) error { + onCall("InitializeModuleData", i, nil) + return nil + }, + StartBlock: func(StartBlockData) error { + onCall("StartBlock", i, nil) + return nil + }, + OnTx: func(TxData) error { + onCall("OnTx", i, nil) + return nil + }, + OnEvent: func(EventData) error { + onCall("OnEvent", i, nil) + return nil + }, + OnKVPair: func(KVPairData) error { + onCall("OnKVPair", i, nil) + return nil + }, + OnObjectUpdate: func(ObjectUpdateData) error { + onCall("OnObjectUpdate", i, nil) + return nil + }, + Commit: func(CommitData) error { + onCall("Commit", i, nil) + return nil + }, + } +} - for i := range calls { - if calls[i] != expected[i] { - t.Errorf("expected %q, got %q", expected[i], calls[i]) - } +func checkExpectedCallOrder(t *testing.T, actual, expected []string) { + if len(actual) != len(expected) { + t.Fatalf("expected %d calls, got %d", len(expected), len(actual)) + } + + for i := range actual { + if actual[i] != expected[i] { + t.Errorf("expected %q, got %q", expected[i], actual[i]) } - }) + } } From d5b55a97a4bb2d79dc587c06d7146064d28f4433 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Wed, 3 Jul 2024 23:14:43 +0200 Subject: [PATCH 14/29] testing WIP --- schema/appdata/async.go | 9 ++-- schema/appdata/async_test.go | 89 +++++++++++++++++++++++++++++------- schema/appdata/mux_test.go | 20 ++++---- 3 files changed, 89 insertions(+), 29 deletions(-) diff --git a/schema/appdata/async.go b/schema/appdata/async.go index ae98f8b7c04e..a8aeb02676cb 100644 --- a/schema/appdata/async.go +++ b/schema/appdata/async.go @@ -16,9 +16,11 @@ func AsyncListenerMux(listeners []Listener, bufferSize int, doneChan <-chan stru mux := ListenerMux(asyncListeners...) muxCommit := mux.Commit mux.Commit = func(data CommitData) error { - err := muxCommit(data) - if err != nil { - return err + if muxCommit != nil { + err := muxCommit(data) + if err != nil { + return err + } } for _, commitChan := range commitChans { @@ -70,6 +72,7 @@ func AsyncListener(listener Listener, bufferSize int, commitChan chan<- error, d } case <-doneChan: + close(packetChan) return } } diff --git a/schema/appdata/async_test.go b/schema/appdata/async_test.go index 1fb52a38233e..ff2e3918227b 100644 --- a/schema/appdata/async_test.go +++ b/schema/appdata/async_test.go @@ -6,7 +6,67 @@ import ( ) func TestAsyncListenerMux(t *testing.T) { + t.Run("empty", func(t *testing.T) { + listener := AsyncListenerMux([]Listener{{}, {}}, 16, make(chan struct{})) + if listener.InitializeModuleData != nil { + t.Error("expected nil") + } + if listener.StartBlock != nil { + t.Error("expected nil") + } + if listener.OnTx != nil { + t.Error("expected nil") + } + if listener.OnEvent != nil { + t.Error("expected nil") + } + if listener.OnKVPair != nil { + t.Error("expected nil") + } + if listener.OnObjectUpdate != nil { + t.Error("expected nil") + } + + // commit is not expected to be nil + }) + + t.Run("call done", func(t *testing.T) { + doneChan := make(chan struct{}) + var calls1, calls2 []string + listener1 := callCollector(1, func(name string, _ int, _ Packet) { + calls1 = append(calls1, name) + }) + listener2 := callCollector(2, func(name string, _ int, _ Packet) { + calls2 = append(calls2, name) + }) + res := AsyncListenerMux([]Listener{listener1, listener2}, 16, doneChan) + + callAllCallbacksOnces(t, res) + + expectedCalls := []string{ + "InitializeModuleData", + "StartBlock", + "OnTx", + "OnEvent", + "OnKVPair", + "OnObjectUpdate", + "Commit", + } + + checkExpectedCallOrder(t, calls1, expectedCalls) + checkExpectedCallOrder(t, calls2, expectedCalls) + + calls1 = nil + calls2 = nil + + doneChan <- struct{}{} + + callAllCallbacksOnces(t, res) + // + //checkExpectedCallOrder(t, calls1, nil) + //checkExpectedCallOrder(t, calls2, nil) + }) } func TestAsyncListener(t *testing.T) { @@ -14,8 +74,8 @@ func TestAsyncListener(t *testing.T) { commitChan := make(chan error) doneChan := make(chan struct{}) var calls []string - listener := callCollector(1, func(name string, i int, _ Packet) { - calls = append(calls, fmt.Sprintf("%s %d", name, i)) + listener := callCollector(1, func(name string, _ int, _ Packet) { + calls = append(calls, name) }) res := AsyncListener(listener, 16, commitChan, doneChan) @@ -27,13 +87,13 @@ func TestAsyncListener(t *testing.T) { } checkExpectedCallOrder(t, calls, []string{ - "InitializeModuleData 1", - "StartBlock 1", - "OnTx 1", - "OnEvent 1", - "OnKVPair 1", - "OnObjectUpdate 1", - "Commit 1", + "InitializeModuleData", + "StartBlock", + "OnTx", + "OnEvent", + "OnKVPair", + "OnObjectUpdate", + "Commit", }) calls = nil @@ -49,8 +109,8 @@ func TestAsyncListener(t *testing.T) { commitChan := make(chan error) doneChan := make(chan struct{}) var calls []string - listener := callCollector(1, func(name string, i int, _ Packet) { - calls = append(calls, fmt.Sprintf("%s %d", name, i)) + listener := callCollector(1, func(name string, _ int, _ Packet) { + calls = append(calls, name) }) listener.OnKVPair = func(updates KVPairData) error { @@ -66,11 +126,6 @@ func TestAsyncListener(t *testing.T) { t.Fatalf("expected error, got %v", err) } - checkExpectedCallOrder(t, calls, []string{ - "InitializeModuleData 1", - "StartBlock 1", - "OnTx 1", - "OnEvent 1", - }) + checkExpectedCallOrder(t, calls, []string{"InitializeModuleData", "StartBlock", "OnTx", "OnEvent"}) }) } diff --git a/schema/appdata/mux_test.go b/schema/appdata/mux_test.go index 20ed9644d85d..b5e3a95dd569 100644 --- a/schema/appdata/mux_test.go +++ b/schema/appdata/mux_test.go @@ -1,31 +1,33 @@ package appdata import ( + "fmt" "testing" ) func TestListenerMux(t *testing.T) { t.Run("empty", func(t *testing.T) { - res := ListenerMux(Listener{}, Listener{}) - if res.InitializeModuleData != nil { + listener := ListenerMux(Listener{}, Listener{}) + + if listener.InitializeModuleData != nil { t.Error("expected nil") } - if res.StartBlock != nil { + if listener.StartBlock != nil { t.Error("expected nil") } - if res.OnTx != nil { + if listener.OnTx != nil { t.Error("expected nil") } - if res.OnEvent != nil { + if listener.OnEvent != nil { t.Error("expected nil") } - if res.OnKVPair != nil { + if listener.OnKVPair != nil { t.Error("expected nil") } - if res.OnObjectUpdate != nil { + if listener.OnObjectUpdate != nil { t.Error("expected nil") } - if res.Commit != nil { + if listener.Commit != nil { t.Error("expected nil") } }) @@ -33,7 +35,7 @@ func TestListenerMux(t *testing.T) { t.Run("all called once", func(t *testing.T) { var calls []string onCall := func(name string, i int, _ Packet) { - calls = append(calls, name) + calls = append(calls, fmt.Sprintf("%s %d", name, i)) } res := ListenerMux(callCollector(1, onCall), callCollector(2, onCall)) From ac04ca0707d660fb2d039cae88c61df283f88c59 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Thu, 4 Jul 2024 12:27:11 +0200 Subject: [PATCH 15/29] add module filter --- schema/appdata/filter.go | 42 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 schema/appdata/filter.go diff --git a/schema/appdata/filter.go b/schema/appdata/filter.go new file mode 100644 index 000000000000..7c5f47c813c5 --- /dev/null +++ b/schema/appdata/filter.go @@ -0,0 +1,42 @@ +package appdata + +// ModuleFilter returns an updated listener that filters state updates based on the module name. +func ModuleFilter(listener Listener, filter func(moduleName string) bool) Listener { + if initModData := listener.InitializeModuleData; initModData != nil { + listener.InitializeModuleData = func(data ModuleInitializationData) error { + if !filter(data.ModuleName) { + return nil + } + + return initModData(data) + } + } + + if onKVPair := listener.OnKVPair; onKVPair != nil { + listener.OnKVPair = func(data KVPairData) error { + for _, update := range data.Updates { + if !filter(update.ModuleName) { + continue + } + + if err := onKVPair(KVPairData{Updates: []ModuleKVPairUpdate{update}}); err != nil { + return err + } + } + + return nil + } + } + + if onObjectUpdate := listener.OnObjectUpdate; onObjectUpdate != nil { + listener.OnObjectUpdate = func(data ObjectUpdateData) error { + if !filter(data.ModuleName) { + return nil + } + + return onObjectUpdate(data) + } + } + + return listener +} From 09b0c97424c51f0598066680584d9a8e9030646b Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Thu, 4 Jul 2024 13:31:32 +0200 Subject: [PATCH 16/29] feat(schema): decoding middleware --- schema/decoding/middleware.go | 106 ++++++++++++++++++++++++++++++++++ schema/decoding/resolver.go | 71 +++++++++++++++++++++++ schema/decoding/sync.go | 63 ++++++++++++++++++++ 3 files changed, 240 insertions(+) create mode 100644 schema/decoding/middleware.go create mode 100644 schema/decoding/resolver.go create mode 100644 schema/decoding/sync.go diff --git a/schema/decoding/middleware.go b/schema/decoding/middleware.go new file mode 100644 index 000000000000..57c0783c6281 --- /dev/null +++ b/schema/decoding/middleware.go @@ -0,0 +1,106 @@ +package decoding + +import ( + "cosmossdk.io/schema" + "cosmossdk.io/schema/appdata" +) + +type MiddlewareOptions struct { + ModuleFilter func(moduleName string) bool +} + +// Middleware decodes raw data passed to the listener as kv-updates into decoded object updates. Module initialization +// is done lazily as modules are encountered in the kv-update stream. +func Middleware(target appdata.Listener, resolver DecoderResolver, opts MiddlewareOptions) (appdata.Listener, error) { + initializeModuleData := target.InitializeModuleData + onObjectUpdate := target.OnObjectUpdate + + // no-op if not listening to decoded data + if initializeModuleData == nil && onObjectUpdate == nil { + return target, nil + } + + onKVPair := target.OnKVPair + + moduleCodecs := map[string]*schema.ModuleCodec{} + + target.OnKVPair = func(data appdata.KVPairData) error { + // first forward kv pair updates + if onKVPair != nil { + err := onKVPair(data) + if err != nil { + return err + } + } + + for _, kvUpdate := range data.Updates { + // look for an existing codec + pcdc, ok := moduleCodecs[kvUpdate.ModuleName] + if !ok { + if opts.ModuleFilter != nil && !opts.ModuleFilter(kvUpdate.ModuleName) { + // we don't care about this module so store nil and continue + moduleCodecs[kvUpdate.ModuleName] = nil + continue + } + + // look for a new codec + cdc, found, err := resolver.LookupDecoder(kvUpdate.ModuleName) + if err != nil { + return err + } + + if !found { + // store nil to indicate we've seen this module and don't have a codec + // and keep processing the kv updates + moduleCodecs[kvUpdate.ModuleName] = nil + continue + } + + pcdc = &cdc + moduleCodecs[kvUpdate.ModuleName] = pcdc + + if initializeModuleData != nil { + err = initializeModuleData(appdata.ModuleInitializationData{ + ModuleName: kvUpdate.ModuleName, + Schema: cdc.Schema, + }) + if err != nil { + return err + } + } + } + + if pcdc == nil { + // we've already seen this module and can't decode + continue + } + + if onObjectUpdate == nil || pcdc.KVDecoder == nil { + // not listening to updates or can't decode so continue + continue + } + + updates, err := pcdc.KVDecoder(kvUpdate.Update) + if err != nil { + return err + } + + if len(updates) == 0 { + // no updates + continue + } + + err = target.OnObjectUpdate(appdata.ObjectUpdateData{ + ModuleName: kvUpdate.ModuleName, + Updates: updates, + }) + if err != nil { + return err + } + } + + return nil + } + + return target, nil +} diff --git a/schema/decoding/resolver.go b/schema/decoding/resolver.go new file mode 100644 index 000000000000..a09eeafe82b9 --- /dev/null +++ b/schema/decoding/resolver.go @@ -0,0 +1,71 @@ +package decoding + +import ( + "sort" + + "cosmossdk.io/schema" +) + +type DecoderResolver interface { + // IterateAll iterates over all module decoders which should be initialized at startup. + IterateAll(func(moduleName string, cdc schema.ModuleCodec) error) error + + // LookupDecoder allows for resolving decoders dynamically. For instance, some module-like + // things may come into existence dynamically (like x/accounts or EVM or WASM contracts). + // The first time the manager sees one of these appearing in KV-store writes, it will + // lookup a decoder for it and cache it for future use. The manager will also perform + // a catch-up sync before passing any new writes to ensure that all historical state has + // been synced if there is any This check will only happen the first time a module is seen + // by the manager in a given process (a process restart will cause this check to happen again). + LookupDecoder(moduleName string) (decoder schema.ModuleCodec, found bool, err error) +} + +type moduleSetDecoderResolver struct { + moduleSet map[string]interface{} +} + +// ModuleSetDecoderResolver returns DecoderResolver that will discover modules implementing +// DecodeableModule in the provided module set. +func ModuleSetDecoderResolver(moduleSet map[string]interface{}) DecoderResolver { + return &moduleSetDecoderResolver{ + moduleSet: moduleSet, + } +} + +func (a moduleSetDecoderResolver) IterateAll(f func(string, schema.ModuleCodec) error) error { + keys := make([]string, 0, len(a.moduleSet)) + for k := range a.moduleSet { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + module := a.moduleSet[k] + dm, ok := module.(schema.HasModuleCodec) + if ok { + decoder, err := dm.ModuleCodec() + if err != nil { + return err + } + err = f(k, decoder) + if err != nil { + return err + } + } + } + return nil +} + +func (a moduleSetDecoderResolver) LookupDecoder(moduleName string) (schema.ModuleCodec, bool, error) { + mod, ok := a.moduleSet[moduleName] + if !ok { + return schema.ModuleCodec{}, false, nil + } + + dm, ok := mod.(schema.HasModuleCodec) + if !ok { + return schema.ModuleCodec{}, false, nil + } + + decoder, err := dm.ModuleCodec() + return decoder, true, err +} diff --git a/schema/decoding/sync.go b/schema/decoding/sync.go new file mode 100644 index 000000000000..a1af8fa24efd --- /dev/null +++ b/schema/decoding/sync.go @@ -0,0 +1,63 @@ +package decoding + +import ( + "cosmossdk.io/schema" + "cosmossdk.io/schema/appdata" +) + +// SyncSource is an interface that allows indexers to start indexing modules with pre-existing state. +type SyncSource interface { + + // IterateAllKVPairs iterates over all key-value pairs for a given module. + IterateAllKVPairs(moduleName string, fn func(key, value []byte) error) error +} + +// SyncOptions are the options for Sync. +type SyncOptions struct { + ModuleFilter func(moduleName string) bool +} + +// Sync synchronizes existing state from the sync source to the listener using the resolver to decode data. +func Sync(listener appdata.Listener, source SyncSource, resolver DecoderResolver, opts SyncOptions) error { + initializeModuleData := listener.InitializeModuleData + onObjectUpdate := listener.OnObjectUpdate + + // no-op if not listening to decoded data + if initializeModuleData == nil && onObjectUpdate == nil { + return nil + } + + return resolver.IterateAll(func(moduleName string, cdc schema.ModuleCodec) error { + if opts.ModuleFilter != nil && !opts.ModuleFilter(moduleName) { + // ignore this module + return nil + } + + if initializeModuleData != nil { + err := initializeModuleData(appdata.ModuleInitializationData{ + ModuleName: moduleName, + Schema: cdc.Schema, + }) + if err != nil { + return err + } + } + + if onObjectUpdate == nil || cdc.KVDecoder == nil { + return nil + } + + return source.IterateAllKVPairs(moduleName, func(key, value []byte) error { + updates, err := cdc.KVDecoder(schema.KVPairUpdate{Key: key, Value: value}) + if err != nil { + return err + } + + if len(updates) == 0 { + return nil + } + + return onObjectUpdate(appdata.ObjectUpdateData{ModuleName: moduleName, Updates: updates}) + }) + }) +} From 7be101eade7776cf4619818e94d3768ec40bcb1d Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Thu, 4 Jul 2024 21:37:07 +0200 Subject: [PATCH 17/29] WIP --- schema/indexing/indexer.go | 36 ++++++++++++++++++++ schema/indexing/manager.go | 70 ++++++++++++++++++++++++++++++++++++++ schema/logutil/logger.go | 35 +++++++++++++++++++ 3 files changed, 141 insertions(+) create mode 100644 schema/indexing/indexer.go create mode 100644 schema/indexing/manager.go create mode 100644 schema/logutil/logger.go diff --git a/schema/indexing/indexer.go b/schema/indexing/indexer.go new file mode 100644 index 000000000000..926392952153 --- /dev/null +++ b/schema/indexing/indexer.go @@ -0,0 +1,36 @@ +package indexing + +import ( + "context" + "fmt" + + "cosmossdk.io/schema/appdata" + "cosmossdk.io/schema/log" +) + +type Indexer interface { + Initialize(context.Context, InitializationData) (InitializationResult, error) +} + +type IndexerResources struct { + Logger log.Logger +} + +type IndexerFactory = func(options map[string]interface{}, resources IndexerResources) (Indexer, error) + +type InitializationData struct{} + +type InitializationResult struct { + Listener appdata.Listener + LastBlockPersisted int64 +} + +func RegisterIndexer(name string, factory IndexerFactory) { + if _, ok := indexerRegistry[name]; ok { + panic(fmt.Sprintf("indexer %s already registered", name)) + } + + indexerRegistry[name] = factory +} + +var indexerRegistry = map[string]IndexerFactory{} diff --git a/schema/indexing/manager.go b/schema/indexing/manager.go new file mode 100644 index 000000000000..c858fbdc4ee8 --- /dev/null +++ b/schema/indexing/manager.go @@ -0,0 +1,70 @@ +package indexing + +import ( + "context" + "fmt" + + "cosmossdk.io/schema/appdata" + "cosmossdk.io/schema/decoding" + "cosmossdk.io/schema/log" +) + +type Options struct { + Context context.Context + Options map[string]interface{} + Resolver decoding.DecoderResolver + SyncSource decoding.SyncSource + Logger log.Logger +} + +func Start(opts Options) (appdata.Listener, error) { + if opts.Logger == nil { + opts.Logger = log.NoopLogger{} + } + + opts.Logger.Info("Starting Indexer Manager") + + resources := IndexerResources{Logger: opts.Logger} + + var indexers []appdata.Listener + ctx := opts.Context + if ctx == nil { + ctx = context.Background() + } + + for indexerName, factory := range indexerRegistry { + indexerOpts, ok := opts.Options[indexerName] + if !ok { + continue + } + + if opts.Logger != nil { + opts.Logger.Info(fmt.Sprintf("Starting Indexer %s", indexerName), "options", indexerOpts) + } + + optsMap, ok := indexerOpts.(map[string]interface{}) + if !ok { + return appdata.Listener{}, fmt.Errorf("invalid indexer options type %T for %s, expected a map", indexerOpts, indexerName) + } + + indexer, err := factory(optsMap, resources) + if err != nil { + return appdata.Listener{}, fmt.Errorf("failed to create indexer %s: %w", indexerName, err) + } + + res, err := indexer.Initialize(ctx, InitializationData{}) + if err != nil { + return appdata.Listener{}, fmt.Errorf("failed to initialize indexer %s: %w", indexerName, err) + } + + indexers = append(indexers, res.Listener) + + // TODO handle last block persisted + } + + return decoding.Middleware(appdata.AsyncListenerMux(indexers, 1024, ctx.Done()), decoding.Options{ + DecoderResolver: opts.Resolver, + SyncSource: opts.SyncSource, + Logger: opts.Logger, + }) +} diff --git a/schema/logutil/logger.go b/schema/logutil/logger.go new file mode 100644 index 000000000000..cb6b34ebfd2b --- /dev/null +++ b/schema/logutil/logger.go @@ -0,0 +1,35 @@ +// Package logutil defines the Logger interface expected by indexer implementations. +// It is implemented by cosmossdk.io/log which is not imported to minimize dependencies. +package logutil + +// Logger is the logger interface expected by indexer implementations. +type Logger interface { + // Info takes a message and a set of key/value pairs and logs with level INFO. + // The key of the tuple must be a string. + Info(msg string, keyVals ...interface{}) + + // Warn takes a message and a set of key/value pairs and logs with level WARN. + // The key of the tuple must be a string. + Warn(msg string, keyVals ...interface{}) + + // Error takes a message and a set of key/value pairs and logs with level ERR. + // The key of the tuple must be a string. + Error(msg string, keyVals ...interface{}) + + // Debug takes a message and a set of key/value pairs and logs with level DEBUG. + // The key of the tuple must be a string. + Debug(msg string, keyVals ...interface{}) +} + +// NoopLogger is a logger that doesn't do anything. +type NoopLogger struct{} + +func (n NoopLogger) Info(string, ...interface{}) {} + +func (n NoopLogger) Warn(string, ...interface{}) {} + +func (n NoopLogger) Error(string, ...interface{}) {} + +func (n NoopLogger) Debug(string, ...interface{}) {} + +var _ Logger = NoopLogger{} From 9600152523a61913dc654b833f62acca8425bca5 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Thu, 4 Jul 2024 21:39:11 +0200 Subject: [PATCH 18/29] WIP --- schema/indexing/manager.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/schema/indexing/manager.go b/schema/indexing/manager.go index c858fbdc4ee8..e726a968c42c 100644 --- a/schema/indexing/manager.go +++ b/schema/indexing/manager.go @@ -6,7 +6,7 @@ import ( "cosmossdk.io/schema/appdata" "cosmossdk.io/schema/decoding" - "cosmossdk.io/schema/log" + "cosmossdk.io/schema/logutil" ) type Options struct { @@ -14,12 +14,12 @@ type Options struct { Options map[string]interface{} Resolver decoding.DecoderResolver SyncSource decoding.SyncSource - Logger log.Logger + Logger logutil.Logger } func Start(opts Options) (appdata.Listener, error) { if opts.Logger == nil { - opts.Logger = log.NoopLogger{} + opts.Logger = logutil.NoopLogger{} } opts.Logger.Info("Starting Indexer Manager") @@ -62,9 +62,9 @@ func Start(opts Options) (appdata.Listener, error) { // TODO handle last block persisted } - return decoding.Middleware(appdata.AsyncListenerMux(indexers, 1024, ctx.Done()), decoding.Options{ - DecoderResolver: opts.Resolver, - SyncSource: opts.SyncSource, - Logger: opts.Logger, - }) + return decoding.Middleware( + appdata.AsyncListenerMux(indexers, 1024, ctx.Done()), + opts.Resolver, + decoding.MiddlewareOptions{}, + ) } From 23ff67f82e7692c7f0dcad35a93c8d612336c7df Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Fri, 5 Jul 2024 13:59:31 +0200 Subject: [PATCH 19/29] add manager API --- schema/appdata/async.go | 131 ------------------------------ schema/appdata/async_test.go | 131 ------------------------------ schema/appdata/filter.go | 42 ---------- schema/appdata/mux.go | 121 --------------------------- schema/appdata/mux_test.go | 131 ------------------------------ schema/decoding/middleware.go | 106 ------------------------ schema/decoding/resolver.go | 19 ++--- schema/decoding/resolver_test.go | 125 ++++++++++++++++++++++++++++ schema/decoding/sync.go | 56 +------------ schema/indexer/indexer.go | 77 ++++++++++++++++++ schema/indexer/manager/manager.go | 40 +++++++++ schema/indexer/registry.go | 19 +++++ schema/indexer/registry_test.go | 26 ++++++ schema/indexing/indexer.go | 36 -------- schema/indexing/manager.go | 70 ---------------- 15 files changed, 295 insertions(+), 835 deletions(-) delete mode 100644 schema/appdata/async.go delete mode 100644 schema/appdata/async_test.go delete mode 100644 schema/appdata/filter.go delete mode 100644 schema/appdata/mux.go delete mode 100644 schema/appdata/mux_test.go delete mode 100644 schema/decoding/middleware.go create mode 100644 schema/decoding/resolver_test.go create mode 100644 schema/indexer/indexer.go create mode 100644 schema/indexer/manager/manager.go create mode 100644 schema/indexer/registry.go create mode 100644 schema/indexer/registry_test.go delete mode 100644 schema/indexing/indexer.go delete mode 100644 schema/indexing/manager.go diff --git a/schema/appdata/async.go b/schema/appdata/async.go deleted file mode 100644 index a8aeb02676cb..000000000000 --- a/schema/appdata/async.go +++ /dev/null @@ -1,131 +0,0 @@ -package appdata - -// AsyncListenerMux returns a listener that forwards received events to all the provided listeners asynchronously -// with each listener processing in a separate go routine. All callbacks in the returned listener will return nil -// except for Commit which will return an error or nil once all listeners have processed the commit. The doneChan -// is used to signal that the listeners should stop listening and return. bufferSize is the size of the buffer for the -// channels used to send events to the listeners. -func AsyncListenerMux(listeners []Listener, bufferSize int, doneChan <-chan struct{}) Listener { - asyncListeners := make([]Listener, len(listeners)) - commitChans := make([]chan error, len(listeners)) - for i, l := range listeners { - commitChan := make(chan error) - commitChans[i] = commitChan - asyncListeners[i] = AsyncListener(l, bufferSize, commitChan, doneChan) - } - mux := ListenerMux(asyncListeners...) - muxCommit := mux.Commit - mux.Commit = func(data CommitData) error { - if muxCommit != nil { - err := muxCommit(data) - if err != nil { - return err - } - } - - for _, commitChan := range commitChans { - err := <-commitChan - if err != nil { - return err - } - } - return nil - } - - return mux -} - -// AsyncListener returns a listener that forwards received events to the provided listener listening in asynchronously -// in a separate go routine. The listener that is returned will return nil for all methods including Commit and -// an error or nil will only be returned in commitChan once the sender has sent commit and the receiving listener has -// processed it. Thus commitChan can be used as a synchronization and error checking mechanism. The go routine -// that is being used for listening will exit when doneChan is closed and no more events will be received by the listener. -// bufferSize is the size of the buffer for the channel that is used to send events to the listener. -// Instead of using AsyncListener directly, it is recommended to use AsyncListenerMux which does coordination directly -// via its Commit callback. -func AsyncListener(listener Listener, bufferSize int, commitChan chan<- error, doneChan <-chan struct{}) Listener { - packetChan := make(chan Packet, bufferSize) - res := Listener{} - - go func() { - var err error - for { - select { - case packet := <-packetChan: - if err != nil { - // if we have an error, don't process any more packets - // and return the error and finish when it's time to commit - if _, ok := packet.(CommitData); ok { - commitChan <- err - return - } - } else { - // process the packet - err = listener.SendPacket(packet) - // if it's a commit - if _, ok := packet.(CommitData); ok { - commitChan <- err - if err != nil { - return - } - } - } - - case <-doneChan: - close(packetChan) - return - } - } - }() - - if listener.InitializeModuleData != nil { - res.InitializeModuleData = func(data ModuleInitializationData) error { - packetChan <- data - return nil - } - } - - if listener.StartBlock != nil { - res.StartBlock = func(data StartBlockData) error { - packetChan <- data - return nil - } - } - - if listener.OnTx != nil { - res.OnTx = func(data TxData) error { - packetChan <- data - return nil - } - } - - if listener.OnEvent != nil { - res.OnEvent = func(data EventData) error { - packetChan <- data - return nil - } - } - - if listener.OnKVPair != nil { - res.OnKVPair = func(data KVPairData) error { - packetChan <- data - return nil - } - } - - if listener.OnObjectUpdate != nil { - res.OnObjectUpdate = func(data ObjectUpdateData) error { - packetChan <- data - return nil - } - } - - if listener.Commit != nil { - res.Commit = func(data CommitData) error { - packetChan <- data - return nil - } - } - - return res -} diff --git a/schema/appdata/async_test.go b/schema/appdata/async_test.go deleted file mode 100644 index ff2e3918227b..000000000000 --- a/schema/appdata/async_test.go +++ /dev/null @@ -1,131 +0,0 @@ -package appdata - -import ( - "fmt" - "testing" -) - -func TestAsyncListenerMux(t *testing.T) { - t.Run("empty", func(t *testing.T) { - listener := AsyncListenerMux([]Listener{{}, {}}, 16, make(chan struct{})) - - if listener.InitializeModuleData != nil { - t.Error("expected nil") - } - if listener.StartBlock != nil { - t.Error("expected nil") - } - if listener.OnTx != nil { - t.Error("expected nil") - } - if listener.OnEvent != nil { - t.Error("expected nil") - } - if listener.OnKVPair != nil { - t.Error("expected nil") - } - if listener.OnObjectUpdate != nil { - t.Error("expected nil") - } - - // commit is not expected to be nil - }) - - t.Run("call done", func(t *testing.T) { - doneChan := make(chan struct{}) - var calls1, calls2 []string - listener1 := callCollector(1, func(name string, _ int, _ Packet) { - calls1 = append(calls1, name) - }) - listener2 := callCollector(2, func(name string, _ int, _ Packet) { - calls2 = append(calls2, name) - }) - res := AsyncListenerMux([]Listener{listener1, listener2}, 16, doneChan) - - callAllCallbacksOnces(t, res) - - expectedCalls := []string{ - "InitializeModuleData", - "StartBlock", - "OnTx", - "OnEvent", - "OnKVPair", - "OnObjectUpdate", - "Commit", - } - - checkExpectedCallOrder(t, calls1, expectedCalls) - checkExpectedCallOrder(t, calls2, expectedCalls) - - calls1 = nil - calls2 = nil - - doneChan <- struct{}{} - - callAllCallbacksOnces(t, res) - // - //checkExpectedCallOrder(t, calls1, nil) - //checkExpectedCallOrder(t, calls2, nil) - }) -} - -func TestAsyncListener(t *testing.T) { - t.Run("call done", func(t *testing.T) { - commitChan := make(chan error) - doneChan := make(chan struct{}) - var calls []string - listener := callCollector(1, func(name string, _ int, _ Packet) { - calls = append(calls, name) - }) - res := AsyncListener(listener, 16, commitChan, doneChan) - - callAllCallbacksOnces(t, res) - - err := <-commitChan - if err != nil { - t.Fatalf("expected nil, got %v", err) - } - - checkExpectedCallOrder(t, calls, []string{ - "InitializeModuleData", - "StartBlock", - "OnTx", - "OnEvent", - "OnKVPair", - "OnObjectUpdate", - "Commit", - }) - - calls = nil - - doneChan <- struct{}{} - - callAllCallbacksOnces(t, res) - - checkExpectedCallOrder(t, calls, nil) - }) - - t.Run("error", func(t *testing.T) { - commitChan := make(chan error) - doneChan := make(chan struct{}) - var calls []string - listener := callCollector(1, func(name string, _ int, _ Packet) { - calls = append(calls, name) - }) - - listener.OnKVPair = func(updates KVPairData) error { - return fmt.Errorf("error") - } - - res := AsyncListener(listener, 16, commitChan, doneChan) - - callAllCallbacksOnces(t, res) - - err := <-commitChan - if err == nil || err.Error() != "error" { - t.Fatalf("expected error, got %v", err) - } - - checkExpectedCallOrder(t, calls, []string{"InitializeModuleData", "StartBlock", "OnTx", "OnEvent"}) - }) -} diff --git a/schema/appdata/filter.go b/schema/appdata/filter.go deleted file mode 100644 index 7c5f47c813c5..000000000000 --- a/schema/appdata/filter.go +++ /dev/null @@ -1,42 +0,0 @@ -package appdata - -// ModuleFilter returns an updated listener that filters state updates based on the module name. -func ModuleFilter(listener Listener, filter func(moduleName string) bool) Listener { - if initModData := listener.InitializeModuleData; initModData != nil { - listener.InitializeModuleData = func(data ModuleInitializationData) error { - if !filter(data.ModuleName) { - return nil - } - - return initModData(data) - } - } - - if onKVPair := listener.OnKVPair; onKVPair != nil { - listener.OnKVPair = func(data KVPairData) error { - for _, update := range data.Updates { - if !filter(update.ModuleName) { - continue - } - - if err := onKVPair(KVPairData{Updates: []ModuleKVPairUpdate{update}}); err != nil { - return err - } - } - - return nil - } - } - - if onObjectUpdate := listener.OnObjectUpdate; onObjectUpdate != nil { - listener.OnObjectUpdate = func(data ObjectUpdateData) error { - if !filter(data.ModuleName) { - return nil - } - - return onObjectUpdate(data) - } - } - - return listener -} diff --git a/schema/appdata/mux.go b/schema/appdata/mux.go deleted file mode 100644 index cc7b662b9128..000000000000 --- a/schema/appdata/mux.go +++ /dev/null @@ -1,121 +0,0 @@ -package appdata - -// ListenerMux returns a listener that forwards received events to all the provided listeners in order. -// A callback is only registered if a non-nil callback is present in at least one of the listeners. -func ListenerMux(listeners ...Listener) Listener { - mux := Listener{} - - for _, l := range listeners { - if l.InitializeModuleData != nil { - mux.InitializeModuleData = func(data ModuleInitializationData) error { - for _, l := range listeners { - if l.InitializeModuleData != nil { - if err := l.InitializeModuleData(data); err != nil { - return err - } - } - } - return nil - } - break - } - } - - for _, l := range listeners { - if l.StartBlock != nil { - mux.StartBlock = func(data StartBlockData) error { - for _, l := range listeners { - if l.StartBlock != nil { - if err := l.StartBlock(data); err != nil { - return err - } - } - } - return nil - } - break - } - } - - for _, l := range listeners { - if l.OnTx != nil { - mux.OnTx = func(data TxData) error { - for _, l := range listeners { - if l.OnTx != nil { - if err := l.OnTx(data); err != nil { - return err - } - } - } - return nil - } - break - } - } - - for _, listener := range listeners { - if listener.OnEvent != nil { - mux.OnEvent = func(data EventData) error { - for _, l := range listeners { - if l.OnEvent != nil { - if err := l.OnEvent(data); err != nil { - return err - } - } - } - return nil - } - break - } - } - - for _, listener := range listeners { - if listener.OnKVPair != nil { - mux.OnKVPair = func(data KVPairData) error { - for _, l := range listeners { - if l.OnKVPair != nil { - if err := l.OnKVPair(data); err != nil { - return err - } - } - } - return nil - } - break - } - } - - for _, listener := range listeners { - if listener.OnObjectUpdate != nil { - mux.OnObjectUpdate = func(data ObjectUpdateData) error { - for _, l := range listeners { - if l.OnObjectUpdate != nil { - if err := l.OnObjectUpdate(data); err != nil { - return err - } - } - } - return nil - } - break - } - } - - for _, listener := range listeners { - if listener.Commit != nil { - mux.Commit = func(data CommitData) error { - for _, l := range listeners { - if l.Commit != nil { - if err := l.Commit(data); err != nil { - return err - } - } - } - return nil - } - break - } - } - - return mux -} diff --git a/schema/appdata/mux_test.go b/schema/appdata/mux_test.go deleted file mode 100644 index b5e3a95dd569..000000000000 --- a/schema/appdata/mux_test.go +++ /dev/null @@ -1,131 +0,0 @@ -package appdata - -import ( - "fmt" - "testing" -) - -func TestListenerMux(t *testing.T) { - t.Run("empty", func(t *testing.T) { - listener := ListenerMux(Listener{}, Listener{}) - - if listener.InitializeModuleData != nil { - t.Error("expected nil") - } - if listener.StartBlock != nil { - t.Error("expected nil") - } - if listener.OnTx != nil { - t.Error("expected nil") - } - if listener.OnEvent != nil { - t.Error("expected nil") - } - if listener.OnKVPair != nil { - t.Error("expected nil") - } - if listener.OnObjectUpdate != nil { - t.Error("expected nil") - } - if listener.Commit != nil { - t.Error("expected nil") - } - }) - - t.Run("all called once", func(t *testing.T) { - var calls []string - onCall := func(name string, i int, _ Packet) { - calls = append(calls, fmt.Sprintf("%s %d", name, i)) - } - - res := ListenerMux(callCollector(1, onCall), callCollector(2, onCall)) - - callAllCallbacksOnces(t, res) - - checkExpectedCallOrder(t, calls, []string{ - "InitializeModuleData 1", - "InitializeModuleData 2", - "StartBlock 1", - "StartBlock 2", - "OnTx 1", - "OnTx 2", - "OnEvent 1", - "OnEvent 2", - "OnKVPair 1", - "OnKVPair 2", - "OnObjectUpdate 1", - "OnObjectUpdate 2", - "Commit 1", - "Commit 2", - }) - }) -} - -func callAllCallbacksOnces(t *testing.T, listener Listener) { - if err := listener.InitializeModuleData(ModuleInitializationData{}); err != nil { - t.Error(err) - } - if err := listener.StartBlock(StartBlockData{}); err != nil { - t.Error(err) - } - if err := listener.OnTx(TxData{}); err != nil { - t.Error(err) - } - if err := listener.OnEvent(EventData{}); err != nil { - t.Error(err) - } - if err := listener.OnKVPair(KVPairData{}); err != nil { - t.Error(err) - } - if err := listener.OnObjectUpdate(ObjectUpdateData{}); err != nil { - t.Error(err) - } - if err := listener.Commit(CommitData{}); err != nil { - t.Error(err) - } -} - -func callCollector(i int, onCall func(string, int, Packet)) Listener { - return Listener{ - InitializeModuleData: func(ModuleInitializationData) error { - onCall("InitializeModuleData", i, nil) - return nil - }, - StartBlock: func(StartBlockData) error { - onCall("StartBlock", i, nil) - return nil - }, - OnTx: func(TxData) error { - onCall("OnTx", i, nil) - return nil - }, - OnEvent: func(EventData) error { - onCall("OnEvent", i, nil) - return nil - }, - OnKVPair: func(KVPairData) error { - onCall("OnKVPair", i, nil) - return nil - }, - OnObjectUpdate: func(ObjectUpdateData) error { - onCall("OnObjectUpdate", i, nil) - return nil - }, - Commit: func(CommitData) error { - onCall("Commit", i, nil) - return nil - }, - } -} - -func checkExpectedCallOrder(t *testing.T, actual, expected []string) { - if len(actual) != len(expected) { - t.Fatalf("expected %d calls, got %d", len(expected), len(actual)) - } - - for i := range actual { - if actual[i] != expected[i] { - t.Errorf("expected %q, got %q", expected[i], actual[i]) - } - } -} diff --git a/schema/decoding/middleware.go b/schema/decoding/middleware.go deleted file mode 100644 index 57c0783c6281..000000000000 --- a/schema/decoding/middleware.go +++ /dev/null @@ -1,106 +0,0 @@ -package decoding - -import ( - "cosmossdk.io/schema" - "cosmossdk.io/schema/appdata" -) - -type MiddlewareOptions struct { - ModuleFilter func(moduleName string) bool -} - -// Middleware decodes raw data passed to the listener as kv-updates into decoded object updates. Module initialization -// is done lazily as modules are encountered in the kv-update stream. -func Middleware(target appdata.Listener, resolver DecoderResolver, opts MiddlewareOptions) (appdata.Listener, error) { - initializeModuleData := target.InitializeModuleData - onObjectUpdate := target.OnObjectUpdate - - // no-op if not listening to decoded data - if initializeModuleData == nil && onObjectUpdate == nil { - return target, nil - } - - onKVPair := target.OnKVPair - - moduleCodecs := map[string]*schema.ModuleCodec{} - - target.OnKVPair = func(data appdata.KVPairData) error { - // first forward kv pair updates - if onKVPair != nil { - err := onKVPair(data) - if err != nil { - return err - } - } - - for _, kvUpdate := range data.Updates { - // look for an existing codec - pcdc, ok := moduleCodecs[kvUpdate.ModuleName] - if !ok { - if opts.ModuleFilter != nil && !opts.ModuleFilter(kvUpdate.ModuleName) { - // we don't care about this module so store nil and continue - moduleCodecs[kvUpdate.ModuleName] = nil - continue - } - - // look for a new codec - cdc, found, err := resolver.LookupDecoder(kvUpdate.ModuleName) - if err != nil { - return err - } - - if !found { - // store nil to indicate we've seen this module and don't have a codec - // and keep processing the kv updates - moduleCodecs[kvUpdate.ModuleName] = nil - continue - } - - pcdc = &cdc - moduleCodecs[kvUpdate.ModuleName] = pcdc - - if initializeModuleData != nil { - err = initializeModuleData(appdata.ModuleInitializationData{ - ModuleName: kvUpdate.ModuleName, - Schema: cdc.Schema, - }) - if err != nil { - return err - } - } - } - - if pcdc == nil { - // we've already seen this module and can't decode - continue - } - - if onObjectUpdate == nil || pcdc.KVDecoder == nil { - // not listening to updates or can't decode so continue - continue - } - - updates, err := pcdc.KVDecoder(kvUpdate.Update) - if err != nil { - return err - } - - if len(updates) == 0 { - // no updates - continue - } - - err = target.OnObjectUpdate(appdata.ObjectUpdateData{ - ModuleName: kvUpdate.ModuleName, - Updates: updates, - }) - if err != nil { - return err - } - } - - return nil - } - - return target, nil -} diff --git a/schema/decoding/resolver.go b/schema/decoding/resolver.go index a09eeafe82b9..db0ec0bb1726 100644 --- a/schema/decoding/resolver.go +++ b/schema/decoding/resolver.go @@ -6,24 +6,15 @@ import ( "cosmossdk.io/schema" ) +// DecoderResolver is an interface that allows indexers to discover and use module decoders. type DecoderResolver interface { - // IterateAll iterates over all module decoders which should be initialized at startup. + // IterateAll iterates over all available module decoders. IterateAll(func(moduleName string, cdc schema.ModuleCodec) error) error - // LookupDecoder allows for resolving decoders dynamically. For instance, some module-like - // things may come into existence dynamically (like x/accounts or EVM or WASM contracts). - // The first time the manager sees one of these appearing in KV-store writes, it will - // lookup a decoder for it and cache it for future use. The manager will also perform - // a catch-up sync before passing any new writes to ensure that all historical state has - // been synced if there is any This check will only happen the first time a module is seen - // by the manager in a given process (a process restart will cause this check to happen again). + // LookupDecoder looks up a specific module decoder. LookupDecoder(moduleName string) (decoder schema.ModuleCodec, found bool, err error) } -type moduleSetDecoderResolver struct { - moduleSet map[string]interface{} -} - // ModuleSetDecoderResolver returns DecoderResolver that will discover modules implementing // DecodeableModule in the provided module set. func ModuleSetDecoderResolver(moduleSet map[string]interface{}) DecoderResolver { @@ -32,6 +23,10 @@ func ModuleSetDecoderResolver(moduleSet map[string]interface{}) DecoderResolver } } +type moduleSetDecoderResolver struct { + moduleSet map[string]interface{} +} + func (a moduleSetDecoderResolver) IterateAll(f func(string, schema.ModuleCodec) error) error { keys := make([]string, 0, len(a.moduleSet)) for k := range a.moduleSet { diff --git a/schema/decoding/resolver_test.go b/schema/decoding/resolver_test.go new file mode 100644 index 000000000000..3cad85694863 --- /dev/null +++ b/schema/decoding/resolver_test.go @@ -0,0 +1,125 @@ +package decoding + +import ( + "fmt" + "testing" + + "cosmossdk.io/schema" +) + +type modA struct{} + +func (m modA) ModuleCodec() (schema.ModuleCodec, error) { + return schema.ModuleCodec{ + Schema: schema.ModuleSchema{ObjectTypes: []schema.ObjectType{{Name: "A"}}}, + }, nil +} + +type modB struct{} + +func (m modB) ModuleCodec() (schema.ModuleCodec, error) { + return schema.ModuleCodec{ + Schema: schema.ModuleSchema{ObjectTypes: []schema.ObjectType{{Name: "B"}}}, + }, nil +} + +type modC struct{} + +var moduleSet = map[string]interface{}{ + "modA": modA{}, + "modB": modB{}, + "modC": modC{}, +} + +var resolver = ModuleSetDecoderResolver(moduleSet) + +func TestModuleSetDecoderResolver_IterateAll(t *testing.T) { + objectTypes := map[string]bool{} + err := resolver.IterateAll(func(moduleName string, cdc schema.ModuleCodec) error { + objectTypes[cdc.Schema.ObjectTypes[0].Name] = true + return nil + }) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if len(objectTypes) != 2 { + t.Fatalf("expected 2 object types, got %d", len(objectTypes)) + } + + if !objectTypes["A"] { + t.Fatalf("expected object type A") + } + + if !objectTypes["B"] { + t.Fatalf("expected object type B") + } +} + +func TestModuleSetDecoderResolver_LookupDecoder(t *testing.T) { + decoder, found, err := resolver.LookupDecoder("modA") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if !found { + t.Fatalf("expected to find decoder for modA") + } + + if decoder.Schema.ObjectTypes[0].Name != "A" { + t.Fatalf("expected object type A, got %s", decoder.Schema.ObjectTypes[0].Name) + } + + decoder, found, err = resolver.LookupDecoder("modB") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if !found { + t.Fatalf("expected to find decoder for modB") + } + + if decoder.Schema.ObjectTypes[0].Name != "B" { + t.Fatalf("expected object type B, got %s", decoder.Schema.ObjectTypes[0].Name) + } + + decoder, found, err = resolver.LookupDecoder("modC") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if found { + t.Fatalf("expected not to find decoder") + } + + decoder, found, err = resolver.LookupDecoder("modD") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if found { + t.Fatalf("expected not to find decoder") + } +} + +type modD struct{} + +func (m modD) ModuleCodec() (schema.ModuleCodec, error) { + return schema.ModuleCodec{}, fmt.Errorf("an error") +} + +func TestModuleSetDecoderResolver_IterateAll_Error(t *testing.T) { + resolver := ModuleSetDecoderResolver(map[string]interface{}{ + "modD": modD{}, + }) + err := resolver.IterateAll(func(moduleName string, cdc schema.ModuleCodec) error { + if moduleName == "modD" { + t.Fatalf("expected error") + } + return nil + }) + if err == nil { + t.Fatalf("expected error") + } +} diff --git a/schema/decoding/sync.go b/schema/decoding/sync.go index a1af8fa24efd..1487aca38c7c 100644 --- a/schema/decoding/sync.go +++ b/schema/decoding/sync.go @@ -1,63 +1,9 @@ package decoding -import ( - "cosmossdk.io/schema" - "cosmossdk.io/schema/appdata" -) - // SyncSource is an interface that allows indexers to start indexing modules with pre-existing state. +// It should generally be a wrapper around the key-value store. type SyncSource interface { // IterateAllKVPairs iterates over all key-value pairs for a given module. IterateAllKVPairs(moduleName string, fn func(key, value []byte) error) error } - -// SyncOptions are the options for Sync. -type SyncOptions struct { - ModuleFilter func(moduleName string) bool -} - -// Sync synchronizes existing state from the sync source to the listener using the resolver to decode data. -func Sync(listener appdata.Listener, source SyncSource, resolver DecoderResolver, opts SyncOptions) error { - initializeModuleData := listener.InitializeModuleData - onObjectUpdate := listener.OnObjectUpdate - - // no-op if not listening to decoded data - if initializeModuleData == nil && onObjectUpdate == nil { - return nil - } - - return resolver.IterateAll(func(moduleName string, cdc schema.ModuleCodec) error { - if opts.ModuleFilter != nil && !opts.ModuleFilter(moduleName) { - // ignore this module - return nil - } - - if initializeModuleData != nil { - err := initializeModuleData(appdata.ModuleInitializationData{ - ModuleName: moduleName, - Schema: cdc.Schema, - }) - if err != nil { - return err - } - } - - if onObjectUpdate == nil || cdc.KVDecoder == nil { - return nil - } - - return source.IterateAllKVPairs(moduleName, func(key, value []byte) error { - updates, err := cdc.KVDecoder(schema.KVPairUpdate{Key: key, Value: value}) - if err != nil { - return err - } - - if len(updates) == 0 { - return nil - } - - return onObjectUpdate(appdata.ObjectUpdateData{ModuleName: moduleName, Updates: updates}) - }) - }) -} diff --git a/schema/indexer/indexer.go b/schema/indexer/indexer.go new file mode 100644 index 000000000000..653f096a5bc0 --- /dev/null +++ b/schema/indexer/indexer.go @@ -0,0 +1,77 @@ +package indexer + +import ( + "context" + + "cosmossdk.io/schema/appdata" + "cosmossdk.io/schema/logutil" +) + +// Config species the configuration passed to an indexer initialization function. +// It includes both common configuration options related to include or excluding +// parts of the data stream as well as indexer specific options under the config +// subsection. +// +// NOTE: it is an error for an indexer to change its common options after the indexer +// has been initialized because this could result in an inconsistent state. +type Config struct { + // Type is the name of the indexer type as registered with Register. + Type string `json:"type"` + + // Config are the indexer specific config options specified by the user. + Config map[string]interface{} `json:"config"` + + // ExcludeState specifies that the indexer will not receive state updates. + ExcludeState bool `json:"exclude_state"` + + // ExcludeEvents specifies that the indexer will not receive events. + ExcludeEvents bool `json:"exclude_events"` + + // ExcludeTxs specifies that the indexer will not receive transaction's. + ExcludeTxs bool `json:"exclude_txs"` + + // ExcludeBlockHeaders specifies that the indexer will not receive block headers, + // although it will still receive StartBlock and Commit callbacks, just without + // the header data. + ExcludeBlockHeaders bool `json:"exclude_block_headers"` + + // IncludeModules specifies a list of modules whose state the indexer will + // receive state updates for. + // Only one of include or exclude modules should be specified. + IncludeModules []string `json:"include_modules"` + + // ExcludeModules specifies a list of modules whose state the indexer will not + // receive state updates for. + // Only one of include or exclude modules should be specified. + ExcludeModules []string `json:"exclude_modules"` +} + +type InitFunc = func(InitParams) (InitResult, error) + +// InitParams is the input to the indexer initialization function. +type InitParams struct { + // Config is the indexer config. + Config Config + + // Context is the context that the indexer should use to listen for a shutdown signal via Context.Done(). Other + // parameters may also be passed through context from the app if necessary. + Context context.Context + + // Logger is a logger the indexer can use to write log messages. + Logger logutil.Logger +} + +// InitResult is the indexer initialization result and includes the indexer's listener implementation. +type InitResult struct { + // Listener is the indexer's app data listener. + Listener appdata.Listener + + // LastBlockPersisted indicates the last block that the indexer persisted (if it is persisting data). It + // should be 0 if the indexer has no data stored and wants to start syncing state. It should be -1 if the indexer + // does not care to persist state at all and is just listening for some other streaming purpose. If the indexer + // has persisted state and has missed some blocks, a runtime error will occur to prevent the indexer from continuing + // in an invalid state. If an indexer starts indexing after a chain's genesis (returning 0), the indexer manager + // will attempt to perform a catch-up sync of state. Historical events will not be replayed, but an accurate + // representation of the current state at the height at which indexing began can be reproduced. + LastBlockPersisted int64 +} diff --git a/schema/indexer/manager/manager.go b/schema/indexer/manager/manager.go new file mode 100644 index 000000000000..9fb02849b1b7 --- /dev/null +++ b/schema/indexer/manager/manager.go @@ -0,0 +1,40 @@ +package manager + +import ( + "context" + + "cosmossdk.io/schema/appdata" + "cosmossdk.io/schema/decoding" + "cosmossdk.io/schema/logutil" +) + +// Options are the options for starting the indexer manager. +type Options struct { + // Config is the user configuration for all indexing. It should match the IndexingConfig struct and + // the indexer manager will attempt to convert it to that data structure. + Config interface{} + + // Resolver is the decoder resolver that will be used to decode the data. + Resolver decoding.DecoderResolver + + // SyncSource is a representation of the current state of key-value data to be used in a catch-up sync. + // Catch-up syncs will be performed at initialization when necessary. + SyncSource decoding.SyncSource + + // Logger is the logger that indexers can use to write logs. + Logger logutil.Logger + + // Context is the context that indexers should use for shutdown signals via Context.Done(). It can also + // be used to pass down other parameters to indexers if necessary. + Context context.Context +} + +// IndexingConfig is the configuration of all the +type IndexingConfig struct { +} + +// Start starts the indexer manager with the given options. The state machine should write all relevant app data to +// the returned listener. +func Start(opts Options) (appdata.Listener, error) { + panic("TODO: this will be implemented in a follow-up PR, this function is just a stub to demonstrate the API") +} diff --git a/schema/indexer/registry.go b/schema/indexer/registry.go new file mode 100644 index 000000000000..5d33e239f4d3 --- /dev/null +++ b/schema/indexer/registry.go @@ -0,0 +1,19 @@ +package indexer + +import "fmt" + +// Register registers an indexer type with the given initialization function. +func Register(indexerType string, initFunc InitFunc) { + if _, ok := indexerRegistry[indexerType]; ok { + panic(fmt.Sprintf("indexer %s already registered", indexerType)) + } + + indexerRegistry[indexerType] = initFunc +} + +// Lookup returns the initialization function for the given indexer type or nil. +func Lookup(indexerType string) InitFunc { + return indexerRegistry[indexerType] +} + +var indexerRegistry = map[string]InitFunc{} diff --git a/schema/indexer/registry_test.go b/schema/indexer/registry_test.go new file mode 100644 index 000000000000..e57ac7338258 --- /dev/null +++ b/schema/indexer/registry_test.go @@ -0,0 +1,26 @@ +package indexer + +import "testing" + +func TestRegister(t *testing.T) { + Register("test", func(params InitParams) (InitResult, error) { + return InitResult{}, nil + }) + + if Lookup("test") == nil { + t.Fatalf("expected to find indexer") + } + + if Lookup("test2") != nil { + t.Fatalf("expected not to find indexer") + } + + defer func() { + if r := recover(); r == nil { + t.Fatalf("expected to panic") + } + }() + Register("test", func(params InitParams) (InitResult, error) { + return InitResult{}, nil + }) +} diff --git a/schema/indexing/indexer.go b/schema/indexing/indexer.go deleted file mode 100644 index 926392952153..000000000000 --- a/schema/indexing/indexer.go +++ /dev/null @@ -1,36 +0,0 @@ -package indexing - -import ( - "context" - "fmt" - - "cosmossdk.io/schema/appdata" - "cosmossdk.io/schema/log" -) - -type Indexer interface { - Initialize(context.Context, InitializationData) (InitializationResult, error) -} - -type IndexerResources struct { - Logger log.Logger -} - -type IndexerFactory = func(options map[string]interface{}, resources IndexerResources) (Indexer, error) - -type InitializationData struct{} - -type InitializationResult struct { - Listener appdata.Listener - LastBlockPersisted int64 -} - -func RegisterIndexer(name string, factory IndexerFactory) { - if _, ok := indexerRegistry[name]; ok { - panic(fmt.Sprintf("indexer %s already registered", name)) - } - - indexerRegistry[name] = factory -} - -var indexerRegistry = map[string]IndexerFactory{} diff --git a/schema/indexing/manager.go b/schema/indexing/manager.go deleted file mode 100644 index e726a968c42c..000000000000 --- a/schema/indexing/manager.go +++ /dev/null @@ -1,70 +0,0 @@ -package indexing - -import ( - "context" - "fmt" - - "cosmossdk.io/schema/appdata" - "cosmossdk.io/schema/decoding" - "cosmossdk.io/schema/logutil" -) - -type Options struct { - Context context.Context - Options map[string]interface{} - Resolver decoding.DecoderResolver - SyncSource decoding.SyncSource - Logger logutil.Logger -} - -func Start(opts Options) (appdata.Listener, error) { - if opts.Logger == nil { - opts.Logger = logutil.NoopLogger{} - } - - opts.Logger.Info("Starting Indexer Manager") - - resources := IndexerResources{Logger: opts.Logger} - - var indexers []appdata.Listener - ctx := opts.Context - if ctx == nil { - ctx = context.Background() - } - - for indexerName, factory := range indexerRegistry { - indexerOpts, ok := opts.Options[indexerName] - if !ok { - continue - } - - if opts.Logger != nil { - opts.Logger.Info(fmt.Sprintf("Starting Indexer %s", indexerName), "options", indexerOpts) - } - - optsMap, ok := indexerOpts.(map[string]interface{}) - if !ok { - return appdata.Listener{}, fmt.Errorf("invalid indexer options type %T for %s, expected a map", indexerOpts, indexerName) - } - - indexer, err := factory(optsMap, resources) - if err != nil { - return appdata.Listener{}, fmt.Errorf("failed to create indexer %s: %w", indexerName, err) - } - - res, err := indexer.Initialize(ctx, InitializationData{}) - if err != nil { - return appdata.Listener{}, fmt.Errorf("failed to initialize indexer %s: %w", indexerName, err) - } - - indexers = append(indexers, res.Listener) - - // TODO handle last block persisted - } - - return decoding.Middleware( - appdata.AsyncListenerMux(indexers, 1024, ctx.Done()), - opts.Resolver, - decoding.MiddlewareOptions{}, - ) -} From d6cf6558269d5728a2f854a71f4ca484b750b358 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Fri, 5 Jul 2024 14:05:28 +0200 Subject: [PATCH 20/29] update config --- schema/indexer/manager/manager.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/schema/indexer/manager/manager.go b/schema/indexer/manager/manager.go index 9fb02849b1b7..264e29be9eb2 100644 --- a/schema/indexer/manager/manager.go +++ b/schema/indexer/manager/manager.go @@ -5,6 +5,7 @@ import ( "cosmossdk.io/schema/appdata" "cosmossdk.io/schema/decoding" + "cosmossdk.io/schema/indexer" "cosmossdk.io/schema/logutil" ) @@ -29,8 +30,11 @@ type Options struct { Context context.Context } -// IndexingConfig is the configuration of all the +// IndexingConfig is the configuration of the indexing manager and contains the configuration for each indexer target. type IndexingConfig struct { + + // Target is a map of indexer targets to their configuration. + Target map[string]indexer.Config } // Start starts the indexer manager with the given options. The state machine should write all relevant app data to From b5ca35a967a18833cf2af27890799618b28eade8 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Fri, 5 Jul 2024 14:25:59 +0200 Subject: [PATCH 21/29] add README, refactor --- schema/indexer/README.md | 15 +++++++++++ schema/indexer/manager.go | 45 +++++++++++++++++++++++++++++++ schema/indexer/manager/manager.go | 44 ------------------------------ schema/indexer/registry.go | 5 ---- schema/indexer/registry_test.go | 4 +-- 5 files changed, 62 insertions(+), 51 deletions(-) create mode 100644 schema/indexer/README.md create mode 100644 schema/indexer/manager.go delete mode 100644 schema/indexer/manager/manager.go diff --git a/schema/indexer/README.md b/schema/indexer/README.md new file mode 100644 index 000000000000..48e26b74d21e --- /dev/null +++ b/schema/indexer/README.md @@ -0,0 +1,15 @@ +# Indexer Framework + +# Defining an Indexer + +Indexer implementations should be registered with the `indexer.Register` function with a unique type name. Indexers take the configuration options defined by `indexer.Config` which defines a common set of configuration options as well as indexer-specific options under the `config` sub-key. Indexers do not need to manage the common filtering options specified in `Config` - the indexer manager will manage these for the indexer. Indexer implementations just need to return a correct `InitResult` response. + +# Integrating the Indexer Manager + +The indexer manager should be used for managing all indexers and should be integrated directly with applications wishing to support indexing. The `StartManager` function is used to start the manager. The configuration options for the manager and all indexer targets should be passed as the ManagerOptions.Config field and should match the json structure of ManagerConfig. An example configuration might look like this: + +```toml +[indexer.target.postgres] +type = "postgres" +config.database_url = "postgres://user:password@localhost:5432/dbname" +``` diff --git a/schema/indexer/manager.go b/schema/indexer/manager.go new file mode 100644 index 000000000000..9c7723917e14 --- /dev/null +++ b/schema/indexer/manager.go @@ -0,0 +1,45 @@ +package indexer + +import ( + "context" + + "cosmossdk.io/schema/appdata" + "cosmossdk.io/schema/decoding" + "cosmossdk.io/schema/logutil" +) + +// ManagerOptions are the options for starting the indexer manager. +type ManagerOptions struct { + // Config is the user configuration for all indexing. It should generally be an instance of map[string]interface{} + // and match the json structure of ManagerConfig. The manager will attempt to convert it to ManagerConfig. + Config interface{} + + // Resolver is the decoder resolver that will be used to decode the data. It is required. + Resolver decoding.DecoderResolver + + // SyncSource is a representation of the current state of key-value data to be used in a catch-up sync. + // Catch-up syncs will be performed at initialization when necessary. SyncSource is optional but if + // it is omitted, indexers will only be able to start indexing state from genesis. + SyncSource decoding.SyncSource + + // Logger is the logger that indexers can use to write logs. It is optional. + Logger logutil.Logger + + // Context is the context that indexers should use for shutdown signals via Context.Done(). It can also + // be used to pass down other parameters to indexers if necessary. If it is omitted, context.Background + // will be used. + Context context.Context +} + +// ManagerConfig is the configuration of the indexer manager and contains the configuration for each indexer target. +type ManagerConfig struct { + + // Target is a map of named indexer targets to their configuration. + Target map[string]Config +} + +// StartManager starts the indexer manager with the given options. The state machine should write all relevant app data to +// the returned listener. +func StartManager(opts ManagerOptions) (appdata.Listener, error) { + panic("TODO: this will be implemented in a follow-up PR, this function is just a stub to demonstrate the API") +} diff --git a/schema/indexer/manager/manager.go b/schema/indexer/manager/manager.go deleted file mode 100644 index 264e29be9eb2..000000000000 --- a/schema/indexer/manager/manager.go +++ /dev/null @@ -1,44 +0,0 @@ -package manager - -import ( - "context" - - "cosmossdk.io/schema/appdata" - "cosmossdk.io/schema/decoding" - "cosmossdk.io/schema/indexer" - "cosmossdk.io/schema/logutil" -) - -// Options are the options for starting the indexer manager. -type Options struct { - // Config is the user configuration for all indexing. It should match the IndexingConfig struct and - // the indexer manager will attempt to convert it to that data structure. - Config interface{} - - // Resolver is the decoder resolver that will be used to decode the data. - Resolver decoding.DecoderResolver - - // SyncSource is a representation of the current state of key-value data to be used in a catch-up sync. - // Catch-up syncs will be performed at initialization when necessary. - SyncSource decoding.SyncSource - - // Logger is the logger that indexers can use to write logs. - Logger logutil.Logger - - // Context is the context that indexers should use for shutdown signals via Context.Done(). It can also - // be used to pass down other parameters to indexers if necessary. - Context context.Context -} - -// IndexingConfig is the configuration of the indexing manager and contains the configuration for each indexer target. -type IndexingConfig struct { - - // Target is a map of indexer targets to their configuration. - Target map[string]indexer.Config -} - -// Start starts the indexer manager with the given options. The state machine should write all relevant app data to -// the returned listener. -func Start(opts Options) (appdata.Listener, error) { - panic("TODO: this will be implemented in a follow-up PR, this function is just a stub to demonstrate the API") -} diff --git a/schema/indexer/registry.go b/schema/indexer/registry.go index 5d33e239f4d3..445f56876add 100644 --- a/schema/indexer/registry.go +++ b/schema/indexer/registry.go @@ -11,9 +11,4 @@ func Register(indexerType string, initFunc InitFunc) { indexerRegistry[indexerType] = initFunc } -// Lookup returns the initialization function for the given indexer type or nil. -func Lookup(indexerType string) InitFunc { - return indexerRegistry[indexerType] -} - var indexerRegistry = map[string]InitFunc{} diff --git a/schema/indexer/registry_test.go b/schema/indexer/registry_test.go index e57ac7338258..b9f46910c8fd 100644 --- a/schema/indexer/registry_test.go +++ b/schema/indexer/registry_test.go @@ -7,11 +7,11 @@ func TestRegister(t *testing.T) { return InitResult{}, nil }) - if Lookup("test") == nil { + if indexerRegistry["test"] == nil { t.Fatalf("expected to find indexer") } - if Lookup("test2") != nil { + if indexerRegistry["test2"] != nil { t.Fatalf("expected not to find indexer") } From 595df7c873ac8374f1b772dbc3ea7ab971a86441 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Fri, 5 Jul 2024 14:30:04 +0200 Subject: [PATCH 22/29] add baseapp and simapp integration --- baseapp/streaming.go | 82 ++++++++++++++++++++++++++++++++++++++++++++ go.mod | 2 ++ simapp/app_di.go | 19 ++++++++-- simapp/go.mod | 6 ++++ simapp/go.sum | 8 +++++ 5 files changed, 114 insertions(+), 3 deletions(-) diff --git a/baseapp/streaming.go b/baseapp/streaming.go index c978d959aa7c..da49575c4034 100644 --- a/baseapp/streaming.go +++ b/baseapp/streaming.go @@ -1,17 +1,24 @@ package baseapp import ( + "context" "fmt" "sort" "strings" "github.com/spf13/cast" + "cosmossdk.io/schema" + "cosmossdk.io/schema/appdata" + "cosmossdk.io/schema/decoding" + "cosmossdk.io/schema/indexing" "cosmossdk.io/store/streaming" storetypes "cosmossdk.io/store/types" "github.com/cosmos/cosmos-sdk/client/flags" servertypes "github.com/cosmos/cosmos-sdk/server/types" + + abci "github.com/cometbft/cometbft/api/cometbft/abci/v1" ) const ( @@ -22,6 +29,33 @@ const ( StreamingABCIStopNodeOnErrTomlKey = "stop-node-on-err" ) +func (app *BaseApp) EnableIndexer(indexerOpts interface{}, keys map[string]*storetypes.KVStoreKey, appModules map[string]any) error { + optsMap, ok := indexerOpts.(map[string]interface{}) + if !ok { + return fmt.Errorf("invalid indexer options type %T, expected a map", indexerOpts) + } + + listener, err := indexing.Start(indexing.Options{ + Options: optsMap, + Resolver: decoding.ModuleSetDecoderResolver(appModules), + SyncSource: nil, + Logger: app.logger.With("module", "indexer"), + }) + if err != nil { + return err + } + + exposedKeys := exposeStoreKeysSorted([]string{"*"}, keys) + app.cms.AddListeners(exposedKeys) + + app.streamingManager = storetypes.StreamingManager{ + ABCIListeners: []storetypes.ABCIListener{listenerWrapper{listener}}, + StopNodeOnErr: true, + } + + return nil +} + // RegisterStreamingServices registers streaming services with the BaseApp. func (app *BaseApp) RegisterStreamingServices(appOpts servertypes.AppOptions, keys map[string]*storetypes.KVStoreKey) error { // register streaming services @@ -110,3 +144,51 @@ func exposeStoreKeysSorted(keysStr []string, keys map[string]*storetypes.KVStore return exposeStoreKeys } + +type listenerWrapper struct { + listener appdata.Listener +} + +func (p listenerWrapper) ListenFinalizeBlock(_ context.Context, req abci.FinalizeBlockRequest, res abci.FinalizeBlockResponse) error { + if p.listener.StartBlock != nil { + err := p.listener.StartBlock(appdata.StartBlockData{ + Height: uint64(req.Height), + }) + if err != nil { + return err + } + } + + //// TODO txs, events + + return nil +} + +func (p listenerWrapper) ListenCommit(ctx context.Context, res abci.CommitResponse, changeSet []*storetypes.StoreKVPair) error { + if cb := p.listener.OnKVPair; cb != nil { + updates := make([]appdata.ModuleKVPairUpdate, len(changeSet)) + for i, pair := range changeSet { + updates[i] = appdata.ModuleKVPairUpdate{ + ModuleName: pair.StoreKey, + Update: schema.KVPairUpdate{ + Key: pair.Key, + Value: pair.Value, + Delete: pair.Delete, + }, + } + } + err := cb(appdata.KVPairData{Updates: updates}) + if err != nil { + return err + } + } + + if p.listener.Commit != nil { + err := p.listener.Commit(appdata.CommitData{}) + if err != nil { + return err + } + } + + return nil +} diff --git a/go.mod b/go.mod index 1e61e06ced33..d069025c157c 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( cosmossdk.io/errors v1.0.1 cosmossdk.io/log v1.3.1 cosmossdk.io/math v1.3.0 + cosmossdk.io/schema v0.0.0 cosmossdk.io/store v1.1.1-0.20240418092142-896cdf1971bc cosmossdk.io/x/auth v0.0.0-00010101000000-000000000000 cosmossdk.io/x/bank v0.0.0-20240226161501-23359a0b6d91 @@ -190,6 +191,7 @@ replace ( cosmossdk.io/core/testing => ./core/testing cosmossdk.io/depinject => ./depinject cosmossdk.io/log => ./log + cosmossdk.io/schema => ./schema cosmossdk.io/store => ./store cosmossdk.io/x/accounts => ./x/accounts cosmossdk.io/x/auth => ./x/auth diff --git a/simapp/app_di.go b/simapp/app_di.go index ac73c5af53fb..94aacf740c7c 100644 --- a/simapp/app_di.go +++ b/simapp/app_di.go @@ -179,8 +179,10 @@ func NewSimApp( ) ) + var appModules map[string]appmodule.AppModule if err := depinject.Inject(appConfig, &appBuilder, + &appModules, &app.appCodec, &app.legacyAmino, &app.txConfig, @@ -242,9 +244,20 @@ func NewSimApp( app.App = appBuilder.Build(db, traceStore, baseAppOptions...) - // register streaming services - if err := app.RegisterStreamingServices(appOpts, app.kvStoreKeys()); err != nil { - panic(err) + if indexerOpts := appOpts.Get("indexer"); indexerOpts != nil { + moduleSet := map[string]any{} + for modName, mod := range appModules { + moduleSet[modName] = mod + } + err := app.EnableIndexer(indexerOpts, app.kvStoreKeys(), moduleSet) + if err != nil { + panic(err) + } + } else { + // register streaming services + if err := app.RegisterStreamingServices(appOpts, app.kvStoreKeys()); err != nil { + panic(err) + } } /**** Module Options ****/ diff --git a/simapp/go.mod b/simapp/go.mod index 7acfc530b54c..34ad562517bc 100644 --- a/simapp/go.mod +++ b/simapp/go.mod @@ -60,6 +60,7 @@ require ( cloud.google.com/go/storage v1.42.0 // indirect cosmossdk.io/core/testing v0.0.0-00010101000000-000000000000 // indirect cosmossdk.io/errors v1.0.1 // indirect + cosmossdk.io/schema v0.0.0 // indirect filippo.io/edwards25519 v1.1.0 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/99designs/keyring v1.2.2 // indirect @@ -147,6 +148,9 @@ require ( github.com/huandu/skiplist v1.2.0 // indirect github.com/iancoleman/strcase v0.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jmhodges/levigo v1.0.0 // indirect github.com/klauspost/compress v1.17.9 // indirect @@ -245,7 +249,9 @@ replace ( cosmossdk.io/core => ../core cosmossdk.io/core/testing => ../core/testing cosmossdk.io/depinject => ../depinject + cosmossdk.io/indexer/postgres => ../indexer/postgres cosmossdk.io/log => ../log + cosmossdk.io/schema => ../schema cosmossdk.io/store => ../store cosmossdk.io/tools/confix => ../tools/confix cosmossdk.io/x/accounts => ../x/accounts diff --git a/simapp/go.sum b/simapp/go.sum index 69973a085acc..ee740f82d77a 100644 --- a/simapp/go.sum +++ b/simapp/go.sum @@ -594,6 +594,14 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= +github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jhump/protoreflect v1.15.3 h1:6SFRuqU45u9hIZPJAoZ8c28T3nK64BNdp9w6jFonzls= github.com/jhump/protoreflect v1.15.3/go.mod h1:4ORHmSBmlCW8fh3xHmJMGyul1zNqZK4Elxc8qKP+p1k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= From 40e443773988125cefe5c296358a690637be91fa Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Fri, 5 Jul 2024 14:35:59 +0200 Subject: [PATCH 23/29] docs updates --- schema/indexer/README.md | 2 +- schema/indexer/indexer.go | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/schema/indexer/README.md b/schema/indexer/README.md index 48e26b74d21e..9fdec6753a27 100644 --- a/schema/indexer/README.md +++ b/schema/indexer/README.md @@ -6,7 +6,7 @@ Indexer implementations should be registered with the `indexer.Register` functio # Integrating the Indexer Manager -The indexer manager should be used for managing all indexers and should be integrated directly with applications wishing to support indexing. The `StartManager` function is used to start the manager. The configuration options for the manager and all indexer targets should be passed as the ManagerOptions.Config field and should match the json structure of ManagerConfig. An example configuration might look like this: +The indexer manager should be used for managing all indexers and should be integrated directly with applications wishing to support indexing. The `StartManager` function is used to start the manager. The configuration options for the manager and all indexer targets should be passed as the ManagerOptions.Config field and should match the json structure of ManagerConfig. An example configuration section in `app.toml` might look like this: ```toml [indexer.target.postgres] diff --git a/schema/indexer/indexer.go b/schema/indexer/indexer.go index 653f096a5bc0..ba3d0db704fa 100644 --- a/schema/indexer/indexer.go +++ b/schema/indexer/indexer.go @@ -12,8 +12,9 @@ import ( // parts of the data stream as well as indexer specific options under the config // subsection. // -// NOTE: it is an error for an indexer to change its common options after the indexer -// has been initialized because this could result in an inconsistent state. +// NOTE: it is an error for an indexer to change its common options, such as adding +// or removing indexed modules, after the indexer has been initialized because this +// could result in an inconsistent state. type Config struct { // Type is the name of the indexer type as registered with Register. Type string `json:"type"` From afce42b6a5a3c0dcec89636cc2b74dcdccf87afc Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Fri, 5 Jul 2024 14:38:27 +0200 Subject: [PATCH 24/29] docs --- baseapp/streaming.go | 6 +++++- simapp/app_di.go | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/baseapp/streaming.go b/baseapp/streaming.go index da49575c4034..f06a4676aafc 100644 --- a/baseapp/streaming.go +++ b/baseapp/streaming.go @@ -8,10 +8,11 @@ import ( "github.com/spf13/cast" + "cosmossdk.io/schema/indexing" + "cosmossdk.io/schema" "cosmossdk.io/schema/appdata" "cosmossdk.io/schema/decoding" - "cosmossdk.io/schema/indexing" "cosmossdk.io/store/streaming" storetypes "cosmossdk.io/store/types" @@ -29,6 +30,9 @@ const ( StreamingABCIStopNodeOnErrTomlKey = "stop-node-on-err" ) +// EnableIndexer enables the built-in indexer with the provided options (usually from the app.toml indexer key), +// kv-store keys, and app modules. Using the built-in indexer framework is mutually exclusive from using other +// types of streaming listeners. func (app *BaseApp) EnableIndexer(indexerOpts interface{}, keys map[string]*storetypes.KVStoreKey, appModules map[string]any) error { optsMap, ok := indexerOpts.(map[string]interface{}) if !ok { diff --git a/simapp/app_di.go b/simapp/app_di.go index 94aacf740c7c..77cc652a9a52 100644 --- a/simapp/app_di.go +++ b/simapp/app_di.go @@ -245,6 +245,7 @@ func NewSimApp( app.App = appBuilder.Build(db, traceStore, baseAppOptions...) if indexerOpts := appOpts.Get("indexer"); indexerOpts != nil { + // if we have indexer options in app.toml, then enable the built-in indexer framework moduleSet := map[string]any{} for modName, mod := range appModules { moduleSet[modName] = mod @@ -254,7 +255,7 @@ func NewSimApp( panic(err) } } else { - // register streaming services + // register legacy streaming services if we don't have the built-in indexer enabled if err := app.RegisterStreamingServices(appOpts, app.kvStoreKeys()); err != nil { panic(err) } From 6c409794522a5036892357ffdf098769ac9cfe4c Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Fri, 5 Jul 2024 14:44:08 +0200 Subject: [PATCH 25/29] update go.mod --- simapp/go.mod | 1 - 1 file changed, 1 deletion(-) diff --git a/simapp/go.mod b/simapp/go.mod index 34ad562517bc..77b4016f6e29 100644 --- a/simapp/go.mod +++ b/simapp/go.mod @@ -249,7 +249,6 @@ replace ( cosmossdk.io/core => ../core cosmossdk.io/core/testing => ../core/testing cosmossdk.io/depinject => ../depinject - cosmossdk.io/indexer/postgres => ../indexer/postgres cosmossdk.io/log => ../log cosmossdk.io/schema => ../schema cosmossdk.io/store => ../store From 86ec5bd263b8111d8f77b70c88c33c6c84c56b27 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Fri, 5 Jul 2024 14:46:20 +0200 Subject: [PATCH 26/29] fixes --- baseapp/streaming.go | 11 +++-------- simapp/go.mod | 3 --- simapp/go.sum | 8 -------- 3 files changed, 3 insertions(+), 19 deletions(-) diff --git a/baseapp/streaming.go b/baseapp/streaming.go index f06a4676aafc..445c6dcc0798 100644 --- a/baseapp/streaming.go +++ b/baseapp/streaming.go @@ -8,7 +8,7 @@ import ( "github.com/spf13/cast" - "cosmossdk.io/schema/indexing" + "cosmossdk.io/schema/indexer" "cosmossdk.io/schema" "cosmossdk.io/schema/appdata" @@ -34,13 +34,8 @@ const ( // kv-store keys, and app modules. Using the built-in indexer framework is mutually exclusive from using other // types of streaming listeners. func (app *BaseApp) EnableIndexer(indexerOpts interface{}, keys map[string]*storetypes.KVStoreKey, appModules map[string]any) error { - optsMap, ok := indexerOpts.(map[string]interface{}) - if !ok { - return fmt.Errorf("invalid indexer options type %T, expected a map", indexerOpts) - } - - listener, err := indexing.Start(indexing.Options{ - Options: optsMap, + listener, err := indexer.StartManager(indexer.ManagerOptions{ + Config: indexerOpts, Resolver: decoding.ModuleSetDecoderResolver(appModules), SyncSource: nil, Logger: app.logger.With("module", "indexer"), diff --git a/simapp/go.mod b/simapp/go.mod index 77b4016f6e29..01d924c52be8 100644 --- a/simapp/go.mod +++ b/simapp/go.mod @@ -148,9 +148,6 @@ require ( github.com/huandu/skiplist v1.2.0 // indirect github.com/iancoleman/strcase v0.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect - github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jmhodges/levigo v1.0.0 // indirect github.com/klauspost/compress v1.17.9 // indirect diff --git a/simapp/go.sum b/simapp/go.sum index ee740f82d77a..69973a085acc 100644 --- a/simapp/go.sum +++ b/simapp/go.sum @@ -594,14 +594,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= -github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= -github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= -github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= -github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jhump/protoreflect v1.15.3 h1:6SFRuqU45u9hIZPJAoZ8c28T3nK64BNdp9w6jFonzls= github.com/jhump/protoreflect v1.15.3/go.mod h1:4ORHmSBmlCW8fh3xHmJMGyul1zNqZK4Elxc8qKP+p1k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= From cbc8a95395a0d7b0dcdfca26f63af7771de47d8c Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 9 Jul 2024 12:45:12 +0200 Subject: [PATCH 27/29] lint fix --- baseapp/streaming.go | 6 ++---- schema/decoding/resolver_test.go | 1 - schema/decoding/sync.go | 1 - schema/indexer/manager.go | 1 - 4 files changed, 2 insertions(+), 7 deletions(-) diff --git a/baseapp/streaming.go b/baseapp/streaming.go index 445c6dcc0798..9c5c3d5d1742 100644 --- a/baseapp/streaming.go +++ b/baseapp/streaming.go @@ -6,20 +6,18 @@ import ( "sort" "strings" + abci "github.com/cometbft/cometbft/api/cometbft/abci/v1" "github.com/spf13/cast" - "cosmossdk.io/schema/indexer" - "cosmossdk.io/schema" "cosmossdk.io/schema/appdata" "cosmossdk.io/schema/decoding" + "cosmossdk.io/schema/indexer" "cosmossdk.io/store/streaming" storetypes "cosmossdk.io/store/types" "github.com/cosmos/cosmos-sdk/client/flags" servertypes "github.com/cosmos/cosmos-sdk/server/types" - - abci "github.com/cometbft/cometbft/api/cometbft/abci/v1" ) const ( diff --git a/schema/decoding/resolver_test.go b/schema/decoding/resolver_test.go index 3cad85694863..ecea614d1999 100644 --- a/schema/decoding/resolver_test.go +++ b/schema/decoding/resolver_test.go @@ -39,7 +39,6 @@ func TestModuleSetDecoderResolver_IterateAll(t *testing.T) { objectTypes[cdc.Schema.ObjectTypes[0].Name] = true return nil }) - if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/schema/decoding/sync.go b/schema/decoding/sync.go index 1487aca38c7c..85e6b4d74ba0 100644 --- a/schema/decoding/sync.go +++ b/schema/decoding/sync.go @@ -3,7 +3,6 @@ package decoding // SyncSource is an interface that allows indexers to start indexing modules with pre-existing state. // It should generally be a wrapper around the key-value store. type SyncSource interface { - // IterateAllKVPairs iterates over all key-value pairs for a given module. IterateAllKVPairs(moduleName string, fn func(key, value []byte) error) error } diff --git a/schema/indexer/manager.go b/schema/indexer/manager.go index 9c7723917e14..5a7e39faad0a 100644 --- a/schema/indexer/manager.go +++ b/schema/indexer/manager.go @@ -33,7 +33,6 @@ type ManagerOptions struct { // ManagerConfig is the configuration of the indexer manager and contains the configuration for each indexer target. type ManagerConfig struct { - // Target is a map of named indexer targets to their configuration. Target map[string]Config } From 7fc0c03e0d3deef50429cb25040f9cb3b47a3ff0 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 9 Jul 2024 12:59:09 +0200 Subject: [PATCH 28/29] go mod tidy --- client/v2/go.mod | 6 +++++- server/v2/cometbft/go.mod | 2 ++ server/v2/go.mod | 1 + simapp/v2/go.mod | 2 ++ tests/go.mod | 2 ++ x/accounts/defaults/lockup/go.mod | 2 ++ x/accounts/go.mod | 6 +++++- x/auth/go.mod | 2 ++ x/authz/go.mod | 6 +++++- x/bank/go.mod | 6 +++++- x/circuit/go.mod | 2 ++ x/consensus/go.mod | 2 ++ x/distribution/go.mod | 2 ++ x/epochs/go.mod | 2 ++ x/evidence/go.mod | 2 ++ x/feegrant/go.mod | 6 +++++- x/gov/go.mod | 6 +++++- x/group/go.mod | 2 ++ x/mint/go.mod | 2 ++ x/nft/go.mod | 2 ++ x/params/go.mod | 2 ++ x/protocolpool/go.mod | 2 ++ x/slashing/go.mod | 2 ++ x/staking/go.mod | 6 +++++- x/upgrade/go.mod | 2 ++ 25 files changed, 70 insertions(+), 7 deletions(-) diff --git a/client/v2/go.mod b/client/v2/go.mod index 97c1c76d7852..3c3e9ad8f953 100644 --- a/client/v2/go.mod +++ b/client/v2/go.mod @@ -170,7 +170,10 @@ require ( pgregory.net/rapid v1.1.0 // indirect ) -require github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect +require ( + cosmossdk.io/schema v0.0.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect +) replace github.com/cosmos/cosmos-sdk => ./../../ @@ -181,6 +184,7 @@ replace ( cosmossdk.io/core/testing => ../../core/testing cosmossdk.io/depinject => ./../../depinject cosmossdk.io/log => ./../../log + cosmossdk.io/schema => ./../../schema cosmossdk.io/store => ./../../store cosmossdk.io/x/accounts => ./../../x/accounts cosmossdk.io/x/auth => ./../../x/auth diff --git a/server/v2/cometbft/go.mod b/server/v2/cometbft/go.mod index 6e3509c7c406..e2e19e6442f0 100644 --- a/server/v2/cometbft/go.mod +++ b/server/v2/cometbft/go.mod @@ -8,6 +8,7 @@ replace ( cosmossdk.io/core/testing => ../../../core/testing cosmossdk.io/depinject => ../../../depinject cosmossdk.io/log => ../../../log + cosmossdk.io/schema => ../../../schema cosmossdk.io/server/v2 => ../ cosmossdk.io/server/v2/appmanager => ../appmanager cosmossdk.io/store => ../../../store @@ -53,6 +54,7 @@ require ( cosmossdk.io/depinject v1.0.0-alpha.4 // indirect cosmossdk.io/log v1.3.1 // indirect cosmossdk.io/math v1.3.0 // indirect + cosmossdk.io/schema v0.0.0 // indirect cosmossdk.io/store v1.1.1-0.20240418092142-896cdf1971bc // indirect cosmossdk.io/x/bank v0.0.0-20240226161501-23359a0b6d91 // indirect cosmossdk.io/x/staking v0.0.0-00010101000000-000000000000 // indirect diff --git a/server/v2/go.mod b/server/v2/go.mod index 9a7210318e07..0a73d099c41a 100644 --- a/server/v2/go.mod +++ b/server/v2/go.mod @@ -7,6 +7,7 @@ replace ( cosmossdk.io/core => ../../core cosmossdk.io/depinject => ../../depinject cosmossdk.io/log => ../../log + cosmossdk.io/schema => ../../schema cosmossdk.io/server/v2/appmanager => ./appmanager cosmossdk.io/server/v2/stf => ./stf cosmossdk.io/x/tx => ../../x/tx diff --git a/simapp/v2/go.mod b/simapp/v2/go.mod index 7f8309ceab17..b3a476344404 100644 --- a/simapp/v2/go.mod +++ b/simapp/v2/go.mod @@ -62,6 +62,7 @@ require ( cloud.google.com/go/storage v1.42.0 // indirect cosmossdk.io/core/testing v0.0.0-00010101000000-000000000000 // indirect cosmossdk.io/errors v1.0.1 // indirect + cosmossdk.io/schema v0.0.0 // indirect cosmossdk.io/server/v2/appmanager v0.0.0-00010101000000-000000000000 // indirect cosmossdk.io/server/v2/stf v0.0.0-00010101000000-000000000000 // indirect cosmossdk.io/store v1.1.1-0.20240418092142-896cdf1971bc // indirect @@ -251,6 +252,7 @@ replace ( cosmossdk.io/collections => ../../collections cosmossdk.io/core => ../../core cosmossdk.io/depinject => ../../depinject + cosmossdk.io/schema => ../../schema cosmossdk.io/tools/confix => ../../tools/confix cosmossdk.io/x/accounts => ../../x/accounts cosmossdk.io/x/accounts/defaults/lockup => ../../x/accounts/defaults/lockup diff --git a/tests/go.mod b/tests/go.mod index 20ae5e3ae144..b9fa1b4bc9cf 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -64,6 +64,7 @@ require ( cloud.google.com/go/storage v1.42.0 // indirect cosmossdk.io/client/v2 v2.0.0-20230630094428-02b760776860 // indirect cosmossdk.io/core/testing v0.0.0-00010101000000-000000000000 // indirect + cosmossdk.io/schema v0.0.0 // indirect cosmossdk.io/x/circuit v0.0.0-20230613133644-0a778132a60f // indirect cosmossdk.io/x/epochs v0.0.0-20240522060652-a1ae4c3e0337 // indirect filippo.io/edwards25519 v1.1.0 // indirect @@ -241,6 +242,7 @@ replace ( cosmossdk.io/core/testing => ../core/testing cosmossdk.io/depinject => ../depinject cosmossdk.io/log => ../log + cosmossdk.io/schema => ../schema cosmossdk.io/store => ../store cosmossdk.io/x/accounts => ../x/accounts cosmossdk.io/x/accounts/defaults/lockup => ../x/accounts/defaults/lockup diff --git a/x/accounts/defaults/lockup/go.mod b/x/accounts/defaults/lockup/go.mod index c8789e951229..eef1094c6ce8 100644 --- a/x/accounts/defaults/lockup/go.mod +++ b/x/accounts/defaults/lockup/go.mod @@ -19,6 +19,7 @@ require ( buf.build/gen/go/cometbft/cometbft/protocolbuffers/go v1.34.2-20240701160653-fedbb9acfd2f.2 // indirect cosmossdk.io/core/testing v0.0.0-00010101000000-000000000000 // indirect cosmossdk.io/depinject v1.0.0-alpha.4 // indirect + cosmossdk.io/schema v0.0.0 // indirect github.com/cometbft/cometbft/api v1.0.0-rc.1 // indirect github.com/cosmos/crypto v0.1.1 // indirect github.com/dgraph-io/badger/v4 v4.2.0 // indirect @@ -177,6 +178,7 @@ replace ( cosmossdk.io/core/testing => ../../../../core/testing cosmossdk.io/depinject => ../../../../depinject cosmossdk.io/log => ../../../../log + cosmossdk.io/schema => ../../../../schema cosmossdk.io/x/accounts => ../../. cosmossdk.io/x/auth => ../../../auth cosmossdk.io/x/bank => ../../../bank diff --git a/x/accounts/go.mod b/x/accounts/go.mod index b2ca8c8b5592..6408a0e7dc33 100644 --- a/x/accounts/go.mod +++ b/x/accounts/go.mod @@ -21,7 +21,10 @@ require ( require github.com/golang/mock v1.6.0 // indirect -require github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect +require ( + cosmossdk.io/schema v0.0.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect +) require ( buf.build/gen/go/cometbft/cometbft/protocolbuffers/go v1.34.2-20240701160653-fedbb9acfd2f.2 // indirect @@ -182,6 +185,7 @@ replace ( cosmossdk.io/core/testing => ../../core/testing cosmossdk.io/depinject => ../../depinject cosmossdk.io/log => ../../log + cosmossdk.io/schema => ../../schema cosmossdk.io/x/auth => ../auth cosmossdk.io/x/bank => ../bank cosmossdk.io/x/consensus => ../consensus diff --git a/x/auth/go.mod b/x/auth/go.mod index 5546bd71de46..1d5adf28f81d 100644 --- a/x/auth/go.mod +++ b/x/auth/go.mod @@ -38,6 +38,7 @@ require ( buf.build/gen/go/cometbft/cometbft/protocolbuffers/go v1.34.2-20240701160653-fedbb9acfd2f.2 // indirect buf.build/gen/go/cosmos/gogo-proto/protocolbuffers/go v1.34.2-20240130113600-88ef6483f90f.2 // indirect cosmossdk.io/log v1.3.1 // indirect + cosmossdk.io/schema v0.0.0 // indirect cosmossdk.io/x/bank v0.0.0-20240226161501-23359a0b6d91 // indirect cosmossdk.io/x/staking v0.0.0-00010101000000-000000000000 // indirect filippo.io/edwards25519 v1.1.0 // indirect @@ -178,6 +179,7 @@ replace ( cosmossdk.io/core/testing => ../../core/testing cosmossdk.io/depinject => ../../depinject cosmossdk.io/log => ../../log + cosmossdk.io/schema => ../../schema cosmossdk.io/x/accounts => ../accounts cosmossdk.io/x/bank => ../bank cosmossdk.io/x/consensus => ../consensus diff --git a/x/authz/go.mod b/x/authz/go.mod index f76a67cac84b..62614342a819 100644 --- a/x/authz/go.mod +++ b/x/authz/go.mod @@ -167,7 +167,10 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -require github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect +require ( + cosmossdk.io/schema v0.0.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect +) replace github.com/cosmos/cosmos-sdk => ../../. @@ -179,6 +182,7 @@ replace ( cosmossdk.io/core/testing => ../../core/testing cosmossdk.io/depinject => ../../depinject cosmossdk.io/log => ../../log + cosmossdk.io/schema => ../../schema cosmossdk.io/x/accounts => ../accounts cosmossdk.io/x/auth => ../auth cosmossdk.io/x/bank => ../bank diff --git a/x/bank/go.mod b/x/bank/go.mod index b6a0be5f191c..f61da35636a2 100644 --- a/x/bank/go.mod +++ b/x/bank/go.mod @@ -166,7 +166,10 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -require github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect +require ( + cosmossdk.io/schema v0.0.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect +) replace github.com/cosmos/cosmos-sdk => ../../. @@ -178,6 +181,7 @@ replace ( cosmossdk.io/core/testing => ../../core/testing cosmossdk.io/depinject => ../../depinject cosmossdk.io/log => ../../log + cosmossdk.io/schema => ../../schema cosmossdk.io/x/accounts => ../accounts cosmossdk.io/x/auth => ../auth cosmossdk.io/x/consensus => ../consensus diff --git a/x/circuit/go.mod b/x/circuit/go.mod index 18714f7e8ac4..e6705e023aa9 100644 --- a/x/circuit/go.mod +++ b/x/circuit/go.mod @@ -24,6 +24,7 @@ require ( buf.build/gen/go/cosmos/gogo-proto/protocolbuffers/go v1.34.2-20240130113600-88ef6483f90f.2 // indirect cosmossdk.io/log v1.3.1 // indirect cosmossdk.io/math v1.3.0 // indirect + cosmossdk.io/schema v0.0.0 // indirect cosmossdk.io/x/bank v0.0.0-20240226161501-23359a0b6d91 // indirect cosmossdk.io/x/consensus v0.0.0-00010101000000-000000000000 // indirect cosmossdk.io/x/staking v0.0.0-00010101000000-000000000000 // indirect @@ -177,6 +178,7 @@ replace ( cosmossdk.io/core/testing => ../../core/testing cosmossdk.io/depinject => ../../depinject cosmossdk.io/log => ../../log + cosmossdk.io/schema => ../../schema cosmossdk.io/x/accounts => ../accounts cosmossdk.io/x/auth => ../auth cosmossdk.io/x/bank => ../bank diff --git a/x/consensus/go.mod b/x/consensus/go.mod index 6adbd307ae80..64dc7bf7eb2a 100644 --- a/x/consensus/go.mod +++ b/x/consensus/go.mod @@ -26,6 +26,7 @@ require ( cosmossdk.io/errors v1.0.1 // indirect cosmossdk.io/log v1.3.1 // indirect cosmossdk.io/math v1.3.0 // indirect + cosmossdk.io/schema v0.0.0 // indirect cosmossdk.io/x/auth v0.0.0-00010101000000-000000000000 // indirect cosmossdk.io/x/bank v0.0.0-20240226161501-23359a0b6d91 // indirect cosmossdk.io/x/staking v0.0.0-00010101000000-000000000000 // indirect @@ -174,6 +175,7 @@ replace ( cosmossdk.io/core/testing => ../../core/testing cosmossdk.io/depinject => ../../depinject cosmossdk.io/log => ../../log + cosmossdk.io/schema => ../../schema cosmossdk.io/x/accounts => ../accounts cosmossdk.io/x/auth => ../auth cosmossdk.io/x/bank => ../bank diff --git a/x/distribution/go.mod b/x/distribution/go.mod index 8a0e9df7c681..dc870f41f747 100644 --- a/x/distribution/go.mod +++ b/x/distribution/go.mod @@ -30,6 +30,7 @@ require ( ) require ( + cosmossdk.io/schema v0.0.0 // indirect github.com/cockroachdb/errors v1.11.1 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect @@ -182,6 +183,7 @@ replace ( cosmossdk.io/core/testing => ../../core/testing cosmossdk.io/depinject => ../../depinject cosmossdk.io/log => ../../log + cosmossdk.io/schema => ../../schema cosmossdk.io/x/accounts => ../accounts cosmossdk.io/x/auth => ../auth cosmossdk.io/x/bank => ../bank diff --git a/x/epochs/go.mod b/x/epochs/go.mod index 7aaf3ebad844..2be559903268 100644 --- a/x/epochs/go.mod +++ b/x/epochs/go.mod @@ -165,6 +165,7 @@ require ( require ( cosmossdk.io/log v1.3.1 // indirect + cosmossdk.io/schema v0.0.0 // indirect cosmossdk.io/x/consensus v0.0.0-00010101000000-000000000000 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect @@ -179,6 +180,7 @@ replace ( cosmossdk.io/core/testing => ../../core/testing cosmossdk.io/depinject => ../../depinject cosmossdk.io/log => ../../log + cosmossdk.io/schema => ../../schema cosmossdk.io/x/accounts => ../accounts cosmossdk.io/x/auth => ../auth cosmossdk.io/x/bank => ../bank diff --git a/x/evidence/go.mod b/x/evidence/go.mod index 2e45731e33c4..3202a5a72deb 100644 --- a/x/evidence/go.mod +++ b/x/evidence/go.mod @@ -28,6 +28,7 @@ require ( buf.build/gen/go/cometbft/cometbft/protocolbuffers/go v1.34.2-20240701160653-fedbb9acfd2f.2 // indirect buf.build/gen/go/cosmos/gogo-proto/protocolbuffers/go v1.34.2-20240130113600-88ef6483f90f.2 // indirect cosmossdk.io/log v1.3.1 // indirect + cosmossdk.io/schema v0.0.0 // indirect cosmossdk.io/x/auth v0.0.0-00010101000000-000000000000 // indirect cosmossdk.io/x/bank v0.0.0-20240226161501-23359a0b6d91 // indirect cosmossdk.io/x/staking v0.0.0-00010101000000-000000000000 // indirect @@ -177,6 +178,7 @@ replace ( cosmossdk.io/core/testing => ../../core/testing cosmossdk.io/depinject => ../../depinject cosmossdk.io/log => ../../log + cosmossdk.io/schema => ../../schema cosmossdk.io/x/accounts => ../accounts cosmossdk.io/x/auth => ../auth cosmossdk.io/x/bank => ../bank diff --git a/x/feegrant/go.mod b/x/feegrant/go.mod index 0082f43338db..9f8b3315fca8 100644 --- a/x/feegrant/go.mod +++ b/x/feegrant/go.mod @@ -171,7 +171,10 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -require github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect +require ( + cosmossdk.io/schema v0.0.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect +) replace github.com/cosmos/cosmos-sdk => ../../. @@ -183,6 +186,7 @@ replace ( cosmossdk.io/core/testing => ../../core/testing cosmossdk.io/depinject => ../../depinject cosmossdk.io/log => ../../log + cosmossdk.io/schema => ../../schema cosmossdk.io/x/accounts => ../accounts cosmossdk.io/x/auth => ../auth cosmossdk.io/x/bank => ../bank diff --git a/x/gov/go.mod b/x/gov/go.mod index 92efbf8c3c80..9a4c46ea4bd4 100644 --- a/x/gov/go.mod +++ b/x/gov/go.mod @@ -170,7 +170,10 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -require github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect +require ( + cosmossdk.io/schema v0.0.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect +) replace github.com/cosmos/cosmos-sdk => ../../. @@ -182,6 +185,7 @@ replace ( cosmossdk.io/core/testing => ../../core/testing cosmossdk.io/depinject => ../../depinject cosmossdk.io/log => ../../log + cosmossdk.io/schema => ../../schema cosmossdk.io/x/accounts => ../accounts cosmossdk.io/x/auth => ../auth cosmossdk.io/x/bank => ../bank diff --git a/x/group/go.mod b/x/group/go.mod index 37eccf21f606..dd490ce5dbc1 100644 --- a/x/group/go.mod +++ b/x/group/go.mod @@ -43,6 +43,7 @@ require ( buf.build/gen/go/cosmos/gogo-proto/protocolbuffers/go v1.34.2-20240130113600-88ef6483f90f.2 // indirect cosmossdk.io/collections v0.4.0 // indirect cosmossdk.io/core/testing v0.0.0-00010101000000-000000000000 // indirect + cosmossdk.io/schema v0.0.0 // indirect cosmossdk.io/x/accounts/defaults/lockup v0.0.0-20240417181816-5e7aae0db1f5 // indirect cosmossdk.io/x/epochs v0.0.0-20240522060652-a1ae4c3e0337 // indirect cosmossdk.io/x/tx v0.13.3 // indirect @@ -188,6 +189,7 @@ replace ( cosmossdk.io/core/testing => ../../core/testing cosmossdk.io/depinject => ../../depinject cosmossdk.io/log => ../../log + cosmossdk.io/schema => ../../schema cosmossdk.io/x/accounts => ../accounts cosmossdk.io/x/auth => ../auth cosmossdk.io/x/authz => ../authz diff --git a/x/mint/go.mod b/x/mint/go.mod index f286acecd279..92d629616826 100644 --- a/x/mint/go.mod +++ b/x/mint/go.mod @@ -159,6 +159,7 @@ require ( require ( buf.build/gen/go/cometbft/cometbft/protocolbuffers/go v1.34.2-20240701160653-fedbb9acfd2f.2 // indirect + cosmossdk.io/schema v0.0.0 // indirect cosmossdk.io/x/consensus v0.0.0-00010101000000-000000000000 // indirect github.com/cometbft/cometbft/api v1.0.0-rc.1 // indirect github.com/cosmos/crypto v0.1.1 // indirect @@ -181,6 +182,7 @@ replace ( cosmossdk.io/core/testing => ../../core/testing cosmossdk.io/depinject => ../../depinject cosmossdk.io/log => ../../log + cosmossdk.io/schema => ../../schema cosmossdk.io/x/accounts => ../accounts cosmossdk.io/x/auth => ../auth cosmossdk.io/x/bank => ../bank diff --git a/x/nft/go.mod b/x/nft/go.mod index 10037c6b0749..aae6cb4809cc 100644 --- a/x/nft/go.mod +++ b/x/nft/go.mod @@ -25,6 +25,7 @@ require ( buf.build/gen/go/cometbft/cometbft/protocolbuffers/go v1.34.2-20240701160653-fedbb9acfd2f.2 // indirect buf.build/gen/go/cosmos/gogo-proto/protocolbuffers/go v1.34.2-20240130113600-88ef6483f90f.2 // indirect cosmossdk.io/collections v0.4.0 // indirect + cosmossdk.io/schema v0.0.0 // indirect cosmossdk.io/x/auth v0.0.0-00010101000000-000000000000 // indirect cosmossdk.io/x/bank v0.0.0-20240226161501-23359a0b6d91 // indirect cosmossdk.io/x/consensus v0.0.0-00010101000000-000000000000 // indirect @@ -177,6 +178,7 @@ replace ( cosmossdk.io/core/testing => ../../core/testing cosmossdk.io/depinject => ../../depinject cosmossdk.io/log => ../../log + cosmossdk.io/schema => ../../schema cosmossdk.io/x/accounts => ../accounts cosmossdk.io/x/auth => ../auth cosmossdk.io/x/bank => ../bank diff --git a/x/params/go.mod b/x/params/go.mod index fc95311d927b..458d703a99a0 100644 --- a/x/params/go.mod +++ b/x/params/go.mod @@ -29,6 +29,7 @@ require ( buf.build/gen/go/cometbft/cometbft/protocolbuffers/go v1.34.2-20240701160653-fedbb9acfd2f.2 // indirect buf.build/gen/go/cosmos/gogo-proto/protocolbuffers/go v1.34.2-20240130113600-88ef6483f90f.2 // indirect cosmossdk.io/collections v0.4.0 // indirect + cosmossdk.io/schema v0.0.0 // indirect cosmossdk.io/x/auth v0.0.0-00010101000000-000000000000 // indirect cosmossdk.io/x/bank v0.0.0-20240226161501-23359a0b6d91 // indirect cosmossdk.io/x/consensus v0.0.0-00010101000000-000000000000 // indirect @@ -178,6 +179,7 @@ replace ( cosmossdk.io/core/testing => ../../core/testing cosmossdk.io/depinject => ../../depinject cosmossdk.io/log => ../../log + cosmossdk.io/schema => ../../schema cosmossdk.io/x/accounts => ../accounts cosmossdk.io/x/auth => ../auth cosmossdk.io/x/bank => ../bank diff --git a/x/protocolpool/go.mod b/x/protocolpool/go.mod index ec89c50376d1..09e197cd0eed 100644 --- a/x/protocolpool/go.mod +++ b/x/protocolpool/go.mod @@ -28,6 +28,7 @@ require ( require ( buf.build/gen/go/cometbft/cometbft/protocolbuffers/go v1.34.2-20240701160653-fedbb9acfd2f.2 // indirect buf.build/gen/go/cosmos/gogo-proto/protocolbuffers/go v1.34.2-20240130113600-88ef6483f90f.2 // indirect + cosmossdk.io/schema v0.0.0 // indirect cosmossdk.io/x/bank v0.0.0-20240226161501-23359a0b6d91 // indirect cosmossdk.io/x/consensus v0.0.0-00010101000000-000000000000 // indirect cosmossdk.io/x/staking v0.0.0-00010101000000-000000000000 // indirect @@ -177,6 +178,7 @@ replace ( cosmossdk.io/core/testing => ../../core/testing cosmossdk.io/depinject => ../../depinject cosmossdk.io/log => ../../log + cosmossdk.io/schema => ../../schema cosmossdk.io/x/accounts => ../accounts cosmossdk.io/x/auth => ../auth cosmossdk.io/x/bank => ../bank diff --git a/x/slashing/go.mod b/x/slashing/go.mod index 1d9ebe756497..af4bddc02852 100644 --- a/x/slashing/go.mod +++ b/x/slashing/go.mod @@ -30,6 +30,7 @@ require ( buf.build/gen/go/cometbft/cometbft/protocolbuffers/go v1.34.2-20240701160653-fedbb9acfd2f.2 // indirect buf.build/gen/go/cosmos/gogo-proto/protocolbuffers/go v1.34.2-20240130113600-88ef6483f90f.2 // indirect cosmossdk.io/log v1.3.1 // indirect + cosmossdk.io/schema v0.0.0 // indirect cosmossdk.io/x/bank v0.0.0-20240226161501-23359a0b6d91 // indirect cosmossdk.io/x/consensus v0.0.0-00010101000000-000000000000 // indirect cosmossdk.io/x/tx v0.13.3 // indirect @@ -178,6 +179,7 @@ replace ( cosmossdk.io/core/testing => ../../core/testing cosmossdk.io/depinject => ../../depinject cosmossdk.io/log => ../../log + cosmossdk.io/schema => ../../schema cosmossdk.io/x/accounts => ../accounts cosmossdk.io/x/auth => ../auth cosmossdk.io/x/bank => ../bank diff --git a/x/staking/go.mod b/x/staking/go.mod index 1d077317e32c..a43f71232d90 100644 --- a/x/staking/go.mod +++ b/x/staking/go.mod @@ -169,7 +169,10 @@ require ( go.opencensus.io v0.24.0 // indirect ) -require github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect +require ( + cosmossdk.io/schema v0.0.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect +) replace github.com/cosmos/cosmos-sdk => ../../. @@ -181,6 +184,7 @@ replace ( cosmossdk.io/core/testing => ../../core/testing cosmossdk.io/depinject => ../../depinject cosmossdk.io/log => ../../log + cosmossdk.io/schema => ../../schema cosmossdk.io/x/accounts => ../accounts cosmossdk.io/x/auth => ../auth cosmossdk.io/x/bank => ../bank diff --git a/x/upgrade/go.mod b/x/upgrade/go.mod index 11324a6d7465..091180ed5671 100644 --- a/x/upgrade/go.mod +++ b/x/upgrade/go.mod @@ -44,6 +44,7 @@ require ( cloud.google.com/go/storage v1.42.0 // indirect cosmossdk.io/collections v0.4.0 // indirect cosmossdk.io/math v1.3.0 // indirect + cosmossdk.io/schema v0.0.0 // indirect cosmossdk.io/x/bank v0.0.0-20240226161501-23359a0b6d91 // indirect cosmossdk.io/x/staking v0.0.0-00010101000000-000000000000 // indirect cosmossdk.io/x/tx v0.13.3 // indirect @@ -208,6 +209,7 @@ replace ( cosmossdk.io/core/testing => ../../core/testing cosmossdk.io/depinject => ../../depinject cosmossdk.io/log => ../../log + cosmossdk.io/schema => ../../schema cosmossdk.io/x/accounts => ../accounts cosmossdk.io/x/auth => ../auth cosmossdk.io/x/bank => ../bank From 199d06bbf99cda5aaa2f68fbc4f1e70fbaa168b5 Mon Sep 17 00:00:00 2001 From: Aaron Craelius Date: Tue, 9 Jul 2024 13:02:54 +0200 Subject: [PATCH 29/29] simapp build fix --- simapp/app_di.go | 1 + 1 file changed, 1 insertion(+) diff --git a/simapp/app_di.go b/simapp/app_di.go index 77cc652a9a52..52c9917fd186 100644 --- a/simapp/app_di.go +++ b/simapp/app_di.go @@ -12,6 +12,7 @@ import ( "github.com/spf13/cast" clienthelpers "cosmossdk.io/client/v2/helpers" + "cosmossdk.io/core/appmodule" "cosmossdk.io/core/legacy" "cosmossdk.io/depinject" "cosmossdk.io/log"