diff --git a/packages/eventindexer/.l1.env b/packages/eventindexer/.l1.env index 948bcf31155..ec15d743aa0 100644 --- a/packages/eventindexer/.l1.env +++ b/packages/eventindexer/.l1.env @@ -7,11 +7,10 @@ DATABASE_HOST=localhost:3306 DATABASE_MAX_IDLE_CONNS=50 DATABASE_MAX_OPEN_CONNS=3000 DATABASE_CONN_MAX_LIFETIME_IN_MS=100000 -L1_TAIKO_ADDRESS=0x95fF8D3CE9dcB7455BEB7845143bEA84Fe5C4F6f -BRIDGE_ADDRESS=0x5293Bb897db0B64FFd11E0194984E8c5F1f06178 -RPC_URL=wss://l1ws.jolnir.taiko.xyz +L1_TAIKO_ADDRESS=0x0DCd1Bf9A1b36cE34237eEaFef220932846BCD82 +BRIDGE_ADDRESS=0x9A9f2CCfdE556A7E9Ff0848998Aa4a0CFD8863AE +RPC_URL=wss://l1ws.internal.taiko.xyz CORS_ORIGINS=* BLOCK_BATCH_SIZE=10 CACHE_INTERVAL_IN_SECONDS=60 LAYER=l1 -GENESIS_DATE=2023-09-08 \ No newline at end of file diff --git a/packages/eventindexer/cmd/flags/generator.go b/packages/eventindexer/cmd/flags/generator.go index 9c53b437f50..b9388df43ac 100644 --- a/packages/eventindexer/cmd/flags/generator.go +++ b/packages/eventindexer/cmd/flags/generator.go @@ -10,15 +10,7 @@ var ( Category: generatorCategory, EnvVars: []string{"GENESIS_DATE"}, } - Regenerate = &cli.StringFlag{ - Name: "regenerate", - Usage: "True to delete all existing data and regenerate from genesis, false to not", - Required: false, - Category: generatorCategory, - EnvVars: []string{"REGENERATE"}, - } ) var GeneratorFlags = MergeFlags(CommonFlags, []cli.Flag{ GenesisDate, - Regenerate, }) diff --git a/packages/eventindexer/cmd/flags/indexer.go b/packages/eventindexer/cmd/flags/indexer.go index 37c845599bf..3b19820b05b 100644 --- a/packages/eventindexer/cmd/flags/indexer.go +++ b/packages/eventindexer/cmd/flags/indexer.go @@ -121,7 +121,6 @@ var ( var IndexerFlags = MergeFlags(CommonFlags, []cli.Flag{ RPCUrl, // optional - ETHClientTimeout, L1TaikoAddress, HTTPPort, MetricsHTTPPort, diff --git a/packages/eventindexer/db/db.go b/packages/eventindexer/db/db.go index 3b865643fb7..9223263b3cb 100644 --- a/packages/eventindexer/db/db.go +++ b/packages/eventindexer/db/db.go @@ -46,14 +46,14 @@ func OpenDBConnection(opts DBConnectionOpts) (*DB, error) { dsn := "" if opts.Password == "" { dsn = fmt.Sprintf( - "%v@tcp(%v)/%v?charset=utf8mb4&parseTime=True", + "%v@tcp(%v)/%v?charset=utf8mb4&parseTime=True&loc=Local", opts.Name, opts.Host, opts.Database, ) } else { dsn = fmt.Sprintf( - "%v:%v@tcp(%v)/%v?charset=utf8mb4&parseTime=True", + "%v:%v@tcp(%v)/%v?charset=utf8mb4&parseTime=True&loc=Local", opts.Name, opts.Password, opts.Host, diff --git a/packages/eventindexer/generator/config.go b/packages/eventindexer/generator/config.go index 378b03c24c0..1cf9ec83675 100644 --- a/packages/eventindexer/generator/config.go +++ b/packages/eventindexer/generator/config.go @@ -28,7 +28,6 @@ type Config struct { DatabaseMaxConnLifetime uint64 MetricsHTTPPort uint64 GenesisDate time.Time - Regenerate bool OpenDBFunc func() (DB, error) } @@ -49,7 +48,6 @@ func NewConfigFromCliContext(c *cli.Context) (*Config, error) { DatabaseMaxConnLifetime: c.Uint64(flags.DatabaseConnMaxLifetime.Name), MetricsHTTPPort: c.Uint64(flags.MetricsHTTPPort.Name), GenesisDate: date, - Regenerate: c.Bool(flags.Regenerate.Name), OpenDBFunc: func() (DB, error) { return db.OpenDBConnection(db.DBConnectionOpts{ Name: c.String(flags.DatabaseUsername.Name), diff --git a/packages/eventindexer/generator/config_test.go b/packages/eventindexer/generator/config_test.go index f38f1ed1205..2731d49fb0f 100644 --- a/packages/eventindexer/generator/config_test.go +++ b/packages/eventindexer/generator/config_test.go @@ -32,7 +32,6 @@ func TestNewConfigFromCliContext(t *testing.T) { assert.Equal(t, "dbpass", c.DatabasePassword) assert.Equal(t, "dbname", c.DatabaseName) assert.Equal(t, "dbhost", c.DatabaseHost) - assert.Equal(t, true, c.Regenerate) wantTime, _ := time.Parse("2006-01-02", "2023-07-07") assert.Equal(t, wantTime, c.GenesisDate) @@ -53,6 +52,5 @@ func TestNewConfigFromCliContext(t *testing.T) { "-" + flags.DatabaseHost.Name, "dbhost", "-" + flags.DatabaseName.Name, "dbname", "-" + flags.GenesisDate.Name, "2023-07-07", - "-" + flags.Regenerate.Name, "true", })) } diff --git a/packages/eventindexer/generator/generator.go b/packages/eventindexer/generator/generator.go index b60228f9b4c..0b933732148 100644 --- a/packages/eventindexer/generator/generator.go +++ b/packages/eventindexer/generator/generator.go @@ -4,10 +4,11 @@ import ( "context" "errors" "log/slog" + "strconv" + "syscall" "time" "github.com/ethereum/go-ethereum/common" - "github.com/shopspring/decimal" "github.com/taikoxyz/taiko-mono/packages/eventindexer" "github.com/taikoxyz/taiko-mono/packages/eventindexer/tasks" "github.com/urfave/cli/v2" @@ -23,7 +24,6 @@ var ( type Generator struct { db DB genesisDate time.Time - regenerate bool } func (g *Generator) InitFromCli(ctx context.Context, c *cli.Context) error { @@ -43,7 +43,6 @@ func InitFromConfig(ctx context.Context, g *Generator, cfg *Config) error { g.db = db g.genesisDate = cfg.GenesisDate - g.regenerate = cfg.Regenerate return nil } @@ -53,20 +52,16 @@ func (g *Generator) Name() string { } func (g *Generator) Start() error { - if g.regenerate { - slog.Info("regenerating, deleting existing data") - - if err := g.deleteTimeSeriesData(context.Background()); err != nil { - return err - } - } - slog.Info("generating time series data") if err := g.generateTimeSeriesData(context.Background()); err != nil { return err } + if err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM); err != nil { + return err + } + return nil } @@ -81,15 +76,6 @@ func (g *Generator) Close(ctx context.Context) { } } -func (g *Generator) deleteTimeSeriesData(ctx context.Context) error { - deleteStmt := "DELETE FROM time_series_data;" - if err := g.db.GormDB().Exec(deleteStmt).Error; err != nil { - return err - } - - return nil -} - // generateTimeSeriesData iterates over each task and generates time series data. func (g *Generator) generateTimeSeriesData(ctx context.Context) error { for _, task := range tasks.Tasks { @@ -107,25 +93,19 @@ func (g *Generator) generateTimeSeriesData(ctx context.Context) error { func (g *Generator) generateByTask(ctx context.Context, task string) error { slog.Info("generating for task", "task", task) - startingDate, err := g.getStartingDateByTask(ctx, task) + latestDate, err := g.getLatestDateByTask(ctx, task) if err != nil { return err } currentDate := g.getCurrentDate() - if startingDate.Compare(currentDate) == 0 { - slog.Info( - "data already generated up-to-date for task", - "task", task, - "date", startingDate.Format("2006-01-02"), - "currentDate", currentDate.Format("2006-01-02"), - ) - + if latestDate.AddDate(0, 0, 1).Compare(currentDate) == 0 { + slog.Info("data already generated up-to-date for task", "task", task, "date", latestDate.Format("2006-01-02")) return nil } // Loop through each date from latestDate to currentDate - for d := startingDate; d.Before(currentDate); d = d.AddDate(0, 0, 1) { + for d := latestDate; d.Before(currentDate); d = d.AddDate(0, 0, 1) { slog.Info("Processing", "task", task, "date", d.Format("2006-01-02"), "currentDate", currentDate.Format("2006-01-02")) result, err := g.queryByTask(task, d) @@ -134,7 +114,7 @@ func (g *Generator) generateByTask(ctx context.Context, task string) error { return err } - slog.Info("Query successful", "task", task, "date", d.Format("2006-01-02"), "result", result.String()) + slog.Info("Query successful", "task", task, "date", d.Format("2006-01-02"), "result", result) insertStmt := ` INSERT INTO time_series_data(task, value, date) @@ -152,12 +132,12 @@ func (g *Generator) generateByTask(ctx context.Context, task string) error { return nil } -// getStartingDateByTask returns first required time series data, one after the latest date entry, -// or the genesis date. -func (g *Generator) getStartingDateByTask(ctx context.Context, task string) (time.Time, error) { +// getLatestDateByTask returns the last time time series data has been generated +// for the given task. +func (g *Generator) getLatestDateByTask(ctx context.Context, task string) (time.Time, error) { var latestDateString string - var nextRequiredDate time.Time + var latestDate time.Time q := `SELECT date FROM time_series_data WHERE task = ? ORDER BY date DESC LIMIT 1;` @@ -166,207 +146,178 @@ func (g *Generator) getStartingDateByTask(ctx context.Context, task string) (tim slog.Info("latestDateString", "task", task, "date", latestDateString) if err != nil || latestDateString == "" { - nextRequiredDate = g.genesisDate + latestDate = g.genesisDate } else { - latestDate, err := time.Parse("2006-01-02", latestDateString) - if err != nil { - return time.Time{}, err - } + latestDate, err = time.Parse("2006-01-02", latestDateString) + } - nextRequiredDate = latestDate.AddDate(0, 0, 1) + if err != nil { + return time.Time{}, err } - slog.Info("next required date for task", "task", task, "nextRequiredDate", nextRequiredDate.Format("2006-01-02")) + slog.Info("latest date for task", "task", task, "latestDate", latestDate.Format("2006-01-02")) - return nextRequiredDate, nil + return latestDate, nil } // getCurrentDate returns the current date in YYYY-MM-DD format func (g *Generator) getCurrentDate() time.Time { // Get current date - currentTime := time.Now().UTC() + currentTime := time.Now() currentDate := time.Date(currentTime.Year(), currentTime.Month(), currentTime.Day(), 0, 0, 0, 0, time.UTC) return currentDate } -// nolint: funlen, gocognit +// nolint: funlen // queryByTask runs a database query which should return result data based on the // task -func (g *Generator) queryByTask(task string, date time.Time) (decimal.Decimal, error) { +func (g *Generator) queryByTask(task string, date time.Time) (string, error) { dateString := date.Format("2006-01-02") - var result decimal.Decimal + var result string var err error switch task { - case tasks.ProposerRewardsPerDay: - query := "SELECT COALESCE(SUM(proposer_reward), 0) FROM events WHERE event = ? AND DATE(transacted_at) = ?" - err = g.db.GormDB(). - Raw(query, eventindexer.EventNameBlockProposed, dateString). - Scan(&result).Error - - case tasks.TotalProposerRewards: - var dailyProposerRewards decimal.NullDecimal - - query := "SELECT COALESCE(SUM(proposer_reward), 0) FROM events WHERE event = ? AND DATE(transacted_at) = ?" - err = g.db.GormDB(). - Raw(query, eventindexer.EventNameBlockProposed, dateString). - Scan(&dailyProposerRewards).Error - - tsdResult, err := g.previousDayTsdResultByTask(task, date) - if err != nil { - return result, err - } - - result = tsdResult.Decimal.Add(dailyProposerRewards.Decimal) - - case tasks.TotalProofRewards: - var dailyProofRewards decimal.NullDecimal - - query := "SELECT COALESCE(SUM(proof_reward), 0) FROM events WHERE event = ? AND DATE(transacted_at) = ?" - err = g.db.GormDB(). - Raw(query, eventindexer.EventNameBlockProposed, dateString). - Scan(&dailyProofRewards).Error - - tsdResult, err := g.previousDayTsdResultByTask(task, date) - if err != nil { - return result, err - } - - result = tsdResult.Decimal.Add(dailyProofRewards.Decimal) - case tasks.ProofRewardsPerDay: - query := "SELECT COALESCE(SUM(proof_reward), 0) FROM events WHERE event = ? AND DATE(transacted_at) = ?" - err = g.db.GormDB(). - Raw(query, eventindexer.EventNameBlockProposed, dateString). - Scan(&result).Error case tasks.BridgeMessagesSentPerDay: err = g.eventCount(task, date, eventindexer.EventNameMessageSent, &result) case tasks.TotalBridgeMessagesSent: - var dailyMsgSentCount decimal.NullDecimal + var dailyMsgSentCount int err = g.eventCount(task, date, eventindexer.EventNameMessageSent, &dailyMsgSentCount) if err != nil { - return result, err + return "", err } tsdResult, err := g.previousDayTsdResultByTask(task, date) if err != nil { - return result, err + return "", err } - result = tsdResult.Decimal.Add(dailyMsgSentCount.Decimal) + result = strconv.Itoa(dailyMsgSentCount + tsdResult) case tasks.ProposeBlockTxPerDay: err = g.eventCount(task, date, eventindexer.EventNameBlockProposed, &result) case tasks.TotalProposeBlockTx: - var dailyProposerCount decimal.NullDecimal + var dailyProposerCount int err = g.eventCount(task, date, eventindexer.EventNameBlockProposed, &dailyProposerCount) if err != nil { - return result, err + return "", err } tsdResult, err := g.previousDayTsdResultByTask(task, date) if err != nil { - return result, err + return "", err } - result = tsdResult.Decimal.Add(dailyProposerCount.Decimal) + result = strconv.Itoa(dailyProposerCount + tsdResult) case tasks.UniqueProposersPerDay: query := "SELECT COUNT(DISTINCT address) FROM events WHERE event = ? AND DATE(transacted_at) = ?" err = g.db.GormDB(). - Raw(query, eventindexer.EventNameBlockProposed, dateString). + Raw(query, eventindexer.EventNameBlockProposed, date). Scan(&result).Error case tasks.TotalUniqueProposers: - query := `SELECT COUNT(DISTINCT address) FROM events WHERE event = ?` + var dailyProposerCount int + + query := `SELECT COUNT(DISTINCT address) FROM events WHERE event = ? AND DATE(transacted_at) = ?` - err = g.db.GormDB().Raw( - query, - eventindexer.EventNameBlockProposed, - ).Scan(&result).Error + err = g.db.GormDB().Raw(query, eventindexer.EventNameBlockProposed, dateString).Scan(&dailyProposerCount).Error if err != nil { - return result, err + return "", err } + + tsdResult, err := g.previousDayTsdResultByTask(task, date) + if err != nil { + return "", err + } + + result = strconv.Itoa(dailyProposerCount + tsdResult) case tasks.UniqueProversPerDay: query := "SELECT COUNT(DISTINCT address) FROM events WHERE event = ? AND DATE(transacted_at) = ?" err = g.db.GormDB(). - Raw(query, eventindexer.EventNameBlockProven, dateString). + Raw(query, eventindexer.EventNameBlockProven, date). Scan(&result).Error case tasks.TotalUniqueProvers: - query := `SELECT COUNT(DISTINCT address) FROM events WHERE event = ?` + var dailyProposerCount int + + query := `SELECT COUNT(DISTINCT address) FROM events WHERE event = ? AND DATE(transacted_at) = ?` - err = g.db.GormDB().Raw( - query, - eventindexer.EventNameBlockProven, - ).Scan(&result).Error + err = g.db.GormDB().Raw(query, eventindexer.EventNameBlockProven, dateString).Scan(&dailyProposerCount).Error if err != nil { - return result, err + return "", err } + + tsdResult, err := g.previousDayTsdResultByTask(task, date) + if err != nil { + return "", err + } + + result = strconv.Itoa(dailyProposerCount + tsdResult) case tasks.ProveBlockTxPerDay: query := "SELECT COUNT(*) FROM events WHERE event = ? AND DATE(transacted_at) = ?" err = g.db.GormDB(). - Raw(query, eventindexer.EventNameBlockProven, dateString). + Raw(query, eventindexer.EventNameBlockProven, date). Scan(&result).Error case tasks.TotalProveBlockTx: - var dailyProveBlockCount decimal.NullDecimal + var dailyProposerCount int query := `SELECT COUNT(*) FROM events WHERE event = ? AND DATE(transacted_at) = ?` - err = g.db.GormDB().Raw(query, eventindexer.EventNameBlockProven, dateString).Scan(&dailyProveBlockCount).Error + err = g.db.GormDB().Raw(query, eventindexer.EventNameBlockProven, dateString).Scan(&dailyProposerCount).Error if err != nil { - return result, err + return "", err } tsdResult, err := g.previousDayTsdResultByTask(task, date) if err != nil { - return result, err + return "", err } - result = tsdResult.Decimal.Add(dailyProveBlockCount.Decimal) + result = strconv.Itoa(dailyProposerCount + tsdResult) case tasks.AccountsPerDay: query := `SELECT COUNT(*) FROM accounts WHERE DATE(transacted_at) = ?` err = g.db.GormDB().Raw(query, dateString).Scan(&result).Error case tasks.TotalAccounts: - var dailyAccountsCount decimal.NullDecimal + var dailyAccountsCount int query := `SELECT COUNT(*) FROM accounts WHERE DATE(transacted_at) = ?` err = g.db.GormDB().Raw(query, dateString).Scan(&dailyAccountsCount).Error if err != nil { - return result, err + return "", err } tsdResult, err := g.previousDayTsdResultByTask(task, date) if err != nil { - return result, err + return "", err } - result = tsdResult.Decimal.Add(dailyAccountsCount.Decimal) + result = strconv.Itoa(dailyAccountsCount + tsdResult) case tasks.BlocksPerDay: query := `SELECT COUNT(*) FROM blocks WHERE DATE(transacted_at) = ?` err = g.db.GormDB().Raw(query, dateString).Scan(&result).Error case tasks.TotalBlocks: - var dailyBlockCount decimal.NullDecimal + var dailyBlockCount int query := `SELECT COUNT(*) FROM blocks WHERE DATE(transacted_at) = ?` err = g.db.GormDB().Raw(query, dateString).Scan(&dailyBlockCount).Error if err != nil { - return result, err + return "", err } tsdResult, err := g.previousDayTsdResultByTask(task, date) if err != nil { - return result, err + return "", err } - result = tsdResult.Decimal.Add(dailyBlockCount.Decimal) + result = strconv.Itoa(dailyBlockCount + tsdResult) case tasks.TransactionsPerDay: query := `SELECT COUNT(*) FROM transactions WHERE DATE(transacted_at) = ?` err = g.db.GormDB().Raw(query, dateString).Scan(&result).Error case tasks.TotalTransactions: - var dailyTxCount decimal.NullDecimal + var dailyTxCount int // get current days txs, get previous entry for the time series data, add them together. @@ -374,41 +325,41 @@ func (g *Generator) queryByTask(task string, date time.Time) (decimal.Decimal, e err = g.db.GormDB().Raw(query, dateString).Scan(&dailyTxCount).Error if err != nil { - return result, err + return "", err } tsdResult, err := g.previousDayTsdResultByTask(task, date) if err != nil { - return result, err + return "", err } - result = tsdResult.Decimal.Add(dailyTxCount.Decimal) + result = strconv.Itoa(dailyTxCount + tsdResult) case tasks.ContractDeploymentsPerDay: query := `SELECT COUNT(*) FROM transactions WHERE DATE(transacted_at) = ? AND contract_address != ?` - err = g.db.GormDB().Raw(query, dateString, ZeroAddress.Hex()).Scan(&result).Error + err = g.db.GormDB().Raw(query, dateString, ZeroAddress).Scan(&result).Error case tasks.TotalContractDeployments: - var dailyContractCount decimal.NullDecimal + var dailyContractCount int // get current days txs, get previous entry for the time series data, add them together. query := `SELECT COUNT(*) FROM transactions WHERE DATE(transacted_at) = ? AND contract_address != ?` - err = g.db.GormDB().Raw(query, dateString, ZeroAddress.Hex()).Scan(&dailyContractCount).Error + err = g.db.GormDB().Raw(query, dateString, ZeroAddress).Scan(&dailyContractCount).Error if err != nil { - return result, err + return "", err } tsdResult, err := g.previousDayTsdResultByTask(task, date) if err != nil { - return result, err + return "", err } - result = tsdResult.Decimal.Add(dailyContractCount.Decimal) + result = strconv.Itoa(dailyContractCount + tsdResult) default: - return result, errors.New("task not supported") + return "", errors.New("task not supported") } if err != nil { - return result, err + return "", err } return result, nil @@ -416,21 +367,14 @@ func (g *Generator) queryByTask(task string, date time.Time) (decimal.Decimal, e // previousDayTsdResultByTask returns the previous day's time series data, based on // task and time passed in. -func (g *Generator) previousDayTsdResultByTask(task string, date time.Time) (decimal.NullDecimal, error) { - var tsdResult decimal.NullDecimal +func (g *Generator) previousDayTsdResultByTask(task string, date time.Time) (int, error) { + var tsdResult int tsdQuery := `SELECT value FROM time_series_data WHERE task = ? AND date = ?` err := g.db.GormDB().Raw(tsdQuery, task, date.AddDate(0, 0, -1).Format("2006-01-02")).Scan(&tsdResult).Error if err != nil { - return tsdResult, err - } - - if !tsdResult.Valid { - return decimal.NullDecimal{ - Valid: true, - Decimal: decimal.Zero, - }, nil + return 0, err } return tsdResult, nil @@ -442,6 +386,6 @@ func (g *Generator) eventCount(task string, date time.Time, event string, result query := "SELECT COUNT(*) FROM events WHERE event = ? AND DATE(transacted_at) = ?" return g.db.GormDB(). - Raw(query, event, date.Format("2006-01-02")). + Raw(query, event, date). Scan(result).Error } diff --git a/packages/eventindexer/indexer/config.go b/packages/eventindexer/indexer/config.go index a46bd5c4afe..10279990797 100644 --- a/packages/eventindexer/indexer/config.go +++ b/packages/eventindexer/indexer/config.go @@ -36,7 +36,7 @@ type Config struct { SwapAddresses []common.Address CORSOrigins []string BlockBatchSize uint64 - SubscriptionBackoff uint64 + Subscriptionbackoff uint64 SyncMode SyncMode WatchMode WatchMode IndexNFTs bool @@ -78,7 +78,7 @@ func NewConfigFromCliContext(c *cli.Context) (*Config, error) { SwapAddresses: swaps, CORSOrigins: cors, BlockBatchSize: c.Uint64(flags.BlockBatchSize.Name), - SubscriptionBackoff: c.Uint64(flags.SubscriptionBackoff.Name), + Subscriptionbackoff: c.Uint64(flags.SubscriptionBackoff.Name), RPCUrl: c.String(flags.RPCUrl.Name), WatchMode: WatchMode(c.String(flags.WatchMode.Name)), SyncMode: SyncMode(c.String(flags.SyncMode.Name)), diff --git a/packages/eventindexer/indexer/config_test.go b/packages/eventindexer/indexer/config_test.go index 8513c8a5b41..cc59a482030 100644 --- a/packages/eventindexer/indexer/config_test.go +++ b/packages/eventindexer/indexer/config_test.go @@ -10,22 +10,11 @@ import ( ) var ( - httpPort = "1000" - metricsHttpPort = "1001" - l1TaikoAddress = "0x63FaC9201494f0bd17B9892B9fae4d52fe3BD377" - bridgeAddress = "0x73FaC9201494f0bd17B9892B9fae4d52fe3BD377" - swapAddresses = "0x33FaC9201494f0bd17B9892B9fae4d52fe3BD377,0x13FaC9201494f0bd17B9892B9fae4d52fe3BD377" - corsOrigins = "http://localhost:3000,http://localhost:3001" - databaseMaxIdleConns = "10" - databaseMaxOpenConns = "10" - databaseMaxConnLifetime = "30" - ethClientTimeout = "30" - blockBatchSize = "100" - subscriptionBackoff = "30" - syncMode = "sync" - watchMode = "filter" - indexNFTs = "true" - layer = "l1" + httpPort = "1000" + metricsHttpPort = "1001" + l1TaikoAddress = "0x63FaC9201494f0bd17B9892B9fae4d52fe3BD377" + bridgeAddress = "0x73FaC9201494f0bd17B9892B9fae4d52fe3BD377" + swapAddresses = "0x33FaC9201494f0bd17B9892B9fae4d52fe3BD377,0x13FaC9201494f0bd17B9892B9fae4d52fe3BD377" ) func setupApp() *cli.App { @@ -55,17 +44,6 @@ func TestNewConfigFromCliContext(t *testing.T) { assert.Equal(t, uint64(1001), c.MetricsHTTPPort) assert.Equal(t, common.HexToAddress(l1TaikoAddress), c.L1TaikoAddress) assert.Equal(t, common.HexToAddress(bridgeAddress), c.BridgeAddress) - assert.Equal(t, uint64(10), c.DatabaseMaxIdleConns) - assert.Equal(t, uint64(10), c.DatabaseMaxOpenConns) - assert.Equal(t, uint64(30), c.DatabaseMaxConnLifetime) - assert.Equal(t, uint64(30), c.ETHClientTimeout) - assert.Equal(t, uint64(100), c.BlockBatchSize) - assert.Equal(t, uint64(30), c.SubscriptionBackoff) - assert.Equal(t, SyncMode(syncMode), c.SyncMode) - assert.Equal(t, WatchMode(watchMode), c.WatchMode) - assert.Equal(t, true, c.IndexNFTs) - assert.Equal(t, layer, c.Layer) - assert.NotNil(t, c.OpenDBFunc) // assert.Nil(t, InitFromConfig(context.Background(), new(Indexer), c)) @@ -84,16 +62,5 @@ func TestNewConfigFromCliContext(t *testing.T) { "-" + flags.SwapAddresses.Name, swapAddresses, "-" + flags.HTTPPort.Name, httpPort, "-" + flags.MetricsHTTPPort.Name, metricsHttpPort, - "-" + flags.CORSOrigins.Name, corsOrigins, - "-" + flags.DatabaseMaxIdleConns.Name, databaseMaxIdleConns, - "-" + flags.DatabaseMaxOpenConns.Name, databaseMaxOpenConns, - "-" + flags.DatabaseConnMaxLifetime.Name, databaseMaxConnLifetime, - "-" + flags.ETHClientTimeout.Name, ethClientTimeout, - "-" + flags.BlockBatchSize.Name, blockBatchSize, - "-" + flags.SubscriptionBackoff.Name, subscriptionBackoff, - "-" + flags.SyncMode.Name, syncMode, - "-" + flags.WatchMode.Name, watchMode, - "-" + flags.IndexNFTs.Name, indexNFTs, - "-" + flags.Layer.Name, layer, })) } diff --git a/packages/eventindexer/indexer/indexer.go b/packages/eventindexer/indexer/indexer.go index 31d6ca2a6fe..e1e8a53cd69 100644 --- a/packages/eventindexer/indexer/indexer.go +++ b/packages/eventindexer/indexer/indexer.go @@ -229,7 +229,7 @@ func InitFromConfig(ctx context.Context, i *Indexer, cfg *Config) error { i.bridge = bridgeContract i.swaps = swapContracts i.blockBatchSize = cfg.BlockBatchSize - i.subscriptionBackoff = time.Duration(cfg.SubscriptionBackoff) * time.Second + i.subscriptionBackoff = time.Duration(cfg.Subscriptionbackoff) * time.Second i.srv = srv i.httpPort = cfg.HTTPPort i.wg = &sync.WaitGroup{} diff --git a/packages/eventindexer/indexer/save_block_proposed_event.go b/packages/eventindexer/indexer/save_block_proposed_event.go index 6564efcef1f..8c5c0e8bb8d 100644 --- a/packages/eventindexer/indexer/save_block_proposed_event.go +++ b/packages/eventindexer/indexer/save_block_proposed_event.go @@ -75,16 +75,6 @@ func (indxr *Indexer) saveBlockProposedEvent( return errors.Wrap(err, "indxr.ethClient.BlockByNumber") } - proposerReward, err := indxr.updateAverageProposerReward(ctx, event) - if err != nil { - return errors.Wrap(err, "indxr.updateAverageProposerReward") - } - - proverReward, err := indxr.updateAverageProverReward(ctx, event) - if err != nil { - return errors.Wrap(err, "indxr.updateAverageProposerReward") - } - _, err = indxr.eventRepo.Save(ctx, eventindexer.SaveEventOpts{ Name: eventindexer.EventNameBlockProposed, Data: string(marshaled), @@ -93,10 +83,7 @@ func (indxr *Indexer) saveBlockProposedEvent( Address: sender.Hex(), BlockID: &blockID, AssignedProver: &assignedProver, - TransactedAt: time.Unix(int64(block.Time()), 0).UTC(), - Amount: event.Reward, - ProposerReward: proposerReward, - ProofReward: proverReward, + TransactedAt: time.Unix(int64(block.Time()), 0), }) if err != nil { return errors.Wrap(err, "indxr.eventRepo.Save") diff --git a/packages/eventindexer/migrations/20230906203931_create_time_series_data_table.sql b/packages/eventindexer/migrations/20230906203931_create_time_series_data_table.sql index b41b7cf0bcc..64bd23e013e 100644 --- a/packages/eventindexer/migrations/20230906203931_create_time_series_data_table.sql +++ b/packages/eventindexer/migrations/20230906203931_create_time_series_data_table.sql @@ -3,7 +3,7 @@ CREATE TABLE IF NOT EXISTS time_series_data ( id int NOT NULL PRIMARY KEY AUTO_INCREMENT, task VARCHAR(40) NOT NULL, - value DECIMAL(65, 0) NOT NULL, + value VARCHAR(100) NOT NULL, date VARCHAR(20) NOT NULL, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP , updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, diff --git a/packages/eventindexer/repo/block_test.go b/packages/eventindexer/repo/block_test.go deleted file mode 100644 index eec8cfbfbb4..00000000000 --- a/packages/eventindexer/repo/block_test.go +++ /dev/null @@ -1,102 +0,0 @@ -package repo - -import ( - "context" - "math/big" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/stretchr/testify/assert" - "github.com/taikoxyz/taiko-mono/packages/eventindexer" - "github.com/taikoxyz/taiko-mono/packages/eventindexer/db" -) - -func Test_NewBlockRepo(t *testing.T) { - tests := []struct { - name string - db eventindexer.DB - wantErr error - }{ - { - "success", - &db.DB{}, - nil, - }, - { - "noDb", - nil, - eventindexer.ErrNoDB, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - _, err := NewBlockRepository(tt.db) - assert.Equal(t, tt.wantErr, err) - }) - } -} - -func TestIntegration_RawBlock_Save(t *testing.T) { - db, close, err := testMysql(t) - assert.Equal(t, nil, err) - - defer close() - - blockRepo, err := NewBlockRepository(db) - assert.Equal(t, nil, err) - - header := &types.Header{ - ParentHash: common.HexToHash("0x3a537c89809712367218bb171b3b1c46aa95df3dee7200ae9dc78f4052024068"), - UncleHash: common.HexToHash("0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347"), - Coinbase: common.HexToAddress("0x0000000000000000000000000000000000000000"), - Root: common.HexToHash("0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347"), - TxHash: common.HexToHash("0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347"), - ReceiptHash: common.HexToHash("0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347"), - Bloom: types.Bloom{}, - Difficulty: new(big.Int).SetInt64(2), - Number: new(big.Int).SetInt64(1), - GasLimit: 100000, - GasUsed: 2000, - Time: 1234, - Extra: []byte{0x7f}, - MixDigest: common.HexToHash("0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347"), - Nonce: types.BlockNonce{0x13}, - BaseFee: big.NewInt(10), - } - b := types.NewBlockWithHeader(header) - - genesisHeader := &types.Header{ - Time: 0, - } - - b2 := types.NewBlockWithHeader(genesisHeader) - - tests := []struct { - name string - block *types.Block - chainID *big.Int - wantErr error - }{ - { - "success", - b, - big.NewInt(0), - nil, - }, - { - "genesis", - b2, - big.NewInt(0), - nil, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := blockRepo.Save(context.Background(), tt.block, tt.chainID) - assert.Equal(t, tt.wantErr, err) - }) - } -} diff --git a/packages/eventindexer/repo/chart_repo.go b/packages/eventindexer/repo/chart_repo.go new file mode 100644 index 00000000000..e1326c4a35c --- /dev/null +++ b/packages/eventindexer/repo/chart_repo.go @@ -0,0 +1,56 @@ +package repo + +import ( + "context" + + "github.com/taikoxyz/taiko-mono/packages/eventindexer" + "gorm.io/gorm" +) + +type ChartRepository struct { + db eventindexer.DB +} + +func NewChartRepository(db eventindexer.DB) (*ChartRepository, error) { + if db == nil { + return nil, eventindexer.ErrNoDB + } + + return &ChartRepository{ + db: db, + }, nil +} + +func (r *ChartRepository) getDB() *gorm.DB { + return r.db.GormDB().Table("time_series_data") +} + +func (r *ChartRepository) Find( + ctx context.Context, + task string, + start string, + end string, +) (*eventindexer.ChartResponse, error) { + q := `SELECT * FROM time_series_data + WHERE task = ? AND date BETWEEN ? AND ? + ORDER BY date;` + + var tsd []*eventindexer.TimeSeriesData + + if err := r.getDB().Raw(q, task, start, end).Scan(&tsd).Error; err != nil { + return nil, err + } + + chart := &eventindexer.ChartResponse{ + Chart: make([]eventindexer.ChartItem, 0), + } + + for _, d := range tsd { + chart.Chart = append(chart.Chart, eventindexer.ChartItem{ + Date: d.Date, + Value: d.Value, + }) + } + + return chart, nil +} diff --git a/packages/eventindexer/tasks/tasks.go b/packages/eventindexer/tasks/tasks.go index 62e4bc5cab5..971a8ffb717 100644 --- a/packages/eventindexer/tasks/tasks.go +++ b/packages/eventindexer/tasks/tasks.go @@ -19,10 +19,6 @@ var ( TotalProposeBlockTx = "total-propose-block-tx" BridgeMessagesSentPerDay = "bridge-messages-sent-per-day" TotalBridgeMessagesSent = "total-bridge-messages-sent" - TotalProofRewards = "total-proof-rewards" - ProofRewardsPerDay = "proof-rewards-per-day" - TotalProposerRewards = "total-proposer-rewards" - ProposerRewardsPerDay = "proposer-rewards-per-day" ) var Tasks = []string{ @@ -44,8 +40,4 @@ var Tasks = []string{ TotalProposeBlockTx, BridgeMessagesSentPerDay, TotalBridgeMessagesSent, - TotalProofRewards, - ProofRewardsPerDay, - TotalProposerRewards, - ProposerRewardsPerDay, } diff --git a/packages/eventindexer/time_series_data.go b/packages/eventindexer/time_series_data.go index 35c7dd8ec84..4d593b901b0 100644 --- a/packages/eventindexer/time_series_data.go +++ b/packages/eventindexer/time_series_data.go @@ -1,15 +1,11 @@ package eventindexer -import ( - "time" - - "github.com/shopspring/decimal" -) +import "time" type TimeSeriesData struct { ID int Task string - Value decimal.NullDecimal + Value string Date string CreatedAt time.Time UpdatedAt time.Time