diff --git a/Makefile b/Makefile index ec41bcd2..a7d1234a 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ ADDLICENSE_CMD=go run github.com/google/addlicense ADDLICENCE_SCRIPT=${ADDLICENSE_CMD} -c "Coinbase, Inc." -l "apache" -v GOLINES_CMD=go run github.com/segmentio/golines GOVERALLS_CMD=go run github.com/mattn/goveralls -TEST_SCRIPT=go test -v ./internal/... +TEST_SCRIPT=go test -v ./internal/... ./configuration/... deps: go get ./... @@ -39,7 +39,7 @@ check-license: ${ADDLICENCE_SCRIPT} -check . shorten-lines: - ${GOLINES_CMD} -w --shorten-comments internal cmd + ${GOLINES_CMD} -w --shorten-comments internal cmd configuration salus: docker run --rm -t -v ${PWD}:/home/repo coinbase/salus diff --git a/README.md b/README.md index c59e9bde..9a750847 100644 --- a/README.md +++ b/README.md @@ -41,17 +41,25 @@ Usage: rosetta-cli [command] Available Commands: - check Check the correctness of a Rosetta Data API Server - create:configuration Generate a static configuration file for the Asserter - help Help about any command - version Print rosetta-cli version - view:account View an account balance - view:block View a block - view:network View network status + check:construction Check the correctness of a Rosetta Construction API Implementation + check:data Check the correctness of a Rosetta Data API Implementation + configuration:create Create a default configuration file at the provided path + configuration:validate Validate the correctness of a configuration file at the provided path + help Help about any command + utils:asserter-configuration Generate a static configuration file for the Asserter + version Print rosetta-cli version + view:account View an account balance + view:block View a block + view:network View network status Flags: - -h, --help help for rosetta-cli - --server-url string base URL for a Rosetta server (default "http://localhost:8080") + --configuration-file string Configuration file that provides connection and test settings. + If you would like to generate a starter configuration file (populated + with the defaults), run rosetta-cli configuration:create. + + Any fields not populated in the configuration file will be populated with + default values. + -h, --help help for rosetta-cli Use "rosetta-cli [command] --help" for more information about a command. ``` @@ -67,10 +75,15 @@ Flags: -h, --help help for version Global Flags: - --server-url string base URL for a Rosetta server (default "http://localhost:8080") + --configuration-file string Configuration file that provides connection and test settings. + If you would like to generate a starter configuration file (populated + with the defaults), run rosetta-cli configuration:create. + + Any fields not populated in the configuration file will be populated with + default values. ``` -### check +### check:data ``` Check all server responses are properly constructed, that there are no duplicate blocks and transactions, that blocks can be processed @@ -78,7 +91,7 @@ from genesis to the current block (re-orgs handled automatically), and that computed balance changes are equal to balance changes reported by the node. When re-running this command, it will start where it left off if you specify -some --data-dir. Otherwise, it will create a new temporary directory and start +some data directory. Otherwise, it will create a new temporary directory and start again from the genesis block. If you want to discard some number of blocks populate the --start flag with some block index. Starting from a given index can be useful to debug a small range of blocks for issues but it is highly @@ -87,59 +100,41 @@ are performed. By default, account balances are looked up at specific heights (instead of only at the current block). If your node does not support this functionality -set --lookup-balance-by-block to false. This will make reconciliation much +set historical balance disabled to true. This will make reconciliation much less efficient but it will still work. If check fails due to an INACTIVE reconciliation error (balance changed without any corresponding operation), the cli will automatically try to find the block -missing an operation. If --lookup-balance-by-block is not enabled, this automatic +missing an operation. If historical balance disabled is true, this automatic debugging tool does not work. -To debug an INACTIVE account reconciliation error without --lookup-balance-by-block, set the ---interesting-accounts flag to the absolute path of a JSON file containing +To debug an INACTIVE account reconciliation error without historical balance lookup, +set the interesting accunts to the path of a JSON file containing accounts that will be actively checked for balance changes at each block. This will return an error at the block where a balance change occurred with no corresponding operations. If your blockchain has a genesis allocation of funds and you set ---lookup-balance-by-block to false, you must provide an +historical balance disabled to true, you must provide an absolute path to a JSON file containing initial balances with the ---bootstrap-balances flag. You can look at the examples folder for an example +bootstrap balance config. You can look at the examples folder for an example of what one of these files looks like. Usage: - rosetta-cli check [flags] + rosetta-cli check:data [flags] Flags: - --active-reconciliation-concurrency uint concurrency to use while fetching accounts during active reconciliation (default 8) - --block-concurrency uint concurrency to use while fetching blocks (default 8) - --bootstrap-balances string Absolute path to a file used to bootstrap balances before starting syncing. - Populating this value after beginning syncing will return an error. - --data-dir string folder used to store logs and any data used to perform validation - --end int block index to stop syncing (default -1) - --exempt-accounts string Absolute path to a file listing all accounts to exempt from balance - tracking and reconciliation. Look at the examples directory for an example of - how to structure this file. - --halt-on-reconciliation-error Determines if block processing should halt on a reconciliation - error. It can be beneficial to collect all reconciliation errors or silence - reconciliation errors during development. (default true) - -h, --help help for check - --inactive-reconciliation-concurrency uint concurrency to use while fetching accounts during inactive reconciliation (default 4) - --inactive-reconciliation-frequency uint the number of blocks to wait between inactive reconiliations on each account (default 250) - --interesting-accounts string Absolute path to a file listing all accounts to check on each block. Look - at the examples directory for an example of how to structure this file. - --log-balance-changes log balance changes - --log-blocks log processed blocks - --log-reconciliations log balance reconciliations - --log-transactions log processed transactions - --lookup-balance-by-block When set to true, balances are looked up at the block where a balance - change occurred instead of at the current block. Blockchains that do not support - historical balance lookup should set this to false. (default true) - --start int block index to start syncing (default -1) - --transaction-concurrency uint concurrency to use while fetching transactions (if required) (default 16) + --end int block index to stop syncing (default -1) + -h, --help help for check:data + --start int block index to start syncing (default -1) Global Flags: - --server-url string base URL for a Rosetta server (default "http://localhost:8080") + --configuration-file string Configuration file that provides connection and test settings. + If you would like to generate a starter configuration file (populated + with the defaults), run rosetta-cli configuration:create. + + Any fields not populated in the configuration file will be populated with + default values. ``` #### Status Codes @@ -147,27 +142,23 @@ If there are no issues found while running `check`, it will exit with a `0` stat If there are any issues, it will exit with a `1` status code. It can be useful to run this command as an integration test for any changes to your implementation. -### create:configuration +### configuration:create ``` -In production deployments, it is useful to initialize the response -Asserter (https://github.com/coinbase/rosetta-sdk-go/tree/master/asserter) using -a static configuration instead of intializing a configuration dynamically -from the node. This allows a client to error on new types/statuses that may -have been added in an update instead of silently erroring. - -To use this command, simply provide an absolute path as the argument for where -the configuration file should be saved (in JSON). Populate the optional ---server-url flag with the url of the server to generate the configuration -from. +Check the correctness of a Rosetta Construction API Implementation Usage: - rosetta-cli create:configuration [flags] + rosetta-cli check:construction [flags] Flags: - -h, --help help for create:configuration + -h, --help help for check:construction Global Flags: - --server-url string base URL for a Rosetta server (default "http://localhost:8080") + --configuration-file string Configuration file that provides connection and test settings. + If you would like to generate a starter configuration file (populated + with the defaults), run rosetta-cli configuration:create. + + Any fields not populated in the configuration file will be populated with + default values. ``` ### view:network @@ -186,7 +177,12 @@ Flags: -h, --help help for view:network Global Flags: - --server-url string base URL for a Rosetta server (default "http://localhost:8080") + --configuration-file string Configuration file that provides connection and test settings. + If you would like to generate a starter configuration file (populated + with the defaults), run rosetta-cli configuration:create. + + Any fields not populated in the configuration file will be populated with + default values. ``` ### view:account @@ -207,7 +203,12 @@ Flags: -h, --help help for view:account Global Flags: - --server-url string base URL for a Rosetta server (default "http://localhost:8080") + --configuration-file string Configuration file that provides connection and test settings. + If you would like to generate a starter configuration file (populated + with the defaults), run rosetta-cli configuration:create. + + Any fields not populated in the configuration file will be populated with + default values. ``` ### view:block @@ -229,7 +230,38 @@ Flags: -h, --help help for view:block Global Flags: - --server-url string base URL for a Rosetta server (default "http://localhost:8080") + --configuration-file string Configuration file that provides connection and test settings. + If you would like to generate a starter configuration file (populated + with the defaults), run rosetta-cli configuration:create. + + Any fields not populated in the configuration file will be populated with + default values. +``` + +### utils:asserter-configuration +``` +In production deployments, it is useful to initialize the response +Asserter (https://github.com/coinbase/rosetta-sdk-go/tree/master/asserter) using +a static configuration instead of intializing a configuration dynamically +from the node. This allows a client to error on new types/statuses that may +have been added in an update instead of silently erroring. + +To use this command, simply provide an absolute path as the argument for where +the configuration file should be saved (in JSON). + +Usage: + rosetta-cli utils:asserter-configuration [flags] + +Flags: + -h, --help help for utils:asserter-configuration + +Global Flags: + --configuration-file string Configuration file that provides connection and test settings. + If you would like to generate a starter configuration file (populated + with the defaults), run rosetta-cli configuration:create. + + Any fields not populated in the configuration file will be populated with + default values. ``` ## Development diff --git a/cmd/check_construction.go b/cmd/check_construction.go new file mode 100644 index 00000000..dbc663e3 --- /dev/null +++ b/cmd/check_construction.go @@ -0,0 +1,34 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "log" + + "github.com/spf13/cobra" +) + +var ( + checkConstructionCmd = &cobra.Command{ + Use: "check:construction", + Short: "Check the correctness of a Rosetta Construction API Implementation", + Run: runCheckConstructionCmd, + } +) + +func runCheckConstructionCmd(cmd *cobra.Command, args []string) { + // ensureDataDirectoryExists() + log.Fatal("not implemented!") +} diff --git a/cmd/check.go b/cmd/check_data.go similarity index 66% rename from cmd/check.go rename to cmd/check_data.go index 06fd498b..ebf1bf76 100644 --- a/cmd/check.go +++ b/cmd/check_data.go @@ -16,15 +16,12 @@ package cmd import ( "context" - "encoding/json" "errors" "fmt" - "io/ioutil" "log" "math/big" "os" "os/signal" - "path" "syscall" "time" @@ -66,16 +63,16 @@ const ( ) var ( - checkCmd = &cobra.Command{ - Use: "check", - Short: "Check the correctness of a Rosetta Data API Server", + checkDataCmd = &cobra.Command{ + Use: "check:data", + Short: "Check the correctness of a Rosetta Data API Implementation", Long: `Check all server responses are properly constructed, that there are no duplicate blocks and transactions, that blocks can be processed from genesis to the current block (re-orgs handled automatically), and that computed balance changes are equal to balance changes reported by the node. When re-running this command, it will start where it left off if you specify -some --data-dir. Otherwise, it will create a new temporary directory and start +some data directory. Otherwise, it will create a new temporary directory and start again from the genesis block. If you want to discard some number of blocks populate the --start flag with some block index. Starting from a given index can be useful to debug a small range of blocks for issues but it is highly @@ -84,216 +81,52 @@ are performed. By default, account balances are looked up at specific heights (instead of only at the current block). If your node does not support this functionality -set --lookup-balance-by-block to false. This will make reconciliation much +set historical balance disabled to true. This will make reconciliation much less efficient but it will still work. If check fails due to an INACTIVE reconciliation error (balance changed without any corresponding operation), the cli will automatically try to find the block -missing an operation. If --lookup-balance-by-block is not enabled, this automatic +missing an operation. If historical balance disabled is true, this automatic debugging tool does not work. -To debug an INACTIVE account reconciliation error without --lookup-balance-by-block, set the ---interesting-accounts flag to the absolute path of a JSON file containing +To debug an INACTIVE account reconciliation error without historical balance lookup, +set the interesting accunts to the path of a JSON file containing accounts that will be actively checked for balance changes at each block. This will return an error at the block where a balance change occurred with no corresponding operations. If your blockchain has a genesis allocation of funds and you set ---lookup-balance-by-block to false, you must provide an +historical balance disabled to true, you must provide an absolute path to a JSON file containing initial balances with the ---bootstrap-balances flag. You can look at the examples folder for an example +bootstrap balance config. You can look at the examples folder for an example of what one of these files looks like.`, - Run: runCheckCmd, + Run: runCheckDataCmd, } - // BootstrapBalances is a path to a file used to bootstrap - // balances before starting syncing. Populating this value - // after beginning syncing will return an error. - BootstrapBalances string - - // LookupBalanceByBlock determines if balances are looked up - // at the block where a balance change occurred instead of at the current - // block. Blockchains that do not support historical balance lookup - // should set this to false. - LookupBalanceByBlock bool - - // DataDir is a folder used to store logs - // and any data used to perform validation. - DataDir string - // StartIndex is the block index to start syncing. StartIndex int64 // EndIndex is the block index to stop syncing. EndIndex int64 - // BlockConcurrency is the concurrency to use - // while fetching blocks. - BlockConcurrency uint64 - - // TransactionConcurrency is the concurrency to use - // while fetching transactions (if required). - TransactionConcurrency uint64 - - // ActiveReconciliationConcurrency is the concurrency to use - // while fetching accounts during active reconciliation. - ActiveReconciliationConcurrency uint64 - - // InactiveReconciliationConcurrency is the concurrency to use - // while fetching accounts during inactive reconciliation. - InactiveReconciliationConcurrency uint64 - - // InactiveReconciliationFrequency is the number of blocks - // to wait between inactive reconiliations on each account. - InactiveReconciliationFrequency uint64 - - // LogBlocks determines if blocks are - // logged. - LogBlocks bool - - // LogTransactions determines if transactions are - // logged. - LogTransactions bool - - // LogBalanceChanges determines if balance changes are - // logged. - LogBalanceChanges bool - - // LogReconciliations determines if reconciliations are - // logged. - LogReconciliations bool - - // HaltOnReconciliationError determines if processing - // should stop when encountering a reconciliation error. - // It can be beneficial to collect all reconciliation errors - // during development. - HaltOnReconciliationError bool - - // ExemptFile is an absolute path to a file listing all accounts - // to exempt from balance tracking and reconciliation. - ExemptFile string - - // InterestingFile is an absolute path to a file listing all accounts - // to actively reconcile on each block (if there are no operations - // present for the account, the reconciler asserts a balance change of 0). - InterestingFile string - // signalReceived is set to true when a signal causes us to exit. This makes // determining the error message to show on exit much more easy. signalReceived = false ) func init() { - checkCmd.Flags().StringVar( - &DataDir, - "data-dir", - "", - "folder used to store logs and any data used to perform validation", - ) - checkCmd.Flags().Int64Var( + checkDataCmd.Flags().Int64Var( &StartIndex, "start", -1, "block index to start syncing", ) - checkCmd.Flags().Int64Var( + checkDataCmd.Flags().Int64Var( &EndIndex, "end", -1, "block index to stop syncing", ) - checkCmd.Flags().Uint64Var( - &BlockConcurrency, - "block-concurrency", - 8, - "concurrency to use while fetching blocks", - ) - checkCmd.Flags().Uint64Var( - &TransactionConcurrency, - "transaction-concurrency", - 16, - "concurrency to use while fetching transactions (if required)", - ) - checkCmd.Flags().Uint64Var( - &ActiveReconciliationConcurrency, - "active-reconciliation-concurrency", - 8, - "concurrency to use while fetching accounts during active reconciliation", - ) - checkCmd.Flags().Uint64Var( - &InactiveReconciliationConcurrency, - "inactive-reconciliation-concurrency", - 4, - "concurrency to use while fetching accounts during inactive reconciliation", - ) - checkCmd.Flags().Uint64Var( - &InactiveReconciliationFrequency, - "inactive-reconciliation-frequency", - 250, - "the number of blocks to wait between inactive reconiliations on each account", - ) - checkCmd.Flags().BoolVar( - &LogBlocks, - "log-blocks", - false, - "log processed blocks", - ) - checkCmd.Flags().BoolVar( - &LogTransactions, - "log-transactions", - false, - "log processed transactions", - ) - checkCmd.Flags().BoolVar( - &LogBalanceChanges, - "log-balance-changes", - false, - "log balance changes", - ) - checkCmd.Flags().BoolVar( - &LogReconciliations, - "log-reconciliations", - false, - "log balance reconciliations", - ) - checkCmd.Flags().BoolVar( - &HaltOnReconciliationError, - "halt-on-reconciliation-error", - true, - `Determines if block processing should halt on a reconciliation -error. It can be beneficial to collect all reconciliation errors or silence -reconciliation errors during development.`, - ) - checkCmd.Flags().StringVar( - &ExemptFile, - "exempt-accounts", - "", - `Absolute path to a file listing all accounts to exempt from balance -tracking and reconciliation. Look at the examples directory for an example of -how to structure this file.`, - ) - checkCmd.Flags().StringVar( - &BootstrapBalances, - "bootstrap-balances", - "", - `Absolute path to a file used to bootstrap balances before starting syncing. -Populating this value after beginning syncing will return an error.`, - ) - checkCmd.Flags().BoolVar( - &LookupBalanceByBlock, - "lookup-balance-by-block", - true, - `When set to true, balances are looked up at the block where a balance -change occurred instead of at the current block. Blockchains that do not support -historical balance lookup should set this to false.`, - ) - checkCmd.Flags().StringVar( - &InterestingFile, - "interesting-accounts", - "", - `Absolute path to a file listing all accounts to check on each block. Look -at the examples directory for an example of how to structure this file.`, - ) } // loadAccounts is a utility function to parse the []*reconciler.AccountCurrency @@ -303,14 +136,9 @@ func loadAccounts(filePath string) ([]*reconciler.AccountCurrency, error) { return []*reconciler.AccountCurrency{}, nil } - accountsRaw, err := ioutil.ReadFile(path.Clean(filePath)) - if err != nil { - return nil, err - } - accounts := []*reconciler.AccountCurrency{} - if err := json.Unmarshal(accountsRaw, &accounts); err != nil { - return nil, err + if err := utils.LoadAndParse(filePath, &accounts); err != nil { + return nil, fmt.Errorf("%w: unable to open account file", err) } log.Printf( @@ -334,9 +162,9 @@ func findMissingOps( endIndex int64, ) (*types.BlockIdentifier, error) { fetcher := fetcher.New( - ServerURL, - fetcher.WithBlockConcurrency(BlockConcurrency), - fetcher.WithTransactionConcurrency(TransactionConcurrency), + Config.Data.OnlineURL, + fetcher.WithBlockConcurrency(Config.Data.BlockConcurrency), + fetcher.WithTransactionConcurrency(Config.Data.TransactionConcurrency), fetcher.WithRetryElapsedTime(ExtendedRetryElapsedTime), ) @@ -375,7 +203,7 @@ func findMissingOps( blockStorageHelper := processor.NewBlockStorageHelper( primaryNetwork, fetcher, - LookupBalanceByBlock, + !Config.Data.HistoricalBalanceDisabled, nil, ) @@ -409,7 +237,7 @@ func findMissingOps( // Do not do any inactive lookups when looking for the block with missing // operations. reconciler.WithInactiveConcurrency(0), - reconciler.WithLookupBalanceByBlock(LookupBalanceByBlock), + reconciler.WithLookupBalanceByBlock(!Config.Data.HistoricalBalanceDisabled), reconciler.WithInterestingAccounts([]*reconciler.AccountCurrency{accountCurrency}), ) @@ -496,23 +324,24 @@ func findMissingOps( return reconcilerHandler.ActiveFailureBlock, nil } -func runCheckCmd(cmd *cobra.Command, args []string) { +func runCheckDataCmd(cmd *cobra.Command, args []string) { + ensureDataDirectoryExists() ctx, cancel := context.WithCancel(context.Background()) - exemptAccounts, err := loadAccounts(ExemptFile) + exemptAccounts, err := loadAccounts(Config.Data.ExemptAccounts) if err != nil { log.Fatal(fmt.Errorf("%w: unable to load exempt accounts", err)) } - interestingAccounts, err := loadAccounts(InterestingFile) + interestingAccounts, err := loadAccounts(Config.Data.InterestingAccounts) if err != nil { log.Fatal(fmt.Errorf("%w: unable to load interesting accounts", err)) } fetcher := fetcher.New( - ServerURL, - fetcher.WithBlockConcurrency(BlockConcurrency), - fetcher.WithTransactionConcurrency(TransactionConcurrency), + Config.Data.OnlineURL, + fetcher.WithBlockConcurrency(Config.Data.BlockConcurrency), + fetcher.WithTransactionConcurrency(Config.Data.TransactionConcurrency), fetcher.WithRetryElapsedTime(ExtendedRetryElapsedTime), ) @@ -522,18 +351,7 @@ func runCheckCmd(cmd *cobra.Command, args []string) { log.Fatal(fmt.Errorf("%w: unable to initialize asserter", err)) } - // If data directory is not specified, we use a temporary directory - // and delete its contents when execution is complete. - if len(DataDir) == 0 { - tmpDir, err := utils.CreateTempDir() - if err != nil { - log.Fatal(fmt.Errorf("%w: unable to create temporary directory", err)) - } - defer utils.RemoveTempDir(tmpDir) - - DataDir = tmpDir - } - localStore, err := storage.NewBadgerStorage(ctx, DataDir) + localStore, err := storage.NewBadgerStorage(ctx, Config.Data.DataDirectory) if err != nil { log.Fatal(fmt.Errorf("%w: unable to initialize database", err)) } @@ -543,27 +361,27 @@ func runCheckCmd(cmd *cobra.Command, args []string) { logger := logger.NewLogger( counterStorage, - DataDir, - LogBlocks, - LogTransactions, - LogBalanceChanges, - LogReconciliations, + Config.Data.DataDirectory, + Config.Data.LogBlocks, + Config.Data.LogTransactions, + Config.Data.LogBalanceChanges, + Config.Data.LogReconciliations, ) blockStorageHelper := processor.NewBlockStorageHelper( primaryNetwork, fetcher, - LookupBalanceByBlock, + !Config.Data.HistoricalBalanceDisabled, exemptAccounts, ) blockStorage := storage.NewBlockStorage(localStore, blockStorageHelper) // Bootstrap balances if provided - if len(BootstrapBalances) > 0 { + if len(Config.Data.BootstrapBalances) > 0 { err = blockStorage.BootstrapBalances( ctx, - BootstrapBalances, + Config.Data.BootstrapBalances, networkStatus.GenesisBlockIdentifier, ) if err != nil { @@ -595,7 +413,7 @@ func runCheckCmd(cmd *cobra.Command, args []string) { reconcilerHandler := processor.NewReconcilerHandler( logger, - HaltOnReconciliationError, + !Config.Data.IgnoreReconciliationError, ) r := reconciler.New( @@ -603,13 +421,13 @@ func runCheckCmd(cmd *cobra.Command, args []string) { reconcilerHelper, reconcilerHandler, fetcher, - reconciler.WithActiveConcurrency(int(ActiveReconciliationConcurrency)), - reconciler.WithInactiveConcurrency(int(InactiveReconciliationConcurrency)), - reconciler.WithLookupBalanceByBlock(LookupBalanceByBlock), + reconciler.WithActiveConcurrency(int(Config.Data.ActiveReconciliationConcurrency)), + reconciler.WithInactiveConcurrency(int(Config.Data.InactiveReconciliationConcurrency)), + reconciler.WithLookupBalanceByBlock(!Config.Data.HistoricalBalanceDisabled), reconciler.WithInterestingAccounts(interestingAccounts), reconciler.WithSeenAccounts(seenAccounts), - reconciler.WithDebugLogging(LogReconciliations), - reconciler.WithInactiveFrequency(int64(InactiveReconciliationFrequency)), + reconciler.WithDebugLogging(Config.Data.LogReconciliations), + reconciler.WithInactiveFrequency(int64(Config.Data.InactiveReconciliationFrequency)), ) syncerHandler := processor.NewSyncerHandler( @@ -725,7 +543,7 @@ func handleCheckResult( os.Exit(1) } - if !LookupBalanceByBlock { + if Config.Data.HistoricalBalanceDisabled { color.Red( "Can't find the block missing operations automatically, please enable --lookup-balance-by-block", ) diff --git a/cmd/configuration_create.go b/cmd/configuration_create.go new file mode 100644 index 00000000..4f4ddf30 --- /dev/null +++ b/cmd/configuration_create.go @@ -0,0 +1,39 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "log" + + "github.com/coinbase/rosetta-cli/configuration" + "github.com/coinbase/rosetta-cli/internal/utils" + + "github.com/spf13/cobra" +) + +var ( + configurationCreateCmd = &cobra.Command{ + Use: "configuration:create", + Short: "Create a default configuration file at the provided path", + Run: runConfigurationCreateCmd, + Args: cobra.ExactArgs(1), + } +) + +func runConfigurationCreateCmd(cmd *cobra.Command, args []string) { + if err := utils.SerializeAndWrite(args[0], configuration.DefaultConfiguration()); err != nil { + log.Fatalf("%s: unable to save configuration file to %s", err.Error(), args[0]) + } +} diff --git a/cmd/configuration_validate.go b/cmd/configuration_validate.go new file mode 100644 index 00000000..dc70ae22 --- /dev/null +++ b/cmd/configuration_validate.go @@ -0,0 +1,41 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "log" + + "github.com/coinbase/rosetta-cli/configuration" + + "github.com/spf13/cobra" +) + +var ( + configurationValidateCmd = &cobra.Command{ + Use: "configuration:validate", + Short: "Validate the correctness of a configuration file at the provided path", + Run: runConfigurationValidateCmd, + Args: cobra.ExactArgs(1), + } +) + +func runConfigurationValidateCmd(cmd *cobra.Command, args []string) { + _, err := configuration.LoadConfiguration(args[0]) + if err != nil { + log.Fatalf("%s: unable to save configuration file to %s", err.Error(), args[0]) + } + + log.Println("Configuration file validated!") +} diff --git a/cmd/root.go b/cmd/root.go index 42231f47..9e8bcb36 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -16,6 +16,10 @@ package cmd import ( "fmt" + "log" + + "github.com/coinbase/rosetta-cli/configuration" + "github.com/coinbase/rosetta-cli/internal/utils" "github.com/spf13/cobra" ) @@ -26,9 +30,12 @@ var ( Short: "CLI for the Rosetta API", } - // ServerURL is the base URL for a Rosetta - // server to validate. - ServerURL string + configurationFile string + + // Config is the populated *configuration.Configuration from + // the configurationFile. If none is provided, this is set + // to the default settings. + Config *configuration.Configuration ) // Execute handles all invocations of the @@ -38,25 +45,67 @@ func Execute() error { } func init() { + cobra.OnInitialize(initConfig) + rootCmd.PersistentFlags().StringVar( - &ServerURL, - "server-url", - "http://localhost:8080", - "base URL for a Rosetta server", + &configurationFile, + "configuration-file", + "", + `Configuration file that provides connection and test settings. +If you would like to generate a starter configuration file (populated +with the defaults), run rosetta-cli configuration:create. + +Any fields not populated in the configuration file will be populated with +default values.`, ) + rootCmd.AddCommand(versionCmd) - rootCmd.AddCommand(checkCmd) + // Configuration Commands + rootCmd.AddCommand(configurationCreateCmd) + rootCmd.AddCommand(configurationValidateCmd) + + // Check commands + rootCmd.AddCommand(checkDataCmd) + rootCmd.AddCommand(checkConstructionCmd) + + // View Commands rootCmd.AddCommand(viewBlockCmd) rootCmd.AddCommand(viewAccountCmd) rootCmd.AddCommand(viewNetworkCmd) - rootCmd.AddCommand(createConfigurationCmd) - rootCmd.AddCommand(versionCmd) + + // Utils + rootCmd.AddCommand(utilsAsserterConfigurationCmd) +} + +func initConfig() { + var err error + if len(configurationFile) == 0 { + Config = configuration.DefaultConfiguration() + } else { + Config, err = configuration.LoadConfiguration(configurationFile) + } + if err != nil { + log.Fatalf("%s: unable to load configuration", err.Error()) + } +} + +func ensureDataDirectoryExists() { + // If data directory is not specified, we use a temporary directory + // and delete its contents when execution is complete. + if len(Config.Data.DataDirectory) == 0 { + tmpDir, err := utils.CreateTempDir() + if err != nil { + log.Fatalf("%s: unable to create temporary directory", err.Error()) + } + + Config.Data.DataDirectory = tmpDir + } } var versionCmd = &cobra.Command{ Use: "version", Short: "Print rosetta-cli version", Run: func(cmd *cobra.Command, args []string) { - fmt.Println("v0.3.1") + fmt.Println("v0.3.2") }, } diff --git a/cmd/create_configuration.go b/cmd/utils_asserter_configuration.go similarity index 69% rename from cmd/create_configuration.go rename to cmd/utils_asserter_configuration.go index fd66cc62..cf5597d2 100644 --- a/cmd/create_configuration.go +++ b/cmd/utils_asserter_configuration.go @@ -16,26 +16,17 @@ package cmd import ( "context" - "fmt" - "io/ioutil" "log" - "os" - "path" + + "github.com/coinbase/rosetta-cli/internal/utils" "github.com/coinbase/rosetta-sdk-go/fetcher" - "github.com/coinbase/rosetta-sdk-go/types" "github.com/spf13/cobra" ) -const ( - // fileMode 0600 indicates that the user/owner can read and write - // but can't execute. - fileMode = 0600 -) - var ( - createConfigurationCmd = &cobra.Command{ - Use: "create:configuration", + utilsAsserterConfigurationCmd = &cobra.Command{ + Use: "utils:asserter-configuration", Short: "Generate a static configuration file for the Asserter", Long: `In production deployments, it is useful to initialize the response Asserter (https://github.com/coinbase/rosetta-sdk-go/tree/master/asserter) using @@ -44,9 +35,7 @@ from the node. This allows a client to error on new types/statuses that may have been added in an update instead of silently erroring. To use this command, simply provide an absolute path as the argument for where -the configuration file should be saved (in JSON). Populate the optional ---server-url flag with the url of the server to generate the configuration -from.`, +the configuration file should be saved (in JSON).`, Run: runCreateConfigurationCmd, Args: cobra.ExactArgs(1), } @@ -57,24 +46,21 @@ func runCreateConfigurationCmd(cmd *cobra.Command, args []string) { // Create a new fetcher newFetcher := fetcher.New( - ServerURL, + Config.Data.OnlineURL, ) // Initialize the fetcher's asserter _, _, err := newFetcher.InitializeAsserter(ctx) if err != nil { - log.Fatal(err) + log.Fatalf("%s: failed to initialize asserter", err.Error()) } configuration, err := newFetcher.Asserter.ClientConfiguration() if err != nil { - log.Fatal(fmt.Errorf("%w: unable to generate spec", err)) + log.Fatalf("%s: unable to generate spec", err.Error()) } - specString := types.PrettyPrintStruct(configuration) - log.Printf("Spec File: %s\n", specString) - - if err := ioutil.WriteFile(path.Clean(args[0]), []byte(specString), os.FileMode(fileMode)); err != nil { - log.Fatal(err) + if err := utils.SerializeAndWrite(args[0], configuration); err != nil { + log.Fatalf("%s: unable to serialize asserter configuration", err.Error()) } } diff --git a/cmd/view_account.go b/cmd/view_account.go index 994d413d..e756043b 100644 --- a/cmd/view_account.go +++ b/cmd/view_account.go @@ -58,7 +58,7 @@ func runViewAccountCmd(cmd *cobra.Command, args []string) { // Create a new fetcher newFetcher := fetcher.New( - ServerURL, + Config.Data.OnlineURL, ) // Initialize the fetcher's asserter diff --git a/cmd/view_block.go b/cmd/view_block.go index 15427cfc..0e5b910a 100644 --- a/cmd/view_block.go +++ b/cmd/view_block.go @@ -53,7 +53,7 @@ func runViewBlockCmd(cmd *cobra.Command, args []string) { // Create a new fetcher newFetcher := fetcher.New( - ServerURL, + Config.Data.OnlineURL, ) // Initialize the fetcher's asserter diff --git a/cmd/view_network.go b/cmd/view_network.go index 45b328e0..038d068a 100644 --- a/cmd/view_network.go +++ b/cmd/view_network.go @@ -41,7 +41,7 @@ not formatted correctly.`, func runViewNetworkCmd(cmd *cobra.Command, args []string) { ctx := context.Background() - f := fetcher.New(ServerURL) + f := fetcher.New(Config.Data.OnlineURL) // Attempt to fetch network list networkList, err := f.NetworkListRetry(ctx, nil) diff --git a/configuration/configuration.go b/configuration/configuration.go new file mode 100644 index 00000000..f6b0a32c --- /dev/null +++ b/configuration/configuration.go @@ -0,0 +1,435 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package configuration + +import ( + "fmt" + "log" + "math/big" + + "github.com/coinbase/rosetta-cli/internal/scenario" + "github.com/coinbase/rosetta-cli/internal/utils" + + "github.com/coinbase/rosetta-sdk-go/asserter" + "github.com/coinbase/rosetta-sdk-go/types" +) + +// AccountingModel is a type representing possible accounting models +// in the Construction API. +type AccountingModel string + +const ( + // AccountModel is for account-based blockchains. + AccountModel AccountingModel = "account" + + // UtxoModel is for UTXO-based blockchains. + UtxoModel AccountingModel = "utxo" +) + +// Default Configuration Values +const ( + DefaultURL = "http://localhost:8080" + DefaultMinimumBalance = "0" + DefaultMaximumFee = "50000" + DefaultCurveType = types.Secp256k1 + DefaultAccountingModel = AccountModel + DefaultBlockConcurrency = 8 + DefaultTransactionConcurrency = 16 + DefaultActiveReconciliationConcurrency = 16 + DefaultInactiveReconciliationConcurrency = 4 + DefaultInactiveReconciliationFrequency = 250 + + // ETH Defaults + EthereumBlockchain = "Ethereum" + EthereumNetwork = "Ropsten" + EthereumTransfer = "transfer" + EthereumSymbol = "ETH" + EthereumDecimals = 18 +) + +// Default Configuration Values +var ( + DefaultNetwork = &types.NetworkIdentifier{ + Blockchain: EthereumBlockchain, + Network: EthereumNetwork, + } + DefaultCurrency = &types.Currency{ + Symbol: EthereumSymbol, + Decimals: EthereumDecimals, + } + DefaultTransferScenario = []*types.Operation{ + { + OperationIdentifier: &types.OperationIdentifier{ + Index: 0, + }, + Account: &types.AccountIdentifier{ + Address: scenario.Sender, + }, + Type: EthereumTransfer, + Amount: &types.Amount{ + Value: scenario.SenderValue, + }, + }, + { + OperationIdentifier: &types.OperationIdentifier{ + Index: 1, + }, + RelatedOperations: []*types.OperationIdentifier{ + { + Index: 0, + }, + }, + Account: &types.AccountIdentifier{ + Address: scenario.Recipient, + }, + Type: EthereumTransfer, + Amount: &types.Amount{ + Value: scenario.RecipientValue, + }, + }, + } +) + +// TODO: Add support for sophisticated end conditions +// (https://github.com/coinbase/rosetta-cli/issues/66) + +// ConstructionConfiguration contains all configurations +// to run check:construction. +type ConstructionConfiguration struct { + // Network is the *types.NetworkIdentifier where transactions should + // be constructed and where blocks should be synced to monitor + // for broadcast success. + Network *types.NetworkIdentifier `json:"network"` + + // OnlineURL is the URL of a Rosetta API implementation in "online mode". + // default: http://localhost:8080 + OnlineURL string `json:"online_url"` + + // OfflineURL is the URL of a Rosetta API implementation in "online mode". + // default: http://localhost:8080 + OfflineURL string `json:"offline_url"` + + // Currency is the *types.Currency to track and use for transactions. + // default: {Symbol: "ETH", Decimals: 18} + Currency *types.Currency `json:"currency"` + + // MinimumBalance is balance at a particular address + // that is not considered spendable. + // default: "0" + MinimumBalance string `json:"minimum_balance"` + + // MaximumFee is the maximum fee that could be used + // to send a transaction. The sendable balance + // of any address is calculated as balance - minimum_balance - maximum_fee. + // default: "10000000" + MaximumFee string `json:"maximum_fee"` + + // CurveType is the curve to use when generating a *keys.KeyPair. + // default: "secp256k1" + CurveType types.CurveType `json:"curve_type"` + + // AccountingModel is the type of acccount model to use for + // testing (account vs UTXO). + // default: "account" + AccountingModel AccountingModel `json:"accounting_model"` + + // TransferScenario contains a slice of operations that + // indicate how to perform a transfer on a blockchain. In the future + // this will be expanded to support all kinds of construction scenarios (like + // staking or governance). + // default: ETH transfer + TransferScenario []*types.Operation `json:"transfer_scenario"` +} + +// DefaultConstructionConfiguration returns the *ConstructionConfiguration +// used for testing Ethereum transfers on Ropsten. +func DefaultConstructionConfiguration() *ConstructionConfiguration { + return &ConstructionConfiguration{ + Network: DefaultNetwork, + OnlineURL: DefaultURL, + OfflineURL: DefaultURL, + Currency: DefaultCurrency, + MinimumBalance: DefaultMinimumBalance, + MaximumFee: DefaultMaximumFee, + CurveType: DefaultCurveType, + AccountingModel: DefaultAccountingModel, + TransferScenario: DefaultTransferScenario, + } +} + +// DefaultDataConfiguration returns the default *DataConfiguration +// for running `check:data`. +func DefaultDataConfiguration() *DataConfiguration { + return &DataConfiguration{ + OnlineURL: DefaultURL, + BlockConcurrency: DefaultBlockConcurrency, + TransactionConcurrency: DefaultTransactionConcurrency, + ActiveReconciliationConcurrency: DefaultActiveReconciliationConcurrency, + InactiveReconciliationConcurrency: DefaultInactiveReconciliationConcurrency, + InactiveReconciliationFrequency: DefaultInactiveReconciliationFrequency, + } +} + +// DefaultConfiguration returns a *Configuration with the +// DefaultConstructionConfiguration and DefaultDataConfiguration. +func DefaultConfiguration() *Configuration { + return &Configuration{ + Construction: DefaultConstructionConfiguration(), + Data: DefaultDataConfiguration(), + } +} + +// DataConfiguration contains all configurations to run check:data. +// TODO: Add configurable timeout (https://github.com/coinbase/rosetta-cli/issues/64) +type DataConfiguration struct { + // OnlineURL is the URL of a Rosetta API implementation in "online mode". + // default: http://localhost:8080 + OnlineURL string `json:"online_url"` + + // DataDirectory is a folder used to store logs and any data used to perform validation. + // default: "" + DataDirectory string `json:"data_directory"` + + // BlockConcurrency is the concurrency to use while fetching blocks. + // default: 8 + BlockConcurrency uint64 `json:"block_concurrency"` + + // TransactionConcurrency is the concurrency to use while fetching transactions (if required). + // default: 16 + TransactionConcurrency uint64 `json:"transaction_concurrency"` + + // ActiveReconciliationConcurrency is the concurrency to use while fetching accounts + // during active reconciliation. + // default: 8 + ActiveReconciliationConcurrency uint64 `json:"active_reconciliation_concurrency"` + + // InactiveReconciliationConcurrency is the concurrency to use while fetching accounts + // during inactive reconciliation. + // default: 4 + InactiveReconciliationConcurrency uint64 `json:"inactive_reconciliation_concurrency"` + + // InactiveReconciliationFrequency is the number of blocks to wait between + // inactive reconiliations on each account. + // default: 250 + InactiveReconciliationFrequency uint64 `json:"inactive_reconciliation_frequency"` + + // LogBlocks is a boolean indicating whether to log processed blocks. + // default: false + LogBlocks bool `json:"log_blocks"` + + // LogTransactions is a boolean indicating whether to log processed transactions. + // default: false + LogTransactions bool `json:"log_transactions"` + + // LogBalanceChanges is a boolean indicating whether to log all balance changes. + // default: false + LogBalanceChanges bool `json:"log_balance_changes"` + + // LogReconciliations is a boolean indicating whether to log all reconciliations. + // default: false + LogReconciliations bool `json:"log_reconciliations"` + + // IgnoreReconciliationError determines if block processing should halt on a reconciliation + // error. It can be beneficial to collect all reconciliation errors or silence + // reconciliation errors during development. + // default: false + IgnoreReconciliationError bool `json:"ignore_reconciliation_error"` + + // ExemptAccounts is a path to a file listing all accounts to exempt from balance + // tracking and reconciliation. Look at the examples directory for an example of + // how to structure this file. + // default: "" + ExemptAccounts string `json:"exempt_accounts"` + + // BootstrapBalances is a path to a file used to bootstrap balances + // before starting syncing. If this value is populated after beginning syncing, + // it will be ignored. + // default: "" + BootstrapBalances string `json:"bootstrap_balances"` + + // HistoricalBalanceDisabled is a boolean that dictates how balance lookup is performed. + // When set to true, balances are looked up at the block where a balance + // change occurred instead of at the current block. Blockchains that do not support + // historical balance lookup should set this to false. + // default: false + HistoricalBalanceDisabled bool `json:"historical_balance_disabled"` + + // InterestingAccounts is a path to a file listing all accounts to check on each block. Look + // at the examples directory for an example of how to structure this file. + // default: "" + InterestingAccounts string `json:"interesting_accounts"` +} + +// Configuration contains all configuration settings for running +// check:data or check:construction. +type Configuration struct { + Construction *ConstructionConfiguration `json:"construction"` + Data *DataConfiguration `json:"data"` +} + +func populateConstructionMissingFields( + constructionConfig *ConstructionConfiguration, +) *ConstructionConfiguration { + if constructionConfig == nil { + return DefaultConstructionConfiguration() + } + + if constructionConfig.Network == nil { + constructionConfig.Network = DefaultNetwork + } + + if len(constructionConfig.OnlineURL) == 0 { + constructionConfig.OnlineURL = DefaultURL + } + + if len(constructionConfig.OfflineURL) == 0 { + constructionConfig.OfflineURL = DefaultURL + } + + if constructionConfig.Currency == nil { + constructionConfig.Currency = DefaultCurrency + } + + if len(constructionConfig.MinimumBalance) == 0 { + constructionConfig.MinimumBalance = DefaultMinimumBalance + } + + if len(constructionConfig.MaximumFee) == 0 { + constructionConfig.MaximumFee = DefaultMaximumFee + } + + if len(constructionConfig.CurveType) == 0 { + constructionConfig.CurveType = DefaultCurveType + } + + if len(constructionConfig.AccountingModel) == 0 { + constructionConfig.AccountingModel = DefaultAccountingModel + } + + if len(constructionConfig.TransferScenario) == 0 { + constructionConfig.TransferScenario = DefaultTransferScenario + } + + return constructionConfig +} + +func populateDataMissingFields(dataConfig *DataConfiguration) *DataConfiguration { + if dataConfig == nil { + return DefaultDataConfiguration() + } + + if len(dataConfig.OnlineURL) == 0 { + dataConfig.OnlineURL = DefaultURL + } + + if dataConfig.BlockConcurrency == 0 { + dataConfig.BlockConcurrency = DefaultBlockConcurrency + } + + if dataConfig.TransactionConcurrency == 0 { + dataConfig.TransactionConcurrency = DefaultTransactionConcurrency + } + + if dataConfig.ActiveReconciliationConcurrency == 0 { + dataConfig.ActiveReconciliationConcurrency = DefaultActiveReconciliationConcurrency + } + + if dataConfig.InactiveReconciliationConcurrency == 0 { + dataConfig.InactiveReconciliationConcurrency = DefaultInactiveReconciliationConcurrency + } + + if dataConfig.InactiveReconciliationFrequency == 0 { + dataConfig.InactiveReconciliationFrequency = DefaultInactiveReconciliationFrequency + } + + return dataConfig +} + +func populateMissingFields(config *Configuration) *Configuration { + if config == nil { + return DefaultConfiguration() + } + + config.Construction = populateConstructionMissingFields(config.Construction) + config.Data = populateDataMissingFields(config.Data) + + return config +} + +func checkStringUint(input string) error { + val, ok := new(big.Int).SetString(input, 10) + if !ok { + return fmt.Errorf("%s is not an integer", input) + } + + if val.Sign() == -1 { + return fmt.Errorf("%s must not be negative", input) + } + + return nil +} + +func assertConstructionConfiguration(config *ConstructionConfiguration) error { + if err := asserter.NetworkIdentifier(config.Network); err != nil { + return fmt.Errorf("%w: invalid network identifier", err) + } + + // TODO: add asserter.Currency method + if err := asserter.Amount(&types.Amount{Value: "0", Currency: config.Currency}); err != nil { + return fmt.Errorf("%w: invalid currency", err) + } + + switch config.AccountingModel { + case AccountModel, UtxoModel: + default: + return fmt.Errorf("accounting model %s not supported", config.AccountingModel) + } + + if err := asserter.CurveType(config.CurveType); err != nil { + return fmt.Errorf("%w: invalid curve type", err) + } + + if err := checkStringUint(config.MinimumBalance); err != nil { + return fmt.Errorf("%w: invalid value for MinimumBalance", err) + } + + if err := checkStringUint(config.MaximumFee); err != nil { + return fmt.Errorf("%w: invalid value for MaximumFee", err) + } + + return nil +} + +// LoadConfiguration returns a parsed and asserted Configuration for running +// tests. +func LoadConfiguration(filePath string) (*Configuration, error) { + var configRaw Configuration + if err := utils.LoadAndParse(filePath, &configRaw); err != nil { + return nil, fmt.Errorf("%w: unable to open configuration file", err) + } + + config := populateMissingFields(&configRaw) + + if err := assertConstructionConfiguration(config.Construction); err != nil { + return nil, fmt.Errorf("%w: invalid construction configuration", err) + } + + log.Printf( + "loaded configuration file: %s\n", + filePath, + ) + + return config, nil +} diff --git a/configuration/configuration_test.go b/configuration/configuration_test.go new file mode 100644 index 00000000..c378a40c --- /dev/null +++ b/configuration/configuration_test.go @@ -0,0 +1,155 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package configuration + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/coinbase/rosetta-cli/internal/utils" + + "github.com/coinbase/rosetta-sdk-go/types" + "github.com/stretchr/testify/assert" +) + +var ( + whackyConfig = &Configuration{ + Construction: &ConstructionConfiguration{ + Network: &types.NetworkIdentifier{ + Blockchain: "sweet", + Network: "sweeter", + }, + OnlineURL: "http://hasudhasjkdk", + OfflineURL: "https://ashdjaksdkjshdk", + Currency: &types.Currency{ + Symbol: "FIRE", + Decimals: 100, + }, + MinimumBalance: "1002", + MaximumFee: "1", + CurveType: types.Edwards25519, + AccountingModel: UtxoModel, + TransferScenario: DefaultTransferScenario, + }, + Data: &DataConfiguration{ + OnlineURL: "https://asjdlkasjdklajsdlkj", + BlockConcurrency: 12, + TransactionConcurrency: 2, + ActiveReconciliationConcurrency: 100, + InactiveReconciliationConcurrency: 2938, + InactiveReconciliationFrequency: 3, + }, + } + invalidNetwork = &Configuration{ + Construction: &ConstructionConfiguration{ + Network: &types.NetworkIdentifier{ + Blockchain: "?", + }, + }, + } + invalidCurrency = &Configuration{ + Construction: &ConstructionConfiguration{ + Currency: &types.Currency{ + Decimals: 12, + }, + }, + } + invalidCurve = &Configuration{ + Construction: &ConstructionConfiguration{ + CurveType: "hello", + }, + } + invalidAccountingModel = &Configuration{ + Construction: &ConstructionConfiguration{ + AccountingModel: "hello", + }, + } + invalidMinimumBalance = &Configuration{ + Construction: &ConstructionConfiguration{ + MinimumBalance: "-1000", + }, + } + invalidMaximumFee = &Configuration{ + Construction: &ConstructionConfiguration{ + MaximumFee: "hello", + }, + } +) + +func TestLoadConfiguration(t *testing.T) { + var tests = map[string]struct { + provided *Configuration + expected *Configuration + + err bool + }{ + "nothing provided": { + provided: &Configuration{}, + expected: DefaultConfiguration(), + }, + "no overwrite": { + provided: whackyConfig, + expected: whackyConfig, + }, + "invalid network": { + provided: invalidNetwork, + err: true, + }, + "invalid currency": { + provided: invalidCurrency, + err: true, + }, + "invalid curve type": { + provided: invalidCurve, + err: true, + }, + "invalid accounting model": { + provided: invalidAccountingModel, + err: true, + }, + "invalid minimum balance": { + provided: invalidMinimumBalance, + err: true, + }, + "invalid maximum fee": { + provided: invalidMaximumFee, + err: true, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + // Write configuration file to tempdir + tmpfile, err := ioutil.TempFile("", "test.json") + assert.NoError(t, err) + defer os.Remove(tmpfile.Name()) + + err = utils.SerializeAndWrite(tmpfile.Name(), test.provided) + assert.NoError(t, err) + + // Check if expected fields populated + config, err := LoadConfiguration(tmpfile.Name()) + if test.err { + assert.Error(t, err) + assert.Nil(t, config) + } else { + assert.NoError(t, err) + assert.Equal(t, test.expected, config) + } + assert.NoError(t, tmpfile.Close()) + }) + } +} diff --git a/go.sum b/go.sum index 3e6b20c2..25432635 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,7 @@ github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpi github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= +github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= @@ -354,6 +355,7 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= diff --git a/internal/logger/logger.go b/internal/logger/logger.go index 31267cd0..b17e07b7 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -23,6 +23,7 @@ import ( "path" "github.com/coinbase/rosetta-cli/internal/storage" + "github.com/coinbase/rosetta-cli/internal/utils" "github.com/coinbase/rosetta-sdk-go/parser" "github.com/coinbase/rosetta-sdk-go/reconciler" @@ -55,10 +56,6 @@ const ( // removeEvent is printed in a stream // when an event is orphaned. removeEvent = "Remove" - - // logFilePermissions specifies that the user can - // read and write the file. - logFilePermissions = 0600 ) // Logger contains all logic to record validator output @@ -155,7 +152,7 @@ func (l *Logger) AddBlockStream( f, err := os.OpenFile( path.Join(l.logDir, blockStreamFile), os.O_APPEND|os.O_CREATE|os.O_WRONLY, - logFilePermissions, + os.FileMode(utils.DefaultFilePermissions), ) if err != nil { return err @@ -191,7 +188,7 @@ func (l *Logger) RemoveBlockStream( f, err := os.OpenFile( path.Join(l.logDir, blockStreamFile), os.O_APPEND|os.O_CREATE|os.O_WRONLY, - logFilePermissions, + os.FileMode(utils.DefaultFilePermissions), ) if err != nil { return err @@ -225,7 +222,7 @@ func (l *Logger) TransactionStream( f, err := os.OpenFile( path.Join(l.logDir, transactionStreamFile), os.O_APPEND|os.O_CREATE|os.O_WRONLY, - logFilePermissions, + os.FileMode(utils.DefaultFilePermissions), ) if err != nil { return err @@ -293,7 +290,7 @@ func (l *Logger) BalanceStream( f, err := os.OpenFile( path.Join(l.logDir, balanceStreamFile), os.O_APPEND|os.O_CREATE|os.O_WRONLY, - logFilePermissions, + os.FileMode(utils.DefaultFilePermissions), ) if err != nil { return err @@ -335,7 +332,7 @@ func (l *Logger) ReconcileSuccessStream( f, err := os.OpenFile( path.Join(l.logDir, reconcileSuccessStreamFile), os.O_APPEND|os.O_CREATE|os.O_WRONLY, - logFilePermissions, + os.FileMode(utils.DefaultFilePermissions), ) if err != nil { return err @@ -406,7 +403,7 @@ func (l *Logger) ReconcileFailureStream( f, err := os.OpenFile( path.Join(l.logDir, reconcileFailureStreamFile), os.O_APPEND|os.O_CREATE|os.O_WRONLY, - logFilePermissions, + os.FileMode(utils.DefaultFilePermissions), ) if err != nil { return err diff --git a/internal/utils/utils.go b/internal/utils/utils.go index f00821e4..386de043 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -15,14 +15,23 @@ package utils import ( + "encoding/json" + "fmt" "io/ioutil" "log" "os" + "path" "github.com/coinbase/rosetta-sdk-go/types" "github.com/fatih/color" ) +const ( + // DefaultFilePermissions specifies that the user can + // read and write the file. + DefaultFilePermissions = 0600 +) + // CreateTempDir creates a directory in // /tmp for usage within testing. func CreateTempDir() (string, error) { @@ -48,3 +57,33 @@ func RemoveTempDir(dir string) { func Equal(a interface{}, b interface{}) bool { return types.Hash(a) == types.Hash(b) } + +// SerializeAndWrite attempts to serialize the provided object +// into a file at filePath. +func SerializeAndWrite(filePath string, object interface{}) error { + err := ioutil.WriteFile( + filePath, + []byte(types.PrettyPrintStruct(object)), + os.FileMode(DefaultFilePermissions), + ) + if err != nil { + return fmt.Errorf("%w: unable to write to file path %s", err, filePath) + } + + return nil +} + +// LoadAndParse reads the file at the provided path +// and attempts to unmarshal it into output. +func LoadAndParse(filePath string, output interface{}) error { + bytes, err := ioutil.ReadFile(path.Clean(filePath)) + if err != nil { + return fmt.Errorf("%w: unable to load file %s", err, filePath) + } + + if err := json.Unmarshal(bytes, &output); err != nil { + return fmt.Errorf("%w: unable to unmarshal", err) + } + + return nil +}