goal: Add export and import options for genesis files during network creation

cmd/goal/network.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/algorand/go-algorand/cmd/util/datadir"
 	"github.com/algorand/go-algorand/config"
+	"github.com/algorand/go-algorand/gen"
 	"github.com/algorand/go-algorand/netdeploy"
 	"github.com/algorand/go-algorand/util"
 )
@@ -40,26 +41,41 @@ var noImportKeys bool
 var noClean bool
 var devModeOverride bool
 var startOnCreation bool
+var genesisDir string
 
 func init() {
 	networkCmd.AddCommand(networkCreateCmd)
 	networkCmd.PersistentFlags().StringVarP(&networkRootDir, "rootdir", "r", "", "Root directory for the private network directories")
-	networkCmd.MarkPersistentFlagRequired("rootdir")
 
 	networkCreateCmd.Flags().StringVarP(&networkName, "network", "n", "", "Specify the name to use for the private network")
 	networkCreateCmd.Flags().StringVarP(&networkTemplateFile, "template", "t", "", "Specify the path to the template file for the network")
 	networkCreateCmd.Flags().BoolVarP(&noImportKeys, "noimportkeys", "K", false, "Do not import root keys when creating the network (by default will import)")
 	networkCreateCmd.Flags().BoolVar(&noClean, "noclean", false, "Prevents auto-cleanup on error - for diagnosing problems")
 	networkCreateCmd.Flags().BoolVar(&devModeOverride, "devMode", false, "Forces the configuration to enable DevMode, returns an error if the template is not compatible with DevMode.")
 	networkCreateCmd.Flags().BoolVarP(&startOnCreation, "start", "s", false, "Automatically start the network after creating it.")
+	networkCreateCmd.Flags().StringVarP(&genesisDir, "genesisdir", "g", "", "Specify the path to the directory with existing genesis.json, root and partkeys to import into the network directory. By default, the genesis.json and keys will be generated on start. This should only be used on private networks.")
+	networkCreateCmd.MarkFlagRequired("rootdir")
 
+	networkCmd.AddCommand(networkStartCmd)
 	networkStartCmd.Flags().StringVarP(&startNode, "node", "n", "", "Specify the name of a specific node to start")
+	networkStartCmd.MarkFlagRequired("rootdir")
 
-	networkCmd.AddCommand(networkStartCmd)
 	networkCmd.AddCommand(networkRestartCmd)
+	networkRestartCmd.MarkFlagRequired("rootdir")
+
 	networkCmd.AddCommand(networkStopCmd)
+	networkStopCmd.MarkFlagRequired("rootdir")
+
 	networkCmd.AddCommand(networkStatusCmd)
+	networkStatusCmd.MarkFlagRequired("rootdir")
+
 	networkCmd.AddCommand(networkDeleteCmd)
+	networkDeleteCmd.MarkFlagRequired("rootdir")
+
+	networkCmd.AddCommand(networkGenesisCmd)
+	networkGenesisCmd.Flags().StringVarP(&networkTemplateFile, "template", "t", "", "Specify the path to the template file for the network")
+	networkGenesisCmd.Flags().StringVarP(&genesisDir, "genesisdir", "g", "", "Specify the path to the directory to export genesis.json, root and partkey files. This should only be used on private networks.")
+	networkStopCmd.MarkFlagRequired("genesisdir")
 }
 
 var networkCmd = &cobra.Command{
@@ -112,6 +128,18 @@ var networkCreateCmd = &cobra.Command{
 			reportErrorf(infoNetworkAlreadyExists, networkRootDir)
 		}
 
+		// If genesisDir is specified, copy files over
+		if genesisDir != "" {
+			genesisDir, err = filepath.Abs(genesisDir)
+			if err != nil {
+				panic(err)
+			}
+			err = util.CopyFolder(genesisDir, networkRootDir)
+			if err != nil {
+				panic(err)
+			}
+		}
+
 		binDir, err := util.ExeDir()
 		if err != nil {
 			panic(err)
@@ -246,3 +274,60 @@ var networkDeleteCmd = &cobra.Command{
 		reportInfof(infoNetworkDeleted, networkRootDir)
 	},
 }
+
+var networkGenesisCmd = &cobra.Command{
+	Use:   "genesis",
+	Short: "Creates the genesis.json, root and participation keys for a wallet",
+	Args:  validateNoPosArgsFn,
+	Run: func(cmd *cobra.Command, _ []string) {
+		var err error
+		networkRootDir, err = filepath.Abs(networkRootDir)
+		if err != nil {
+			panic(err)
+		}
+
+		var templateReader io.Reader
+
+		if networkTemplateFile == "" {
+			templateReader = strings.NewReader(defaultNetworkTemplate)
+		} else {
+			networkTemplateFile, err = filepath.Abs(networkTemplateFile)
+			if err != nil {
+				panic(err)
+			}
+			file, osErr := os.Open(networkTemplateFile)
+			if osErr != nil {
+				reportErrorf(errorCreateNetwork, osErr)
+			}
+
+			defer file.Close()
+			templateReader = file
+		}
+
+		// Make sure target directory does not exist or is empty
+		if util.FileExists(networkRootDir) && !util.IsEmpty(networkRootDir) {
+			reportErrorf(infoNetworkAlreadyExists, networkRootDir)
+		}
+
+		var template netdeploy.NetworkTemplate
+		err = netdeploy.LoadTemplateFromReader(templateReader, &template)
+		if err != nil {
+			reportErrorf("Error in loading template: %v\n", err)
+		}
+
+		dataDir := datadir.MaybeSingleDataDir()
+		var consensus config.ConsensusProtocols
+		if dataDir != "" {
+			// try to load the consensus from there. If there is none, we can just use the built in one.
+			consensus, _ = config.PreloadConfigurableConsensusProtocols(dataDir)
+		}
+		if err = template.Validate(); err != nil {
+			reportErrorf("Error in template validation: %v\n", err)
+		}
+
+		err = gen.GenerateGenesisFiles(template.Genesis, consensus, networkRootDir, os.Stdout)
+		if err != nil {
+			reportErrorf("Cannot write genesis files: %s", err)
+		}
+	},
+}

netdeploy/network.go

@@ -70,7 +70,7 @@ func CreateNetworkFromTemplate(name, rootDir string, templateReader io.Reader, b
 	var err error
 	template := defaultNetworkTemplate
 
-	err = loadTemplateFromReader(templateReader, &template)
+	err = LoadTemplateFromReader(templateReader, &template)
 
 	if err == nil {
 		if overrideDevMode {
@@ -100,7 +100,7 @@ func CreateNetworkFromTemplate(name, rootDir string, templateReader io.Reader, b
 		return n, err
 	}
 	template.Consensus = consensus
-	err = template.generateGenesisAndWallets(rootDir, n.cfg.Name, binDir)
+	err = template.generateGenesisAndWallets(rootDir, n.cfg.Name)
 	if err != nil {
 		return n, err
 	}

netdeploy/networkTemplate.go

@@ -47,7 +47,7 @@ var defaultNetworkTemplate = NetworkTemplate{
 	Genesis: gen.DefaultGenesis,
 }
 
-func (t NetworkTemplate) generateGenesisAndWallets(targetFolder, networkName, binDir string) error {
+func (t NetworkTemplate) generateGenesisAndWallets(targetFolder, networkName string) error {
 	genesisData := t.Genesis
 	genesisData.NetworkName = networkName
 	mergedConsensus := config.Consensus.Merge(t.Consensus)
@@ -165,11 +165,12 @@ func loadTemplate(templateFile string) (NetworkTemplate, error) {
 	}
 	defer f.Close()
 
-	err = loadTemplateFromReader(f, &template)
+	err = LoadTemplateFromReader(f, &template)
 	return template, err
 }
 
-func loadTemplateFromReader(reader io.Reader, template *NetworkTemplate) error {
+// LoadTemplateFromReader loads and decodes a network template
+func LoadTemplateFromReader(reader io.Reader, template *NetworkTemplate) error {
 
 	if runtime.GOARCH == "arm" || runtime.GOARCH == "arm64" {
 		// for arm machines, use smaller key dilution

netdeploy/networkTemplates_test.go

@@ -69,9 +69,8 @@ func TestGenerateGenesis(t *testing.T) {
 
 	targetFolder := t.TempDir()
 	networkName := "testGenGen"
-	binDir := os.ExpandEnv("${GOPATH}/bin")
 
-	err := template.generateGenesisAndWallets(targetFolder, networkName, binDir)
+	err := template.generateGenesisAndWallets(targetFolder, networkName)
 	a.NoError(err)
 	_, err = os.Stat(filepath.Join(targetFolder, config.GenesisJSONFile))
 	fileExists := err == nil

test/scripts/test_private_network.sh

@@ -2,7 +2,10 @@
 echo "######################################################################"
 echo "  test_private_network"
 echo "######################################################################"
+
 set -e
+set -o pipefail
+set -o nounset
 
 # Suppress telemetry reporting for tests
 export ALGOTEST=1
@@ -11,9 +14,12 @@ export GOPATH=$(go env GOPATH)
 SRCPATH=$(pwd)
 
 NETROOTPATH=${SRCPATH}/tmp/test_private_network
+GENFILESPATH=${SRCPATH}/tmp/test_private_network_genesis_files
+
 # purge if it already exists
 ${GOPATH}/bin/goal network delete -r ${NETROOTPATH} || true
 rm -rf ${NETROOTPATH}
+rm -rf ${GENFILESPATH}
 
 ${GOPATH}/bin/goal network create -r ${NETROOTPATH} -n net1 -t ./test/testdata/nettemplates/TwoNodes50Each.json
 
@@ -35,6 +41,26 @@ ${GOPATH}/bin/goal network stop -r ${NETROOTPATH}
 
 ${GOPATH}/bin/goal network delete -r ${NETROOTPATH}
 
+# Test that genesis generation works correctly
+${GOPATH}/bin/goal network genesis -r ${GENFILESPATH} -t ./test/testdata/nettemplates/TwoNodes50Each.json
+# Try importing genesis files from same template -- should reuse the root and partkey files
+${GOPATH}/bin/goal network create -r ${NETROOTPATH} -n net1 -t ./test/testdata/nettemplates/TwoNodes50Each.json --genesisdir ${GENFILESPATH}
+
+${GOPATH}/bin/goal network start -r ${NETROOTPATH}
+
+${GOPATH}/bin/goal network stop -r ${NETROOTPATH}
+
+${GOPATH}/bin/goal network delete -r ${NETROOTPATH}
+
+# Creating genesis files should fail if directory is not empty
+# Capture output upon failure
+RES=$(${GOPATH}/bin/goal network genesis -r ${GENFILESPATH} 2>&1 || true)
+EXPERROR="already exists and is not empty"
+if [[ $RES != *"${EXPERROR}"* ]]; then
+    date '+test_private_network FAIL goal network genesis did not fail even though specified directory was not empty %Y%m%d_%H%M%S'
+    exit 1
+fi
+
 echo "----------------------------------------------------------------------"
 echo "  DONE: test_private_network"
 echo "----------------------------------------------------------------------"