From ad7fee7e296c1881d2feae87949e1af121eeac9c Mon Sep 17 00:00:00 2001 From: Libby Kent Date: Tue, 27 Oct 2020 21:49:15 -0400 Subject: [PATCH 1/4] [qctl] add support for external nodes. * add external_nodes to configyaml used by qctl. * add support for listing the external nodes in a network: qctl ls extnodes --all --- qctl/configyaml.go | 10 ++- qctl/nodecmd.go | 212 +++++++++++++++++++++++++++++++++++++++++---- qctl/qctl.go | 1 + 3 files changed, 204 insertions(+), 19 deletions(-) diff --git a/qctl/configyaml.go b/qctl/configyaml.go index 0595608..8feb5a2 100644 --- a/qctl/configyaml.go +++ b/qctl/configyaml.go @@ -48,6 +48,13 @@ type NodeEntry struct { GethEntry GethEntry `yaml:"geth"` } +type ExternalNodeEntry struct { + NodeUserIdent string `yaml:"Node_UserIdent"` + EnodeUrl string `yaml:"Enode_Url"` + TmUrl string `yaml:"Tm_Url"` + NodekeyAddress string `yaml:"Node_Acct_Addr,omitempty"` +} + type Prometheus struct { //#monitor_params_geth: --metrics --metrics.expensive --pprof --pprofaddr=0.0.0.0 //monitorParamsGeth string `yaml:"monitor_params_geth"` @@ -87,7 +94,8 @@ type QConfig struct { Cakeshop Cakeshop `yaml:"cakeshop,omitempty"` K8s K8s `yaml:"k8s,omitempty"` - Nodes []NodeEntry + Nodes []NodeEntry + ExternalNodes []ExternalNodeEntry `yaml:"external_nodes,omitempty"` } func GetYamlConfig() QConfig { diff --git a/qctl/nodecmd.go b/qctl/nodecmd.go index 1c4488f..4a4fc6c 100644 --- a/qctl/nodecmd.go +++ b/qctl/nodecmd.go @@ -738,11 +738,6 @@ var ( isGethParams = true } - // get the current directory path, we'll use this in case the config file passed in was a relative path. - pwdCmd := exec.Command("pwd") - b, _ := runCmd(pwdCmd) - pwd := strings.TrimSpace(b.String()) - if configFile == "" { c.App.Run([]string{"qctl", "help", "node"}) @@ -761,6 +756,11 @@ var ( return cli.Exit("--config flag must be set to the fullpath of your config file.", 3) } + // get the current directory path, we'll use this in case the config file passed in was a relative path. + pwdCmd := exec.Command("pwd") + b, _ := runCmd(pwdCmd) + pwd := strings.TrimSpace(b.String()) + // the config file must exist or this is an error. if fileExists(configFile) { // check if config file is full path or relative path. @@ -769,7 +769,7 @@ var ( } } else { - c.App.Run([]string{"qctl", "help", "init"}) + c.App.Run([]string{"qctl", "help", "node"}) return cli.Exit(fmt.Sprintf("ConfigFile must exist! Given configFile [%v]", configFile), 3) } if !isBare { @@ -802,12 +802,13 @@ var ( } for i := 0; i < len(configFileYaml.Nodes); i++ { - if nodeName == configFileYaml.Nodes[i].NodeUserIdent || nodeName == "" { // node name not set always show node + currentNode := configFileYaml.Nodes[i] + if nodeName == currentNode.NodeUserIdent || nodeName == "" { // node name not set always show node nodeFound = true if asExternal { // qctl ls nodes --asexternal -b --node-ip=$(minikube ip) // qctl ls urls --node quorum-node1 --tm -bare - tmUrlCmd := exec.Command("qctl", "ls", "urls", "--node="+configFileYaml.Nodes[i].NodeUserIdent, "--type=nodeport", "--tm", "--bare", "--node-ip="+nodeip) + tmUrlCmd := exec.Command("qctl", "ls", "urls", "--node="+currentNode.NodeUserIdent, "--type=nodeport", "--tm", "--bare", "--node-ip="+nodeip) //fmt.Println(cmd.String()) res, err := runCmd(tmUrlCmd) if err != nil { @@ -815,7 +816,7 @@ var ( } tmurl := strings.TrimSpace(res.String()) - p2pCmd := exec.Command("qctl", "ls", "urls", "--node="+configFileYaml.Nodes[i].NodeUserIdent, "--type=nodeport", "--p2p", "--bare", "--node-ip="+nodeip) + p2pCmd := exec.Command("qctl", "ls", "urls", "--node="+currentNode.NodeUserIdent, "--type=nodeport", "--p2p", "--bare", "--node-ip="+nodeip) //fmt.Println(cmd.String()) res, err = runCmd(p2pCmd) if err != nil { @@ -826,10 +827,10 @@ var ( // kc get configMap quorum-node1-nodekey-address-config -o jsonpath='{.data.nodekey}' // try to get the node key address (ibft) nodeKeyAddrCmd := exec.Command("kubectl", "get", "configMap", - configFileYaml.Nodes[i].NodeUserIdent+"-nodekey-address-config", "-o=jsonpath='{.data.nodekey}'") + currentNode.NodeUserIdent+"-nodekey-address-config", "-o=jsonpath='{.data.nodekey}'") res, err = runCmd(nodeKeyAddrCmd) nodekeyAddress := "" - if err != nil && configFileYaml.Nodes[i].QuorumEntry.Quorum.Consensus == IstanbulConsensus { + if err != nil && currentNode.QuorumEntry.Quorum.Consensus == IstanbulConsensus { red.Println(fmt.Sprintf(" issue getting the nodekey-address for node %s", configFileYaml.Nodes[i].NodeUserIdent)) red.Println(nodeKeyAddrCmd.String()) log.Fatal(err) @@ -838,12 +839,12 @@ var ( nodekeyAddress = strings.TrimSpace(nodekeyAddress) } //fmt.Println("nodekeyAddress", nodekeyAddress) - displayNodeAsExternal(k8sdir, tmurl, nodekeyAddress, p2pUrl, configFileYaml.Nodes[i], true, isConsensus) + displayNodeAsExternal(k8sdir, currentNode, tmurl, nodekeyAddress, p2pUrl, true) } else { if isBare { // show the bare version, cleaner for scripts. - displayNodeBare(k8sdir, configFileYaml.Nodes[i], isName, isKeyDir, isConsensus, isQuorumVersion, isTmName, isTmVersion, isEnodeUrl, isQuorumImageFull, isTmImageFull, isGethParams) + displayNodeBare(k8sdir, currentNode, isName, isKeyDir, isConsensus, isQuorumVersion, isTmName, isTmVersion, isEnodeUrl, isQuorumImageFull, isTmImageFull, isGethParams) } else { - displayNode(k8sdir, configFileYaml.Nodes[i], isName, isKeyDir, isConsensus, isQuorumVersion, isTmName, isTmVersion, isEnodeUrl, isQuorumImageFull, isTmImageFull, isGethParams) + displayNode(k8sdir, currentNode, isName, isKeyDir, isConsensus, isQuorumVersion, isTmName, isTmVersion, isEnodeUrl, isQuorumImageFull, isTmImageFull, isGethParams) } } } @@ -861,6 +862,150 @@ var ( return nil }, } + externalNodeListCommand = cli.Command{ + Name: "external-node", + Usage: "list external node(s) info", + Aliases: []string{"extnode", "extnodes", "external-nodes"}, + ArgsUsage: "NodeName", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "config, c", + Usage: "Load configuration from `FULL_PATH_FILE`", + EnvVars: []string{"QUBE_CONFIG"}, + Required: true, + }, + &cli.BoolFlag{ + Name: "all", + Usage: "display all node values", + }, + &cli.BoolFlag{ + Name: "name", + Usage: "display the name of the node", + }, + &cli.BoolFlag{ + Name: "enodeurl", + Aliases: []string{"enode"}, + Usage: "display the enodeurl of the external node", + }, + &cli.BoolFlag{ + Name: "tmurl", + Usage: "display the transaction manager url for the external node", + }, //Node_Acct_Addr + &cli.BoolFlag{ + Name: "nodekey-addr", + Usage: "display the nodekey address for the external node (ibft)", + }, + &cli.BoolFlag{ + Name: "bare", + Aliases: []string{"b"}, + Usage: "display the minimum output, useful for scripts / automation", + }, + }, + Action: func(c *cli.Context) error { + // potentially show only this node + nodeName := c.Args().First() + nodeFound := true + if nodeName != "" { // if the user request a specific node, we want to make sure we let them know it was found or not. + nodeFound = false + } + isName := c.Bool("name") + isEnodeUrl := c.Bool("enodeurl") + isTmUrl := c.Bool("tmurl") + isNodeKeyAddr := c.Bool("nodekey-addr") + isAll := c.Bool("all") + isBare := c.Bool("bare") + + configFile := c.String("config") + // if no flags are set, show the names + if !isName && !isEnodeUrl && !isTmUrl && !isNodeKeyAddr && !isAll { + isAll = true + } + // set all values to true + if isAll { + isName = true + isEnodeUrl = true + isTmUrl = true + isNodeKeyAddr = true + } + + if configFile == "" { + c.App.Run([]string{"qctl", "help", "node"}) + + // QUBE_CONFIG or flag + fmt.Println() + + fmt.Println() + red.Println(" --config flag must be provided.") + red.Println(" or ") + red.Println(" QUBE_CONFIG environment variable needs to be set to your config file.") + fmt.Println() + red.Println(" If you need to generate a qubernetes.yaml config use the command: ") + fmt.Println() + green.Println(" qctl generate config") + fmt.Println() + return cli.Exit("--config flag must be set to the fullpath of your config file.", 3) + } + + // get the current directory path, we'll use this in case the config file passed in was a relative path. + pwdCmd := exec.Command("pwd") + b, _ := runCmd(pwdCmd) + pwd := strings.TrimSpace(b.String()) + + // the config file must exist or this is an error. + if fileExists(configFile) { + // check if config file is full path or relative path. + if !strings.HasPrefix(configFile, "/") { + configFile = pwd + "/" + configFile + } + } else { + c.App.Run([]string{"qctl", "help", "external-node"}) + return cli.Exit(fmt.Sprintf("ConfigFile must exist! Given configFile [%v]", configFile), 3) + } + if !isBare { + fmt.Println() + green.Println(" Using config file:") + fmt.Println() + fmt.Println(" " + configFile) + fmt.Println() + fmt.Println("*****************************************************************************************") + fmt.Println() + } + + configFileYaml, err := LoadYamlConfig(configFile) + if err != nil { + log.Fatal("config file [%v] could not be loaded into the valid qubernetes yaml. err: [%v]", configFile, err) + } + currentNum := len(configFileYaml.Nodes) + if !isBare { + fmt.Printf("config currently has %d external nodes \n", currentNum) + fmt.Println() + fmt.Println("external_nodes:") + } + + for i := 0; i < len(configFileYaml.ExternalNodes); i++ { + currentExternalNode := configFileYaml.ExternalNodes[i] + if nodeName == currentExternalNode.NodeUserIdent || nodeName == "" { // node name not set always show node + nodeFound = true + if isBare { + displayExternalNodeBare(currentExternalNode, isName, isEnodeUrl, isTmUrl, isNodeKeyAddr) + } else { + displayExternalNode(currentExternalNode, isName, isEnodeUrl, isTmUrl, isNodeKeyAddr) + } + } + } + // if the nodename was specified, but not found in the config, list the names of the nodes for the user. + if !nodeFound { + fmt.Println() + red.Println(fmt.Sprintf(" External node name [%s] not found in config file ", nodeName)) + fmt.Println() + fmt.Println(fmt.Sprintf(" External Node Names are: ")) + for i := 0; i < len(configFileYaml.ExternalNodes); i++ { + fmt.Println(fmt.Sprintf(" [%s]", configFileYaml.ExternalNodes[i].NodeUserIdent)) + } + } + return nil + }, + } ) func createNodeEntry(nodeName, nodeKeyDir, consensus, quorumVersion, txManager, tmVersion, quorumImageFull string) NodeEntry { @@ -985,13 +1130,10 @@ func displayNodeBare(k8sdir string, nodeEntry NodeEntry, name, consensus, keydir } } -func displayNodeAsExternal(k8sdir string, tmurl string, nodekeyAddress string, p2pUrl string, nodeEntry NodeEntry, name, consensus bool) { +func displayNodeAsExternal(k8sdir string, nodeEntry NodeEntry, tmurl string, nodekeyAddress string, p2pUrl string, name bool) { if name { fmt.Println("- Node_UserIdent: ", nodeEntry.NodeUserIdent) } - if consensus { - fmt.Println(nodeEntry.QuorumEntry.Quorum.Consensus) - } fmt.Println(" Tm_Url: ", tmurl) // need the tm URL that is addressable from outside the cluster (ingress or nodeport). // need the enodeURL of the node, that is addressable from outside the cluster (ingress or nodeport). @@ -1010,6 +1152,40 @@ func displayNodeAsExternal(k8sdir string, tmurl string, nodekeyAddress string, p // Acct_PubKey?? } +func displayExternalNode(extNode ExternalNodeEntry, isName, isEnodeUrl, isTmUrl, isNodekeyAddress bool) { + if isName { + fmt.Println("- Node_UserIdent: ", extNode.NodeUserIdent) + } + if isTmUrl { + fmt.Println(" Tm_Url: ", extNode.TmUrl) + } + if isEnodeUrl { + fmt.Println(" Enode_Url:", extNode.EnodeUrl) + } + // if IBFT need the Node_Acct_Addr + if isNodekeyAddress && extNode.NodekeyAddress != "" { + fmt.Println(" Node_Acct_Addr:", "\""+extNode.NodekeyAddress+"\"") + } + // Acct_PubKey?? +} + +func displayExternalNodeBare(extNode ExternalNodeEntry, isName, isEnodeUrl, isTmUrl, isNodekeyAddress bool) { + if isName { + fmt.Println(extNode.NodeUserIdent) + } + if isTmUrl { + fmt.Println(extNode.TmUrl) + } + if isEnodeUrl { + fmt.Println(extNode.EnodeUrl) + } + // if IBFT need the Node_Acct_Addr + if isNodekeyAddress && extNode.NodekeyAddress != "" { + fmt.Println("\"" + extNode.NodekeyAddress + "\"") + } + // Acct_PubKey?? +} + // stop node should just remove the deployment, and not delete any resources or persistent data. func stopNode(nodeName string) error { // TODO: should there be a separate delete and remove node? where remove only removes it from the cluster, but delete removes all traces? diff --git a/qctl/qctl.go b/qctl/qctl.go index f782ebf..e95465f 100644 --- a/qctl/qctl.go +++ b/qctl/qctl.go @@ -95,6 +95,7 @@ func main() { Usage: "options for listing resources", Subcommands: []*cli.Command{ &nodeListCommand, + &externalNodeListCommand, &allListCommand, &urlGetCommand, &describeConfigCommand, From 5526dd173abfeeffbde3ec8db93a4f2c3a21ddfe Mon Sep 17 00:00:00 2001 From: Libby Kent Date: Tue, 27 Oct 2020 23:38:41 -0400 Subject: [PATCH 2/4] [qctl] support for adding an external node. qctl add extnode --enode=enode://12343@1.2.3.4:7000 --tmurl=http://1.2.3.4:9000 --nodekeyaddr="0x1234334" * TODO: find a way to preserve double quotes in gopkg.in/yaml.v2, or use another yaml parsing library, need to preserve quotes for hex nodekey address in qubernetes config, or else the hex number is evaluated. currently wrapping in single quotes (ugly), as this is preserved, e.g. '"0x92392CAF837B94BC8541D0baE2db41BE2B999F39"'. --- qctl/nodecmd.go | 145 ++++++++++++++++++++++++++++++++++++++++++++++-- qctl/qctl.go | 1 + 2 files changed, 141 insertions(+), 5 deletions(-) diff --git a/qctl/nodecmd.go b/qctl/nodecmd.go index 4a4fc6c..9dfbf77 100644 --- a/qctl/nodecmd.go +++ b/qctl/nodecmd.go @@ -443,6 +443,141 @@ var ( return nil }, } + //qctl add extnode --enode=enode://12343@1.2.3.4:7000 --tmurl=http://1.2.3.4:9000 --nodekeyaddr="0x1234334" + externalNodeAddCommand = cli.Command{ + Name: "external-node", + Usage: "add new external node", + Aliases: []string{"extnode", "extnodes", "external-nodes"}, + ArgsUsage: "UniqueNodeName", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "config, c", + Usage: "Load configuration from `FULL_PATH_FILE`", + EnvVars: []string{"QUBE_CONFIG"}, + }, + // TODO: set default to Node-name-key-dir + &cli.StringFlag{ + Name: "enode", + Aliases: []string{"enodeurl"}, + Usage: "enode url of the external node to add (p2p portion must be reachable).", + Required: true, + }, + &cli.StringFlag{ + Name: "tmurl", + Aliases: []string{"tm"}, + Usage: "transaction manager url of the external node to add (must be reachable).", + Required: true, + }, + &cli.StringFlag{ + Name: "nodekeyaddr", + Aliases: []string{"nkaddr", "na"}, + Usage: "Nodekey address required for ibft only.", + }, + }, + Action: func(c *cli.Context) error { + name := c.Args().First() + // node name argument is required to update a node + if name == "" { + c.App.Run([]string{"qctl", "help", "external-node"}) + red.Println(" required argument: Unique NodeName of the external node you wish to add.") + return cli.Exit(" required argument: Unique NodeName of external node you wish to add.", 3) + } + + enode := c.String("enode") + tmurl := c.String("tmurl") + nodekeyaddr := c.String("nodekeyaddr") + + configFile := c.String("config") + if configFile == "" { + c.App.Run([]string{"qctl", "help", "init"}) + + // QUBE_CONFIG or flag + fmt.Println() + + fmt.Println() + red.Println(" --config flag must be provided.") + red.Println(" or ") + red.Println(" QUBE_CONFIG environment variable needs to be set to your config file.") + fmt.Println() + red.Println(" If you need to generate a qubernetes.yaml config use the command: ") + fmt.Println() + green.Println(" qctl generate config") + fmt.Println() + return cli.Exit("--config flag must be set to the fullpath of your config file.", 3) + } + fmt.Println() + green.Println(" Using config file:") + fmt.Println() + fmt.Println(" " + configFile) + fmt.Println() + fmt.Println("*****************************************************************************************") + fmt.Println() + + // get the current directory path, we'll use this in case the config file passed in was a relative path. + pwdCmd := exec.Command("pwd") + b, _ := runCmd(pwdCmd) + pwd := strings.TrimSpace(b.String()) + // the config file must exist or this is an error. + if fileExists(configFile) { + // check if config file is full path or relative path. + if !strings.HasPrefix(configFile, "/") { + configFile = pwd + "/" + configFile + } + + } else { + c.App.Run([]string{"qctl", "help", "init"}) + return cli.Exit(fmt.Sprintf("ConfigFile must exist! Given configFile [%v]", configFile), 3) + } + configFileYaml, err := LoadYamlConfig(configFile) + // check if the name is already taken + for i := 0; i < len(configFileYaml.ExternalNodes); i++ { + externalNode := configFileYaml.ExternalNodes[i] + // need to preserve the quotes " around the hex address 0x the yaml parser strips these out when it parses it. + // if the quotes are removed, the yaml parser then evaluates the hex number :( + // https://github.com/mikefarah/yq/issues/19 + // FIXME: this will yield an entry like '"0x92392CAF837B94BC8541D0baE2db41BE2B999F39"' which will be added to the istanbul-validator.toml. + // better to preserve the double quotes, and not use sed awk when adding nodes as validators. + configFileYaml.ExternalNodes[i].NodekeyAddress = "\"" + externalNode.NodekeyAddress + "\"" + if name == externalNode.NodeUserIdent { + red.Println(fmt.Sprintf("External node name [%s] already exist!", name)) + displayExternalNode(externalNode, true, true, true, true) + red.Println(fmt.Sprintf("External node name [%s] exists", name)) + return cli.Exit(fmt.Sprintf("External node name [%s] exists, External node names must be unique", name), 3) + } + } + if err != nil { + log.Fatal("config file [%v] could not be loaded into the valid qubernetes yaml. err: [%v]", configFile, err) + } + + fmt.Println(fmt.Sprintf("Adding external node [%s] ", name)) + currentNumExtNodes := len(configFileYaml.ExternalNodes) + fmt.Println(fmt.Sprintf("config currently has %d nodes", currentNumExtNodes)) + externalNodeEntry := ExternalNodeEntry{ + NodeUserIdent: name, + EnodeUrl: enode, + TmUrl: tmurl, + } + if nodekeyaddr != "" { + externalNodeEntry.NodekeyAddress = nodekeyaddr + } + configFileYaml.ExternalNodes = append(configFileYaml.ExternalNodes, externalNodeEntry) + fmt.Println() + green.Println("Adding External Node: ") + displayExternalNode(externalNodeEntry, true, true, true, true) + // write file back + WriteYamlConfig(configFileYaml, configFile) + fmt.Println("The external node(s) have been added to the config file [%s]", configFile) + fmt.Println("Next, generate (update) the additional node resources for quorum and k8s:") + fmt.Println() + fmt.Println("**********************************************************************************************") + fmt.Println() + green.Println(fmt.Sprintf(" $> qctl generate network --update")) + fmt.Println() + fmt.Println("**********************************************************************************************") + + return nil + }, + } // TODO: consolidate this and add node nodeUpdateCommand = cli.Command{ Name: "node", @@ -814,7 +949,7 @@ var ( if err != nil { log.Fatal(err) } - tmurl := strings.TrimSpace(res.String()) + tmUrl := strings.TrimSpace(res.String()) p2pCmd := exec.Command("qctl", "ls", "urls", "--node="+currentNode.NodeUserIdent, "--type=nodeport", "--p2p", "--bare", "--node-ip="+nodeip) //fmt.Println(cmd.String()) @@ -839,7 +974,7 @@ var ( nodekeyAddress = strings.TrimSpace(nodekeyAddress) } //fmt.Println("nodekeyAddress", nodekeyAddress) - displayNodeAsExternal(k8sdir, currentNode, tmurl, nodekeyAddress, p2pUrl, true) + displayNodeAsExternal(k8sdir, currentNode, p2pUrl, tmUrl, nodekeyAddress, true) } else { if isBare { // show the bare version, cleaner for scripts. displayNodeBare(k8sdir, currentNode, isName, isKeyDir, isConsensus, isQuorumVersion, isTmName, isTmVersion, isEnodeUrl, isQuorumImageFull, isTmImageFull, isGethParams) @@ -1130,21 +1265,21 @@ func displayNodeBare(k8sdir string, nodeEntry NodeEntry, name, consensus, keydir } } -func displayNodeAsExternal(k8sdir string, nodeEntry NodeEntry, tmurl string, nodekeyAddress string, p2pUrl string, name bool) { +func displayNodeAsExternal(k8sdir string, nodeEntry NodeEntry, p2pUrl string, tmUrl string, nodekeyAddress string, name bool) { if name { fmt.Println("- Node_UserIdent: ", nodeEntry.NodeUserIdent) } - fmt.Println(" Tm_Url: ", tmurl) // need the tm URL that is addressable from outside the cluster (ingress or nodeport). // need the enodeURL of the node, that is addressable from outside the cluster (ingress or nodeport). if k8sdir == "" { - red.Println("Set --k8sdir flag or QUBE_K8S_DIR env in order to display enodeurl") + red.Println("Set --k8sdir flag or QUBE_K8S_DIR env in order to display enodeUrl") } else { enodeUrl := getEnodeUrl(nodeEntry.NodeUserIdent, k8sdir) // replace the internal dns addressable p2p @quorum-node1:30303? with an external p2p URL (nodeport) enodeUrl = strings.ReplaceAll(enodeUrl, nodeEntry.NodeUserIdent+":"+DefaultP2PPort, p2pUrl) fmt.Println(" Enode_Url:", enodeUrl) } + fmt.Println(" Tm_Url: ", tmUrl) // if IBFT need the Node_Acct_Addr if nodekeyAddress != "" { fmt.Println(" Node_Acct_Addr:", "\""+nodekeyAddress+"\"") diff --git a/qctl/qctl.go b/qctl/qctl.go index e95465f..b251f36 100644 --- a/qctl/qctl.go +++ b/qctl/qctl.go @@ -108,6 +108,7 @@ func main() { Usage: "options for adding resources", Subcommands: []*cli.Command{ &nodeAddCommand, + &externalNodeAddCommand, &cakeshopAddCommand, &monitorAddCommand, }, From 2542d324ddac85ce4d5f0a9296a0943b4ad6485b Mon Sep 17 00:00:00 2001 From: Libby Kent Date: Wed, 28 Oct 2020 00:02:27 -0400 Subject: [PATCH 3/4] [qctl] minor clean up --k8s-dir -> --k8sdir. --- qctl/networkcmd.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/qctl/networkcmd.go b/qctl/networkcmd.go index a7d2abc..af74d78 100644 --- a/qctl/networkcmd.go +++ b/qctl/networkcmd.go @@ -16,7 +16,7 @@ var ( Usage: "delete a quorum k8s network given the dir holding the k8s yaml resources.", Flags: []cli.Flag{ &cli.StringFlag{ - Name: "k8s-dir", + Name: "k8sdir", Usage: "the path of the dir containing the K8s resource yaml.", EnvVars: []string{"QUBE_K8S_DIR"}, Required: true, @@ -32,7 +32,7 @@ var ( Usage: "create a quorum k8s network given the dir holding the k8s yaml resources.", Flags: []cli.Flag{ &cli.StringFlag{ - Name: "k8s-dir", + Name: "k8sdir", Usage: "the path of the dir containing the K8s resource yaml.", EnvVars: []string{"QUBE_K8S_DIR"}, Required: true, @@ -74,7 +74,7 @@ var ( Usage: "list the status of the running network.", Flags: []cli.Flag{ &cli.StringFlag{ - Name: "k8s-dir", + Name: "k8sdir", Usage: "the path of the dir containing the K8s resource yaml.", EnvVars: []string{"QUBE_K8S_DIR"}, Required: true, @@ -294,10 +294,10 @@ var ( ) func k8sCreateDeleteCluster(c *cli.Context, action string) error { - k8sdir := c.String("k8s-dir") + k8sdir := c.String("k8sdir") // if the passed in k8s dir does not exit, tell the user and do not proceed. if _, err := os.Stat(k8sdir); os.IsNotExist(err) { - log.Error("the --k8s-dir [%v] does not exist!", k8sdir) + log.Error("the --k8sdir [%v] does not exist!", k8sdir) return err } namespace := c.String("namespace") From f7c089a413e2b7f490b1dab9da5e8f1c28ab9c57 Mon Sep 17 00:00:00 2001 From: Libby Kent Date: Wed, 28 Oct 2020 01:49:52 -0400 Subject: [PATCH 4/4] [qct] delete extnodes, nodekeyaddr yaml fix. * support deleting external nodes: qctl delete external-node * allow external node Node_Acct_Addr to be set as a hex without quotes or a string hex in the qubernetes yaml. cover both cases in istanbul-validator.toml.erb. --- qctl/configyaml.go | 12 +- qctl/nodecmd.go | 134 ++++++++++++++++--- qctl/qctl.go | 1 + templates/quorum/istanbul-validator.toml.erb | 17 ++- 4 files changed, 139 insertions(+), 25 deletions(-) diff --git a/qctl/configyaml.go b/qctl/configyaml.go index 8feb5a2..fa60888 100644 --- a/qctl/configyaml.go +++ b/qctl/configyaml.go @@ -5,7 +5,7 @@ import ( "os" log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" + yaml "gopkg.in/yaml.v2" ) var ( @@ -49,9 +49,13 @@ type NodeEntry struct { } type ExternalNodeEntry struct { - NodeUserIdent string `yaml:"Node_UserIdent"` - EnodeUrl string `yaml:"Enode_Url"` - TmUrl string `yaml:"Tm_Url"` + NodeUserIdent string `yaml:"Node_UserIdent"` + EnodeUrl string `yaml:"Enode_Url"` + TmUrl string `yaml:"Tm_Url"` + // must be set in the yaml without quotes. + // The hex number will be evaluted to a BigNum and + // template/istanbul-validator.toml.erb will convert back to hex + // https://github.com/mikefarah/yq/issues/19 NodekeyAddress string `yaml:"Node_Acct_Addr,omitempty"` } diff --git a/qctl/nodecmd.go b/qctl/nodecmd.go index 9dfbf77..8c1585c 100644 --- a/qctl/nodecmd.go +++ b/qctl/nodecmd.go @@ -75,7 +75,7 @@ var ( pwd := strings.TrimSpace(b.String()) if configFile == "" { - c.App.Run([]string{"qctl", "help", "init"}) + c.App.Run([]string{"qctl", "help", "node"}) fmt.Println() fmt.Println() red.Println(" --config flag must be provided.") @@ -103,7 +103,7 @@ var ( } } else { - c.App.Run([]string{"qctl", "help", "init"}) + c.App.Run([]string{"qctl", "help", "node"}) return cli.Exit(fmt.Sprintf("ConfigFile must exist! Given configFile [%v]", configFile), 3) } configFileYaml, err := LoadYamlConfig(configFile) @@ -184,7 +184,109 @@ var ( return nil }, } + // qctl delete extnode quorum-node5 + externalNodeDeleteCommand = cli.Command{ + Name: "external-node", + Aliases: []string{"extnode", "extnodes"}, + Usage: "delete external node from config.", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "config, c", + Usage: "Load configuration from `FULL_PATH_FILE`", + EnvVars: []string{"QUBE_CONFIG"}, + Required: true, + }, + &cli.StringFlag{ // this is only required if the user wants to delete the generated (k8s/quorum) resources as well. + Name: "k8sdir", + Usage: "The k8sdir (usually out) containing the output k8s resources", + EnvVars: []string{"QUBE_K8S_DIR"}, + }, + }, + Action: func(c *cli.Context) error { + if c.Args().Len() < 1 { + c.App.Run([]string{"qctl", "help", "delete", "node"}) + return cli.Exit("wrong number of arguments", 2) + } + nodeName := c.Args().First() + fmt.Println("Delete external node " + nodeName) + // TODO: abstract this away as it is used in multiple places now. + configFile := c.String("config") + //k8sdir := c.String("k8sdir") + // get the current directory path, we'll use this in case the config file passed in was a relative path. + pwdCmd := exec.Command("pwd") + b, _ := runCmd(pwdCmd) + pwd := strings.TrimSpace(b.String()) + + if configFile == "" { + c.App.Run([]string{"qctl", "help", "external-node"}) + fmt.Println() + fmt.Println() + red.Println(" --config flag must be provided.") + red.Println(" or ") + red.Println(" QUBE_CONFIG environment variable needs to be set to your config file.") + fmt.Println() + red.Println(" If you need to generate a qubernetes.yaml config use the command: ") + fmt.Println() + green.Println(" qctl generate config") + fmt.Println() + return cli.Exit("--config flag must be set to the fullpath of your config file.", 3) + } + fmt.Println() + green.Println(" Using config file:") + fmt.Println() + fmt.Println(" " + configFile) + fmt.Println() + fmt.Println("*****************************************************************************************") + fmt.Println() + // the config file must exist or this is an error. + if fileExists(configFile) { + // check if config file is full path or relative path. + if !strings.HasPrefix(configFile, "/") { + configFile = pwd + "/" + configFile + } + } else { + c.App.Run([]string{"qctl", "help", "external-node"}) + return cli.Exit(fmt.Sprintf("ConfigFile must exist! Given configFile [%v]", configFile), 3) + } + configFileYaml, err := LoadYamlConfig(configFile) + if err != nil { + log.Fatal("config file [%v] could not be loaded into the valid qubernetes yaml. err: [%v]", configFile, err) + } + currentNum := len(configFileYaml.ExternalNodes) + fmt.Printf("config currently has %d external nodes \n", currentNum) + fmt.Println() + var nodeToDelete ExternalNodeEntry + for i := 0; i <= len(configFileYaml.ExternalNodes); i++ { + //displayNode(k8sdir, configFileYaml.Nodes[i], isName, isKeyDir, isConsensus, isQuorumVersion, isTmName, isTmVersion, isEnodeUrl, isQuorumImageFull) + if configFileYaml.ExternalNodes[i].NodeUserIdent == nodeName { + nodeToDelete = configFileYaml.ExternalNodes[i] + // remove the external node from the qubernetes config. + configFileYaml.ExternalNodes = append(configFileYaml.ExternalNodes[:i], configFileYaml.ExternalNodes[i+1:]...) + // do we want to remove the external node from the network here? or have the user run the steps separately? + // regenerate the resource without the node + // > qctl generate network --k8sdir=k8sdir --update + // deploy the new resources without the external node + // > qctl deploy network --k8sdir=k8sdir --wait + } + } + // write file back + WriteYamlConfig(configFileYaml, configFile) + green.Println(fmt.Sprintf(" Deleted external node [%s] from the config", nodeToDelete.NodeUserIdent)) + green.Println(" Next Steps : ") + green.Println(" 1. regenerate the resources without the external nodes.") + green.Println(" 2. Deploy the resources to the network.") + fmt.Print() + green.Println(" Run the next steps by entering the commands below") + fmt.Println("**********************************************************************************************") + fmt.Println() + green.Println(fmt.Sprintf(" $> qctl generate network --update")) + green.Println(fmt.Sprintf(" $> qctl deploy network --wait")) + fmt.Println() + fmt.Println("**********************************************************************************************") + return nil + }, + } /* * stops the give node, stopping will only remove the deployment from the K8s cluster, it will not remove any other * associated resources, such as the PVC (persistent volume claim) therefore maintaining the state of the node. The @@ -219,7 +321,7 @@ var ( pwd := strings.TrimSpace(b.String()) if configFile == "" { - c.App.Run([]string{"qctl", "help", "init"}) + c.App.Run([]string{"qctl", "help", "node"}) fmt.Println() fmt.Println() red.Println(" --config flag must be provided.") @@ -247,7 +349,7 @@ var ( } } else { - c.App.Run([]string{"qctl", "help", "init"}) + c.App.Run([]string{"qctl", "help", "node"}) return cli.Exit(fmt.Sprintf("ConfigFile must exist! Given configFile [%v]", configFile), 3) } configFileYaml, err := LoadYamlConfig(configFile) @@ -359,7 +461,7 @@ var ( pwd := strings.TrimSpace(b.String()) if configFile == "" { - c.App.Run([]string{"qctl", "help", "init"}) + c.App.Run([]string{"qctl", "help", "node"}) // QUBE_CONFIG or flag fmt.Println() @@ -388,9 +490,8 @@ var ( if !strings.HasPrefix(configFile, "/") { configFile = pwd + "/" + configFile } - } else { - c.App.Run([]string{"qctl", "help", "init"}) + c.App.Run([]string{"qctl", "help", "node"}) return cli.Exit(fmt.Sprintf("ConfigFile must exist! Given configFile [%v]", configFile), 3) } configFileYaml, err := LoadYamlConfig(configFile) @@ -443,7 +544,7 @@ var ( return nil }, } - //qctl add extnode --enode=enode://12343@1.2.3.4:7000 --tmurl=http://1.2.3.4:9000 --nodekeyaddr="0x1234334" + //qctl add extnode --enode="enode://12343@1.2.3.4:7000" --tmurl=http://1.2.3.4:9000 --nodekeyaddr=0x1234334 externalNodeAddCommand = cli.Command{ Name: "external-node", Usage: "add new external node", @@ -489,7 +590,7 @@ var ( configFile := c.String("config") if configFile == "" { - c.App.Run([]string{"qctl", "help", "init"}) + c.App.Run([]string{"qctl", "help", "external-node"}) // QUBE_CONFIG or flag fmt.Println() @@ -525,19 +626,13 @@ var ( } } else { - c.App.Run([]string{"qctl", "help", "init"}) + c.App.Run([]string{"qctl", "help", "external-node"}) return cli.Exit(fmt.Sprintf("ConfigFile must exist! Given configFile [%v]", configFile), 3) } configFileYaml, err := LoadYamlConfig(configFile) // check if the name is already taken for i := 0; i < len(configFileYaml.ExternalNodes); i++ { externalNode := configFileYaml.ExternalNodes[i] - // need to preserve the quotes " around the hex address 0x the yaml parser strips these out when it parses it. - // if the quotes are removed, the yaml parser then evaluates the hex number :( - // https://github.com/mikefarah/yq/issues/19 - // FIXME: this will yield an entry like '"0x92392CAF837B94BC8541D0baE2db41BE2B999F39"' which will be added to the istanbul-validator.toml. - // better to preserve the double quotes, and not use sed awk when adding nodes as validators. - configFileYaml.ExternalNodes[i].NodekeyAddress = "\"" + externalNode.NodekeyAddress + "\"" if name == externalNode.NodeUserIdent { red.Println(fmt.Sprintf("External node name [%s] already exist!", name)) displayExternalNode(externalNode, true, true, true, true) @@ -655,7 +750,7 @@ var ( pwd := strings.TrimSpace(b.String()) if configFile == "" { - c.App.Run([]string{"qctl", "help", "init"}) + c.App.Run([]string{"qctl", "help", "node"}) // QUBE_CONFIG or flag fmt.Println() @@ -686,7 +781,7 @@ var ( } } else { - c.App.Run([]string{"qctl", "help", "init"}) + c.App.Run([]string{"qctl", "help", "node"}) return cli.Exit(fmt.Sprintf("ConfigFile must exist! Given configFile [%v]", configFile), 3) } configFileYaml, err := LoadYamlConfig(configFile) @@ -754,6 +849,7 @@ var ( } // qctl ls node --name --consensus --quorumversion // qctl ls node --name --consensus --quorumversion --tmversion --tmname + // qctl ls nodes --asexternal -b --node-ip=$(minikube ip) nodeListCommand = cli.Command{ Name: "node", Usage: "list nodes info", @@ -1282,7 +1378,7 @@ func displayNodeAsExternal(k8sdir string, nodeEntry NodeEntry, p2pUrl string, tm fmt.Println(" Tm_Url: ", tmUrl) // if IBFT need the Node_Acct_Addr if nodekeyAddress != "" { - fmt.Println(" Node_Acct_Addr:", "\""+nodekeyAddress+"\"") + fmt.Println(" Node_Acct_Addr:", nodekeyAddress) } // Acct_PubKey?? } diff --git a/qctl/qctl.go b/qctl/qctl.go index b251f36..a70623e 100644 --- a/qctl/qctl.go +++ b/qctl/qctl.go @@ -54,6 +54,7 @@ func main() { &networkDeleteCommand, // TODO: Think this through a bit, hard delete vs soft delete, etc. &nodeDeleteCommand, + &externalNodeDeleteCommand, &cakeshopDeleteCommand, &monitorDeleteCommand, }, diff --git a/templates/quorum/istanbul-validator.toml.erb b/templates/quorum/istanbul-validator.toml.erb index 2516ace..23b18e4 100644 --- a/templates/quorum/istanbul-validator.toml.erb +++ b/templates/quorum/istanbul-validator.toml.erb @@ -26,8 +26,21 @@ validators = [ end -%> <%- if @external_nodes -%> -<%- @external_nodes.each do |extnode| -%> -"<%=extnode["Node_Acct_Addr"] %>", +<%- @external_nodes.each do |extnode| + # this is to get around a yaml parsing issue, qctl uses github.com/urfave/cli/v2 which does not preserve double quotes, + # and if a hex value is stored in yaml without it will be evaluated to a Bignum, Node_Acct_Addr must be set as hex numbers + # without quotes, e.g.0xDEe962299b55dA670fC70702e626ac9009AE3aC7, and then converted back to hex here. + nodekeyAddr = extnode["Node_Acct_Addr"] + # check if the nodekey address is not a hex string, if not we assume it has been converted to a BigNum + # and convert it back to a hex. + nodekeyAddrStr = nodekeyAddr.to_s + if nodekeyAddrStr[0..1] != "0x" + nodekeyAddr = nodekeyAddr.to_s(16) + nodekeyAddr = "0x" + nodekeyAddr + end + puts("external node account for Istanbul: " + nodekeyAddr) + -%> +"<%=nodekeyAddr %>", <%- end -%> <%- end -%> ] \ No newline at end of file