add: keep ENR crawl option

cmd/nebula/cmd_crawl.go

@@ -31,13 +31,14 @@ var crawlConfig = &config.Crawl{
 	WriteWorkerCount: 10,
 	CrawlLimit:       0,
 	PersistNeighbors: false,
-	CheckExposed:     false,
 	FilePathUdgerDB:  "",
 	Network:          string(config.NetworkIPFS),
 	BootstrapPeers:   cli.NewStringSlice(),
 	Protocols:        cli.NewStringSlice(string(kaddht.ProtocolDHT)),
 	AddrTrackTypeStr: "public",
 	AddrDialTypeStr:  "public",
+	KeepENR:          false,
+	CheckExposed:     false,
 }
 
 // CrawlCommand contains the crawl sub-command configuration.
@@ -138,14 +139,6 @@ var CrawlCommand = &cli.Command{
 			Value:       crawlConfig.PersistNeighbors,
 			Destination: &crawlConfig.PersistNeighbors,
 		},
-		&cli.BoolFlag{
-			Name:        "check-exposed",
-			Usage:       "Whether to check if the Kubo API is exposed. Checking also includes crawling the API.",
-			EnvVars:     []string{"NEBULA_CRAWL_CHECK_EXPOSED"},
-			Value:       crawlConfig.CheckExposed,
-			Destination: &crawlConfig.CheckExposed,
-			Category:    flagCategoryNetwork,
-		},
 		&cli.StringFlag{
 			Name:        "addr-track-type",
 			Usage:       "Which type addresses should be stored to the database (private, public, any)",
@@ -167,6 +160,22 @@ var CrawlCommand = &cli.Command{
 			Value:       crawlConfig.Network,
 			Destination: &crawlConfig.Network,
 		},
+		&cli.BoolFlag{
+			Name:        "check-exposed",
+			Usage:       "IPFS/AMINO: Whether to check if the Kubo API is exposed. Checking also includes crawling the API.",
+			EnvVars:     []string{"NEBULA_CRAWL_CHECK_EXPOSED"},
+			Value:       crawlConfig.CheckExposed,
+			Destination: &crawlConfig.CheckExposed,
+			Category:    flagCategoryNetwork,
+		},
+		&cli.BoolFlag{
+			Name:        "keep-enr",
+			Usage:       "ETHEREUM_CONSENSUS: Whether to keep the full ENR.",
+			EnvVars:     []string{"NEBULA_CRAWL_KEEP_ENR"},
+			Value:       crawlConfig.KeepENR,
+			Destination: &crawlConfig.KeepENR,
+			Category:    flagCategoryNetwork,
+		},
 	},
 }
 
@@ -227,6 +236,7 @@ func CrawlAction(c *cli.Context) error {
 			BootstrapPeerStrs: cfg.BootstrapPeers.Value(),
 			AddrDialType:      cfg.AddrDialType(),
 			AddrTrackType:     cfg.AddrTrackType(),
+			KeepENR:           crawlConfig.KeepENR,
 		}
 
 		// init the crawl driver

pkg/config/config.go

@@ -188,9 +188,6 @@ type Crawl struct {
 	// Whether to persist all k-bucket entries
 	PersistNeighbors bool
 
-	// Whether to check if the Kubo API is exposed
-	CheckExposed bool
-
 	// File path to the udger datbase
 	FilePathUdgerDB string
 
@@ -202,6 +199,12 @@ type Crawl struct {
 
 	// Which type of addresses should Nebula try to dial (private, public, both)
 	AddrDialTypeStr string
+
+	// Whether to check if the Kubo API is exposed
+	CheckExposed bool
+
+	// Whether to keep the full enr record alongside all parsed kv-pairs
+	KeepENR bool
 }
 
 func (c *Crawl) AddrTrackType() AddrType {

pkg/discv5/crawler.go

@@ -27,6 +27,7 @@ import (
 type CrawlerConfig struct {
 	DialTimeout  time.Duration
 	AddrDialType config.AddrType
+	KeepENR      bool
 }
 
 type Crawler struct {
@@ -112,6 +113,10 @@ func (c *Crawler) PeerProperties(node *enode.Node) json.RawMessage {
 		properties["opstack_version"] = enrEntryOpStack.Version
 	}
 
+	if c.cfg.KeepENR {
+		properties["enr"] = node.String()
+	}
+
 	data, err := json.Marshal(properties)
 	if err != nil {
 		log.WithError(err).WithField("properties", properties).Warnln("Could not marshal peer properties")

pkg/discv5/driver_crawler.go

@@ -115,12 +115,14 @@ type CrawlDriverConfig struct {
 	BootstrapPeerStrs []string
 	AddrDialType      config.AddrType
 	AddrTrackType     config.AddrType
+	KeepENR           bool
 }
 
 func (cfg *CrawlDriverConfig) CrawlerConfig() *CrawlerConfig {
 	return &CrawlerConfig{
 		DialTimeout:  cfg.DialTimeout,
 		AddrDialType: cfg.AddrDialType,
+		KeepENR:      cfg.KeepENR,
 	}
 }