PKI: 3924 handle stateproof in rest api goal

libgoal/libgoal.go

@@ -978,8 +978,6 @@ func (c *Client) AddParticipationKey(keyfile string) (resp generated.PostPartici
 	}
 
 	return algod.PostParticipationKey(data)
-
-	// PKI TODO: Install state proof keys here.
 }
 
 // GetParticipationKeys gets the currently installed participation keys.

node/node.go

@@ -403,12 +403,7 @@ func (node *AlgorandFullNode) Start() {
 
 // startMonitoringRoutines starts the internal monitoring routines used by the node.
 func (node *AlgorandFullNode) startMonitoringRoutines() {
-	node.monitoringRoutinesWaitGroup.Add(3)
-
-	// PKI TODO: Remove this with #2596
-	// Periodically check for new participation keys
-	go node.checkForParticipationKeys(node.ctx.Done())
-
+	node.monitoringRoutinesWaitGroup.Add(2)
 	go node.txPoolGaugeThread(node.ctx.Done())
 	// Delete old participation keys
 	go node.oldKeyDeletionThread(node.ctx.Done())
@@ -781,24 +776,6 @@ func ensureParticipationDB(genesisDir string, log logging.Logger) (account.Parti
 	return account.MakeParticipationRegistry(accessor, log)
 }
 
-// Reload participation keys from disk periodically
-func (node *AlgorandFullNode) checkForParticipationKeys(done <-chan struct{}) {
-	defer node.monitoringRoutinesWaitGroup.Done()
-	ticker := time.NewTicker(node.config.ParticipationKeysRefreshInterval)
-	for {
-		select {
-		case <-ticker.C:
-			err := node.loadParticipationKeys()
-			if err != nil {
-				node.log.Errorf("Could not refresh participation keys: %v", err)
-			}
-		case <-done:
-			ticker.Stop()
-			return
-		}
-	}
-}
-
 // ListParticipationKeys returns all participation keys currently installed on the node
 func (node *AlgorandFullNode) ListParticipationKeys() (partKeys []account.ParticipationRecord, err error) {
 	return node.accountManager.Registry().GetAll(), nil
@@ -916,7 +893,7 @@ func (node *AlgorandFullNode) InstallParticipationKey(partKeyBinary []byte) (acc
 	}
 	defer inputdb.Close()
 
-	partkey, err := account.RestoreParticipation(inputdb)
+	partkey, err := account.RestoreParticipationWithSecrets(inputdb)
 	if err != nil {
 		return account.ParticipationID{}, err
 	}
@@ -931,22 +908,15 @@ func (node *AlgorandFullNode) InstallParticipationKey(partKeyBinary []byte) (acc
 	if !added {
 		return account.ParticipationID{}, fmt.Errorf("ParticipationRegistry: cannot register duplicate participation key")
 	}
-	err = node.accountManager.Registry().Flush(participationRegistryFlushMaxWaitDuration)
+
+	err = insertStateProofToRegistry(partkey, node)
 	if err != nil {
 		return account.ParticipationID{}, err
 	}
 
-	newFilename := config.PartKeyFilename(partkey.ID().String(), uint64(partkey.FirstValid), uint64(partkey.LastValid))
-	newFullyQualifiedFilename := filepath.Join(outDir, filepath.Base(newFilename))
-
-	if _, err = os.Stat(newFullyQualifiedFilename); os.IsExist(err) {
-		return account.ParticipationID{}, fmt.Errorf("KeyInstallation: cannot register duplicate participation key")
-	}
-
-	err = os.Rename(fullyQualifiedTempFile, newFullyQualifiedFilename)
-
+	err = node.accountManager.Registry().Flush(participationRegistryFlushMaxWaitDuration)
 	if err != nil {
-		return account.ParticipationID{}, nil
+		return account.ParticipationID{}, err
 	}
 
 	return partkey.ID(), nil

test/e2e-go/features/participation/overlappingParticipationKeys_test.go

@@ -39,6 +39,17 @@ import (
 	"github.com/algorand/go-algorand/util/db"
 )
 
+// TestOverlappingParticipationKeys is a test that "overlaps" participation keys across
+// various nodes. Keys are installed in a rotating fashion across the nodes where:
+// ((Network Round - 1) Mod 10) = nodeIdx and nodeIdx is used to pull out from an
+// "array" of nodes similar to {Node1, Node2, Node3} etc.  The Mod 10 simply pulls the
+// "digit" from the number:
+//    Round: 13 -> 13 - 1 = 12 ->  12 Mod 10 -> 2 -> Node3 with nodeIdx == 2
+//
+// The keys are overlapped in the sense that a key is registered to a node and
+// "overlaps" with other installed keys that are also valid.  Meaning there might be:
+// PKI 1 (Valid 3-15) and PKI 2 (Valid 13-25) and PKI 3 (Valid 23-35) all installed
+// on the same node
 func TestOverlappingParticipationKeys(t *testing.T) {
 	partitiontest.PartitionTest(t)
 	defer fixtures.ShutdownSynchronizedTest(t)
@@ -50,6 +61,7 @@ func TestOverlappingParticipationKeys(t *testing.T) {
 	shortPartKeysProtocol := config.Consensus[protocol.ConsensusCurrentVersion]
 	shortPartKeysProtocol.ApprovedUpgrades = map[protocol.ConsensusVersion]uint64{}
 	// keys round = current - 2 * (2 * 1) (see selector.go)
+	//  --> return r.SubSaturate(basics.Round(2 * cparams.SeedRefreshInterval * cparams.SeedLookback))
 	// new keys must exist at least 4 rounds prior use
 	shortPartKeysProtocol.SeedLookback = 2
 	shortPartKeysProtocol.SeedRefreshInterval = 1
@@ -68,13 +80,6 @@ func TestOverlappingParticipationKeys(t *testing.T) {
 	defer fixture.Shutdown()
 
 	accountsNum := len(fixture.NodeDataDirs())
-	for _, dataDir := range fixture.NodeDataDirs() {
-		cfg, err := config.LoadConfigFromDisk(dataDir)
-		a.NoError(err)
-		cfg.ParticipationKeysRefreshInterval = 500 * time.Millisecond
-		err = cfg.SaveToDisk(dataDir)
-		a.NoError(err)
-	}
 
 	genesis, err := bookkeeping.LoadGenesisFromFile(filepath.Join(fixture.PrimaryDataDir(), "genesis.json"))
 	a.NoError(err)
@@ -89,28 +94,63 @@ func TestOverlappingParticipationKeys(t *testing.T) {
 			continue
 		}
 		acctIdx := (round - 1) % 10
+
+		// Prepare the registration keys ahead of time.  Note that the + 10 is because we use Mod 10
+
+		// These variables control when the transaction will be sent out to be valid from.
+		// These variables will also be the name of the file produced EXCEPT
+		// prepareParticipationKey() will add 2 to the txStartRound for the filename.
+		// so the file for round 1 will be 3.15
+		// For round 11 (the next round that Mod 10 will index to 1), that means the filename will be
+		// 13.25 which results in a 2 round overlap
 		txStartRound := round
 		txEndRound := txStartRound + 10 + 4
+		// The registration variables here control when the participation key will actually be valid from
+		// For round 1, that means from 1-16 (one round of overlap)
+		// For round 11 (the next round that Mod 10 will index to 1), that means the 11-26
 		regStartRound := round
 		regEndRound := regStartRound + 11 + 4
+
 		err = prepareParticipationKey(a, &fixture, acctIdx, txStartRound, txEndRound, regStartRound, regEndRound, genesisHash, rootKeys, regTransactions, config.Consensus[protocol.ConsensusCurrentVersion])
 		a.NoError(err)
 	}
 
 	fixture.Start()
 	currentRound := uint64(0)
 	fixture.AlgodClient = fixture.GetAlgodClientForController(fixture.NC)
+
+	// ******** IMPORTANT ********
+	// It is CRITICAL that this for loop NOT BLOCK.
+	// This loop assumes that it stays current with the round of the network.
+	// Remember: this test is running while the network is advancing rounds in parallel
+	// If this test blocks for more than a couple seconds, then the network round count will have advanced
+	// farther than the current "currentRound" variable.  This will mean that the "addParticipationKey" function
+	// will NOT install the participation key in time for the shortened SeedLookback variable resulting
+	// in a network stall and a test failure
 	for {
 		err := fixture.WaitForRoundWithTimeout(currentRound + 1)
 		a.NoError(err)
+
+		// A sanity check that makes sure that the round of the network is the same as our
+		// current round variable
+		sts, err := fixture.GetAlgodClientForController(fixture.NC).Status()
+		a.NoError(err, "the network stalled, see test comments and review node.log in each nodes data directory for details.")
+		a.Equal(sts.LastRound, currentRound+1)
+
 		currentRound++
 		if (currentRound-1)%10 < uint64(accountsNum) {
 			acctIdx := (currentRound - 1) % 10
+
+			// We do a plus two because the filenames were stored with a plus 2
 			startRound := currentRound + 2 // +2 and -2 below to balance, start/end must match in part key file name
 			endRound := startRound + 10 + 4 - 2
+
 			regStartRound := currentRound
 			regEndRound := regStartRound + 11 + 4
 
+			// This cannot block! (See above)
+			// We pull the files from the disk according to their start round end round filenames
+			// and install them as well as send out a transaction
 			pk, err := addParticipationKey(a, &fixture, acctIdx, startRound, endRound, regTransactions)
 			a.NoError(err)
 			t.Logf("[.] Round %d, Added reg key for node %d range [%d..%d] %s\n", currentRound, acctIdx, regStartRound, regEndRound, hex.EncodeToString(pk[:8]))
@@ -128,17 +168,20 @@ func TestOverlappingParticipationKeys(t *testing.T) {
 func addParticipationKey(a *require.Assertions, fixture *fixtures.RestClientFixture, acctNum uint64, startRound, endRound uint64, regTransactions map[int]transactions.SignedTxn) (crypto.OneTimeSignatureVerifier, error) {
 	dataDir := fixture.NodeDataDirs()[acctNum]
 	nc := fixture.GetNodeControllerForDataDir(dataDir)
-	genesisDir, err := nc.GetGenesisDir()
 
 	partKeyName := filepath.Join(dataDir, config.PartKeyFilename("Wallet", startRound, endRound))
-	partKeyNameTarget := filepath.Join(genesisDir, config.PartKeyFilename("Wallet", startRound, endRound))
 
-	// make the rename in the background to ensure it won't take too long. We have ~4 rounds to complete this.
-	go os.Rename(partKeyName, partKeyNameTarget)
+	// This function can take more than a couple seconds, we can't have this function block so
+	// we wrap it in a go routine
+	go func() {
+		clientController := fixture.GetLibGoalClientFromNodeController(nc)
+		_, err := clientController.AddParticipationKey(partKeyName)
+		a.NoError(err)
+	}()
 
 	signedTxn := regTransactions[int(startRound-2)]
 	a.NotEmpty(signedTxn.Sig)
-	_, err = fixture.GetAlgodClientForController(nc).SendRawTransaction(signedTxn)
+	_, err := fixture.GetAlgodClientForController(nc).SendRawTransaction(signedTxn)
 	a.NoError(err)
 	return signedTxn.Txn.KeyregTxnFields.VotePK, err
 }