Merge branch 'rel/beta' into relstable3.6.2

.circleci/config.yml

@@ -138,7 +138,9 @@ workflows:
             - << matrix.platform >>_build
           filters:
             <<: *filters-nightly
-          context: slack-secrets
+          context:
+            - slack-secrets
+            - aws-secrets
 
       - tests_verification_job:
           name: << matrix.platform >>_<< matrix.job_type >>_verification
@@ -196,7 +198,7 @@ commands:
           shell: bash.exe
           command: |
             choco install -y msys2 pacman make wget --force
-            choco install -y golang --version=1.14.7 --force
+            choco install -y golang --version=1.16.15 --force
             choco install -y python3 --version=3.7.3 --force
             export msys2='cmd //C RefreshEnv.cmd '
             export msys2+='& set MSYS=winsymlinks:nativestrict '
@@ -335,9 +337,6 @@ commands:
           key: 'go-cache-v2-{{ .Environment.CIRCLE_STAGE }}-{{ .Environment.CIRCLE_BUILD_NUM }}'
           paths:
             - tmp/go-cache
-      - upload_to_buildpulse:
-          platform: << parameters.platform >>
-          path: << parameters.result_path >>/<< parameters.result_subdir>>
 
   upload_coverage:
     description: Collect coverage reports and upload them
@@ -348,40 +347,6 @@ commands:
           command: |
             scripts/travis/upload_coverage.sh || true
 
-  upload_to_buildpulse:
-    description: Collect build reports and upload them
-    parameters:
-      platform:
-        type: string
-      path:
-        type: string
-    steps:
-      - run:
-          name: Send test results to BuildPulse
-          when: always
-          command: |
-            set -e
-            if ! ls << parameters.path >>/*/*.xml &> /dev/null; then exit 0; fi
-            sed -i"" -e 's/classname="/classname="<< parameters.platform >>-/' << parameters.path >>/*/*.xml
-            case "<< parameters.platform >>" in
-              arm64)
-                URL=https://github.com/buildpulse/test-reporter/releases/download/v0.21.0-pre/test-reporter-linux-arm64
-                SUM=53f94c29ad162c2b9ebb1f4a2f967f5262c0459ee4a0c34332977d8c89aafc18
-                ;;
-              amd64)
-                URL=https://github.com/buildpulse/test-reporter/releases/download/v0.21.0-pre/test-reporter-linux-amd64
-                SUM=4655e54d756580c0de0112cab488e6e08d0af75e9fc8caea2d63f9e13be8beb5
-                ;;
-              mac_amd64)
-                URL=https://github.com/buildpulse/test-reporter/releases/download/v0.21.0-pre/test-reporter-darwin-amd64
-                SUM=2f9e20a6f683c80f35d04e36bc57ecfe605bb48fee5a1b8d8f7c45094028eea3
-                ;;
-            esac
-            curl -fsSL --retry 3 --retry-connrefused $URL > ./buildpulse-test-reporter
-            echo "$SUM *buildpulse-test-reporter" | shasum -a 256 -c --status
-            chmod +x ./buildpulse-test-reporter
-            ./buildpulse-test-reporter submit << parameters.path >> --account-id 23182699 --repository-id 191266671 || true
-
   generic_integration:
     description: Run integration tests from build workspace, for re-use by diferent architectures
     parameters:
@@ -432,6 +397,7 @@ commands:
             export PARTITION_ID=${CIRCLE_NODE_INDEX}
             export PARALLEL_FLAG="-p 1"
             test/scripts/run_integration_tests.sh
+
       - store_artifacts:
           path: << parameters.result_path >>
           destination: test-results
@@ -441,9 +407,6 @@ commands:
           root: << parameters.result_path >>
           paths:
             - << parameters.result_subdir >>
-      - upload_to_buildpulse:
-          platform: << parameters.platform >>
-          path: << parameters.result_path >>/<< parameters.result_subdir>>
 
   tests_verification_command:
     description: Check if all tests were run at least once and only once across all parallel runs
@@ -651,6 +614,7 @@ jobs:
     working_directory: << pipeline.parameters.build_dir >>/project
     environment:
       E2E_TEST_FILTER: "SCRIPTS"
+      E2E_PLATFORM: << parameters.platform >>
     steps:
       - prepare_build_dir
       - prepare_go

.github/workflows/build.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Install golang
         uses: actions/setup-go@v2
         with:
-          go-version: '1.14.7'
+          go-version: '1.16.15'
       - name: Build Test
         run: |
           export ALGORAND_DEADLOCK=enable

.github/workflows/reviewdog.yml

@@ -44,7 +44,7 @@ jobs:
       - name: Install specific golang
         uses: actions/setup-go@v2
         with:
-          go-version: '1.16.6'
+          go-version: '1.16.15'
       - name: Create folders for golangci-lint
         run: mkdir -p cicdtmp/golangci-lint
       - name: Check if custom golangci-lint is already built

Makefile

@@ -8,7 +8,6 @@ else
 export GOPATH := $(shell go env GOPATH)
 GOPATH1 := $(firstword $(subst :, ,$(GOPATH)))
 endif
-export GO111MODULE	:= on
 export GOPROXY := direct
 SRCPATH     := $(shell pwd)
 ARCH        := $(shell ./scripts/archtype.sh)

agreement/actions.go

@@ -237,6 +237,8 @@ func (a ensureAction) do(ctx context.Context, s *Service) {
 			Round:        uint64(a.Certificate.Round),
 			ValidatedAt:  a.Payload.validatedAt,
 			PreValidated: true,
+			PropBufLen:   uint64(len(s.demux.rawProposals)),
+			VoteBufLen:   uint64(len(s.demux.rawVotes)),
 		})
 		s.Ledger.EnsureValidatedBlock(a.Payload.ve, a.Certificate)
 	} else {
@@ -249,6 +251,8 @@ func (a ensureAction) do(ctx context.Context, s *Service) {
 			Round:        uint64(a.Certificate.Round),
 			ValidatedAt:  a.Payload.validatedAt,
 			PreValidated: false,
+			PropBufLen:   uint64(len(s.demux.rawProposals)),
+			VoteBufLen:   uint64(len(s.demux.rawVotes)),
 		})
 		s.Ledger.EnsureBlock(block, a.Certificate)
 	}

agreement/cryptoVerifier.go

@@ -22,8 +22,12 @@ import (
 
 	"github.com/algorand/go-algorand/logging"
 	"github.com/algorand/go-algorand/protocol"
+	"github.com/algorand/go-algorand/util/metrics"
 )
 
+var voteVerifierOutFullCounter = metrics.MakeCounter(
+	metrics.MetricName{Name: "algod_agreement_vote_verifier_responses_dropped", Description: "Number of voteVerifier responses dropped due to full channel"})
+
 // TODO put these in config
 const (
 	voteParallelism     = 16
@@ -210,6 +214,7 @@ func (c *poolCryptoVerifier) voteFillWorker(toBundleWait chan<- bundleFuture) {
 				select {
 				case c.votes.out <- asyncVerifyVoteResponse{index: votereq.TaskIndex, err: err, cancelled: true}:
 				default:
+					voteVerifierOutFullCounter.Inc(nil)
 					c.log.Infof("poolCryptoVerifier.voteFillWorker unable to write failed enqueue response to output channel")
 				}
 			}

agreement/pseudonode.go

@@ -29,6 +29,7 @@ import (
 	"github.com/algorand/go-algorand/logging/logspec"
 	"github.com/algorand/go-algorand/logging/telemetryspec"
 	"github.com/algorand/go-algorand/protocol"
+	"github.com/algorand/go-algorand/util/metrics"
 )
 
 // TODO put these in config
@@ -43,6 +44,9 @@ var errPseudonodeVerifierClosedChannel = errors.New("crypto verifier closed the
 var errPseudonodeNoVotes = errors.New("no valid participation keys to generate votes for given round")
 var errPseudonodeNoProposals = errors.New("no valid participation keys to generate proposals for given round")
 
+var pseudonodeBacklogFullByType = metrics.NewTagCounter("algod_agreement_pseudonode_tasks_dropped_{TAG}", "Number of pseudonode tasks dropped per type")
+var pseudonodeResultTimeoutsByType = metrics.NewTagCounter("algod_agreement_pseudonode_tasks_timeouts_{TAG}", "Number of pseudonode task result timeouts per type")
+
 // A pseudonode creates proposals and votes with a KeyManager which holds participation keys.
 //
 // It constructs these messages as if they arrived from an external source and were verified.
@@ -176,6 +180,7 @@ func (n asyncPseudonode) MakeProposals(ctx context.Context, r round, p period) (
 		return proposalTask.outputChannel(), nil
 	default:
 		proposalTask.close()
+		pseudonodeBacklogFullByType.Add("proposal", 1)
 		return nil, fmt.Errorf("unable to make proposal for (%d, %d): %w", r, p, errPseudonodeBacklogFull)
 	}
 }
@@ -193,6 +198,7 @@ func (n asyncPseudonode) MakeVotes(ctx context.Context, r round, p period, s ste
 		return proposalTask.outputChannel(), nil
 	default:
 		proposalTask.close()
+		pseudonodeBacklogFullByType.Add("vote", 1)
 		return nil, fmt.Errorf("unable to make vote for (%d, %d, %d): %w", r, p, s, errPseudonodeBacklogFull)
 	}
 }
@@ -474,6 +480,7 @@ verifiedVotesLoop:
 				return
 			case <-outputTimeout:
 				// we've been waiting for too long for this vote to be written to the output.
+				pseudonodeResultTimeoutsByType.Add("vote", 1)
 				t.node.log.Warnf("pseudonode.makeVotes: unable to write vote to output channel for round %d, period %d", t.round, t.period)
 				outputTimeout = nil
 			}
@@ -577,6 +584,7 @@ verifiedVotesLoop:
 				return
 			case <-outputTimeout:
 				// we've been waiting for too long for this vote to be written to the output.
+				pseudonodeResultTimeoutsByType.Add("pvote", 1)
 				t.node.log.Warnf("pseudonode.makeProposals: unable to write proposal vote to output channel for round %d, period %d", t.round, t.period)
 				outputTimeout = nil
 			}
@@ -597,6 +605,7 @@ verifiedPayloadsLoop:
 				return
 			case <-outputTimeout:
 				// we've been waiting for too long for this vote to be written to the output.
+				pseudonodeResultTimeoutsByType.Add("ppayload", 1)
 				t.node.log.Warnf("pseudonode.makeProposals: unable to write proposal payload to output channel for round %d, period %d", t.round, t.period)
 				outputTimeout = nil
 			}

buildnumber.dat

@@ -1 +1 @@
-1
+2

cmd/algocfg/getCommand.go

@@ -51,29 +51,29 @@ var getCmd = &cobra.Command{
 				return
 			}
 
-			val, err := getObjectProperty(cfg, getParameterArg)
+			val, err := serializeObjectProperty(cfg, getParameterArg)
 			if err != nil {
 				reportWarnf("Error retrieving property '%s' - %s", getParameterArg, err)
 				anyError = true
 				return
 			}
 
-			fmt.Printf("%s", val)
+			fmt.Print(val)
 		})
 		if anyError {
 			os.Exit(1)
 		}
 	},
 }
 
-func getObjectProperty(object interface{}, property string) (ret interface{}, err error) {
+func serializeObjectProperty(object interface{}, property string) (ret string, err error) {
 	v := reflect.ValueOf(object)
 	val := reflect.Indirect(v)
 	f := val.FieldByName(property)
 
 	if !f.IsValid() {
-		return object, fmt.Errorf("unknown property named '%s'", property)
+		return "", fmt.Errorf("unknown property named '%s'", property)
 	}
 
-	return f.Interface(), nil
+	return fmt.Sprintf("%v", f.Interface()), nil
 }

util/metrics/stringGauge_test.go → cmd/algocfg/getCommand_test.go

@@ -14,42 +14,51 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with go-algorand.  If not, see <https://www.gnu.org/licenses/>.
 
-package metrics
+package main
 
 import (
-	"strings"
+	"fmt"
 	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
 
 	"github.com/algorand/go-algorand/test/partitiontest"
-	"github.com/stretchr/testify/require"
 )
 
-func hasKey(data map[string]string, key string) bool {
-	_, ok := data[key]
-	return ok
-}
-
-func TestMetricStringGauge(t *testing.T) {
+func TestPrint(t *testing.T) {
 	partitiontest.PartitionTest(t)
 
-	stringGauge := MakeStringGauge()
-	stringGauge.Set("number-key", "1")
-	stringGauge.Set("string-key", "value")
-
-	results := make(map[string]string)
-	DefaultRegistry().AddMetrics(results)
-
-	// values are populated
-	require.Equal(t, 2, len(results))
-	require.True(t, hasKey(results, "number-key"))
-	require.Equal(t, "1", results["number-key"])
-	require.True(t, hasKey(results, "string-key"))
-	require.Equal(t, "value", results["string-key"])
-
-	// not included in string builder
-	buf := strings.Builder{}
-	DefaultRegistry().WriteMetrics(&buf, "not used")
-	require.Equal(t, "", buf.String())
-
-	stringGauge.Deregister(nil)
+	testcases := []struct {
+		Input    interface{}
+		expected string
+	}{
+		{
+			Input:    "string",
+			expected: "string",
+		},
+		{
+			Input:    uint64(1234),
+			expected: "1234",
+		},
+		{
+			Input:    int64(-1234),
+			expected: "-1234",
+		},
+		{
+			Input:    true,
+			expected: "true",
+		},
+		{
+			Input:    time.Second,
+			expected: "1s",
+		},
+	}
+	for i, tc := range testcases {
+		t.Run(fmt.Sprintf("test %d", i), func(t *testing.T) {
+			ret, err := serializeObjectProperty(tc, "Input")
+			assert.NoError(t, err)
+			assert.Equal(t, tc.expected, ret)
+		})
+	}
 }

cmd/algod/main.go

@@ -50,10 +50,11 @@ var versionCheck = flag.Bool("v", false, "Display and write current build versio
 var branchCheck = flag.Bool("b", false, "Display the git branch behind the build")
 var channelCheck = flag.Bool("c", false, "Display and release channel behind the build")
 var initAndExit = flag.Bool("x", false, "Initialize the ledger and exit")
+var logToStdout = flag.Bool("o", false, "Write to stdout instead of node.log by overriding config.LogSizeLimit to 0")
 var peerOverride = flag.String("p", "", "Override phonebook with peer ip:port (or semicolon separated list: ip:port;ip:port;ip:port...)")
 var listenIP = flag.String("l", "", "Override config.EndpointAddress (REST listening address) with ip:port")
 var sessionGUID = flag.String("s", "", "Telemetry Session GUID to use")
-var telemetryOverride = flag.String("t", "", `Override telemetry setting if supported (Use "true", "false", "0" or "1"`)
+var telemetryOverride = flag.String("t", "", `Override telemetry setting if supported (Use "true", "false", "0" or "1")`)
 var seed = flag.String("seed", "", "input to math/rand.Seed()")
 
 func main() {
@@ -84,12 +85,12 @@ func run() int {
 	}
 
 	version := config.GetCurrentVersion()
-	heartbeatGauge := metrics.MakeStringGauge()
-	heartbeatGauge.Set("version", version.String())
-	heartbeatGauge.Set("version-num", strconv.FormatUint(version.AsUInt64(), 10))
-	heartbeatGauge.Set("channel", version.Channel)
-	heartbeatGauge.Set("branch", version.Branch)
-	heartbeatGauge.Set("commit-hash", version.GetCommitHash())
+	var baseHeartbeatEvent telemetryspec.HeartbeatEventDetails
+	baseHeartbeatEvent.Info.Version = version.String()
+	baseHeartbeatEvent.Info.VersionNum = strconv.FormatUint(version.AsUInt64(), 10)
+	baseHeartbeatEvent.Info.Channel = version.Channel
+	baseHeartbeatEvent.Info.Branch = version.Branch
+	baseHeartbeatEvent.Info.CommitHash = version.GetCommitHash()
 
 	if *branchCheck {
 		fmt.Println(config.Branch)
@@ -292,6 +293,10 @@ func run() int {
 		}
 	}
 
+	if logToStdout != nil && *logToStdout {
+		cfg.LogSizeLimit = 0
+	}
+
 	err = s.Initialize(cfg, phonebookAddresses, string(genesisText))
 	if err != nil {
 		fmt.Fprintln(os.Stderr, err)
@@ -339,12 +344,11 @@ func run() int {
 			defer ticker.Stop()
 
 			sendHeartbeat := func() {
-				values := make(map[string]string)
+				values := make(map[string]float64)
 				metrics.DefaultRegistry().AddMetrics(values)
 
-				heartbeatDetails := telemetryspec.HeartbeatEventDetails{
-					Metrics: values,
-				}
+				heartbeatDetails := baseHeartbeatEvent
+				heartbeatDetails.Metrics = values
 
 				log.EventWithDetails(telemetryspec.ApplicationState, telemetryspec.HeartbeatEvent, heartbeatDetails)
 			}