Skip to content

Commit

Permalink
Merge branch 'master' into import-partkeys
Browse files Browse the repository at this point in the history
  • Loading branch information
algochoi committed Aug 31, 2023
2 parents e054b23 + b0cf5c8 commit a45062f
Show file tree
Hide file tree
Showing 243 changed files with 64,846 additions and 7,897 deletions.
3 changes: 1 addition & 2 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -457,7 +457,7 @@ commands:
shell: bash.exe
command: |
choco install -y msys2 pacman make wget --force
choco install -y golang --version=1.20.5 --force
choco install -y golang --version=$(./scripts/get_golang_version.sh) --force
choco install -y python3 --version=3.7.3 --force
export msys2='cmd //C RefreshEnv.cmd '
export msys2+='& set MSYS=winsymlinks:nativestrict '
Expand Down Expand Up @@ -602,7 +602,6 @@ commands:
export PACKAGE_NAMES=$(echo $PACKAGES | tr -d '\n')
export PARTITION_TOTAL=${CIRCLE_NODE_TOTAL}
export PARTITION_ID=${CIRCLE_NODE_INDEX}
export GOEXPERIMENT="none"
gotestsum --format standard-verbose --junitfile << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/results.xml --jsonfile << parameters.result_path >>/<< parameters.result_subdir >>/${CIRCLE_NODE_INDEX}/testresults.json -- --tags "sqlite_unlock_notify sqlite_omit_load_extension" << parameters.short_test_flag >> -race -timeout 1h -coverprofile=coverage.txt -covermode=atomic -p 1 $PACKAGE_NAMES
- store_artifacts:
path: << parameters.result_path >>
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/reviewdog.yml
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ jobs:
uses: actions/[email protected]
with:
path: cicdtmp/golangci-lint/golangci-lint-cgo
key: cicd-golangci-lint-cgo-v0.0.2
key: cicd-golangci-lint-cgo-v0.0.2-${{ env.GO_VERSION }}

- name: Build custom golangci-lint with CGO_ENABLED
if: steps.cache-golangci-lint.outputs.cache-hit != 'true'
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ crypto/libs

# doc intermediates
data/transactions/logic/*.md
!data/transactions/logic/TEAL_opcodes*.md

*.pem

Expand Down
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
FROM ubuntu:20.04 as builder

ARG GO_VERSION="1.20.5"
ARG GO_VERSION="1.20.7"

ARG CHANNEL
ARG URL
Expand Down
5 changes: 0 additions & 5 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -69,11 +69,6 @@ ifeq ($(SHORT_PART_PERIOD), 1)
export SHORT_PART_PERIOD_FLAG := -s
endif

# Disable go experiments during build as of go 1.20.5 due to
# https://github.com/golang/go/issues/60825
# Likely fix: https://go-review.googlesource.com/c/go/+/503937/6/src/runtime/race_arm64.s
export GOEXPERIMENT=none

GOTAGS := --tags "$(GOTAGSLIST)"
GOTRIMPATH := $(shell GOPATH=$(GOPATH) && go help build | grep -q .-trimpath && echo -trimpath)

Expand Down
42 changes: 26 additions & 16 deletions agreement/actions.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package agreement
import (
"context"
"fmt"
"time"

"github.com/algorand/go-algorand/logging/logspec"
"github.com/algorand/go-algorand/logging/telemetryspec"
Expand Down Expand Up @@ -209,6 +210,11 @@ type ensureAction struct {
Payload proposal
// the certificate proving commitment
Certificate Certificate

// The time that the winning proposal-vote was validated, relative to the beginning of the round
voteValidatedAt time.Duration
// The dynamic filter timeout calculated for this round, even if not enabled, for reporting to telemetry.
dynamicFilterTimeout time.Duration
}

func (a ensureAction) t() actionType {
Expand All @@ -231,29 +237,33 @@ func (a ensureAction) do(ctx context.Context, s *Service) {
logEvent.Type = logspec.RoundConcluded
s.log.with(logEvent).Infof("committed round %d with pre-validated block %v", a.Certificate.Round, a.Certificate.Proposal)
s.log.EventWithDetails(telemetryspec.Agreement, telemetryspec.BlockAcceptedEvent, telemetryspec.BlockAcceptedEventDetails{
Address: a.Certificate.Proposal.OriginalProposer.String(),
Hash: a.Certificate.Proposal.BlockDigest.String(),
Round: uint64(a.Certificate.Round),
ValidatedAt: a.Payload.validatedAt,
ReceivedAt: a.Payload.receivedAt,
PreValidated: true,
PropBufLen: uint64(len(s.demux.rawProposals)),
VoteBufLen: uint64(len(s.demux.rawVotes)),
Address: a.Certificate.Proposal.OriginalProposer.String(),
Hash: a.Certificate.Proposal.BlockDigest.String(),
Round: uint64(a.Certificate.Round),
ValidatedAt: a.Payload.validatedAt,
ReceivedAt: a.Payload.receivedAt,
VoteValidatedAt: a.voteValidatedAt,
DynamicFilterTimeout: a.dynamicFilterTimeout,
PreValidated: true,
PropBufLen: uint64(len(s.demux.rawProposals)),
VoteBufLen: uint64(len(s.demux.rawVotes)),
})
s.Ledger.EnsureValidatedBlock(a.Payload.ve, a.Certificate)
} else {
block := a.Payload.Block
logEvent.Type = logspec.RoundConcluded
s.log.with(logEvent).Infof("committed round %d with block %v", a.Certificate.Round, a.Certificate.Proposal)
s.log.EventWithDetails(telemetryspec.Agreement, telemetryspec.BlockAcceptedEvent, telemetryspec.BlockAcceptedEventDetails{
Address: a.Certificate.Proposal.OriginalProposer.String(),
Hash: a.Certificate.Proposal.BlockDigest.String(),
Round: uint64(a.Certificate.Round),
ValidatedAt: a.Payload.validatedAt,
ReceivedAt: a.Payload.receivedAt,
PreValidated: false,
PropBufLen: uint64(len(s.demux.rawProposals)),
VoteBufLen: uint64(len(s.demux.rawVotes)),
Address: a.Certificate.Proposal.OriginalProposer.String(),
Hash: a.Certificate.Proposal.BlockDigest.String(),
Round: uint64(a.Certificate.Round),
ValidatedAt: a.Payload.validatedAt,
ReceivedAt: a.Payload.receivedAt,
VoteValidatedAt: a.voteValidatedAt,
DynamicFilterTimeout: a.dynamicFilterTimeout,
PreValidated: false,
PropBufLen: uint64(len(s.demux.rawProposals)),
VoteBufLen: uint64(len(s.demux.rawVotes)),
})
s.Ledger.EnsureBlock(block, a.Certificate)
}
Expand Down
9 changes: 4 additions & 5 deletions agreement/agreementtest/simulate.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ import (
"github.com/algorand/go-algorand/crypto"
"github.com/algorand/go-algorand/data/basics"
"github.com/algorand/go-algorand/logging"
"github.com/algorand/go-algorand/protocol"
"github.com/algorand/go-algorand/util/db"
"github.com/algorand/go-algorand/util/timers"
)
Expand All @@ -52,15 +51,15 @@ func makeInstant() *instant {
return i
}

func (i *instant) Decode([]byte) (timers.Clock, error) {
func (i *instant) Decode([]byte) (timers.Clock[agreement.TimeoutType], error) {
return i, nil
}

func (i *instant) Encode() []byte {
return nil
}

func (i *instant) TimeoutAt(d time.Duration) <-chan time.Time {
func (i *instant) TimeoutAt(d time.Duration, timeoutType agreement.TimeoutType) <-chan time.Time {
ta := make(chan time.Time)
select {
case <-i.timeoutAtCalled:
Expand All @@ -69,13 +68,13 @@ func (i *instant) TimeoutAt(d time.Duration) <-chan time.Time {
return ta
}

if d == agreement.FilterTimeout(0, protocol.ConsensusCurrentVersion) && !i.HasPending("pseudonode") {
if timeoutType == agreement.TimeoutFilter && !i.HasPending("pseudonode") {
close(ta)
}
return ta
}

func (i *instant) Zero() timers.Clock {
func (i *instant) Zero() timers.Clock[agreement.TimeoutType] {
i.Z0 <- struct{}{}
// pause here until runRound is called
i.Z1 <- struct{}{}
Expand Down
83 changes: 83 additions & 0 deletions agreement/credentialArrivalHistory.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// go-algorand is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with go-algorand. If not, see <https://www.gnu.org/licenses/>.

package agreement

import (
"sort"
"time"
)

// credentialArrivalHistory maintains a circular buffer of time.Duration samples.
type credentialArrivalHistory struct {
history []time.Duration
writePtr int
full bool
}

func makeCredentialArrivalHistory(size int) credentialArrivalHistory {
if size < 0 {
panic("can't create CredentialArrivalHistory with negative size")
}
history := credentialArrivalHistory{history: make([]time.Duration, size)}
history.reset()
return history
}

// store saves a new sample into the circular buffer.
// If the buffer is full, it overwrites the oldest sample.
func (history *credentialArrivalHistory) store(sample time.Duration) {
if len(history.history) == 0 {
return
}

history.history[history.writePtr] = sample
history.writePtr++
if history.writePtr == len(history.history) {
history.full = true
history.writePtr = 0
}
}

// reset marks the history buffer as empty
func (history *credentialArrivalHistory) reset() {
history.writePtr = 0
history.full = false
}

// isFull checks if the circular buffer has been fully populated at least once.
func (history *credentialArrivalHistory) isFull() bool {
return history.full
}

// orderStatistics returns the idx'th time duration in the sorted history array.
// It assumes that history is full and the idx is within the array bounds, and
// panics if either of these assumptions doesn't hold.
func (history *credentialArrivalHistory) orderStatistics(idx int) time.Duration {
if !history.isFull() {
panic("history not full")
}
if idx < 0 || idx >= len(history.history) {
panic("index out of bounds")
}

// if history.history is long, then we could optimize this function to use
// the linear time order statistics algorithm.
sortedArrivals := make([]time.Duration, len(history.history))
copy(sortedArrivals[:], history.history[:])
sort.Slice(sortedArrivals, func(i, j int) bool { return sortedArrivals[i] < sortedArrivals[j] })
return sortedArrivals[idx]
}
129 changes: 129 additions & 0 deletions agreement/credentialArrivalHistory_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
// Copyright (C) 2019-2023 Algorand, Inc.
// This file is part of go-algorand
//
// go-algorand is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// go-algorand is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with go-algorand. If not, see <https://www.gnu.org/licenses/>.

package agreement

import (
"testing"
"time"

"github.com/algorand/go-algorand/test/partitiontest"
"github.com/stretchr/testify/require"
)

func TestCredentialHistoryStore(t *testing.T) {
partitiontest.PartitionTest(t)

size := 5
buffer := makeCredentialArrivalHistory(size)
// last store call overwrites the first one
for i := 0; i < size+1; i++ {
buffer.store(time.Duration(i))
}

require.True(t, buffer.isFull())
require.Equal(t, time.Duration(size), buffer.history[0])
for i := 1; i < size; i++ {
require.Equal(t, time.Duration(i), buffer.history[i])
}
}

func TestCredentialHistoryReset(t *testing.T) {
partitiontest.PartitionTest(t)

size := 5
buffer := makeCredentialArrivalHistory(size)
// last store call overwrites the first one
for i := 0; i < size+1; i++ {
buffer.store(time.Duration(i))
}

require.Equal(t, time.Duration(size), buffer.history[0])
for i := 1; i < size; i++ {
require.Equal(t, time.Duration(i), buffer.history[i])
}
require.True(t, buffer.isFull())
buffer.reset()
require.False(t, buffer.isFull())
buffer.store(time.Duration(100))
require.Equal(t, time.Duration(100), buffer.history[0])
}

func TestCredentialHistoryIsFull(t *testing.T) {
partitiontest.PartitionTest(t)
var buffer credentialArrivalHistory
require.False(t, buffer.isFull())

size := 5
buffer = makeCredentialArrivalHistory(size)
require.False(t, buffer.isFull())

for i := 1; i < size+10; i++ {
buffer.store(time.Duration(i))
if i < size {
require.False(t, buffer.isFull())
} else {
require.True(t, buffer.isFull())
}
}

// reset the buffer and then fill it again
buffer.reset()
require.False(t, buffer.isFull())

for i := 1; i < size+10; i++ {
buffer.store(time.Duration(i))
if i < size {
require.False(t, buffer.isFull())
} else {
require.True(t, buffer.isFull())
}
}
}

func TestCredentialHisotyZeroSize(t *testing.T) {
partitiontest.PartitionTest(t)

var buffer credentialArrivalHistory
require.False(t, buffer.isFull())

size := 0
buffer = makeCredentialArrivalHistory(size)
require.False(t, buffer.isFull())

// trying to store new samples won't panic but the history is never full
for i := 0; i < size+10; i++ {
buffer.store(time.Duration(i))
require.False(t, buffer.isFull())
}
}

func TestOrderStatistics(t *testing.T) {
partitiontest.PartitionTest(t)

size := 5
buffer := makeCredentialArrivalHistory(size)
require.False(t, buffer.isFull())

for i := 0; i < size; i++ {
buffer.store(time.Duration(size - i))
}
require.True(t, buffer.isFull())

for i := 0; i < size; i++ {
require.Equal(t, time.Duration(i+1), buffer.orderStatistics(i))
}
}
Loading

0 comments on commit a45062f

Please sign in to comment.