Merge master into feature/1000-assets (#2224)

Merge master into feature/1000-assets

.gitignore

@@ -8,12 +8,6 @@ coverage.html
 *.cdv
 *.cdv.archive
 
-# swagger
-swagger.json
-swagger.json.validated
-kmdSwaggerWrappers.go
-bundledSpecInject.go
-
 # Exclude GoLand files
 .idea/
 

.travis.yml

@@ -32,6 +32,11 @@ jobs:
         - travis_retry scripts/travis/build_test.sh
 
     - stage: build_pr
+      os: linux
+      name: Ubuntu AMD64 CodeGen Verification
+      script:
+        - scripts/travis/codegen_verification.sh
+    - # same stage, parallel job
       os: linux
       name: Ubuntu AMD64 Build
       script:

Makefile

@@ -81,9 +81,6 @@ fmt:
 fix: build
 	$(GOPATH1)/bin/algofix */
 
-fixcheck: build
-	$(GOPATH1)/bin/algofix -error */
-
 lint: deps
 	$(GOPATH1)/bin/golint ./...
 
@@ -148,7 +145,7 @@ $(ALGOD_API_SWAGGER_SPEC): $(ALGOD_API_FILES) crypto/libs/$(OS_TYPE)/$(ARCH)/lib
 		PATH=$(GOPATH1)/bin:$$PATH \
 		go generate ./...
 
-$(ALGOD_API_SWAGGER_INJECT): $(ALGOD_API_SWAGGER_SPEC) $(ALGOD_API_SWAGGER_SPEC).validated
+$(ALGOD_API_SWAGGER_INJECT): deps $(ALGOD_API_SWAGGER_SPEC) $(ALGOD_API_SWAGGER_SPEC).validated
 	./daemon/algod/api/server/lib/bundle_swagger_json.sh
 
 # Regenerate kmd swagger spec files
@@ -175,35 +172,45 @@ $(KMD_API_SWAGGER_SPEC): $(KMD_API_FILES) crypto/libs/$(OS_TYPE)/$(ARCH)/lib/lib
 		touch $@; \
 	fi
 
-$(KMD_API_SWAGGER_INJECT): $(KMD_API_SWAGGER_SPEC) $(KMD_API_SWAGGER_SPEC).validated
+$(KMD_API_SWAGGER_INJECT): deps $(KMD_API_SWAGGER_SPEC) $(KMD_API_SWAGGER_SPEC).validated
 	./daemon/kmd/lib/kmdapi/bundle_swagger_json.sh
 
+# generated files we should make sure we clean
+GENERATED_FILES := \
+	$(ALGOD_API_SWAGGER_INJECT) \
+	$(KMD_API_SWAGGER_INJECT) \
+	$(ALGOD_API_SWAGGER_SPEC) $(ALGOD_API_SWAGGER_SPEC).validated \
+	$(KMD_API_SWAGGER_SPEC) $(KMD_API_SWAGGER_SPEC).validated
+
+rebuild_swagger: deps
+	rm -f $(GENERATED_FILES)
+	# we need to invoke the make here since we want to ensure that the deletion and re-creating are sequential
+	make $(KMD_API_SWAGGER_INJECT) $(ALGOD_API_SWAGGER_INJECT)
+
 # develop
 
-build: buildsrc gen
+build: buildsrc
 
 # We're making an empty file in the go-cache dir to
 # get around a bug in go build where it will fail
 # to cache binaries from time to time on empty NFS
 # dirs
-buildsrc: check-go-version crypto/libs/$(OS_TYPE)/$(ARCH)/lib/libsodium.a node_exporter NONGO_BIN deps $(ALGOD_API_SWAGGER_INJECT) $(KMD_API_SWAGGER_INJECT)
+buildsrc: check-go-version crypto/libs/$(OS_TYPE)/$(ARCH)/lib/libsodium.a node_exporter NONGO_BIN
 	mkdir -p tmp/go-cache && \
 	touch tmp/go-cache/file.txt && \
 	GOCACHE=$(SRCPATH)/tmp/go-cache go install $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./...
 
 check-go-version:
 	./scripts/check_golang_version.sh build
 
-SOURCES_RACE := github.com/algorand/go-algorand/cmd/kmd
-
 ## Build binaries with the race detector enabled in them.
 ## This allows us to run e2e tests with race detection.
 ## We overwrite bin-race/kmd with a non -race version due to
 ## the incredible performance impact of -race on Scrypt.
 build-race: build
 	@mkdir -p $(GOPATH1)/bin-race
 	GOBIN=$(GOPATH1)/bin-race go install $(GOTRIMPATH) $(GOTAGS) -race -ldflags="$(GOLDFLAGS)" ./...
-	GOBIN=$(GOPATH1)/bin-race go install $(GOTRIMPATH) $(GOTAGS) -ldflags="$(GOLDFLAGS)" $(SOURCES_RACE)
+	cp $(GOPATH1)/bin/kmd $(GOPATH1)/bin-race
 
 NONGO_BIN_FILES=$(GOPATH1)/bin/find-nodes.sh $(GOPATH1)/bin/update.sh $(GOPATH1)/bin/COPYING $(GOPATH1)/bin/ddconfig.sh
 
@@ -238,28 +245,20 @@ integration: build-race
 
 testall: fulltest integration
 
-# generated files we should make sure we clean
-GENERATED_FILES := daemon/algod/api/bundledSpecInject.go \
-	daemon/algod/api/lib/bundledSpecInject.go \
-	daemon/kmd/lib/kmdapi/bundledSpecInject.go \
-	$(ALGOD_API_SWAGGER_SPEC) $(ALGOD_API_SWAGGER_SPEC).validated \
-	$(KMD_API_SWAGGER_SPEC) $(KMD_API_SWAGGER_SPEC).validated
-
 clean:
 	go clean -i ./...
 	rm -f $(GOPATH1)/bin/node_exporter
-	rm -f $(GENERATED_FILES)
 	cd crypto/libsodium-fork && \
 		test ! -e Makefile || make clean
 	rm -rf crypto/lib
 	rm -rf crypto/libs
 	rm -rf crypto/copies
+	rm -rf ./gen/devnet ./gen/mainnetnet ./gen/testnet
 
 # clean without crypto
 cleango:
 	go clean -i ./...
 	rm -f $(GOPATH1)/bin/node_exporter
-	rm -f $(GENERATED_FILES)
 
 # assign the phony target node_exporter the dependency of the actual executable.
 node_exporter: $(GOPATH1)/bin/node_exporter
@@ -312,7 +311,7 @@ dump: $(addprefix gen/,$(addsuffix /genesis.dump, $(NETWORKS)))
 install: build
 	scripts/dev_install.sh -p $(GOPATH1)/bin
 
-.PHONY: default fmt vet lint check_shell sanity cover prof deps build test fulltest shorttest clean cleango deploy node_exporter install %gen gen NONGO_BIN check-go-version
+.PHONY: default fmt vet lint check_shell sanity cover prof deps build test fulltest shorttest clean cleango deploy node_exporter install %gen gen NONGO_BIN check-go-version rebuild_swagger
 
 ###### TARGETS FOR CICD PROCESS ######
 include ./scripts/release/mule/Makefile.mule

README.md

@@ -22,8 +22,10 @@ the [official Go documentation website](https://golang.org/doc/).
 ### Linux / OSX ###
 
 We currently strive to support Debian based distributions with Ubuntu 18.04
-being our official release target. Our core engineering team uses Linux and OSX,
-so both environments are well supported for development.
+being our official release target.
+Building on Arch Linux works as well.
+Our core engineering team uses Linux and OSX, so both environments are well
+supported for development.
 
 OSX only: [Homebrew (brew)](https://brew.sh) must be installed before
 continuing. [Here](https://docs.brew.sh/Installation) are the installation
@@ -34,6 +36,7 @@ Initial environment setup:
 git clone https://github.com/algorand/go-algorand
 cd go-algorand
 ./scripts/configure_dev.sh
+./scripts/buildtools/install_buildtools.sh
 ```
 
 At this point you are ready to build go-algorand. We use `make` and have a

agreement/abstractions.go

@@ -225,13 +225,10 @@ type LedgerWriter interface {
 
 // A KeyManager stores and deletes participation keys.
 type KeyManager interface {
-	// Keys returns an immutable array of participation intervals to
-	// participating accounts.
-	Keys() []account.Participation
-
-	// HasLiveKeys returns true if we have any Participation
-	// keys valid for the specified round range (inclusive)
-	HasLiveKeys(from, to basics.Round) bool
+	// VotingKeys returns an immutable array of voting keys that are
+	// valid for the provided votingRound, and were available at
+	// keysRound.
+	VotingKeys(votingRound, keysRound basics.Round) []account.Participation
 }
 
 // MessageHandle is an ID referring to a specific message.

agreement/agreementtest/keyManager.go

@@ -24,24 +24,15 @@ import (
 // SimpleKeyManager provides a simple implementation of a KeyManager.
 type SimpleKeyManager []account.Participation
 
-// Keys implements KeyManager.Keys.
-func (m SimpleKeyManager) Keys() []account.Participation {
+// VotingKeys implements KeyManager.VotingKeys.
+func (m SimpleKeyManager) VotingKeys(votingRound, _ basics.Round) []account.Participation {
 	var km []account.Participation
 	for _, acc := range m {
-		km = append(km, acc)
-	}
-	return km
-}
-
-// HasLiveKeys returns true if we have any Participation
-// keys valid for the specified round range (inclusive)
-func (m SimpleKeyManager) HasLiveKeys(from, to basics.Round) bool {
-	for _, acc := range m {
-		if acc.OverlapsInterval(from, to) {
-			return true
+		if acc.OverlapsInterval(votingRound, votingRound) {
+			km = append(km, acc)
 		}
 	}
-	return false
+	return km
 }
 
 // DeleteOldKeys implements KeyManager.DeleteOldKeys.

agreement/cryptoVerifier_test.go

@@ -317,7 +317,8 @@ func BenchmarkCryptoVerifierProposalVertification(b *testing.B) {
 	}
 
 	Period := period(0)
-	participation := pn.getParticipations("BenchmarkCryptoVerifierProposalVertification", ledger.NextRound())
+	pn.loadRoundParticipationKeys(ledger.NextRound())
+	participation := pn.participationKeys
 
 	proposals, _ := pn.makeProposals(ledger.NextRound(), Period, participation)
 

agreement/fuzzer/fuzzer_test.go

@@ -84,7 +84,7 @@ func MakeFuzzer(config FuzzerConfig) *Fuzzer {
 		crashAccessors:   make([]db.Accessor, config.NodesCount),
 		accounts:         make([]account.Participation, config.NodesCount),
 		balances:         make(map[basics.Address]basics.AccountData),
-		accountAccessors: make([]db.Accessor, config.NodesCount*2),
+		accountAccessors: make([]db.Accessor, config.NodesCount),
 		ledgers:          make([]*testLedger, config.NodesCount),
 		agreementParams:  make([]agreement.Parameters, config.NodesCount),
 		tickGranularity:  time.Millisecond * 300,
@@ -196,7 +196,7 @@ func (n *Fuzzer) initAccountsAndBalances(rootSeed []byte, onlineNodes []bool) er
 		if err != nil {
 			return err
 		}
-		n.accountAccessors[i*2+0] = rootAccess
+		n.accountAccessors[i] = rootAccess
 
 		seed = sha256.Sum256(seed[:])
 		root, err := account.ImportRoot(rootAccess, seed)

agreement/fuzzer/keyManager_test.go

@@ -23,15 +23,12 @@ import (
 
 type simpleKeyManager []account.Participation
 
-func (m simpleKeyManager) Keys() []account.Participation {
-	return m
-}
-
-func (m simpleKeyManager) HasLiveKeys(from, to basics.Round) bool {
+func (m simpleKeyManager) VotingKeys(votingRound, _ basics.Round) []account.Participation {
+	var km []account.Participation
 	for _, acc := range m {
-		if acc.OverlapsInterval(from, to) {
-			return true
+		if acc.OverlapsInterval(votingRound, votingRound) {
+			km = append(km, acc)
 		}
 	}
-	return false
+	return km
 }

agreement/player.go

@@ -554,22 +554,35 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a
 			return append(actions, ignoreAction(e, ef.(payloadProcessedEvent).Err))
 		case payloadPipelined:
 			ep := ef.(payloadProcessedEvent)
+
+			up := e.Input.UnauthenticatedProposal
+			uv := ef.(payloadProcessedEvent).Vote.u()
+
+			// relay proposal if it has been pipelined
+			ra := relayAction(e, protocol.ProposalPayloadTag, compoundMessage{Proposal: up, Vote: uv})
+
 			if ep.Round == p.Round {
-				return append(actions, verifyPayloadAction(e, ep.Round, ep.Period, ep.Pinned))
+				vpa := verifyPayloadAction(e, ep.Round, ep.Period, ep.Pinned)
+				return append(actions, vpa, ra)
 			}
-		}
 
-		var uv unauthenticatedVote
-		switch ef.t() {
-		case payloadPipelined, payloadAccepted:
-			uv = ef.(payloadProcessedEvent).Vote.u()
-		case proposalCommittable:
-			uv = ef.(committableEvent).Vote.u()
+			actions = append(actions, ra)
 		}
-		up := e.Input.UnauthenticatedProposal
 
-		a := relayAction(e, protocol.ProposalPayloadTag, compoundMessage{Proposal: up, Vote: uv})
-		actions = append(actions, a)
+		// relay as the proposer
+		if e.Input.MessageHandle == nil {
+			var uv unauthenticatedVote
+			switch ef.t() {
+			case payloadPipelined, payloadAccepted:
+				uv = ef.(payloadProcessedEvent).Vote.u()
+			case proposalCommittable:
+				uv = ef.(committableEvent).Vote.u()
+			}
+			up := e.Input.UnauthenticatedProposal
+
+			a := relayAction(e, protocol.ProposalPayloadTag, compoundMessage{Proposal: up, Vote: uv})
+			actions = append(actions, a)
+		}
 
 		// If the payload is valid, check it against any received cert threshold.
 		// Of course, this should only trigger for payloadVerified case.