Merge pull request algorand#2304 from Algo-devops-service/relbeta2.7.0

go-algorand 2.7.0-beta
tsachiherman · Jun 17, 2021 · d57586a · d57586a
2 parents 34f7106 + 78a221c
commit d57586a
Show file tree

Hide file tree

Showing 236 changed files with 22,830 additions and 15,749 deletions.
diff --git a/.gitignore b/.gitignore
@@ -3,17 +3,16 @@
 *.out
 *.prof
 coverage.html
+coverage.txt
+cover.out
+cpu.out
+mem.out
+mutex.out
 
 # cadaver
 *.cdv
 *.cdv.archive
 
-# swagger
-swagger.json
-swagger.json.validated
-kmdSwaggerWrappers.go
-bundledSpecInject.go
-
 # Exclude GoLand files
 .idea/
 

diff --git a/.travis.yml b/.travis.yml
@@ -1,7 +1,6 @@
 os: linux
 dist: bionic
-go_import_path: github.com/algorand/go-algorand
-language: go
+language: minimal
 
 # Don't build tags
 if: tag IS blank
@@ -32,6 +31,11 @@ jobs:
         - travis_retry scripts/travis/build_test.sh
 
     - stage: build_pr
+      os: linux
+      name: Ubuntu AMD64 CodeGen Verification
+      script:
+        - scripts/travis/codegen_verification.sh
+    - # same stage, parallel job
       os: linux
       name: Ubuntu AMD64 Build
       script:
@@ -88,6 +92,11 @@ jobs:
          - travis_retry $mingw64 scripts/travis/build_test.sh
 
     - stage: build_release
+      os: linux
+      name: Ubuntu AMD64 CodeGen Verification
+      script:
+        - scripts/travis/codegen_verification.sh
+    - # same stage, parallel job
       os: linux
       name: Ubuntu AMD64 Build
       script:
@@ -98,11 +107,17 @@ jobs:
       script:
         - travis_retry ./scripts/travis/integration_test.sh
     - # same stage, parallel job
-      os: osx
-      osx_image: xcode11
-      name: MacOS AMD64 Build
+      name: External ARM64 Build
+      os: linux
+      env:
+        - BUILD_TYPE: "external_build"
+        - TARGET_PLATFORM: "linux-arm64"
+      addons:
+        apt:
+          packages:
+            - awscli
       script:
-        - travis_retry scripts/travis/build_test.sh
+        - travis_retry scripts/travis/external_build.sh ./scripts/travis/build_test.sh
     - # same stage, parallel job
       name: External ARM64 Integration Test
       os: linux
@@ -115,6 +130,18 @@ jobs:
             - awscli
       script:
         - scripts/travis/external_build.sh ./scripts/travis/integration_test.sh
+    - # same stage, parallel job
+      os: osx
+      osx_image: xcode11
+      name: MacOS AMD64 Build
+      script:
+        - travis_retry scripts/travis/build_test.sh
+    - # same stage, parallel job
+      os: osx
+      osx_image: xcode11
+      name: MacOS AMD64 Integration Test
+      script:
+        - travis_retry ./scripts/travis/integration_test.sh
     - # same stage, parallel job
        os: windows
        name: Windows x64 Build
@@ -148,18 +175,6 @@ jobs:
             - awscli
       script:
         - scripts/travis/external_build.sh ./scripts/travis/deploy_packages.sh
-    - # same stage, parallel job
-      name: External ARM Deploy
-      os: linux
-      env:
-        - BUILD_TYPE: "external_build"
-        - TARGET_PLATFORM: "linux-arm"
-      addons:
-        apt:
-          packages:
-            - awscli
-      script:
-        - scripts/travis/external_build.sh ./scripts/travis/deploy_packages.sh
 
 # Don't rebuild libsodium every time
 cache:
@@ -202,6 +217,9 @@ before_cache:
    esac
    docker save -o $HOME/docker_cache/images.tar $(docker images -a -q)
 
+after_success:
+  - scripts/travis/upload_coverage.sh || true
+
 addons:
   apt:
     packages:
@@ -219,3 +237,7 @@ addons:
 notifications:
   slack:
     secure: lBomhenfQvDzpUGxLadga188zavtGRr05GVWI6iI8ll/uwXewij8ginQ716Axg3pJvvRMNCuTNEmg0+fVlHf0JxTG3qWG35qOUexAHGcmimPD4ugmVkufua8Oc8Nl1LRSdWvbwIKvrAU1HDA37M5tCdEOm+Sf/OL72iVWbXhlHRvhdeiKHkbkcpI/15dOH4RpXuvPeYBrgcB2Ixs7gTSVyIAU458aT0JORu7expElQ+iBP9Kqze1V4dvjJyzHTgE01z2nyAm2Xpko447y//fYX2e20iDBE5S/GukaK/TeB3pD+FoC+Z2R7Gg1qA7a/gdGFbVgP12zWmzNaKVwY1YOvx8IIwP13XutioAE7bQbnHI1zGlQ+jxk3odWKZPhZSN/YvWD3i7g/xVksaTo++zwon2UlZ7x8LmKg1kOMeA3o2TUlzjQrwZZ4cB+4HqYHEqVdsSAadaMhKK82XE1OtGImdrIX1Y/02X2F/myJXwaE4Ap8smULHWGghCL/ALad+hdyE0K4/2SYDcoaOONgX9+VgaSEtYDO00Bt4/GMcuM+gwILsfCA9ZQuxCgIFEL1QWwDud8aGU1qF20yaimDlstlLnZXsv3Oli6r/CEEUVXltI5dkJyRd/jTlqWGjFGL8agGg34EPLP24Vn4h4QcIzoMfGWXkz2yrd6i0fvMAPXds=
+env:
+  global:
+    secure: YMLx+QaBPgg97tVn2efZJSqwPRxybX9oR6Xe9oXUD4fEiVbOIv2BAsLP+MzSJGaFC+VXKha83kyDbAZnatDXTJYsBPOUgmIZDObQnktueR1v3TCKn3aw14G+/w6S4eQL+c7uW3idPE+KP3VWixjrBpWTHEfnZdnH9+qWeIWzy8A8UEAYPY/HXyDmwt5ceCJoITtdsuiuZNO2o7NGN8q+NP5A3ii8f5UyDa9/krxuA+x1e193wSGRdGcqxexM7zIq0uMhBF+2nOk4RAsDsqU33NpQw+dB1VJMta1XF+P6A7m16UKUjllTyOfUPGD068jKobVyWzbvZ79G2FZXGZ7HGE0711JZco3YenwtxXP8oGLgaIds6McoJnn3rbxe1i+PjRX09IXNQM/dNp53uxrtyi1y8ZEjzHjbb6z3rSOZrzaBhdVxBFR9Oc1Ek7i8MS1tlRlH58+U+Z1WPTvbcIQtFA5HmmTcDdBRkZtPC8bpVLHOZTGUcple9k39VtOGuSCvAAfHKENhieCaw/bBXX4bxAHAfiI0NEmdDrULDJLApWjtRCrUIqIqkdsJmtiZhE+iX1FKR9CP5lBXCAwDlPB3XOe0bSmiiGQ1/sf/1CovqOBg/vLwCpjQE0IZPofCiDZZdZbv4n4aCOjiAqhS8MjJ/LfLyUuAtPmQG5g47FzS1K4=
+
diff --git a/Makefile b/Makefile
@@ -29,6 +29,13 @@ GOTAGSLIST          := sqlite_unlock_notify sqlite_omit_load_extension
 # e.g. make GOTAGSCUSTOM=msgtrace
 GOTAGSLIST += ${GOTAGSCUSTOM}
 
+# If available, use gotestsum instead of 'go test'.
+ifeq (, $(shell which gotestsum))
+export GOTESTCOMMAND=go test
+else
+export GOTESTCOMMAND=gotestsum --format pkgname --jsonfile testresults.json --
+endif
+
 ifeq ($(UNAME), Linux)
 EXTLDFLAGS := -static-libstdc++ -static-libgcc
 ifeq ($(ARCH), amd64)
@@ -68,7 +75,7 @@ GOLDFLAGS := $(GOLDFLAGS_BASE) \
 UNIT_TEST_SOURCES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && go list ./... | grep -v /go-algorand/test/ ))
 ALGOD_API_PACKAGES := $(sort $(shell GOPATH=$(GOPATH) && GO111MODULE=off && cd daemon/algod/api; go list ./... ))
 
-MSGP_GENERATE	:= ./protocol ./crypto ./crypto/compactcert ./data/basics ./data/transactions ./data/committee ./data/bookkeeping ./data/hashable ./auction ./agreement ./rpcs ./node ./ledger ./ledger/ledgercore ./compactcert
+MSGP_GENERATE	:= ./protocol ./crypto ./crypto/compactcert ./data/basics ./data/transactions ./data/committee ./data/bookkeeping ./data/hashable ./agreement ./rpcs ./node ./ledger ./ledger/ledgercore ./compactcert
 
 default: build
 
@@ -81,9 +88,6 @@ fmt:
 fix: build
 	$(GOPATH1)/bin/algofix */
 
-fixcheck: build
-	$(GOPATH1)/bin/algofix -error */
-
 lint: deps
 	$(GOPATH1)/bin/golint ./...
 
@@ -148,7 +152,7 @@ $(ALGOD_API_SWAGGER_SPEC): $(ALGOD_API_FILES) crypto/libs/$(OS_TYPE)/$(ARCH)/lib
 		PATH=$(GOPATH1)/bin:$$PATH \
 		go generate ./...
 
-$(ALGOD_API_SWAGGER_INJECT): $(ALGOD_API_SWAGGER_SPEC) $(ALGOD_API_SWAGGER_SPEC).validated
+$(ALGOD_API_SWAGGER_INJECT): deps $(ALGOD_API_SWAGGER_SPEC) $(ALGOD_API_SWAGGER_SPEC).validated
 	./daemon/algod/api/server/lib/bundle_swagger_json.sh
 
 # Regenerate kmd swagger spec files
@@ -175,35 +179,45 @@ $(KMD_API_SWAGGER_SPEC): $(KMD_API_FILES) crypto/libs/$(OS_TYPE)/$(ARCH)/lib/lib
 		touch $@; \
 	fi
 
-$(KMD_API_SWAGGER_INJECT): $(KMD_API_SWAGGER_SPEC) $(KMD_API_SWAGGER_SPEC).validated
+$(KMD_API_SWAGGER_INJECT): deps $(KMD_API_SWAGGER_SPEC) $(KMD_API_SWAGGER_SPEC).validated
 	./daemon/kmd/lib/kmdapi/bundle_swagger_json.sh
 
+# generated files we should make sure we clean
+GENERATED_FILES := \
+	$(ALGOD_API_SWAGGER_INJECT) \
+	$(KMD_API_SWAGGER_INJECT) \
+	$(ALGOD_API_SWAGGER_SPEC) $(ALGOD_API_SWAGGER_SPEC).validated \
+	$(KMD_API_SWAGGER_SPEC) $(KMD_API_SWAGGER_SPEC).validated
+
+rebuild_swagger: deps
+	rm -f $(GENERATED_FILES)
+	# we need to invoke the make here since we want to ensure that the deletion and re-creating are sequential
+	make $(KMD_API_SWAGGER_INJECT) $(ALGOD_API_SWAGGER_INJECT)
+
 # develop
 
-build: buildsrc gen
+build: buildsrc
 
 # We're making an empty file in the go-cache dir to
 # get around a bug in go build where it will fail
 # to cache binaries from time to time on empty NFS
 # dirs
-buildsrc: check-go-version crypto/libs/$(OS_TYPE)/$(ARCH)/lib/libsodium.a node_exporter NONGO_BIN deps $(ALGOD_API_SWAGGER_INJECT) $(KMD_API_SWAGGER_INJECT)
+buildsrc: check-go-version crypto/libs/$(OS_TYPE)/$(ARCH)/lib/libsodium.a node_exporter NONGO_BIN
 	mkdir -p tmp/go-cache && \
 	touch tmp/go-cache/file.txt && \
 	GOCACHE=$(SRCPATH)/tmp/go-cache go install $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./...
 
 check-go-version:
 	./scripts/check_golang_version.sh build
 
-SOURCES_RACE := github.com/algorand/go-algorand/cmd/kmd
-
 ## Build binaries with the race detector enabled in them.
 ## This allows us to run e2e tests with race detection.
 ## We overwrite bin-race/kmd with a non -race version due to
 ## the incredible performance impact of -race on Scrypt.
 build-race: build
 	@mkdir -p $(GOPATH1)/bin-race
 	GOBIN=$(GOPATH1)/bin-race go install $(GOTRIMPATH) $(GOTAGS) -race -ldflags="$(GOLDFLAGS)" ./...
-	GOBIN=$(GOPATH1)/bin-race go install $(GOTRIMPATH) $(GOTAGS) -ldflags="$(GOLDFLAGS)" $(SOURCES_RACE)
+	cp $(GOPATH1)/bin/kmd $(GOPATH1)/bin-race
 
 NONGO_BIN_FILES=$(GOPATH1)/bin/find-nodes.sh $(GOPATH1)/bin/update.sh $(GOPATH1)/bin/COPYING $(GOPATH1)/bin/ddconfig.sh
 
@@ -221,45 +235,33 @@ $(GOPATH1)/bin/%:
 	cp -f $< $@
 
 test: build
-	go test $(GOTAGS) -race $(UNIT_TEST_SOURCES) -timeout 3600s | logfilter
+	$(GOTESTCOMMAND) $(GOTAGS) -race $(UNIT_TEST_SOURCES) -timeout 1h -coverprofile=coverage.txt -covermode=atomic
 
 fulltest: build-race
-	for PACKAGE_DIRECTORY in $(UNIT_TEST_SOURCES) ; do \
-		go test $(GOTAGS) -timeout 2500s -race $$PACKAGE_DIRECTORY | logfilter; \
-	done
-
-shorttest: build-race $(addprefix short_test_target_, $(UNIT_TEST_SOURCES))
+	$(GOTESTCOMMAND) $(GOTAGS) -race $(UNIT_TEST_SOURCES) -timeout 1h -coverprofile=coverage.txt -covermode=atomic
 
-$(addprefix short_test_target_, $(UNIT_TEST_SOURCES)): build
-	@go test $(GOTAGS) -short -timeout 2500s -race $(subst short_test_target_,,$@) | logfilter
+shorttest: build-race
+	$(GOTESTCOMMAND) $(GOTAGS) -short -race $(UNIT_TEST_SOURCES) -timeout 1h -coverprofile=coverage.txt -covermode=atomic
 
 integration: build-race
 	./test/scripts/run_integration_tests.sh
 
 testall: fulltest integration
 
-# generated files we should make sure we clean
-GENERATED_FILES := daemon/algod/api/bundledSpecInject.go \
-	daemon/algod/api/lib/bundledSpecInject.go \
-	daemon/kmd/lib/kmdapi/bundledSpecInject.go \
-	$(ALGOD_API_SWAGGER_SPEC) $(ALGOD_API_SWAGGER_SPEC).validated \
-	$(KMD_API_SWAGGER_SPEC) $(KMD_API_SWAGGER_SPEC).validated
-
 clean:
 	go clean -i ./...
 	rm -f $(GOPATH1)/bin/node_exporter
-	rm -f $(GENERATED_FILES)
 	cd crypto/libsodium-fork && \
 		test ! -e Makefile || make clean
 	rm -rf crypto/lib
 	rm -rf crypto/libs
 	rm -rf crypto/copies
+	rm -rf ./gen/devnet ./gen/mainnetnet ./gen/testnet
 
 # clean without crypto
 cleango:
 	go clean -i ./...
 	rm -f $(GOPATH1)/bin/node_exporter
-	rm -f $(GENERATED_FILES)
 
 # assign the phony target node_exporter the dependency of the actual executable.
 node_exporter: $(GOPATH1)/bin/node_exporter
@@ -312,7 +314,7 @@ dump: $(addprefix gen/,$(addsuffix /genesis.dump, $(NETWORKS)))
 install: build
 	scripts/dev_install.sh -p $(GOPATH1)/bin
 
-.PHONY: default fmt vet lint check_shell sanity cover prof deps build test fulltest shorttest clean cleango deploy node_exporter install %gen gen NONGO_BIN check-go-version
+.PHONY: default fmt vet lint check_shell sanity cover prof deps build test fulltest shorttest clean cleango deploy node_exporter install %gen gen NONGO_BIN check-go-version rebuild_swagger
 
 ###### TARGETS FOR CICD PROCESS ######
 include ./scripts/release/mule/Makefile.mule

diff --git a/README.md b/README.md
@@ -36,6 +36,7 @@ Initial environment setup:
 git clone https://github.com/algorand/go-algorand
 cd go-algorand
 ./scripts/configure_dev.sh
+./scripts/buildtools/install_buildtools.sh
 ```
 
 At this point you are ready to build go-algorand. We use `make` and have a
@@ -164,8 +165,6 @@ The following packages allow developers to interface with the Algorand system:
   - `libgoal` exports a Go interface useful for developers of Algorand clients.
   - `debug` holds secondary commands which assist developers during debugging.
 
-The `auction` package implements the Algorand auctions.
-
 The following packages contain tools to help Algorand developers deploy networks
 of their own:
 

diff --git a/agreement/player.go b/agreement/player.go
@@ -554,22 +554,35 @@ func (p *player) handleMessageEvent(r routerHandle, e messageEvent) (actions []a
 			return append(actions, ignoreAction(e, ef.(payloadProcessedEvent).Err))
 		case payloadPipelined:
 			ep := ef.(payloadProcessedEvent)
+
+			up := e.Input.UnauthenticatedProposal
+			uv := ef.(payloadProcessedEvent).Vote.u()
+
+			// relay proposal if it has been pipelined
+			ra := relayAction(e, protocol.ProposalPayloadTag, compoundMessage{Proposal: up, Vote: uv})
+
 			if ep.Round == p.Round {
-				return append(actions, verifyPayloadAction(e, ep.Round, ep.Period, ep.Pinned))
+				vpa := verifyPayloadAction(e, ep.Round, ep.Period, ep.Pinned)
+				return append(actions, vpa, ra)
 			}
-		}
 
-		var uv unauthenticatedVote
-		switch ef.t() {
-		case payloadPipelined, payloadAccepted:
-			uv = ef.(payloadProcessedEvent).Vote.u()
-		case proposalCommittable:
-			uv = ef.(committableEvent).Vote.u()
+			actions = append(actions, ra)
 		}
-		up := e.Input.UnauthenticatedProposal
 
-		a := relayAction(e, protocol.ProposalPayloadTag, compoundMessage{Proposal: up, Vote: uv})
-		actions = append(actions, a)
+		// relay as the proposer
+		if e.Input.MessageHandle == nil {
+			var uv unauthenticatedVote
+			switch ef.t() {
+			case payloadPipelined, payloadAccepted:
+				uv = ef.(payloadProcessedEvent).Vote.u()
+			case proposalCommittable:
+				uv = ef.(committableEvent).Vote.u()
+			}
+			up := e.Input.UnauthenticatedProposal
+
+			a := relayAction(e, protocol.ProposalPayloadTag, compoundMessage{Proposal: up, Vote: uv})
+			actions = append(actions, a)
+		}
 
 		// If the payload is valid, check it against any received cert threshold.
 		// Of course, this should only trigger for payloadVerified case.