Merge remote-tracking branch 'origin/add-centos8-rpm' into add-centos…

…8-rpm

.gitignore

@@ -3,6 +3,11 @@
 *.out
 *.prof
 coverage.html
+coverage.txt
+cover.out
+cpu.out
+mem.out
+mutex.out
 
 # cadaver
 *.cdv

.travis.yml

@@ -1,7 +1,6 @@
 os: linux
 dist: bionic
-go_import_path: github.com/algorand/go-algorand
-language: go
+language: minimal
 
 # Don't build tags
 if: tag IS blank
@@ -93,6 +92,11 @@ jobs:
          - travis_retry $mingw64 scripts/travis/build_test.sh
 
     - stage: build_release
+      os: linux
+      name: Ubuntu AMD64 CodeGen Verification
+      script:
+        - scripts/travis/codegen_verification.sh
+    - # same stage, parallel job
       os: linux
       name: Ubuntu AMD64 Build
       script:
@@ -103,11 +107,17 @@ jobs:
       script:
         - travis_retry ./scripts/travis/integration_test.sh
     - # same stage, parallel job
-      os: osx
-      osx_image: xcode11
-      name: MacOS AMD64 Build
+      name: External ARM64 Build
+      os: linux
+      env:
+        - BUILD_TYPE: "external_build"
+        - TARGET_PLATFORM: "linux-arm64"
+      addons:
+        apt:
+          packages:
+            - awscli
       script:
-        - travis_retry scripts/travis/build_test.sh
+        - travis_retry scripts/travis/external_build.sh ./scripts/travis/build_test.sh
     - # same stage, parallel job
       name: External ARM64 Integration Test
       os: linux
@@ -120,6 +130,18 @@ jobs:
             - awscli
       script:
         - scripts/travis/external_build.sh ./scripts/travis/integration_test.sh
+    - # same stage, parallel job
+      os: osx
+      osx_image: xcode11
+      name: MacOS AMD64 Build
+      script:
+        - travis_retry scripts/travis/build_test.sh
+    - # same stage, parallel job
+      os: osx
+      osx_image: xcode11
+      name: MacOS AMD64 Integration Test
+      script:
+        - travis_retry ./scripts/travis/integration_test.sh
     - # same stage, parallel job
        os: windows
        name: Windows x64 Build
@@ -153,18 +175,6 @@ jobs:
             - awscli
       script:
         - scripts/travis/external_build.sh ./scripts/travis/deploy_packages.sh
-    - # same stage, parallel job
-      name: External ARM Deploy
-      os: linux
-      env:
-        - BUILD_TYPE: "external_build"
-        - TARGET_PLATFORM: "linux-arm"
-      addons:
-        apt:
-          packages:
-            - awscli
-      script:
-        - scripts/travis/external_build.sh ./scripts/travis/deploy_packages.sh
 
 # Don't rebuild libsodium every time
 cache:
@@ -207,6 +217,9 @@ before_cache:
    esac
    docker save -o $HOME/docker_cache/images.tar $(docker images -a -q)
 
+after_success:
+  - scripts/travis/upload_coverage.sh || true
+
 addons:
   apt:
     packages:
@@ -224,3 +237,7 @@ addons:
 notifications:
   slack:
     secure: lBomhenfQvDzpUGxLadga188zavtGRr05GVWI6iI8ll/uwXewij8ginQ716Axg3pJvvRMNCuTNEmg0+fVlHf0JxTG3qWG35qOUexAHGcmimPD4ugmVkufua8Oc8Nl1LRSdWvbwIKvrAU1HDA37M5tCdEOm+Sf/OL72iVWbXhlHRvhdeiKHkbkcpI/15dOH4RpXuvPeYBrgcB2Ixs7gTSVyIAU458aT0JORu7expElQ+iBP9Kqze1V4dvjJyzHTgE01z2nyAm2Xpko447y//fYX2e20iDBE5S/GukaK/TeB3pD+FoC+Z2R7Gg1qA7a/gdGFbVgP12zWmzNaKVwY1YOvx8IIwP13XutioAE7bQbnHI1zGlQ+jxk3odWKZPhZSN/YvWD3i7g/xVksaTo++zwon2UlZ7x8LmKg1kOMeA3o2TUlzjQrwZZ4cB+4HqYHEqVdsSAadaMhKK82XE1OtGImdrIX1Y/02X2F/myJXwaE4Ap8smULHWGghCL/ALad+hdyE0K4/2SYDcoaOONgX9+VgaSEtYDO00Bt4/GMcuM+gwILsfCA9ZQuxCgIFEL1QWwDud8aGU1qF20yaimDlstlLnZXsv3Oli6r/CEEUVXltI5dkJyRd/jTlqWGjFGL8agGg34EPLP24Vn4h4QcIzoMfGWXkz2yrd6i0fvMAPXds=
+env:
+  global:
+    secure: YMLx+QaBPgg97tVn2efZJSqwPRxybX9oR6Xe9oXUD4fEiVbOIv2BAsLP+MzSJGaFC+VXKha83kyDbAZnatDXTJYsBPOUgmIZDObQnktueR1v3TCKn3aw14G+/w6S4eQL+c7uW3idPE+KP3VWixjrBpWTHEfnZdnH9+qWeIWzy8A8UEAYPY/HXyDmwt5ceCJoITtdsuiuZNO2o7NGN8q+NP5A3ii8f5UyDa9/krxuA+x1e193wSGRdGcqxexM7zIq0uMhBF+2nOk4RAsDsqU33NpQw+dB1VJMta1XF+P6A7m16UKUjllTyOfUPGD068jKobVyWzbvZ79G2FZXGZ7HGE0711JZco3YenwtxXP8oGLgaIds6McoJnn3rbxe1i+PjRX09IXNQM/dNp53uxrtyi1y8ZEjzHjbb6z3rSOZrzaBhdVxBFR9Oc1Ek7i8MS1tlRlH58+U+Z1WPTvbcIQtFA5HmmTcDdBRkZtPC8bpVLHOZTGUcple9k39VtOGuSCvAAfHKENhieCaw/bBXX4bxAHAfiI0NEmdDrULDJLApWjtRCrUIqIqkdsJmtiZhE+iX1FKR9CP5lBXCAwDlPB3XOe0bSmiiGQ1/sf/1CovqOBg/vLwCpjQE0IZPofCiDZZdZbv4n4aCOjiAqhS8MjJ/LfLyUuAtPmQG5g47FzS1K4=
+

Makefile

@@ -29,6 +29,13 @@ GOTAGSLIST          := sqlite_unlock_notify sqlite_omit_load_extension
 # e.g. make GOTAGSCUSTOM=msgtrace
 GOTAGSLIST += ${GOTAGSCUSTOM}
 
+# If available, use gotestsum instead of 'go test'.
+ifeq (, $(shell which gotestsum))
+export GOTESTCOMMAND=go test
+else
+export GOTESTCOMMAND=gotestsum --format pkgname --jsonfile testresults.json --
+endif
+
 ifeq ($(UNAME), Linux)
 EXTLDFLAGS := -static-libstdc++ -static-libgcc
 ifeq ($(ARCH), amd64)
@@ -228,17 +235,13 @@ $(GOPATH1)/bin/%:
 	cp -f $< $@
 
 test: build
-	go test $(GOTAGS) -race $(UNIT_TEST_SOURCES) -timeout 3600s | logfilter
+	$(GOTESTCOMMAND) $(GOTAGS) -race $(UNIT_TEST_SOURCES) -timeout 1h -coverprofile=coverage.txt -covermode=atomic
 
 fulltest: build-race
-	for PACKAGE_DIRECTORY in $(UNIT_TEST_SOURCES) ; do \
-		go test $(GOTAGS) -timeout 2500s -race $$PACKAGE_DIRECTORY | logfilter; \
-	done
-
-shorttest: build-race $(addprefix short_test_target_, $(UNIT_TEST_SOURCES))
+	$(GOTESTCOMMAND) $(GOTAGS) -race $(UNIT_TEST_SOURCES) -timeout 1h -coverprofile=coverage.txt -covermode=atomic
 
-$(addprefix short_test_target_, $(UNIT_TEST_SOURCES)): build
-	@go test $(GOTAGS) -short -timeout 2500s -race $(subst short_test_target_,,$@) | logfilter
+shorttest: build-race
+	$(GOTESTCOMMAND) $(GOTAGS) -short -race $(UNIT_TEST_SOURCES) -timeout 1h -coverprofile=coverage.txt -covermode=atomic
 
 integration: build-race
 	./test/scripts/run_integration_tests.sh

catchup/service.go

@@ -481,6 +481,9 @@ func (s *Service) periodicSync() {
 	defer close(s.done)
 	// if the catchup is disabled in the config file, just skip it.
 	if s.parallelBlocks != 0 && !s.cfg.DisableNetworking {
+		// The following request might be redundent, but it ensures we wait long enough for the DNS records to be loaded,
+		// which are required for the sync operation.
+		s.net.RequestConnectOutgoing(false, s.ctx.Done())
 		s.sync()
 	}
 	stuckInARow := 0

catchup/service_test.go

@@ -421,98 +421,96 @@ func TestServiceFetchBlocksMalformed(t *testing.T) {
 	//require.True(t, s.fetcherFactory.(*MockedFetcherFactory).fetcher.client.closed)
 }
 
-func TestOnSwitchToUnSupportedProtocol(t *testing.T) {
-	t.Skip("This test is flacky and need to be fixed.")
-
-	// Test the interruption in the initial loop
-	// This cannot happen in practice, but is used to test the code.
-	{
-		lastRoundRemote := 5
-		lastRoundLocal := 0
-		roundWithSwitchOn := 0
-		local, remote := helperTestOnSwitchToUnSupportedProtocol(t, lastRoundRemote, lastRoundLocal, roundWithSwitchOn, 0)
-
-		// Last supported round is 0, but is guaranteed
-		// to stop after 2 rounds.
-
-		// SeedLookback is 2, which allows two parallel fetches.
-		// i.e. rounds 1 and 2 may be simultaneously fetched.
-		require.Less(t, int(local.LastRound()), 3)
-		require.Equal(t, lastRoundRemote, int(remote.LastRound()))
-		remote.Ledger.Close()
-	}
-
-	// Test the interruption in "the rest" loop
-	{
-		lastRoundRemote := 10
-		lastRoundLocal := 7
-		roundWithSwitchOn := 5
-		local, remote := helperTestOnSwitchToUnSupportedProtocol(t, lastRoundRemote, lastRoundLocal, roundWithSwitchOn, 0)
-		for r := 1; r <= lastRoundLocal; r++ {
-			blk, err := local.Block(basics.Round(r))
-			require.NoError(t, err)
-			require.Equal(t, r, int(blk.Round()))
-		}
-		require.Equal(t, lastRoundLocal, int(local.LastRound()))
-		require.Equal(t, lastRoundRemote, int(remote.LastRound()))
-		remote.Ledger.Close()
-	}
-
-	// Test the interruption with short notice (less than
-	// SeedLookback or the number of parallel fetches which in the
-	// test is the same: 2)
-
-	// This can not happen in practice, because there will be
-	// enough rounds for the protocol upgrade notice.
-	{
-		lastRoundRemote := 14
-		lastRoundLocal := 7
-		roundWithSwitchOn := 7
-		local, remote := helperTestOnSwitchToUnSupportedProtocol(t, lastRoundRemote, lastRoundLocal, roundWithSwitchOn, 0)
-		for r := 1; r <= lastRoundLocal; r = r + 1 {
-			blk, err := local.Block(basics.Round(r))
-			require.NoError(t, err)
-			require.Equal(t, r, int(blk.Round()))
-		}
-		// Since round with switch on (7) can be fetched
-		// Simultaneously with round 8, round 8 might also be
-		// fetched.
-		require.Less(t, int(local.LastRound()), lastRoundLocal+2)
-		require.Equal(t, lastRoundRemote, int(remote.LastRound()))
-		remote.Ledger.Close()
-	}
-
-	// Test the interruption with short notice (less than
-	// SeedLookback or the number of parallel fetches which in the
-	// test is the same: 2)
-
-	// This case is a variation of the previous case. This may
-	// happen when the catchup service restart at the round when
-	// an upgrade happens.
-	{
-		lastRoundRemote := 14
-		lastRoundLocal := 7
-		roundWithSwitchOn := 7
-		roundsAlreadyInLocal := 8 // round 0 -> 7
-
-		local, remote := helperTestOnSwitchToUnSupportedProtocol(
-			t,
-			lastRoundRemote,
-			lastRoundLocal,
-			roundWithSwitchOn,
-			roundsAlreadyInLocal)
-
-		for r := 1; r <= lastRoundLocal; r = r + 1 {
-			blk, err := local.Block(basics.Round(r))
-			require.NoError(t, err)
-			require.Equal(t, r, int(blk.Round()))
-		}
-		// Since round with switch on (7) is already in the
-		// ledger, round 8 will not be fetched.
-		require.Equal(t, int(local.LastRound()), lastRoundLocal)
-		require.Equal(t, lastRoundRemote, int(remote.LastRound()))
-		remote.Ledger.Close()
+// Test the interruption in the initial loop
+// This cannot happen in practice, but is used to test the code.
+func TestOnSwitchToUnSupportedProtocol1(t *testing.T) {
+
+	lastRoundRemote := 5
+	lastRoundLocal := 0
+	roundWithSwitchOn := 0
+	local, remote := helperTestOnSwitchToUnSupportedProtocol(t, lastRoundRemote, lastRoundLocal, roundWithSwitchOn, 0)
+
+	// Last supported round is 0, but is guaranteed
+	// to stop after 2 rounds.
+
+	// SeedLookback is 2, which allows two parallel fetches.
+	// i.e. rounds 1 and 2 may be simultaneously fetched.
+	require.Less(t, int(local.LastRound()), 3)
+	require.Equal(t, lastRoundRemote, int(remote.LastRound()))
+	remote.Ledger.Close()
+}
+
+// Test the interruption in "the rest" loop
+func TestOnSwitchToUnSupportedProtocol2(t *testing.T) {
+
+	lastRoundRemote := 10
+	lastRoundLocal := 7
+	roundWithSwitchOn := 5
+	local, remote := helperTestOnSwitchToUnSupportedProtocol(t, lastRoundRemote, lastRoundLocal, roundWithSwitchOn, 0)
+	for r := 1; r <= lastRoundLocal; r++ {
+		blk, err := local.Block(basics.Round(r))
+		require.NoError(t, err)
+		require.Equal(t, r, int(blk.Round()))
+	}
+	require.Equal(t, lastRoundLocal, int(local.LastRound()))
+	require.Equal(t, lastRoundRemote, int(remote.LastRound()))
+	remote.Ledger.Close()
+}
+
+// Test the interruption with short notice (less than
+// SeedLookback or the number of parallel fetches which in the
+// test is the same: 2)
+// This can not happen in practice, because there will be
+// enough rounds for the protocol upgrade notice.
+func TestOnSwitchToUnSupportedProtocol3(t *testing.T) {
+
+	lastRoundRemote := 14
+	lastRoundLocal := 7
+	roundWithSwitchOn := 7
+	local, remote := helperTestOnSwitchToUnSupportedProtocol(t, lastRoundRemote, lastRoundLocal, roundWithSwitchOn, 0)
+	for r := 1; r <= lastRoundLocal; r = r + 1 {
+		blk, err := local.Block(basics.Round(r))
+		require.NoError(t, err)
+		require.Equal(t, r, int(blk.Round()))
+	}
+	// Since round with switch on (7) can be fetched
+	// Simultaneously with round 8, round 8 might also be
+	// fetched.
+	require.Less(t, int(local.LastRound()), lastRoundLocal+2)
+	require.Equal(t, lastRoundRemote, int(remote.LastRound()))
+	remote.Ledger.Close()
+}
+
+// Test the interruption with short notice (less than
+// SeedLookback or the number of parallel fetches which in the
+// test is the same: 2)
+// This case is a variation of the previous case. This may
+// happen when the catchup service restart at the round when
+// an upgrade happens.
+func TestOnSwitchToUnSupportedProtocol4(t *testing.T) {
+
+	lastRoundRemote := 14
+	lastRoundLocal := 7
+	roundWithSwitchOn := 7
+	roundsAlreadyInLocal := 8 // round 0 -> 7
+
+	local, remote := helperTestOnSwitchToUnSupportedProtocol(
+		t,
+		lastRoundRemote,
+		lastRoundLocal,
+		roundWithSwitchOn,
+		roundsAlreadyInLocal)
+
+	for r := 1; r <= lastRoundLocal; r = r + 1 {
+		blk, err := local.Block(basics.Round(r))
+		require.NoError(t, err)
+		require.Equal(t, r, int(blk.Round()))
 	}
+	// Since round with switch on (7) is already in the
+	// ledger, round 8 will not be fetched.
+	require.Equal(t, int(local.LastRound()), lastRoundLocal)
+	require.Equal(t, lastRoundRemote, int(remote.LastRound()))
+	remote.Ledger.Close()
 }
 
 func helperTestOnSwitchToUnSupportedProtocol(
@@ -535,14 +533,16 @@ func helperTestOnSwitchToUnSupportedProtocol(
 	config := defaultConfig
 	config.CatchupParallelBlocks = 2
 
-	remote, _, blk, err := buildTestLedger(t, bookkeeping.Block{}) //mRemote.blocks[0])
+	block1 := mRemote.blocks[1]
+	remote, _, blk, err := buildTestLedger(t, block1)
 	if err != nil {
 		t.Fatal(err)
 		return local, remote
 	}
 	for i := 1; i < lastRoundRemote; i++ {
-		blk.NextProtocolSwitchOn = mRemote.blocks[i].NextProtocolSwitchOn
-		blk.NextProtocol = mRemote.blocks[i].NextProtocol
+		blk.NextProtocolSwitchOn = mRemote.blocks[i+1].NextProtocolSwitchOn
+		blk.NextProtocol = mRemote.blocks[i+1].NextProtocol
+		// Adds blk.BlockHeader.Round + 1
 		addBlocks(t, remote, blk, 1)
 		blk.BlockHeader.Round++
 	}

cmd/goal/clerk.go

@@ -375,6 +375,14 @@ var sendCmd = &cobra.Command{
 			payment.RekeyTo = rekeyTo
 		}
 
+		// ConstructPayment fills in the suggested fee when fee=0. But if the user actually used --fee=0 on the
+		// commandline, we ought to do what they asked (especially now that zero or low fees make sense in
+		// combination with other txns that cover the groups's fee.
+		explicitFee := cmd.Flags().Changed("fee")
+		if explicitFee {
+			payment.Fee = basics.MicroAlgos{Raw: fee}
+		}
+
 		var stx transactions.SignedTxn
 		if lsig.Logic != nil {