Checking CircleCI tests run once and only once

agreement/agreementtest/simulate_test.go

@@ -37,6 +37,7 @@ import (
 	"github.com/algorand/go-algorand/data/committee"
 	"github.com/algorand/go-algorand/logging"
 	"github.com/algorand/go-algorand/protocol"
+	"github.com/algorand/go-algorand/test/partitiontest"
 	"github.com/algorand/go-algorand/util/db"
 )
 
@@ -294,6 +295,8 @@ func (l *testLedger) EnsureDigest(c agreement.Certificate, verifier *agreement.A
 }
 
 func TestSimulate(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
 	f, _ := os.Create(t.Name() + ".log")
 	logging.Base().SetJSONFormatter()
 	logging.Base().SetOutput(f)

daemon/algod/api/server/lib/middlewares/auth_test.go

@@ -21,6 +21,7 @@ import (
 	"net/http"
 	"testing"
 
+	"github.com/algorand/go-algorand/test/partitiontest"
 	"github.com/labstack/echo/v4"
 	"github.com/stretchr/testify/require"
 )
@@ -36,6 +37,8 @@ func success(ctx echo.Context) error {
 }
 
 func TestAuth(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
 	tokens := []string{"token1", "token2"}
 
 	tests := []struct {

data/transactions/logic/doc_test.go

@@ -96,6 +96,8 @@ func TestOpImmediateNote(t *testing.T) {
 }
 
 func TestAllImmediatesDocumented(t *testing.T) {
+	partitiontest.PartitionTest(t)
+
 	for _, op := range OpSpecs {
 		count := len(op.Details.Immediates)
 		note := OpImmediateNote(op.Name)

data/transactions/logic/eval_test.go

@@ -2140,6 +2140,7 @@ len`, 2)
 }
 
 func TestExtractOp(t *testing.T) {
+	partitiontest.PartitionTest(t)
 	t.Parallel()
 	testAccepts(t, "byte 0x123456789abc; extract 1 2; byte 0x3456; ==", 5)
 	testAccepts(t, "byte 0x123456789abc; extract 0 6; byte 0x123456789abc; ==", 5)
@@ -2156,6 +2157,7 @@ func TestExtractOp(t *testing.T) {
 }
 
 func TestExtractFlop(t *testing.T) {
+	partitiontest.PartitionTest(t)
 	t.Parallel()
 	// fails in compiler
 	testProg(t, `byte 0xf000000000000000
@@ -4355,6 +4357,7 @@ func TestBytes(t *testing.T) {
 }
 
 func TestMethod(t *testing.T) {
+	partitiontest.PartitionTest(t)
 	t.Parallel()
 	// Although 'method' is new around the time of v5, it is a
 	// pseudo-op, so it's ok to use it earlier, as it compiles to
@@ -4391,6 +4394,7 @@ func TestDig(t *testing.T) {
 }
 
 func TestCover(t *testing.T) {
+	partitiontest.PartitionTest(t)
 	t.Parallel()
 	testAccepts(t, "int 4; int 3; int 2; int 1; cover 0; int 1; ==; return", 5)
 	testAccepts(t, "int 4; int 3; int 2; int 1; cover 1; int 2; ==; return", 5)
@@ -4401,6 +4405,7 @@ func TestCover(t *testing.T) {
 }
 
 func TestUncover(t *testing.T) {
+	partitiontest.PartitionTest(t)
 	t.Parallel()
 	testAccepts(t, "int 4; int 3; int 2; int 1; uncover 0; int 1; ==; return", 5)
 	testAccepts(t, "int 4; int 3; int 2; int 1; uncover 2; int 3; ==; return", 5)

data/transactions/logic/opcodes_test.go

@@ -81,7 +81,8 @@ func TestOpcodesByVersionReordered(t *testing.T) {
 }
 
 func TestOpcodesByVersion(t *testing.T) {
-	partitiontest.PartitionTest(t)
+	// partitiontest.PartitionTest(t)
+	// has partitioning in the TestOpcodesByVersionReordered()
 
 	// Make a copy of the OpSpecs to check if OpcodesByVersion will change it
 	OpSpecs2 := make([]OpSpec, len(OpSpecs))

network/wsNetwork_test.go

@@ -2053,6 +2053,7 @@ type urlCase struct {
 }
 
 func TestParseHostOrURL(t *testing.T) {
+	partitiontest.PartitionTest(t)
 	urlTestCases := []urlCase{
 		{"localhost:123", url.URL{Scheme: "http", Host: "localhost:123"}},
 		{"http://localhost:123", url.URL{Scheme: "http", Host: "localhost:123"}},

scripts/buildtools/check_tests.py

@@ -0,0 +1,116 @@
+print("===== STARTED RUNNING check_tests.py =====\n")
+
+import json
+import sys
+import argparse
+
+# Arguments parsing / help menu
+parser = argparse.ArgumentParser(description='Check test results for intentionally and unintentionally skipped tests, as well as tests that ran multiple times.')
+parser.add_argument('tests_results_filepath', metavar='RESULTS_FILE',
+    help='json format test results file path (e.g. /tmp/results/testresults.json)')
+args = parser.parse_args()
+
+# Go through the given file one json object at a time, and record into a dict
+AllTestResults = {}
+with open(args.tests_results_filepath) as f:
+    for jsonObj in f:
+        testDict = json.loads(jsonObj)
+        if 'Test' not in testDict:
+            continue
+
+        fullTestName = testDict['Package'] + ' ' + testDict['Test']
+        if fullTestName not in AllTestResults:
+            AllTestResults[fullTestName] = {}
+            AllTestResults[fullTestName]['ran'] = 0
+            AllTestResults[fullTestName]['last_output'] = ''
+            AllTestResults[fullTestName]['skipped_intentionally'] = False
+            AllTestResults[fullTestName]['skipped_reason'] = ''
+
+        # actions can be: output, run, skip, pass
+        if 'pass' in testDict["Action"]:
+            AllTestResults[fullTestName]['ran'] += 1
+        elif 'output' in testDict["Action"]:
+            if '--- SKIP' in testDict['Output'] and 'due to partitioning' not in AllTestResults[fullTestName]['last_output']:
+                AllTestResults[fullTestName]['skipped_intentionally'] = True
+                if '=== RUN' in AllTestResults[fullTestName]['last_output']:
+                    AllTestResults[fullTestName]['skipped_reason'] = 'No reason given. PLEASE CHECK!!!'
+                else:
+                    AllTestResults[fullTestName]['skipped_reason'] = AllTestResults[fullTestName]['last_output'].strip()
+
+            AllTestResults[fullTestName]['last_output'] = testDict['Output']
+
+
+
+f.close()
+
+# === Calculate results ===
+
+# Sort and print messages with colored prefix
+RED_TEXT_COLOR = "\033[0;31m"
+GREEN_TEXT_COLOR = "\033[0;32m"
+YELLOW_TEXT_COLOR = "\033[0;33m"
+NORMAL_TEXT_COLOR = "\033[0;0m"
+def printColor(message, color=NORMAL_TEXT_COLOR):
+    print(f"{color}{message}{NORMAL_TEXT_COLOR}")
+
+# Record error message for sys.exit(errorMessage)
+errorMessage = ''
+
+# Check for tests that ran multiple times
+printColor("=========== RAN MULTIPLE TIMES ===================", YELLOW_TEXT_COLOR)
+listOfMultipleRuns = []
+for x in AllTestResults:
+    if AllTestResults[x]['ran'] > 1:
+        listOfMultipleRuns.append(x + " -- ran " + str(AllTestResults[x]['ran']) + " times. (Can probably be fixed by adding \"partitiontest.PartitionTest()\")")
+countMultipleRuns = len(listOfMultipleRuns) 
+if countMultipleRuns:
+    printColor(f"The above {countMultipleRuns} tests ran multiple times:", RED_TEXT_COLOR)
+    [printColor(f"{x}", RED_TEXT_COLOR) for x in sorted(listOfMultipleRuns)]
+    printColor(f"The above {countMultipleRuns} tests ran multiple times:", RED_TEXT_COLOR)
+else:
+    printColor("All tests that ran, ran only once ... OK", GREEN_TEXT_COLOR)
+printColor("==================================================\n", YELLOW_TEXT_COLOR)
+
+# Check intentionally skipped tests
+printColor("============= INTENTIONALLY SKIPPED ==============", YELLOW_TEXT_COLOR)
+listOfSkippedIntentionally = []
+[listOfSkippedIntentionally.append(x) for x in AllTestResults if (AllTestResults[x]['ran'] == 0 and AllTestResults[x]['skipped_intentionally'] == True )]
+countSkippedIntentionally = len(listOfSkippedIntentionally)
+if countSkippedIntentionally:
+    printColor(f"The following {countSkippedIntentionally} tests were skipped intentionally:", YELLOW_TEXT_COLOR)
+    [printColor(f"{x} -- skipped intentionally (please double check) Reason: {AllTestResults[x]['skipped_reason']}", YELLOW_TEXT_COLOR) for x in sorted(listOfSkippedIntentionally)]
+    printColor(f"The above {countSkippedIntentionally} tests were skipped intentionally:", YELLOW_TEXT_COLOR)
+else:
+    printColor("No tests skipped intentionally.", GREEN_TEXT_COLOR)
+printColor("==================================================\n", YELLOW_TEXT_COLOR)
+
+# Check unintentionally skipped tests (due to partition)
+printColor("============= UNINTENTIONALLY SKIPPED ============", YELLOW_TEXT_COLOR)
+listOfSkippedUnintentionally = []
+[listOfSkippedUnintentionally.append(x) for x in AllTestResults if (AllTestResults[x]['ran'] == 0 and AllTestResults[x]['skipped_intentionally'] == False)]
+countSkippedUnintentionally = len(listOfSkippedUnintentionally)
+if countSkippedUnintentionally:
+    printColor(f"{countSkippedUnintentionally} tests were skipped unintentionally", RED_TEXT_COLOR)
+    [printColor(f"{x} -- skipped unintentionally. (due to partitiontest.PartitionTest() being called twice?)", RED_TEXT_COLOR) for x in sorted(listOfSkippedUnintentionally)]
+    printColor(f"{countSkippedUnintentionally} tests were skipped unintentionally.", RED_TEXT_COLOR)
+    errorMessage += f"{countSkippedUnintentionally} tests were skipped unintentionally";
+else:
+    printColor("No tests skipped unintentionally (due to partitioning).", GREEN_TEXT_COLOR)
+printColor("==================================================\n", YELLOW_TEXT_COLOR)
+
+# === Summary ===
+printColor("==================== SUMMARY =====================", YELLOW_TEXT_COLOR)
+countUniqueTests = len(AllTestResults)
+countTotalSkipped = countSkippedIntentionally + countSkippedUnintentionally
+countRanTests = sum(1 for x in AllTestResults if AllTestResults[x]['ran'] > 0)
+
+printColor(f"Saw {countUniqueTests} unique tests", GREEN_TEXT_COLOR if countUniqueTests != 0 else RED_TEXT_COLOR)
+printColor(f"{countTotalSkipped} total skipped tests", GREEN_TEXT_COLOR if countTotalSkipped == 0 else YELLOW_TEXT_COLOR)
+printColor(f"{countRanTests} tests ran", GREEN_TEXT_COLOR if countRanTests != 0 else RED_TEXT_COLOR)
+printColor(f"{countSkippedIntentionally} tests were skipped intentionally. (They were probably disabled, please double check)", GREEN_TEXT_COLOR if countSkippedIntentionally == 0 else YELLOW_TEXT_COLOR)
+printColor(f"{countSkippedUnintentionally} tests were skipped unintentionally. (Due to partitioning multiple times? maybe due to partitiontest.PartitionTest() being called twice?)", GREEN_TEXT_COLOR if countSkippedUnintentionally == 0 else RED_TEXT_COLOR)
+printColor(f"{countMultipleRuns} tests ran multiple times. (Can probably be fixed by adding \"partitiontest.PartitionTest()\")", GREEN_TEXT_COLOR if countMultipleRuns == 0 else RED_TEXT_COLOR)
+printColor("==================================================\n", YELLOW_TEXT_COLOR)
+
+print("===== FINISHED RUNNING check_tests.py =====")
+sys.exit(0 if not errorMessage else errorMessage)

test/scripts/e2e_go_tests.sh

@@ -7,7 +7,16 @@ set -o pipefail
 
 export GOPATH=$(go env GOPATH)
 export GO111MODULE=on
-GOTESTCOMMAND=${GOTESTCOMMAND:="go test"}
+
+# Needed for now because circleci doesn't use makefile yet
+if [ -z "$(which gotestsum)" ]; then
+    GOTESTCOMMAND=${GOTESTCOMMAND:="go test"}
+else
+    TEST_RESULTS=${TEST_RESULTS:="$(pwd)"}
+    GOTESTCOMMAND=${GOTESTCOMMAND:="gotestsum --format testname --junitfile ${TEST_RESULTS}/results.xml --jsonfile ${TEST_RESULTS}/testresults.json --"}
+fi
+
+echo "GOTESTCOMMAND will be: ${GOTESTCOMMAND}"
 
 # If one or more -t <pattern> are specified, use GOTESTCOMMAND -run <pattern> for each