fix(CI): fix test suite and add CI steps (#1808)

## Problem

Badger tests are run through a bash script called test.sh. It is
currently not working, and there are no CI workflows on the Badger
repository.

## Solution

We update test.sh and add CI workflow steps. Test suite now works for
Linux and Mac (x86, M1). To run the test suite, simply run `make test`.
If you are on Mac, see remark below.

### Why a Makefile?

Badger depends on jemalloc for efficient memory allocation (see
[here](https://dgraph.io/blog/post/manual-memory-management-golang-jemalloc/),
and z package in Ristretto). While Badger can be built without jemalloc,
many users will probably want to benefit from it. The makefile makes it
easy to install the jemalloc dependency with `make jemalloc`. Also now
the test.sh script contains only test related functionality.

This also has the advantage that now the CI workflow only needs
essentially two steps:

```
make dependency
make test
```

### Remarks
- In pb/gen.sh, `go get` is
[deprecated](https://go.dev/doc/go-get-install-deprecation) as a way to
retrieve and install an executable into our $GOBIN, which is what we
want to do here. We use `go install` instead.
- In test.sh, Teamcity flags no longer needed
- In test.sh, Installjemalloc no longer needed as it is in the Makefile,
simplifying the test script
- In test.sh, tests are now run sequentially and not in parallel
(currently broken)
- We remove .travisci because it is unused
- Important note for Mac users: for historical reasons, certain tools on
MacOS are different from the standard GNU tools everyone else uses. One
of these is the `mktemp` command, which is used in the test suite. In
order to get the GNU version of this tool, you can run:
```
brew install coreutils
export PATH="$(brew --prefix)/opt/coreutils/libexec/gnubin:$PATH"
make test
```

This will temporarily modify your path so that the GNU version of mktemp
is called in the script. Without this fix the script will complain that
it doesn't recognize the p flag in `mktemp -d -p .`

#### To-do
- tune lint tests
- add code coverage
- potentially add build target for badger in makefile
- bring test suite to parity with [old teamcity
setup](https://teamcity.dgraph.io/project.html?projectId=Badger)
- prune dependencies

.github/workflows/ci-badger-bank-tests.yml

@@ -0,0 +1,36 @@
+name: ci-badger-bank-tests
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  schedule:
+    - cron: "*/30 * * * *"
+jobs:
+  badger-bank:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Get Go Version
+        run: |
+          #!/bin/bash
+          DEFAULT_VERSION="1.18"
+          GOVERSION=$({ [ -f .go-version ] && cat .go-version; } || echo $DEFAULT_VERSION)
+          echo "GOVERSION=$GOVERSION" >> $GITHUB_ENV
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GOVERSION }}
+      - name: Install Dependencies
+        run: make dependency
+      - name: Install jemalloc
+        run: make jemalloc
+      - name: Install Badger
+        run: cd badger && go install --race --tags=jemalloc .
+      - name: Run Badger Bank Test
+        run: |
+          #!/bin/bash
+          mkdir bank && cd bank
+          badger bank test -v --dir=. -d=20m

.github/workflows/ci-badger-tests.yml

@@ -0,0 +1,29 @@
+name: ci-badger-tests
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  schedule:
+    - cron: "*/30 * * * *"
+jobs:
+  badger-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Get Go Version
+        run: |
+          #!/bin/bash
+          DEFAULT_VERSION="1.18"
+          GOVERSION=$({ [ -f .go-version ] && cat .go-version; } || echo $DEFAULT_VERSION)
+          echo "GOVERSION=$GOVERSION" >> $GITHUB_ENV
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GOVERSION }}
+      - name: Install Dependencies
+        run: make dependency
+      - name: Run Badger Tests
+        run: make test

.github/workflows/ci-golang-lint.yml

@@ -0,0 +1,27 @@
+name: ci-golang-lint
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  schedule:
+    - cron: "*/30 * * * *"
+jobs:
+  go-lint:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: golang-lint
+        env:
+          # prevent OOM
+          GOGC: 10
+        uses: golangci/golangci-lint-action@v2
+        with:
+          # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
+          version: v1.36
+          only-new-issues: true
+          args: --timeout=10m
+          skip-go-installation: true

.go-version

@@ -0,0 +1 @@
+1.18

Makefile

@@ -0,0 +1,55 @@
+#
+# Copyright 2022 Dgraph Labs, Inc. and Contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+USER_ID      = $(shell id -u)
+HAS_JEMALLOC = $(shell test -f /usr/local/lib/libjemalloc.a && echo "jemalloc")
+JEMALLOC_URL = "https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2"
+
+
+.PHONY: all test jemalloc dependency
+
+test: jemalloc
+	@echo "Running Badger tests..."
+	@./test.sh
+
+jemalloc:
+	@if [ -z "$(HAS_JEMALLOC)" ] ; then \
+		mkdir -p /tmp/jemalloc-temp && cd /tmp/jemalloc-temp ; \
+		echo "Downloading jemalloc..." ; \
+		curl -s -L ${JEMALLOC_URL} -o jemalloc.tar.bz2 ; \
+		tar xjf ./jemalloc.tar.bz2 ; \
+		cd jemalloc-5.2.1 ; \
+		./configure --with-jemalloc-prefix='je_' --with-malloc-conf='background_thread:true,metadata_thp:auto'; \
+		make ; \
+		if [ "$(USER_ID)" -eq "0" ]; then \
+			make install ; \
+		else \
+			echo "==== Need sudo access to install jemalloc" ; \
+			sudo make install ; \
+		fi \
+	fi
+
+dependency:
+	@echo "Installing dependencies..."
+	@sudo apt-get update
+	@sudo apt-get -y upgrade
+	@sudo apt-get -y install \
+    	ca-certificates \
+    	curl \
+    	gnupg \
+    	lsb-release \
+    	build-essential \
+    	protobuf-compiler \

pb/gen.sh

@@ -3,6 +3,5 @@
 # Run this script from its directory, so that badgerpb2.proto is where it's expected to
 # be.
 
-# You might need to go get -v github.com/gogo/protobuf/...
-go get -v github.com/gogo/protobuf/protoc-gen-gogofaster
+go install github.com/gogo/protobuf/protoc-gen-gogofaster@latest
 protoc --gogofaster_out=. --gogofaster_opt=paths=source_relative -I=. badgerpb3.proto

test.sh

@@ -9,43 +9,13 @@ go version
 # export packages because the test will run in a sub process.
 export packages=$(go list ./... | grep "github.com/dgraph-io/badger/v3/")
 
-if [[ ! -z "$TEAMCITY_VERSION" ]]; then
-  export GOFLAGS="-json"
-fi
-
-function InstallJemalloc() {
-  pushd .
-  if [ ! -f /usr/local/lib/libjemalloc.a ]; then
-		USER_ID=`id -u`
-    	JEMALLOC_URL="https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2"
-
-		mkdir -p /tmp/jemalloc-temp && cd /tmp/jemalloc-temp ;
-		echo "Downloading jemalloc" ;
-		curl -s -L ${JEMALLOC_URL} -o jemalloc.tar.bz2 ;
-		tar xjf ./jemalloc.tar.bz2 ;
-		cd jemalloc-5.2.1 ;
-		./configure --with-jemalloc-prefix='je_' ;
-		make ;
-		if [ "$USER_ID" -eq "0" ]; then
-			make install ;
-		else
-			echo "==== Need sudo access to install jemalloc" ;
-			sudo make install ;
-		fi
-	fi
-  popd
-}
-
 tags="-tags=jemalloc"
 
 # Ensure that we can compile the binary.
 pushd badger
 go build -v $tags .
 popd
 
-# tags=""
-InstallJemalloc
-
 # Run the memory intensive tests first.
 manual() {
   timeout="-timeout 2m"
@@ -110,11 +80,18 @@ stream() {
     return 1
   fi
   echo "==> DONE stream test"
+  popd
   return 0
 }
 
 export -f stream
 export -f manual
 export -f root
 
-parallel --halt now,fail=1 --progress --line-buffer ::: stream manual root
+# parallel tests currently not working
+# parallel --halt now,fail=1 --progress --line-buffer ::: stream manual root
+
+# run tests in sequence
+root
+stream
+manual