Eric/use mule in jenkins

.travis.yml

@@ -34,105 +34,6 @@ jobs:
         - scripts/travis/build_test.sh
 
     - stage: build_pr
-      os: linux
-      name: Ubuntu AMD64 Build
-      script:
-        - scripts/travis/build_test.sh
-    - # same stage, parallel job
-      os: linux
-      name: Ubuntu AMD64 Integration Test
-      script:
-        - ./scripts/travis/integration_test.sh
-    - # same stage, parallel job
-      name: External ARM64 Build
-      os: linux
-      env:
-        - BUILD_TYPE: "external_build"
-        - TARGET_PLATFORM: "linux-arm64"
-      addons:
-        apt:
-          packages:
-            - awscli
-      script:
-        - scripts/travis/external_build.sh ./scripts/travis/build_test.sh
-    - # same stage, parallel job
-      name: External ARM64 Integration Test
-      os: linux
-      env:
-        - BUILD_TYPE: "external_build"
-        - TARGET_PLATFORM: "linux-arm64"
-      addons:
-        apt:
-          packages:
-            - awscli
-      script:
-        - scripts/travis/external_build.sh ./scripts/travis/integration_test.sh
-    - # same stage, parallel job
-      os: osx
-      osx_image: xcode11
-      name: MacOS AMD64 Build
-      script:
-        - scripts/travis/build_test.sh
-    - # same stage, parallel job
-      os: osx
-      osx_image: xcode11
-      name: MacOS AMD64 Integration Test
-      script:
-        - ./scripts/travis/integration_test.sh
-
-    - stage: build_release
-      os: linux
-      name: Ubuntu AMD64 Build
-      script:
-        - ./scripts/travis/build_test.sh
-    - # same stage, parallel job
-      os: linux
-      name: Ubuntu AMD64 Integration Test
-      script:
-        - ./scripts/travis/integration_test.sh
-    - # same stage, parallel job
-      os: osx
-      osx_image: xcode11
-      name: MacOS AMD64 Build
-      script:
-        - scripts/travis/build_test.sh
-    - # same stage, parallel job
-      name: External ARM64 Integration Test
-      os: linux
-      env:
-        - BUILD_TYPE: "external_build"
-        - TARGET_PLATFORM: "linux-arm64"
-      addons:
-        apt:
-          packages:
-            - awscli
-      script:
-        - scripts/travis/external_build.sh ./scripts/travis/integration_test.sh
-
-    - stage: deploy
-      name: Ubuntu Deploy
-      os: linux
-      script:
-        - scripts/travis/deploy_packages.sh
-        - scripts/travis/test_release.sh
-    - # same stage, parallel job
-      name: MacOS Deploy
-      os: osx
-      osx_image: xcode11
-      script: scripts/travis/deploy_packages.sh
-    - # same stage, parallel job
-      name: External ARM64 Deploy
-      os: linux
-      env:
-        - BUILD_TYPE: "external_build"
-        - TARGET_PLATFORM: "linux-arm64"
-      addons:
-        apt:
-          packages:
-            - awscli
-      script:
-        - scripts/travis/external_build.sh ./scripts/travis/deploy_packages.sh
-    - # same stage, parallel job
       name: External ARM Deploy
       os: linux
       env:

Makefile

@@ -1,18 +1,19 @@
 export GOPATH		:= $(shell go env GOPATH)
 GOPATH1		:= $(firstword $(subst :, ,$(GOPATH)))
 export GO111MODULE	:= on
-export GOPROXY := https://gocenter.io
+export GOPROXY := direct
 
 UNAME		:= $(shell uname)
 SRCPATH     := $(shell pwd)
 ARCH        := $(shell ./scripts/archtype.sh)
 OS_TYPE     := $(shell ./scripts/ostype.sh)
+S3_RELEASE_BUCKET = $$S3_RELEASE_BUCKET
 
 # If build number already set, use it - to ensure same build number across multiple platforms being built
 BUILDNUMBER      ?= $(shell ./scripts/compute_build_number.sh)
 COMMITHASH       := $(shell ./scripts/compute_build_commit.sh)
-BUILDBRANCH      ?= $(shell ./scripts/compute_branch.sh)
-BUILDCHANNEL     ?= $(shell ./scripts/compute_branch_channel.sh $(BUILDBRANCH))
+BUILDBRANCH      := $(shell ./scripts/compute_branch.sh)
+BUILDCHANNEL     := $(shell ./scripts/compute_branch_channel.sh $(BUILDBRANCH))
 DEFAULTNETWORK   ?= $(shell ./scripts/compute_branch_network.sh $(BUILDBRANCH))
 DEFAULT_DEADLOCK ?= $(shell ./scripts/compute_branch_deadlock_default.sh $(BUILDBRANCH))
 
@@ -155,8 +156,14 @@ $(KMD_API_SWAGGER_INJECT): $(KMD_API_SWAGGER_SPEC) $(KMD_API_SWAGGER_SPEC).valid
 
 build: buildsrc gen
 
+# We're making an empty file in the go-cache dir to
+# get around a bug in go build where it will fail
+# to cache binaries from time to time on empty NFS
+# dirs
 buildsrc: crypto/libs/$(OS_TYPE)/$(ARCH)/lib/libsodium.a node_exporter NONGO_BIN deps $(ALGOD_API_SWAGGER_INJECT) $(KMD_API_SWAGGER_INJECT)
-	go install $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./...
+	mkdir -p tmp/go-cache && \
+	touch tmp/go-cache/file.txt && \
+	GOCACHE=$(SRCPATH)/tmp/go-cache go install $(GOTRIMPATH) $(GOTAGS) $(GOBUILDMODE) -ldflags="$(GOLDFLAGS)" ./...
 
 SOURCES_RACE := github.com/algorand/go-algorand/cmd/kmd
 
@@ -233,7 +240,10 @@ node_exporter: $(GOPATH1)/bin/node_exporter
 # The file is was taken from the S3 cloud and it traditionally stored at
 # /travis-build-artifacts-us-ea-1.algorand.network/algorand/node_exporter/latest/node_exporter-stable-linux-x86_64.tar.gz
 $(GOPATH1)/bin/node_exporter:
-	tar -xzvf installer/external/node_exporter-stable-$(shell ./scripts/ostype.sh)-$(shell uname -m | tr '[:upper:]' '[:lower:]').tar.gz -C $(GOPATH1)/bin
+	mkdir -p $(GOPATH1)/bin && \
+	cd $(GOPATH1)/bin && \
+	tar -xzvf $(SRCPATH)/installer/external/node_exporter-stable-$(shell ./scripts/ostype.sh)-$(shell uname -m | tr '[:upper:]' '[:lower:]').tar.gz && \
+	cd -
 
 # deploy
 
@@ -279,3 +289,9 @@ install: build
 ###### TARGETS FOR CICD PROCESS ######
 include ./scripts/release/mule/Makefile.mule
 
+SUPPORTED_ARCHIVE_OS_ARCH = linux/amd64 linux/arm64 linux/arm darwin/amd64
+
+archive:
+	CHANNEL=$(BUILDCHANNEL) \
+	PATH=$(SRCPATH)/tmp/node_pkgs/$(OS_TYPE)/$(ARCH)/$(BUILDCHANNEL)/$(OS_TYPE)-$(ARCH)/bin:$${PATH} \
+	scripts/upload_version.sh $(BUILDCHANNEL) $(SRCPATH)/tmp/node_pkgs $(S3_RELEASE_BUCKET)

docker/build/Dockerfile

@@ -6,7 +6,7 @@ WORKDIR /root
 RUN wget --quiet https://dl.google.com/go/go${GOLANG_VERSION}.linux-amd64.tar.gz && tar -xvf go${GOLANG_VERSION}.linux-amd64.tar.gz && mv go /usr/local
 ENV GOROOT=/usr/local/go \
     GOPATH=$HOME/go \
-    GOPROXY=https://gocenter.io
+    GOPROXY=https://gocenter.io,https://goproxy.io,direct
 RUN mkdir -p $GOPATH/src/github.com/algorand
 WORKDIR $GOPATH/src/github.com/algorand
 COPY ./go-algorand ./go-algorand/

docker/build/Dockerfile-deploy

@@ -6,7 +6,7 @@ WORKDIR /root
 RUN wget --quiet https://dl.google.com/go/go${GOLANG_VERSION}.linux-amd64.tar.gz && tar -xvf go${GOLANG_VERSION}.linux-amd64.tar.gz && mv go /usr/local
 ENV GOROOT=/usr/local/go \
     GOPATH=$HOME/go \
-    GOPROXY=https://gocenter.io
+    GOPROXY=https://gocenter.io,https://goproxy.io,direct
 RUN mkdir -p $GOPATH/src/github.com/algorand
 WORKDIR $GOPATH/src/github.com/algorand
 COPY . ./go-algorand/

docker/build/arm.Dockerfile → docker/build/cicd.alpine.Dockerfile

@@ -24,7 +24,7 @@ RUN apk add dpkg && \
 COPY . $GOPATH/src/github.com/algorand/go-algorand
 WORKDIR $GOPATH/src/github.com/algorand/go-algorand
 ENV GCC_CONFIG="--with-arch=armv6" \
-    GOPROXY=https://gocenter.io
+    GOPROXY=https://gocenter.io,https://goproxy.io,direct
 RUN make ci-deps && make clean
 RUN rm -rf $GOPATH/src/github.com/algorand/go-algorand && \
     mkdir -p $GOPATH/src/github.com/algorand/go-algorand

docker/build/cicd.centos.Dockerfile

@@ -1,12 +1,15 @@
 ARG ARCH="amd64"
 
 FROM ${ARCH}/centos:7
-ENV GOLANG_VERSION 1.12
-ARG ARCH="amd64"
+
 RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && \
     yum update -y && \
-    yum install -y autoconf wget awscli git gnupg2 nfs-utils python36 sqlite3 boost-devel expect jq libtool gcc-c++ libstdc++-devel libstdc++-static rpmdevtools createrepo rpm-sign bzip2 which ShellCheck
+    yum install -y autoconf wget awscli git gnupg2 nfs-utils python3-devel sqlite3 boost-devel expect jq \
+    libtool gcc-c++ libstdc++-devel libstdc++-static rpmdevtools createrepo rpm-sign bzip2 which ShellCheck \
+    libffi-devel openssl-devel
 WORKDIR /root
+ARG ARCH
+ENV GOLANG_VERSION 1.12.17
 RUN wget https://dl.google.com/go/go${GOLANG_VERSION}.linux-${ARCH%v*}.tar.gz \
     && tar -xvf go${GOLANG_VERSION}.linux-${ARCH%v*}.tar.gz && \
     mv go /usr/local
@@ -28,4 +31,3 @@ RUN rm -rf $GOPATH/src/github.com/algorand/go-algorand && \
     mkdir -p $GOPATH/src/github.com/algorand/go-algorand
 RUN echo "vm.max_map_count = 262144" >> /etc/sysctl.conf
 CMD ["/bin/bash"]
-