From 8c052f491fe5e2705ec0af37d2eaf977c7862392 Mon Sep 17 00:00:00 2001
From: qbiqing <bq2134@columbia.edu>
Date: Mon, 12 Jul 2021 18:12:33 +0800
Subject: [PATCH 01/25] add DockerhubApi in clients

---
 '                              | 29 +++++++++++++
 .gitignore                     |  1 +
 Dockerfile-golang              |  8 ++--
 client/client.go               |  3 ++
 client/dockerhub_api_client.go | 17 ++++++++
 config/sample.toml             |  7 +++-
 go.mod                         | 14 ++++++-
 go.sum                         | 77 ++++++++++++++++++++++++++++++++++
 main.go                        | 61 +++++++++++++++++++++++++++
 test.sh                        |  6 +--
 10 files changed, 210 insertions(+), 13 deletions(-)
 create mode 100644 '
 create mode 100644 client/dockerhub_api_client.go

diff --git a/' b/'
new file mode 100644
index 0000000..1b3a518
--- /dev/null
+++ b/'
@@ -0,0 +1,29 @@
+pick 364274a fix Dockerfile-golang
+s c81fb5b fix test.sh
+s 7775366 tmp
+
+# Rebase 05a46fd..7775366 onto 05a46fd (3 commands)
+#
+# Commands:
+# p, pick <commit> = use commit
+# r, reword <commit> = use commit, but edit the commit message
+# e, edit <commit> = use commit, but stop for amending
+# s, squash <commit> = use commit, but meld into previous commit
+# f, fixup <commit> = like "squash", but discard this commit's log message
+# x, exec <command> = run command (the rest of the line) using shell
+# b, break = stop here (continue rebase later with 'git rebase --continue')
+# d, drop <commit> = remove commit
+# l, label <label> = label current HEAD with a name
+# t, reset <label> = reset HEAD to a label
+# m, merge [-C <commit> | -c <commit>] <label> [# <oneline>]
+# .       create a merge commit using the original merge commit's
+# .       message (or the oneline, if no original merge commit was
+# .       specified). Use -c <commit> to reword the commit message.
+#
+# These lines can be re-ordered; they are executed from top to bottom.
+#
+# If you remove a line here THAT COMMIT WILL BE LOST.
+#
+# However, if you remove everything, the rebase will be aborted.
+#
+# Note that empty commits are commented out
diff --git a/.gitignore b/.gitignore
index a9e092f..9a318ce 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
 docker-registry-watcher
+registrywatcher
 *.toml
 !test.toml
 !sample.toml
diff --git a/Dockerfile-golang b/Dockerfile-golang
index 94e78e6..07b4b7c 100644
--- a/Dockerfile-golang
+++ b/Dockerfile-golang
@@ -1,6 +1,6 @@
-FROM golang:1.11-alpine
+FROM golang:1.12-alpine
 
 # Only install hard dependencies
-RUN apk add --no-cache gcc git musl-dev && \
-    go get -v -u golang.org/x/lint/golint && \
-    go get -v -u gotest.tools/gotestsum
+RUN apk add --no-cache gcc git musl-dev; \
+    go get -v -u gotest.tools/gotestsum; \
+    :
diff --git a/client/client.go b/client/client.go
index 08dc83a..e8f92ab 100644
--- a/client/client.go
+++ b/client/client.go
@@ -18,6 +18,7 @@ type Clients struct {
 	NomadClient          *NomadClient
 	DockerRegistryClient *DockerRegistryClient
 	PostgresClient       *PostgresClient
+	DockerhubApi 				 *DockerhubApi
 	DockerTags           sync.Map
 	DigestMap            sync.Map
 
@@ -31,6 +32,7 @@ func SetUpClients(conf *viper.Viper) *Clients {
 		panic(fmt.Errorf("starting postgres client failed: %v", err))
 	}
 	dockerClient := InitializeDockerRegistryClient(conf)
+	dockerhubApi := InitializeDockerhubApi(conf)
 
 	// caching fields
 	dockerTags := sync.Map{}
@@ -43,6 +45,7 @@ func SetUpClients(conf *viper.Viper) *Clients {
 		NomadClient:          InitializeNomadClient(conf),
 		PostgresClient:       postgresClient,
 		DockerRegistryClient: dockerClient,
+		DockerhubApi:         dockerhubApi,
 		DockerTags:           dockerTags,
 		DigestMap:            digestMap,
 	}
diff --git a/client/dockerhub_api_client.go b/client/dockerhub_api_client.go
new file mode 100644
index 0000000..45597be
--- /dev/null
+++ b/client/dockerhub_api_client.go
@@ -0,0 +1,17 @@
+package client
+
+import (
+	"github.com/spf13/viper"
+)
+
+type DockerhubApi struct {
+	url		string
+}
+
+
+func InitializeDockerhubApi(conf *viper.Viper) (*DockerhubApi) {
+	client := DockerhubApi{
+		url: conf.GetString("dockerhub_url"),
+	}
+	return &client
+}
diff --git a/config/sample.toml b/config/sample.toml
index f1c9e0c..42bf44c 100644
--- a/config/sample.toml
+++ b/config/sample.toml
@@ -16,13 +16,16 @@ webhook_url = "$YOUR_SLACK_URL_HERE"
 database_url = "postgresql://registry-watcher:registry-watcher@localhost:5432/registry-watcher?sslmode=disable"
 create_database_schema = true
 
+# Dockerhub API Client
+dockerhub_url = "https://hub.docker.com"
+
 # Tests
 is_test = false
 
 # Docker Registry information (to be interpolated by Nomad)
-[registry_map.codefresh]
+[registry_map.dockerhub]
 registry_scheme = "https"
-registry_domain = "r.cfcr.io"
+registry_domain = "registry-1.docker.io"
 registry_prefix = "some_prefix"
 registry_auth = "$YOUR_AUTH_STRING_HERE"
 
diff --git a/go.mod b/go.mod
index 2ea83a9..f94238d 100644
--- a/go.mod
+++ b/go.mod
@@ -9,6 +9,7 @@ require (
 	github.com/docker/go-connections v0.4.0
 	github.com/docker/go-units v0.4.0 // indirect
 	github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
+	github.com/fatih/color v1.12.0 // indirect
 	github.com/gin-contrib/cors v1.3.0
 	github.com/gin-gonic/gin v1.4.0
 	github.com/gorilla/mux v1.7.1 // indirect
@@ -25,21 +26,30 @@ require (
 	github.com/jmoiron/sqlx v1.2.0
 	github.com/lib/pq v1.1.0
 	github.com/lusis/slack-test v0.0.0-20190426140909-c40012f20018 // indirect
+	github.com/mattn/go-isatty v0.0.13 // indirect
 	github.com/mitchellh/hashstructure v1.0.0 // indirect
 	github.com/nlopes/slack v0.5.0
+	github.com/onsi/ginkgo v1.8.0 // indirect
+	github.com/onsi/gomega v1.4.3 // indirect
 	github.com/opencontainers/go-digest v1.0.0-rc1
 	github.com/opencontainers/image-spec v1.0.1 // indirect
-	github.com/pkg/errors v0.8.1
+	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.4.1 // indirect
 	github.com/spf13/viper v1.3.2
 	github.com/stretchr/testify v1.3.0
 	go.uber.org/atomic v1.4.0 // indirect
 	go.uber.org/multierr v1.1.0 // indirect
 	go.uber.org/zap v1.10.0
+	golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect
+	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
+	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
+	golang.org/x/tools v0.1.5 // indirect
 	google.golang.org/appengine v1.5.0 // indirect
 	google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 // indirect
+	gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect
+	gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect
 	gotest.tools v2.2.0+incompatible // indirect
-	gotest.tools/gotestsum v0.3.4 // indirect
+	gotest.tools/gotestsum v1.6.4 // indirect
 )
 
 replace github.com/ugorji/go v1.1.4 => github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43
diff --git a/go.sum b/go.sum
index 6634482..a5f065e 100644
--- a/go.sum
+++ b/go.sum
@@ -20,6 +20,8 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk=
+github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE=
 github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v0.0.0-20180924202107-a9c061deec0f h1:W4fbqg0JUwy6lLesoJaV/rE0fwAmtdtinMa64X1CEh0=
@@ -35,9 +37,15 @@ github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNE
 github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
+github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
+github.com/fatih/color v1.12.0 h1:mRhaKNwANqRgUBGKmnI5ZxEk7QXmjQeCcuYFMX2bfcc=
+github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/gin-contrib/cors v1.3.0 h1:PolezCc89peu+NgkIWt9OB01Kbzt6IP0J/JvkG6xxlg=
 github.com/gin-contrib/cors v1.3.0/go.mod h1:artPvLlhkF7oG06nK8v3U8TNz6IeX+w1uzCSEId5/Vc=
 github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3 h1:t8FVkw33L+wilf2QiWkw0UV77qRpcH/JHPKGpKa2E8g=
@@ -64,6 +72,10 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCy
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75 h1:f0n1xnMSmBLzVfsMMvriDyA75NB/oBgILX2GcHXIQzY=
 github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75/go.mod h1:g2644b03hfBX9Ov0ZBDgXXens4rxSxmqFBbhvKv2yVA=
 github.com/gorilla/mux v1.7.1 h1:Dw4jY2nghMMRsh1ol8dv1axHkDwMQK2DHerMNJsIpJU=
@@ -136,6 +148,8 @@ github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
 github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ=
+github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
 github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
@@ -156,9 +170,15 @@ github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDe
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc=
 github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA=
+github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4=
 github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA=
@@ -204,6 +224,8 @@ github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
@@ -240,6 +262,8 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT
 github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43 h1:BasDe+IErOQKrMVXab7UayvSlIpiyGwRvuX3EKYY7UA=
 github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
@@ -251,9 +275,18 @@ golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI=
+golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -261,17 +294,28 @@ golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20181102091132-c10e9556a7bc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2 h1:iC0Y6EDq+rhnAePxGvJs2kzUAYcwESqdcGRPzEUfzTU=
 golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c h1:uOCk1iQW6Vc18bnC13MfzScl+wdKBmM9Y9kU7Z83/lw=
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -285,18 +329,48 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpbl
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e h1:nFYrTHrdrAOpShe27kaFHjsqYSEQ0KWqdWLu3xuZJts=
 golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b h1:ag/x1USPSsqHud38I9BAC88qdNLDHHtQ4mlgQIZPPNA=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE=
+golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db h1:6/JqlYfC1CCaLnGceQTI+sDGhC9UBSPAsBqI0Gun6kU=
 golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c=
@@ -332,4 +406,7 @@ gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 gotest.tools/gotestsum v0.3.4 h1:LdVJDg3RHrci4MbupUgSkwPCikz4kTzDHWtUahDAleY=
 gotest.tools/gotestsum v0.3.4/go.mod h1:Mnf3e5FUzXbkCfynWBGOwLssY7gTQgCHObK9tMpAriY=
+gotest.tools/gotestsum v1.6.4 h1:HFkapG0hK/HWiOxWS78SbR/JK5EpbH8hFzUuCvvfbfQ=
+gotest.tools/gotestsum v1.6.4/go.mod h1:fTR9ZhxC/TLAAx2/WMk/m3TkMB9eEI89gdEzhiRVJT8=
+gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/main.go b/main.go
index 6adc928..824c824 100644
--- a/main.go
+++ b/main.go
@@ -70,6 +70,7 @@ func SetUpRouter(conf *viper.Viper, clients *client.Clients) *gin.Engine {
 	r.GET("/tags/:repo_name", handler.GetTagHandler)
 	r.GET("/repos", handler.RepoSummaryHandler)
 	r.GET("/debug/caches", handler.CacheSummaryHandler)
+	r.POST("/tags/:repo_name/listener", handler.DockerhubWebhookListener)
 
 	return r
 }
@@ -108,6 +109,14 @@ type deployBody struct {
 	AutoDeploy *bool   `json:"auto_deploy",omitempty`
 }
 
+type webhookBody struct {
+	PushData *pushData `json:"push_data"`
+}
+
+type pushData struct {
+	Tag *string `json:"tag"`
+}
+
 func (h *Handler) ResetTagHandler(c *gin.Context) {
 
 	pinnedTag := ""
@@ -360,3 +369,55 @@ func (h *Handler) CacheSummaryHandler(c *gin.Context) {
 
 	c.JSON(200, rtn)
 }
+
+func (h *Handler) DockerhubWebhookListener(c *gin.Context) {
+	// check if repoName is valid
+	repoName := c.Param("repo_name")
+	validName := false
+	for _, repo := range h.conf.GetStringSlice("watched_repositories") {
+		if repoName == repo {
+			validName = true
+			break
+		}
+	}
+	if !validName {
+		_, registryDomain, _, _ := utils.ExtractRegistryInfo(h.conf, repoName)
+		c.JSON(400, gin.H{
+			"message": fmt.Sprintf("Error: The specified repo name %s is not inside the docker repository registry %s", repoName, registryDomain),
+		})
+		return
+	}
+
+	pinnedTag, err := h.clients.PostgresClient.GetPinnedTag(repoName)
+	if err != nil {
+		log.LogAppErr(fmt.Sprintf("Couldn't fetch pinned tag while checking whether to deploy for %s", repoName), err)
+		c.JSON(500, gin.H{
+			"message": fmt.Sprintf("Unable to fetch repo %s tag, err: %s", repoName, err),
+		})
+		return
+	}
+
+	var webhookBody webhookBody
+	err = c.BindJSON(&webhookBody)
+	if err != nil {
+		c.JSON(400, gin.H{
+			"message": fmt.Sprintf("Error: %s", err),
+		})
+		return
+	}
+
+	if *(webhookBody.PushData.Tag) == pinnedTag {
+		utils.PostSlackUpdate(h.conf, fmt.Sprintf("Update: the SHA of tag `%s` in `%s` changed. Auto deployment will happen shortly.", pinnedTag, repoName))
+		log.LogAppInfo(fmt.Sprintf("Auto deploying tag %s for repo %s", pinnedTag, repoName))
+
+		h.clients.DeployPinnedTag(h.conf, repoName)
+		h.clients.UpdateCaches(repoName)
+		c.JSON(200, gin.H{
+			"message": fmt.Sprintf("Deploying to %s", pinnedTag),
+		})
+	} else {
+		c.JSON(200, gin.H{
+			"message": fmt.Sprintf("Received push data, branch %s not in deployment", pinnedTag),
+		})
+	}
+}
diff --git a/test.sh b/test.sh
index 841d269..2439d35 100755
--- a/test.sh
+++ b/test.sh
@@ -8,7 +8,7 @@ assert_each_go_file () {
     failed_files=$(find . -type f -iname '*.go' -not -path "./.*/*" -exec $@ {} \;)
     printf "${failed_files}"
 
-    if [[ ${failed_files} ]]; then
+    if [ -n "${failed_files}" ]; then
         printf "\nFAILED!\n"
         exit 1
     fi
@@ -18,10 +18,6 @@ echo "-- gofmt check --"
 assert_each_go_file gofmt -l
 echo "OK"
 
-echo "-- golint check --"
-assert_each_go_file golint
-echo "OK"
-
 echo "-- go unit tests --"
 gotestsum -- -tags=unit ./...
 echo "OK"

From d8e867df5d0806f41a953629cb3f3283250eeea3 Mon Sep 17 00:00:00 2001
From: qbiqing <bq2134@columbia.edu>
Date: Mon, 19 Jul 2021 14:33:58 +0800
Subject: [PATCH 02/25] add Authenticate

---
 client/client.go               |  2 +-
 client/dockerhub_api_client.go | 47 ++++++++++++++++++++++++++++++++--
 2 files changed, 46 insertions(+), 3 deletions(-)

diff --git a/client/client.go b/client/client.go
index e8f92ab..173fc02 100644
--- a/client/client.go
+++ b/client/client.go
@@ -18,7 +18,7 @@ type Clients struct {
 	NomadClient          *NomadClient
 	DockerRegistryClient *DockerRegistryClient
 	PostgresClient       *PostgresClient
-	DockerhubApi 				 *DockerhubApi
+	DockerhubApi         *DockerhubApi
 	DockerTags           sync.Map
 	DigestMap            sync.Map
 
diff --git a/client/dockerhub_api_client.go b/client/dockerhub_api_client.go
index 45597be..384c35d 100644
--- a/client/dockerhub_api_client.go
+++ b/client/dockerhub_api_client.go
@@ -1,17 +1,60 @@
 package client
 
 import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+
+	"github.com/dsaidgovsg/registrywatcher/log"
 	"github.com/spf13/viper"
 )
 
 type DockerhubApi struct {
-	url		string
+	url string
 }
 
+type AuthenticateResp struct {
+	Token string `json:"token"`
+}
 
-func InitializeDockerhubApi(conf *viper.Viper) (*DockerhubApi) {
+func InitializeDockerhubApi(conf *viper.Viper) *DockerhubApi {
 	client := DockerhubApi{
 		url: conf.GetString("dockerhub_url"),
 	}
 	return &client
 }
+
+func (api *DockerhubApi) Authenticate(username string, pw string) (*string, error) {
+	addr := fmt.Sprintf("%s%s", api.url, "/v2/users/login")
+	log.LogAppInfo(fmt.Sprintf("dockerhub.users.login url=%s user=%s", addr, username))
+
+	data := url.Values{}
+	data.Set("username", username)
+	data.Set("password", pw)
+	b := bytes.NewBufferString(data.Encode())
+
+	req, err := http.NewRequest("POST", addr, b)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	var deserialized AuthenticateResp
+	json.Unmarshal(body, &deserialized)
+
+	return &deserialized.Token, nil
+}

From a6351fe667f7f3de859ab50337553184b218dfe5 Mon Sep 17 00:00:00 2001
From: qbiqing <bq2134@columbia.edu>
Date: Mon, 19 Jul 2021 15:58:08 +0800
Subject: [PATCH 03/25] add CheckImageIsCurrent

---
 client/dockerhub_api_client.go | 56 ++++++++++++++++++++++++++++++++--
 config/sample.toml             |  1 +
 2 files changed, 54 insertions(+), 3 deletions(-)

diff --git a/client/dockerhub_api_client.go b/client/dockerhub_api_client.go
index 384c35d..251e0c9 100644
--- a/client/dockerhub_api_client.go
+++ b/client/dockerhub_api_client.go
@@ -9,27 +9,39 @@ import (
 	"net/url"
 
 	"github.com/dsaidgovsg/registrywatcher/log"
+	"github.com/pkg/errors"
 	"github.com/spf13/viper"
 )
 
 type DockerhubApi struct {
-	url string
+	url       string
+	namespace string
 }
 
 type AuthenticateResp struct {
 	Token string `json:"token"`
 }
 
+type CheckImageResp struct {
+	Results []CheckImageResults `json:"results"`
+}
+
+type CheckImageResults struct {
+	Tag       string `json:"tag"`
+	IsCurrent bool   `json:"is_current"`
+}
+
 func InitializeDockerhubApi(conf *viper.Viper) *DockerhubApi {
 	client := DockerhubApi{
-		url: conf.GetString("dockerhub_url"),
+		url:       conf.GetString("dockerhub_url"),
+		namespace: conf.GetString("dockerhub_namespace"),
 	}
 	return &client
 }
 
 func (api *DockerhubApi) Authenticate(username string, pw string) (*string, error) {
 	addr := fmt.Sprintf("%s%s", api.url, "/v2/users/login")
-	log.LogAppInfo(fmt.Sprintf("dockerhub.users.login url=%s user=%s", addr, username))
+	log.LogAppInfo(fmt.Sprintf("dockerhub.users.login url=%s", addr))
 
 	data := url.Values{}
 	data.Set("username", username)
@@ -58,3 +70,41 @@ func (api *DockerhubApi) Authenticate(username string, pw string) (*string, erro
 
 	return &deserialized.Token, nil
 }
+
+func (api *DockerhubApi) CheckImageIsCurrent(repository, digest string, checkTag string, jwt *string) (*bool, error) {
+	endpoint := fmt.Sprintf("/v2/namespaces/%s/repositories/%s/images/%s/tags", api.namespace, repository, digest)
+	addr := fmt.Sprintf("%s%s", api.url, endpoint)
+	log.LogAppInfo(fmt.Sprintf("dockerhub.namespace.repositories.images url=%s", addr))
+
+	req, err := http.NewRequest("GET", addr, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	auth_token := fmt.Sprintf("JWT %s", *jwt)
+	req.Header.Set("Authorization", auth_token)
+	req.Header.Set("Accept", "application/json")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	var deserialized CheckImageResp
+	json.Unmarshal(body, &deserialized)
+
+	for _, item := range deserialized.Results {
+		if item.Tag == checkTag {
+			return &item.IsCurrent, nil
+		}
+	}
+	// image tag does not match the tag to be checked
+	// this throws an error as the stored digest should match the tag
+	return nil, errors.New(fmt.Sprintf("Digest does not have tag %s", checkTag))
+}
diff --git a/config/sample.toml b/config/sample.toml
index 42bf44c..856e123 100644
--- a/config/sample.toml
+++ b/config/sample.toml
@@ -18,6 +18,7 @@ create_database_schema = true
 
 # Dockerhub API Client
 dockerhub_url = "https://hub.docker.com"
+dockerhub_namespace = "namespace"
 
 # Tests
 is_test = false

From 66f60937b7967d8164908ecd654d5c9cc23517b5 Mon Sep 17 00:00:00 2001
From: qbiqing <bq2134@columbia.edu>
Date: Mon, 19 Jul 2021 16:16:52 +0800
Subject: [PATCH 04/25] edit isTagDigestChanged

---
 '                              |  29 -------
 client/client.go               |  28 +++---
 client/dockerhub_api_client.go | 154 ++++++++++++++++++++++++++-------
 client/setup.go                |   4 +-
 config/sample.toml             |   2 +
 5 files changed, 145 insertions(+), 72 deletions(-)
 delete mode 100644 '

diff --git a/' b/'
deleted file mode 100644
index 1b3a518..0000000
--- a/'
+++ /dev/null
@@ -1,29 +0,0 @@
-pick 364274a fix Dockerfile-golang
-s c81fb5b fix test.sh
-s 7775366 tmp
-
-# Rebase 05a46fd..7775366 onto 05a46fd (3 commands)
-#
-# Commands:
-# p, pick <commit> = use commit
-# r, reword <commit> = use commit, but edit the commit message
-# e, edit <commit> = use commit, but stop for amending
-# s, squash <commit> = use commit, but meld into previous commit
-# f, fixup <commit> = like "squash", but discard this commit's log message
-# x, exec <command> = run command (the rest of the line) using shell
-# b, break = stop here (continue rebase later with 'git rebase --continue')
-# d, drop <commit> = remove commit
-# l, label <label> = label current HEAD with a name
-# t, reset <label> = reset HEAD to a label
-# m, merge [-C <commit> | -c <commit>] <label> [# <oneline>]
-# .       create a merge commit using the original merge commit's
-# .       message (or the oneline, if no original merge commit was
-# .       specified). Use -c <commit> to reword the commit message.
-#
-# These lines can be re-ordered; they are executed from top to bottom.
-#
-# If you remove a line here THAT COMMIT WILL BE LOST.
-#
-# However, if you remove everything, the rebase will be aborted.
-#
-# Note that empty commits are commented out
diff --git a/client/client.go b/client/client.go
index 173fc02..f6119e1 100644
--- a/client/client.go
+++ b/client/client.go
@@ -32,7 +32,10 @@ func SetUpClients(conf *viper.Viper) *Clients {
 		panic(fmt.Errorf("starting postgres client failed: %v", err))
 	}
 	dockerClient := InitializeDockerRegistryClient(conf)
-	dockerhubApi := InitializeDockerhubApi(conf)
+	dockerhubApi, err := InitializeDockerhubApi(conf)
+	if err != nil {
+		log.LogAppErr("error initializing dockerhub API client", err)
+	}
 
 	// caching fields
 	dockerTags := sync.Map{}
@@ -165,11 +168,12 @@ func (client *Clients) PopulateCaches(repoName string) {
 	if err != nil {
 		log.LogAppErr(fmt.Sprintf("Couldn't fetch pinned tag while populating cache for %s", repoName), err)
 	}
-	tagDigest, err := client.DockerRegistryClient.GetTagDigest(repoName, pinnedTag)
-	client.updateDigestCache(repoName, tagDigest)
+	tagDigest, err := client.DockerhubApi.GetTagDigestFromApi(repoName, pinnedTag)
 	if err != nil {
-		log.LogAppErr(fmt.Sprintf("Couldn't fetch tag digest from registry while populating cache for %s", repoName), err)
+		log.LogAppErr(fmt.Sprintf("Couldn't fetch tag digest from Dockerhub while populating cache for %s", repoName), err)
+		return
 	}
+	client.updateDigestCache(repoName, *tagDigest)
 }
 
 func (client *Clients) isNewReleaseTagAvailable(repoName string) bool {
@@ -252,18 +256,18 @@ func (client *Clients) isTagDigestChanged(repoName string) (bool, error) {
 		log.LogAppErr(fmt.Sprintf("Couldn't fetch pinned tag while checking if it was changed for %s", repoName), err)
 		return false, err
 	}
-	tagDigest, err := client.DockerRegistryClient.GetTagDigest(repoName, pinnedTag)
+	cachedTagDigest, err := client.GetCachedTagDigest(repoName)
 	if err != nil {
-		log.LogAppErr(fmt.Sprintf("Couldn't fetch tag digest from registry while checking if it was changed for %s", repoName), err)
+		log.LogAppErr(fmt.Sprintf("Couldn't fetch tag digest from cache while checking if it was changed for %s", repoName), err)
 		return false, err
 	}
-	cachedTagDigest, err := client.GetCachedTagDigest(repoName)
+	digestIsCurrent, err := client.DockerhubApi.CheckImageIsCurrent(repoName, cachedTagDigest, pinnedTag)
 	if err != nil {
-		log.LogAppErr(fmt.Sprintf("Couldn't fetch tag digest from cache while checking if it was changed for %s", repoName), err)
+		log.LogAppErr("Couldn't check if tag currently points to cached image digest", err)
 		return false, err
 	}
 
-	return cachedTagDigest != tagDigest, nil
+	return *digestIsCurrent, nil
 }
 
 func (client *Clients) UpdateCaches(repoName string) {
@@ -285,7 +289,7 @@ func (client *Clients) UpdateCaches(repoName string) {
 			log.LogAppErr(fmt.Sprintf("Couldn't fetch pinned tag while updating cache for %s", repoName), err)
 			return
 		}
-		tagDigest, err := client.DockerRegistryClient.GetTagDigest(repoName, pinnedTag)
+		tagDigest, err := client.DockerhubApi.GetTagDigestFromApi(repoName, pinnedTag)
 		if err != nil {
 			log.LogAppErr(fmt.Sprintf("Couldn't fetch tag digest from registry while updating cache for %s", repoName), err)
 			return
@@ -297,9 +301,9 @@ func (client *Clients) UpdateCaches(repoName string) {
 			log.LogAppErr(fmt.Sprintf("Couldn't fetch tag digest from cache while updating cache for %s", repoName), err)
 			return
 		}
-		log.LogAppInfo(fmt.Sprintf("cached digest: %s, new digest: %s", cachedTagDigest, tagDigest))
+		log.LogAppInfo(fmt.Sprintf("cached digest: %s, new digest: %s", cachedTagDigest, *tagDigest))
 
-		client.updateDigestCache(repoName, tagDigest)
+		client.updateDigestCache(repoName, *tagDigest)
 	}
 }
 
diff --git a/client/dockerhub_api_client.go b/client/dockerhub_api_client.go
index 251e0c9..5700102 100644
--- a/client/dockerhub_api_client.go
+++ b/client/dockerhub_api_client.go
@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
-	"net/url"
 
 	"github.com/dsaidgovsg/registrywatcher/log"
 	"github.com/pkg/errors"
@@ -16,6 +15,9 @@ import (
 type DockerhubApi struct {
 	url       string
 	namespace string
+	token     string
+	username  string
+	secret    string
 }
 
 type AuthenticateResp struct {
@@ -23,36 +25,50 @@ type AuthenticateResp struct {
 }
 
 type CheckImageResp struct {
-	Results []CheckImageResults `json:"results"`
+	Results []TagWithStatus `json:"results"`
 }
 
-type CheckImageResults struct {
+type TagWithStatus struct {
 	Tag       string `json:"tag"`
 	IsCurrent bool   `json:"is_current"`
 }
 
-func InitializeDockerhubApi(conf *viper.Viper) *DockerhubApi {
+type GetTagDigestResp struct {
+	Results []GetTagDigestResult `json:"results"`
+}
+
+type GetTagDigestResult struct {
+	Digest string          `json:"digest"`
+	Tags   []TagWithStatus `json:"tags"`
+}
+
+func InitializeDockerhubApi(conf *viper.Viper) (*DockerhubApi, error) {
 	client := DockerhubApi{
 		url:       conf.GetString("dockerhub_url"),
 		namespace: conf.GetString("dockerhub_namespace"),
+		username:  conf.GetString("dockerhub_username"),
+		secret:    conf.GetString("dockerhub_secret"),
+		token:     "",
+	}
+
+	jwt, err := client.Authenticate()
+	if err != nil {
+		return &client, err
 	}
-	return &client
+
+	client.token = *jwt
+	return &client, nil
 }
 
-func (api *DockerhubApi) Authenticate(username string, pw string) (*string, error) {
+func (api *DockerhubApi) Authenticate() (*string, error) {
 	addr := fmt.Sprintf("%s%s", api.url, "/v2/users/login")
+	log.LogAppErr("in authenticate", nil)
 	log.LogAppInfo(fmt.Sprintf("dockerhub.users.login url=%s", addr))
 
-	data := url.Values{}
-	data.Set("username", username)
-	data.Set("password", pw)
-	b := bytes.NewBufferString(data.Encode())
-
-	req, err := http.NewRequest("POST", addr, b)
-	if err != nil {
-		return nil, err
-	}
+	data := map[string]string{"username": api.username, "password": api.secret}
+	jsonData, err := json.Marshal(data)
 
+	req, err := http.NewRequest("POST", addr, bytes.NewBuffer(jsonData))
 	req.Header.Set("Content-Type", "application/json")
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
@@ -61,8 +77,11 @@ func (api *DockerhubApi) Authenticate(username string, pw string) (*string, erro
 
 	defer resp.Body.Close()
 	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return nil, err
+
+	if resp.StatusCode != http.StatusOK {
+		log.LogAppInfo("Error obtaining JWT")
+		errMsg := fmt.Sprintf("Response status %d message %s", resp.StatusCode, string(body))
+		return nil, errors.New(errMsg)
 	}
 
 	var deserialized AuthenticateResp
@@ -71,18 +90,15 @@ func (api *DockerhubApi) Authenticate(username string, pw string) (*string, erro
 	return &deserialized.Token, nil
 }
 
-func (api *DockerhubApi) CheckImageIsCurrent(repository, digest string, checkTag string, jwt *string) (*bool, error) {
-	endpoint := fmt.Sprintf("/v2/namespaces/%s/repositories/%s/images/%s/tags", api.namespace, repository, digest)
+func (api *DockerhubApi) CheckImageIsCurrent(repository, digest string, checkTag string) (
+	*bool, error) {
+	endpoint := fmt.Sprintf("/v2/namespaces/%s/repositories/%s/images/%s/tags",
+		api.namespace, repository, digest)
 	addr := fmt.Sprintf("%s%s", api.url, endpoint)
-	log.LogAppInfo(fmt.Sprintf("dockerhub.namespace.repositories.images url=%s", addr))
+	log.LogAppInfo(fmt.Sprintf("dockerhub check image is current, url=%s", addr))
 
 	req, err := http.NewRequest("GET", addr, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	auth_token := fmt.Sprintf("JWT %s", *jwt)
-	req.Header.Set("Authorization", auth_token)
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", api.token))
 	req.Header.Set("Accept", "application/json")
 
 	resp, err := http.DefaultClient.Do(req)
@@ -90,10 +106,28 @@ func (api *DockerhubApi) CheckImageIsCurrent(repository, digest string, checkTag
 		return nil, err
 	}
 
+	// Obtain JWT if it has expired
+	if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden {
+		log.LogAppInfo("Obtaining new JWT")
+		jwt, err := api.Authenticate()
+		if err != nil {
+			return nil, err
+		}
+
+		api.token = *jwt
+		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", api.token))
+		resp, err = http.DefaultClient.Do(req)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	defer resp.Body.Close()
 	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return nil, err
+
+	if resp.StatusCode != http.StatusOK {
+		errMsg := fmt.Sprintf("Response status %d message %s", resp.StatusCode, string(body))
+		return nil, errors.New(errMsg)
 	}
 
 	var deserialized CheckImageResp
@@ -106,5 +140,67 @@ func (api *DockerhubApi) CheckImageIsCurrent(repository, digest string, checkTag
 	}
 	// image tag does not match the tag to be checked
 	// this throws an error as the stored digest should match the tag
-	return nil, errors.New(fmt.Sprintf("Digest does not have tag %s", checkTag))
+	return nil, errors.New(fmt.Sprintf("Digest %s does not have tag %s", digest, checkTag))
+}
+
+// We can't just use the registry GetTagDigest because the digests from the
+// registry and from the dockerhub API do not match
+func (api *DockerhubApi) GetTagDigestFromApi(repository string, checkTag string) (
+	*string, error) {
+	endpoint := fmt.Sprintf("/v2/namespaces/%s/repositories/%s/images?", api.namespace,
+		repository)
+	queryParams := fmt.Sprintf("currently_tagged=%s&page_size=%s&ordering=%s", "true",
+		"100", "-last_activity")
+
+	addr := fmt.Sprintf("%s%s%s", api.url, endpoint, queryParams)
+	log.LogAppInfo(fmt.Sprintf("dockerhub get tag digest url=%s", addr))
+
+	req, err := http.NewRequest("GET", addr, nil)
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", api.token))
+	req.Header.Set("Accept", "application/json")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	// Obtain JWT if it has expired
+	if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden {
+		log.LogAppInfo("Obtaining new JWT")
+		jwt, err := api.Authenticate()
+		if err != nil {
+			return nil, err
+		}
+
+		api.token = *jwt
+		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", api.token))
+		resp, err = http.DefaultClient.Do(req)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+
+	if resp.StatusCode != http.StatusOK {
+		errMsg := fmt.Sprintf("Response status %d message %s", resp.StatusCode, string(body))
+		return nil, errors.New(errMsg)
+	}
+
+	var deserialized GetTagDigestResp
+	json.Unmarshal(body, &deserialized)
+
+	for _, item := range deserialized.Results {
+		for _, tag := range item.Tags {
+			if tag.Tag == checkTag {
+				if tag.IsCurrent {
+					return &item.Digest, nil
+				}
+			}
+		}
+	}
+
+	// image tag not found in repository's 100 last active tags
+	return nil, errors.New(fmt.Sprintf("Tag %s not found in repository %s", checkTag, repository))
 }
diff --git a/client/setup.go b/client/setup.go
index e0165d3..8b6dafd 100644
--- a/client/setup.go
+++ b/client/setup.go
@@ -28,8 +28,8 @@ func (te *testEngine) printState() {
 	fmt.Println("cached tags", cachedTags)
 
 	pinnedTag, _ := te.Clients.GetFormattedPinnedTag(te.TestRepoName)
-	registryTagDigest, _ := te.Clients.DockerRegistryClient.GetTagDigest(te.TestRepoName, pinnedTag)
-	fmt.Println("registry tag digest", registryTagDigest)
+	registryTagDigest, _ := te.Clients.DockerhubApi.GetTagDigestFromApi(te.TestRepoName, pinnedTag)
+	fmt.Println("registry tag digest", *registryTagDigest)
 
 	cachedTagDigest, _ := te.Clients.GetCachedTagDigest(te.TestRepoName)
 	fmt.Println("cached tag digest", cachedTagDigest)
diff --git a/config/sample.toml b/config/sample.toml
index 856e123..092c00f 100644
--- a/config/sample.toml
+++ b/config/sample.toml
@@ -19,6 +19,8 @@ create_database_schema = true
 # Dockerhub API Client
 dockerhub_url = "https://hub.docker.com"
 dockerhub_namespace = "namespace"
+dockerhub_username = "user"
+dockerhub_secret = "secret"
 
 # Tests
 is_test = false

From f9b39fcada92b243120ff52dfd46ae03de7462be Mon Sep 17 00:00:00 2001
From: qbiqing <bq2134@columbia.edu>
Date: Fri, 23 Jul 2021 15:42:58 +0800
Subject: [PATCH 05/25] fix

---
 client/client.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/client.go b/client/client.go
index f6119e1..33bd83b 100644
--- a/client/client.go
+++ b/client/client.go
@@ -267,7 +267,7 @@ func (client *Clients) isTagDigestChanged(repoName string) (bool, error) {
 		return false, err
 	}
 
-	return *digestIsCurrent, nil
+	return !*digestIsCurrent, nil
 }
 
 func (client *Clients) UpdateCaches(repoName string) {

From 5e076a1f5e4a3c37598ea4ba6cc0d695b2724390 Mon Sep 17 00:00:00 2001
From: qbiqing <bq2134@columbia.edu>
Date: Fri, 23 Jul 2021 17:04:36 +0800
Subject: [PATCH 06/25] amend log

---
 client/dockerhub_api_client.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/dockerhub_api_client.go b/client/dockerhub_api_client.go
index 5700102..5290b0b 100644
--- a/client/dockerhub_api_client.go
+++ b/client/dockerhub_api_client.go
@@ -95,7 +95,7 @@ func (api *DockerhubApi) CheckImageIsCurrent(repository, digest string, checkTag
 	endpoint := fmt.Sprintf("/v2/namespaces/%s/repositories/%s/images/%s/tags",
 		api.namespace, repository, digest)
 	addr := fmt.Sprintf("%s%s", api.url, endpoint)
-	log.LogAppInfo(fmt.Sprintf("dockerhub check image is current, url=%s", addr))
+	log.LogAppInfo(fmt.Sprintf("dockerhub check if image is current, url=%s", addr))
 
 	req, err := http.NewRequest("GET", addr, nil)
 	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", api.token))

From f3a9899f5a6bd474e23374f7adc3f3b7e248a9af Mon Sep 17 00:00:00 2001
From: qbiqing <bq2134@columbia.edu>
Date: Mon, 26 Jul 2021 12:51:08 +0800
Subject: [PATCH 07/25] remove unused GetTagDigest

---
 client/docker_registry_client.go | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/client/docker_registry_client.go b/client/docker_registry_client.go
index 3562d60..37d4104 100644
--- a/client/docker_registry_client.go
+++ b/client/docker_registry_client.go
@@ -62,13 +62,3 @@ func (e *DockerRegistryClient) GetAllTags(repoName string) ([]string, error) {
 	tags, err := repoRegistry.Tags(fmt.Sprintf("%s/%s", registryPrefix, repoName))
 	return tags, err
 }
-
-func (e *DockerRegistryClient) GetTagDigest(repoName, tag string) (string, error) {
-	_, _, registryPrefix, _ := utils.ExtractRegistryInfo(e.conf, repoName)
-	repoRegistry := e.Hubs[repoName]
-	deserializedManifest, err := repoRegistry.ManifestV2(fmt.Sprintf("%s/%s", registryPrefix, repoName), tag)
-	if err != nil {
-		return "", err
-	}
-	return string(deserializedManifest.Manifest.Config.Digest), nil
-}

From 77534cc5c648c1efffacbdc0db391b11a2b1e75e Mon Sep 17 00:00:00 2001
From: qbiqing <bq2134@columbia.edu>
Date: Mon, 26 Jul 2021 15:27:18 +0800
Subject: [PATCH 08/25] add mock server

---
 README.md                      | 15 +++++++---
 client/client.go               |  4 ++-
 client/dockerhub_api_client.go | 12 ++++++++
 client/setup.go                |  9 +++++-
 go.mod                         |  2 +-
 testutils/mock_server.go       | 46 ++++++++++++++++++++++++++++
 testutils/snakeoil/cert.pem    | 34 ++++++++++-----------
 testutils/snakeoil/key.pem     | 55 +++++++++++++++++-----------------
 8 files changed, 126 insertions(+), 51 deletions(-)
 create mode 100644 testutils/mock_server.go

diff --git a/README.md b/README.md
index c1f0ef6..d0ff0c9 100644
--- a/README.md
+++ b/README.md
@@ -116,17 +116,24 @@ You need to make one modification to your docker daemon config (usually at ~/.do
 Add the flag `--insecure-registry localhost:5000` to your docker daemon, documented [here](https://docs.docker.com/registry/insecure/) for testing against an insecure registry.
 
 You may need to update the certificates from time to time.
+
+Find `GOROOT` via
+
+```bash
+go env
 ```
-go run /usr/local/go/src/crypto/tls/generate_cert.go --host localhost,127.0.0.1 --ca
-	mv $(CURDIR)/key.pem $(CURDIR)/testutils/snakeoil/key.pem
-	mv $(CURDIR)/cert.pem $(CURDIR)/testutils/snakeoil/cert.pem
+
+```bash
+go run $GOROOT/src/crypto/tls/generate_cert.go --host localhost,127.0.0.1 --ca
+mv $(CURDIR)/key.pem $(CURDIR)/testutils/snakeoil/key.pem
+mv $(CURDIR)/cert.pem $(CURDIR)/testutils/snakeoil/cert.pem
 ```
 
 ## Tests
 
 Run locally:
 
-```
+```bash
 // run both integration and unit tests
 gotestsum -- -tags="integration unit" ./...
 ```
diff --git a/client/client.go b/client/client.go
index 33bd83b..840e560 100644
--- a/client/client.go
+++ b/client/client.go
@@ -2,6 +2,7 @@ package client
 
 import (
 	"fmt"
+	"net/http/httptest"
 	"reflect"
 	"sort"
 	"sync"
@@ -55,7 +56,7 @@ func SetUpClients(conf *viper.Viper) *Clients {
 	return &clients
 }
 
-func SetUpTestClients(t *testing.T, conf *viper.Viper) *Clients {
+func SetUpTestClients(t *testing.T, conf *viper.Viper, te *httptest.Server) *Clients {
 	postgresClient, err := InitializePostgresClient(conf)
 	if err != nil {
 		panic(fmt.Errorf("starting postgres client failed: %v", err))
@@ -90,6 +91,7 @@ func SetUpTestClients(t *testing.T, conf *viper.Viper) *Clients {
 		NomadServer:          ns,
 		PostgresClient:       postgresClient,
 		DockerRegistryClient: InitializeDockerRegistryClient(conf),
+		DockerhubApi:         InitializeDockerhubTestApi(te),
 		DockerTags:           dockerTags,
 		DigestMap:            digestMap,
 	}
diff --git a/client/dockerhub_api_client.go b/client/dockerhub_api_client.go
index 5290b0b..316b241 100644
--- a/client/dockerhub_api_client.go
+++ b/client/dockerhub_api_client.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"net/http/httptest"
 
 	"github.com/dsaidgovsg/registrywatcher/log"
 	"github.com/pkg/errors"
@@ -204,3 +205,14 @@ func (api *DockerhubApi) GetTagDigestFromApi(repository string, checkTag string)
 	// image tag not found in repository's 100 last active tags
 	return nil, errors.New(fmt.Sprintf("Tag %s not found in repository %s", checkTag, repository))
 }
+
+func InitializeDockerhubTestApi(ts *httptest.Server) *DockerhubApi {
+	client := DockerhubApi{
+		url:       ts.URL,
+		namespace: "namespace",
+		username:  "username",
+		secret:    "secret",
+		token:     "fake token",
+	}
+	return &client
+}
diff --git a/client/setup.go b/client/setup.go
index 8b6dafd..123f1de 100644
--- a/client/setup.go
+++ b/client/setup.go
@@ -2,6 +2,7 @@ package client
 
 import (
 	"fmt"
+	"net/http/httptest"
 	"testing"
 
 	"github.com/dsaidgovsg/registrywatcher/config"
@@ -17,6 +18,7 @@ type testEngine struct {
 	Conf         *viper.Viper
 	helper       *testutils.TestHelper
 	Clients      *Clients
+	MockServer   *httptest.Server
 	TestRepoName string
 }
 
@@ -61,8 +63,11 @@ func SetUpClientTest(t *testing.T) *testEngine {
 	te.containerIDs = append(te.containerIDs, regID)
 	te.containerIDs = append(te.containerIDs, pgID)
 
+	// initialize mock server
+	te.MockServer = testutils.SetUpMockDockerhubServer()
+
 	// initialize the clients
-	te.Clients = SetUpTestClients(t, conf)
+	te.Clients = SetUpTestClients(t, conf, te.MockServer)
 
 	// we use this so much might as well keep it in the struct
 	te.TestRepoName = te.Conf.GetStringSlice("watched_repositories")[0]
@@ -129,6 +134,8 @@ func (te *testEngine) PushNewTag(namedTag, actualTag string) {
 	mockImageName := utils.ConstructImageName(registryDomain, registryPrefix, te.TestRepoName, namedTag)
 	publicImageName := fmt.Sprintf("%s:%s", te.Conf.GetString("base_public_image"), actualTag)
 	err := te.helper.AddImageToRegistry(publicImageName, mockImageName)
+
+	// te.MockServer.ImageTag[actualTag] = namedTag
 	if err != nil {
 		panic(fmt.Errorf("couldn't add image to registry: %v", err))
 	}
diff --git a/go.mod b/go.mod
index f94238d..954ee87 100644
--- a/go.mod
+++ b/go.mod
@@ -12,7 +12,7 @@ require (
 	github.com/fatih/color v1.12.0 // indirect
 	github.com/gin-contrib/cors v1.3.0
 	github.com/gin-gonic/gin v1.4.0
-	github.com/gorilla/mux v1.7.1 // indirect
+	github.com/gorilla/mux v1.7.1
 	github.com/gorilla/websocket v1.4.0 // indirect
 	github.com/hashicorp/consul v1.4.4 // indirect
 	github.com/hashicorp/go-hclog v0.9.2 // indirect
diff --git a/testutils/mock_server.go b/testutils/mock_server.go
new file mode 100644
index 0000000..7156a1c
--- /dev/null
+++ b/testutils/mock_server.go
@@ -0,0 +1,46 @@
+package testutils
+
+import (
+	"fmt"
+	"github.com/gorilla/mux"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+)
+
+func SetUpMockDockerhubServer() *httptest.Server {
+	router := mux.NewRouter()
+	ImageTag := make(map[string]string)
+
+	router.HandleFunc("/v2/users/login", func(res http.ResponseWriter, req *http.Request) {
+		res.Write([]byte(`{"token": "test token"}`))
+	}).Methods("GET")
+
+	router.HandleFunc("/v2/namespaces/{namespace}/repositories/{repo}/images/{digest}/tags",
+		func(res http.ResponseWriter, req *http.Request) {
+			vars := mux.Vars(req)
+			digest := vars["digest"]
+
+			if tag, ok := ImageTag[digest]; ok {
+				res.Write([]byte(fmt.Sprintf(`{"results": [{"tag": "%s", "is_current": true}]}`, tag)))
+			} else {
+				res.Write([]byte(`{"results": nil}`))
+			}
+		}).Methods("GET")
+
+	router.HandleFunc("/v2/namespaces/{namespace}/repositories/{repo}/images",
+		func(res http.ResponseWriter, req *http.Request) {
+			var resSlice []string
+			for image, tag := range ImageTag {
+				imageStr := fmt.Sprintf(`{"digest": "%s", "tags": ["tag": "%s", "is_current": true]}`, image, tag)
+				resSlice = append(resSlice, imageStr)
+			}
+			resString := strings.Join(resSlice, ",")
+			res.Write([]byte(fmt.Sprintf(`{"results": [%s]}`, resString)))
+		})
+
+	ts := httptest.NewServer(router)
+	defer ts.Close()
+
+	return ts
+}
diff --git a/testutils/snakeoil/cert.pem b/testutils/snakeoil/cert.pem
index 19d7724..63a4ad9 100644
--- a/testutils/snakeoil/cert.pem
+++ b/testutils/snakeoil/cert.pem
@@ -1,19 +1,19 @@
 -----BEGIN CERTIFICATE-----
-MIIDAjCCAeqgAwIBAgIQXPx/2Iddx4oM41yA7X8HzDANBgkqhkiG9w0BAQsFADAS
-MRAwDgYDVQQKEwdBY21lIENvMB4XDTIwMDYwNzExNDg0MFoXDTIxMDYwNzExNDg0
-MFowEjEQMA4GA1UEChMHQWNtZSBDbzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
-AQoCggEBAND3WGSWGaZEkGfJ8lZabPLyMUgGHhivTPTbjwGl+3O2BW59/wdJ3q5R
-nD4ilmoKipJwGnlwhwvrHJA17fGgZVWslXqCo1Gq3WYoUZ/bJD6EpZYPqgH6RY5l
-pVsmiY8hCRDNcgfdfl1NbxwkQ3zJgUEUyEg2/BTLQ3Dvr4byUx8yGptBg6tek4t2
-oXk1WYF6N4RVTtfcgys3S0H/F7rpO3e+eK5trMTRQ7m4Y/XFt22J3BujvvQffblJ
-uGHKt6Am/RFiSHyU0k2NHwitCZtBesWvg8zvh/TFLrsC4Ib+YeNNihn535y6HLVq
-GLu9TOgdShjHbsrez9WsCeQfub30NQcCAwEAAaNUMFIwDgYDVR0PAQH/BAQDAgKk
-MBMGA1UdJQQMMAoGCCsGAQUFBwMBMA8GA1UdEwEB/wQFMAMBAf8wGgYDVR0RBBMw
-EYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEBCwUAA4IBAQBU747cFtp1Rg/z
-iB7QM66AzWFO2spWtXEKFkQZXmmsTbI+OltwtyD3GJRiGK4nfQYJErGAed2f8SG4
-N6829rffDdc5EftqhVoqtVrCgLtkdHH9tyKzCTDnPkwAZvDHjfv+gs61G6AoaxXa
-Q0UEy/OFA7eP7FodXms1SQeC0rnm+O/+wRZwzS6cllNpsf2gI/VTJbzJyi+GG0OW
-xoaiwpqYnfN8ESphdrp+W/ECA5IXofenlBuDWHTU9l7agqv8wZsYHaxI5yYNnZi6
-nHPjlT/M5EsZCb/k+/fok0/G6FMc+OXC0uDlbTq8/uKslkTsIbCkZQyjgF+h9Q6h
-7krNzH1J
+MIIDITCCAgmgAwIBAgIQeZpSa+H8ymWFJjpvPaNaUTANBgkqhkiG9w0BAQsFADAS
+MRAwDgYDVQQKEwdBY21lIENvMB4XDTIxMDcyNjA2MTI0MloXDTIyMDcyNjA2MTI0
+MlowEjEQMA4GA1UEChMHQWNtZSBDbzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBAMkqOYxszVgyXQs8LbGfoMPjFdMzmS/AZSxUjY1GUsBF2gKv2FY5CC7o
+SGbrZOpTieLxzmND7Fx/J7IcaablfVOnPymSbqrE/xzBjha3ivCu7fSMUz+yWSSs
+jID2jPd8OShGW+nimrj1r4SnfZZlB1Fch0F2jf1nU35Nlz+1ZhqGWPTZlQoo40qw
+YAAwzD/ypCOwFWtPLCwiht43cE2ymReD/J//ZhJM7wpVTj+8ZbW3wJK6N/Y9oRg/
+Tf9dA6LNkSD/OVFuxkgX6ucOM2mM8o8X6KAirT7ieFdUuVF8JS0dgEsEDdoDyrIL
+iAiJYhTKilv4aFTWUZkvOrYWVOHr2gMCAwEAAaNzMHEwDgYDVR0PAQH/BAQDAgKk
+MBMGA1UdJQQMMAoGCCsGAQUFBwMBMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYE
+FMmAx81t71R+M82Sb3rfc5+7jjjXMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAA
+ATANBgkqhkiG9w0BAQsFAAOCAQEAvzZmkRuUHLwqkfPyTVH99wZS3DWf4AgbJZP2
+PYJpIRJlEqqviUwUqoq5VE0H8VLV6k1nDi/Y3kKOQE0ZhvtP62cqp6Ej+C/fkWtH
+JQSdQudQd5CtAh3FoaHnJrwdNqvJ7lWPxXVZE0eCT6HXJyns2EzYHyXCDkrMi8jJ
+wEdt2W9yOkgtfnotHCV6uuLFab7wmvcnqt4x71YbtfN0Iq3vNx1d9A5i4Qhz4fHT
+xC3icbMBgdHqdKEbFBrPvtQfRVg4huftRNzgGDV+FU9uuM1CCtQ4WlrGpH86RsTW
+ekG9Nezx9v8nHZP9rC1YeruITRlgd4I4/BynhGEsuUD20929Tw==
 -----END CERTIFICATE-----
diff --git a/testutils/snakeoil/key.pem b/testutils/snakeoil/key.pem
index 0ce2090..0ec5368 100644
--- a/testutils/snakeoil/key.pem
+++ b/testutils/snakeoil/key.pem
@@ -1,27 +1,28 @@
------BEGIN RSA PRIVATE KEY-----
-MIIEogIBAAKCAQEA0PdYZJYZpkSQZ8nyVlps8vIxSAYeGK9M9NuPAaX7c7YFbn3/
-B0nerlGcPiKWagqKknAaeXCHC+sckDXt8aBlVayVeoKjUardZihRn9skPoSllg+q
-AfpFjmWlWyaJjyEJEM1yB91+XU1vHCRDfMmBQRTISDb8FMtDcO+vhvJTHzIam0GD
-q16Ti3aheTVZgXo3hFVO19yDKzdLQf8Xuuk7d754rm2sxNFDubhj9cW3bYncG6O+
-9B99uUm4Ycq3oCb9EWJIfJTSTY0fCK0Jm0F6xa+DzO+H9MUuuwLghv5h402KGfnf
-nLoctWoYu71M6B1KGMduyt7P1awJ5B+5vfQ1BwIDAQABAoIBACXuP+VCObWsjGnK
-hmqWT2BqBfgE5EGm0FlB4VtzPYlraOD9jeVMy1bpCWtd69+jujOKlcOlYV4Qd0bW
-AzLw4nQCt79dJzgTw8xfVTGnHZUbjtMGVN23CBko6jL5c86/OF4MbL5wvhdBjwGU
-s4tbKFMf6prxICDGJJ/DYiEkq2Hl6le/KX2/MvxD/QpqEWjD11PIKrB5mXle37Rl
-96l+gPwTAZjuSTAFRlkW0N80oaj7ePVngjEIPK2okrDJyvbSvTA5dceoPCyeDrna
-RAadkHY2UBGf8bM5mX7UNesFr43x1ze3CSzCwyHHXGHRBSiU29ThBs68+Ch4jdf0
-XvNg/AECgYEA8CKBMn4nUWX3IMOsM8mWP1jlRZXXJ+wJx1C360tckcq3PowiyFcj
-VcUoTM/qEdKFYbiIk51MhEnPXkqxQamfEQtPwWQ7DqJ3MDs+Gjzat0kLTMCwxmy7
-qCMUVmDWcnPQYuIBvypfKBdupNsP2lQBwPHeQPeXEM35KLrDi6MYDsECgYEA3sWs
-fgEg3Kuhu/dRzVcGOh1U3jRT/bNgn5bx3h14H2CA/k9mkHOQ3qscs67HJDGLwMf7
-KA53F0Oi+U7ISxUNbki6FMH4PXPGUtQgTd1rmpqfmo6RqjjF8B4XwW2yS6Jq9yci
-oXvpVUXIiILoLP3xFvkvKxNqB5Bgn7r1bMas/ccCgYA8/xbAQ6krFPJ0IR/xu6ft
-iWAbj9cqLdvCBC/jla363nozahc/ulqgY8c706hKRKZazyvqCw+vnqwM8H4cBsD/
-7Fai8hnKOtx5pOK+eWy5F9XF9kAw9ten+tkko851HhP1uiDldRbcsS8KsxZQx/v2
-Dbn92KZHv3olrWMUaiYnQQKBgF+nKQWBnoah+3QmIa21rCrP5VzzwwywNNDRk1D3
-/lgaq8iGd1eXB6FUf6J3bXKRA5znQWRVsbT/ib7aoQ38ICQ2fg/jqZ9ZUtLF2okU
-hduAgP9wMAOTFQqhITogy+nF93/gAfKsree7avuSmfaXoylmHWa1k1VkHNj5AeCO
-/kCJAoGAYTZGxQ97WpRupawLlir3kaPEwAlyltGvISHNweOcnbqIrTFeRXq9vnpj
-jUPVRxI8pS7DULHyOFrp5mpCiyptNCGOOR5DTRU7059XWr2nXloq83U2/zyfvhWC
-mc5mZ0Tpp0QVq3qvBMqH/UpqDSWPtHi9DjL4oy+Ve3oyfr3Mc+Q=
------END RSA PRIVATE KEY-----
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDJKjmMbM1YMl0L
+PC2xn6DD4xXTM5kvwGUsVI2NRlLARdoCr9hWOQgu6Ehm62TqU4ni8c5jQ+xcfyey
+HGmm5X1Tpz8pkm6qxP8cwY4Wt4rwru30jFM/slkkrIyA9oz3fDkoRlvp4pq49a+E
+p32WZQdRXIdBdo39Z1N+TZc/tWYahlj02ZUKKONKsGAAMMw/8qQjsBVrTywsIobe
+N3BNspkXg/yf/2YSTO8KVU4/vGW1t8CSujf2PaEYP03/XQOizZEg/zlRbsZIF+rn
+DjNpjPKPF+igIq0+4nhXVLlRfCUtHYBLBA3aA8qyC4gIiWIUyopb+GhU1lGZLzq2
+FlTh69oDAgMBAAECggEAc0sBCqt9ETRkQUEvSyMa58opww9Kl6SkF8EqE7MfUFV4
+epu/2ML9Hq1AecQl5WsVlv1RdXEDcYvq+1jvcJkMuDodqDzlsngPYrLINdriBcms
+VcFrIRGdp7hBGtSzvE/1DlEs8qJOXP1qqBO6lcy3YfkvB+Tjwj1LDBN6QSg9Kaua
+Y9vx5o41uNvlV9T5R5gwHEYcxtXYvH6bCT71BtQ3UbT8NQX/LIKWUKvnkVQtOqHR
+2FCXmYwwCucUYyB51FXKuEk4rNd32iU93BRTHEBhylcsyJBywX2/2JYv6VK09UQ3
+4KTVOz+dRfo7t+WN+QZemUvgafWUnaQzyb8ygl7uoQKBgQD91RRBJWU2IFk2fkC3
+6DvO64ukZ38WY1BR2vGpnc+ebuiffXTAjYPgPbiCm7sZPboQef7P4MeM1SInQe66
+KfrVjVQIvn3BTH99b+4DfEKlw0CbjkJJhwtsXLzTKipMgSAfTSPablOi2Re4IEp4
+HH11d6ckr2UOSiet/UbWIq2PEwKBgQDK4gF4U4fJOYO6U62lsdOcOyj+cgJ7f3L2
+aaZBb+Z7OK+pQ9BZIE/NkokRzjH8Q6dfQ05WYgBssZY26XQdV6XDXdN2mMnMQmUg
+bvdakW0ujPQywbQJ0i+EfVtaVBgHJAENLnPQRFz+xdMNPV7YiB8Cx0CoGiqFJUyG
++grkIbm3UQKBgF4e3ZTOdfodApeXNw46ebrfUndSvSgjCUxg4A0h7DN2MqG5Fkqz
+7rt+h2U+s6DHZ6/mXJreYYASI8jkxM2Y9V6sOgXIBL9hS0DK+TmnoNrwTIuhs2iq
+P40d0hKG5wpZMv0IUIwVLsQKtaXbq2FgRxEAniSXOOSUkKQOb27Fiw+rAoGBAJUn
+ZLdGibxZKUvxuubm/lCMec3emY6bmzQnZdAT6VlLynDhZyovBBx5D4ZklgEE5+a0
+P8JoGDHQuUOzi+cKidZAg3dcSnd9lIUsr3MJGUCw5dLu72KNANwhoOiejG+re45q
+SUCjEc9INqo0JSeBo3LSc+liG0J5snJFH36qgLORAoGBAOq5NJczg1SOWORyG6L7
+JoQ3mpwtM2PazNUZD0UcDTFJvwkFS8KGXyBfk9si2vUVJOZAZRKBq8K5pIlPF3TD
+CLuoD3HMxwS4LGpvZa/kEZvk3nHROOQegArIigne3Bbo3qIt09jOeTZIBOM0FzeE
+UwiqkMMojrYsL1bHEJMRSMPd
+-----END PRIVATE KEY-----

From ccdfbb633fd3ea510d52f16600d1d20be13ab8d1 Mon Sep 17 00:00:00 2001
From: qbiqing <bq2134@columbia.edu>
Date: Tue, 27 Jul 2021 12:46:07 +0800
Subject: [PATCH 09/25] MockDockerhubServer struct

---
 client/client.go               |  6 +++---
 client/dockerhub_api_client.go |  6 +++---
 client/setup.go                |  5 ++---
 testutils/mock_server.go       | 23 ++++++++++++++++++-----
 4 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/client/client.go b/client/client.go
index 840e560..dca58d8 100644
--- a/client/client.go
+++ b/client/client.go
@@ -2,13 +2,13 @@ package client
 
 import (
 	"fmt"
-	"net/http/httptest"
 	"reflect"
 	"sort"
 	"sync"
 	"testing"
 
 	"github.com/dsaidgovsg/registrywatcher/log"
+	"github.com/dsaidgovsg/registrywatcher/testutils"
 	"github.com/dsaidgovsg/registrywatcher/utils"
 	nomad "github.com/hashicorp/nomad/api"
 	"github.com/hashicorp/nomad/testutil"
@@ -56,7 +56,7 @@ func SetUpClients(conf *viper.Viper) *Clients {
 	return &clients
 }
 
-func SetUpTestClients(t *testing.T, conf *viper.Viper, te *httptest.Server) *Clients {
+func SetUpTestClients(t *testing.T, conf *viper.Viper, mds *testutils.MockDockerhubServer) *Clients {
 	postgresClient, err := InitializePostgresClient(conf)
 	if err != nil {
 		panic(fmt.Errorf("starting postgres client failed: %v", err))
@@ -91,7 +91,7 @@ func SetUpTestClients(t *testing.T, conf *viper.Viper, te *httptest.Server) *Cli
 		NomadServer:          ns,
 		PostgresClient:       postgresClient,
 		DockerRegistryClient: InitializeDockerRegistryClient(conf),
-		DockerhubApi:         InitializeDockerhubTestApi(te),
+		DockerhubApi:         InitializeDockerhubTestApi(mds),
 		DockerTags:           dockerTags,
 		DigestMap:            digestMap,
 	}
diff --git a/client/dockerhub_api_client.go b/client/dockerhub_api_client.go
index 316b241..3dd3457 100644
--- a/client/dockerhub_api_client.go
+++ b/client/dockerhub_api_client.go
@@ -6,9 +6,9 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
-	"net/http/httptest"
 
 	"github.com/dsaidgovsg/registrywatcher/log"
+	"github.com/dsaidgovsg/registrywatcher/testutils"
 	"github.com/pkg/errors"
 	"github.com/spf13/viper"
 )
@@ -206,9 +206,9 @@ func (api *DockerhubApi) GetTagDigestFromApi(repository string, checkTag string)
 	return nil, errors.New(fmt.Sprintf("Tag %s not found in repository %s", checkTag, repository))
 }
 
-func InitializeDockerhubTestApi(ts *httptest.Server) *DockerhubApi {
+func InitializeDockerhubTestApi(mds *testutils.MockDockerhubServer) *DockerhubApi {
 	client := DockerhubApi{
-		url:       ts.URL,
+		url:       mds.Ts.URL,
 		namespace: "namespace",
 		username:  "username",
 		secret:    "secret",
diff --git a/client/setup.go b/client/setup.go
index 123f1de..10cfff2 100644
--- a/client/setup.go
+++ b/client/setup.go
@@ -2,7 +2,6 @@ package client
 
 import (
 	"fmt"
-	"net/http/httptest"
 	"testing"
 
 	"github.com/dsaidgovsg/registrywatcher/config"
@@ -18,7 +17,7 @@ type testEngine struct {
 	Conf         *viper.Viper
 	helper       *testutils.TestHelper
 	Clients      *Clients
-	MockServer   *httptest.Server
+	MockServer   *testutils.MockDockerhubServer
 	TestRepoName string
 }
 
@@ -64,7 +63,7 @@ func SetUpClientTest(t *testing.T) *testEngine {
 	te.containerIDs = append(te.containerIDs, pgID)
 
 	// initialize mock server
-	te.MockServer = testutils.SetUpMockDockerhubServer()
+	te.MockServer = testutils.SetUpMockDockerhubServer(make(map[string]string))
 
 	// initialize the clients
 	te.Clients = SetUpTestClients(t, conf, te.MockServer)
diff --git a/testutils/mock_server.go b/testutils/mock_server.go
index 7156a1c..de67796 100644
--- a/testutils/mock_server.go
+++ b/testutils/mock_server.go
@@ -8,9 +8,13 @@ import (
 	"strings"
 )
 
-func SetUpMockDockerhubServer() *httptest.Server {
+type MockDockerhubServer struct {
+	Ts *httptest.Server
+	ImageTag map[string]string
+}
+
+func SetUpMockDockerhubServer(imageTag map[string]string) *MockDockerhubServer {
 	router := mux.NewRouter()
-	ImageTag := make(map[string]string)
 
 	router.HandleFunc("/v2/users/login", func(res http.ResponseWriter, req *http.Request) {
 		res.Write([]byte(`{"token": "test token"}`))
@@ -21,7 +25,7 @@ func SetUpMockDockerhubServer() *httptest.Server {
 			vars := mux.Vars(req)
 			digest := vars["digest"]
 
-			if tag, ok := ImageTag[digest]; ok {
+			if tag, ok := imageTag[digest]; ok {
 				res.Write([]byte(fmt.Sprintf(`{"results": [{"tag": "%s", "is_current": true}]}`, tag)))
 			} else {
 				res.Write([]byte(`{"results": nil}`))
@@ -31,7 +35,7 @@ func SetUpMockDockerhubServer() *httptest.Server {
 	router.HandleFunc("/v2/namespaces/{namespace}/repositories/{repo}/images",
 		func(res http.ResponseWriter, req *http.Request) {
 			var resSlice []string
-			for image, tag := range ImageTag {
+			for image, tag := range imageTag {
 				imageStr := fmt.Sprintf(`{"digest": "%s", "tags": ["tag": "%s", "is_current": true]}`, image, tag)
 				resSlice = append(resSlice, imageStr)
 			}
@@ -42,5 +46,14 @@ func SetUpMockDockerhubServer() *httptest.Server {
 	ts := httptest.NewServer(router)
 	defer ts.Close()
 
-	return ts
+	mds := MockDockerhubServer{
+		Ts: ts,
+		ImageTag: imageTag,
+	}
+
+	return &mds
+}
+
+func (mds *MockDockerhubServer) PushNewTag (tag string, image string) {
+	mds.ImageTag[image] = tag
 }

From dd17097a081808c585ae25d70ef973ed36e52713 Mon Sep 17 00:00:00 2001
From: qbiqing <bq2134@columbia.edu>
Date: Tue, 27 Jul 2021 14:34:52 +0800
Subject: [PATCH 10/25] tests

---
 client/client.go               |  9 ++--
 client/client_test.go          |  3 +-
 client/dockerhub_api_client.go | 12 ------
 client/setup.go                | 76 ++++++++++++++++++++++++++++++++--
 testutils/mock_server.go       | 59 --------------------------
 5 files changed, 79 insertions(+), 80 deletions(-)
 delete mode 100644 testutils/mock_server.go

diff --git a/client/client.go b/client/client.go
index dca58d8..0b80fd8 100644
--- a/client/client.go
+++ b/client/client.go
@@ -8,7 +8,6 @@ import (
 	"testing"
 
 	"github.com/dsaidgovsg/registrywatcher/log"
-	"github.com/dsaidgovsg/registrywatcher/testutils"
 	"github.com/dsaidgovsg/registrywatcher/utils"
 	nomad "github.com/hashicorp/nomad/api"
 	"github.com/hashicorp/nomad/testutil"
@@ -56,7 +55,7 @@ func SetUpClients(conf *viper.Viper) *Clients {
 	return &clients
 }
 
-func SetUpTestClients(t *testing.T, conf *viper.Viper, mds *testutils.MockDockerhubServer) *Clients {
+func SetUpTestClients(t *testing.T, conf *viper.Viper) *Clients {
 	postgresClient, err := InitializePostgresClient(conf)
 	if err != nil {
 		panic(fmt.Errorf("starting postgres client failed: %v", err))
@@ -91,7 +90,7 @@ func SetUpTestClients(t *testing.T, conf *viper.Viper, mds *testutils.MockDocker
 		NomadServer:          ns,
 		PostgresClient:       postgresClient,
 		DockerRegistryClient: InitializeDockerRegistryClient(conf),
-		DockerhubApi:         InitializeDockerhubTestApi(mds),
+		DockerhubApi:         nil,
 		DockerTags:           dockerTags,
 		DigestMap:            digestMap,
 	}
@@ -329,10 +328,14 @@ func (client *Clients) ShouldDeploy(repoName string) (bool, error) {
 	isDigestChanged, err := client.isTagDigestChanged(repoName)
 	if err != nil {
 		log.LogAppErr(fmt.Sprintf("Couldn't check tag digest changed while checking whether to deploy for %s", repoName), err)
+		fmt.Println("returning at isDigestChanged, ", err)
 		return false, err
 	}
 
 	if (pinnedTag == "" && client.isNewReleaseTagAvailable(repoName)) || isDigestChanged {
+		fmt.Println("returning 1")
+		fmt.Println("isTagDigestChanged", isDigestChanged)
+		fmt.Println("isNewReleaseTagAvailable", client.isNewReleaseTagAvailable(repoName))
 		return true, nil
 	}
 
diff --git a/client/client_test.go b/client/client_test.go
index 5712f0c..ade770b 100644
--- a/client/client_test.go
+++ b/client/client_test.go
@@ -3,9 +3,8 @@
 package client
 
 import (
-	"testing"
-
 	"github.com/stretchr/testify/assert"
+	"testing"
 )
 
 // test ShouldDeploy for a multitude of conditions
diff --git a/client/dockerhub_api_client.go b/client/dockerhub_api_client.go
index 3dd3457..5290b0b 100644
--- a/client/dockerhub_api_client.go
+++ b/client/dockerhub_api_client.go
@@ -8,7 +8,6 @@ import (
 	"net/http"
 
 	"github.com/dsaidgovsg/registrywatcher/log"
-	"github.com/dsaidgovsg/registrywatcher/testutils"
 	"github.com/pkg/errors"
 	"github.com/spf13/viper"
 )
@@ -205,14 +204,3 @@ func (api *DockerhubApi) GetTagDigestFromApi(repository string, checkTag string)
 	// image tag not found in repository's 100 last active tags
 	return nil, errors.New(fmt.Sprintf("Tag %s not found in repository %s", checkTag, repository))
 }
-
-func InitializeDockerhubTestApi(mds *testutils.MockDockerhubServer) *DockerhubApi {
-	client := DockerhubApi{
-		url:       mds.Ts.URL,
-		namespace: "namespace",
-		username:  "username",
-		secret:    "secret",
-		token:     "fake token",
-	}
-	return &client
-}
diff --git a/client/setup.go b/client/setup.go
index 10cfff2..2866fe4 100644
--- a/client/setup.go
+++ b/client/setup.go
@@ -10,6 +10,11 @@ import (
 	"github.com/dsaidgovsg/registrywatcher/utils"
 	nomad "github.com/hashicorp/nomad/api"
 	"github.com/spf13/viper"
+
+	"encoding/json"
+	"github.com/gorilla/mux"
+	"net/http"
+	"net/http/httptest"
 )
 
 type testEngine struct {
@@ -17,7 +22,8 @@ type testEngine struct {
 	Conf         *viper.Viper
 	helper       *testutils.TestHelper
 	Clients      *Clients
-	MockServer   *testutils.MockDockerhubServer
+	ImageTagMap  map[string][]string
+	Ts           *httptest.Server
 	TestRepoName string
 }
 
@@ -63,14 +69,73 @@ func SetUpClientTest(t *testing.T) *testEngine {
 	te.containerIDs = append(te.containerIDs, pgID)
 
 	// initialize mock server
-	te.MockServer = testutils.SetUpMockDockerhubServer(make(map[string]string))
+	te.ImageTagMap = make(map[string][]string)
 
 	// initialize the clients
-	te.Clients = SetUpTestClients(t, conf, te.MockServer)
+	te.Clients = SetUpTestClients(t, conf)
 
 	// we use this so much might as well keep it in the struct
 	te.TestRepoName = te.Conf.GetStringSlice("watched_repositories")[0]
 
+	// set up mock Dockerhub API and imageTag store
+	router := mux.NewRouter()
+
+	router.HandleFunc("/v2/users/login", func(res http.ResponseWriter, req *http.Request) {
+		res.Write([]byte(`{"token": "test token"}`))
+	}).Methods("GET")
+
+	router.HandleFunc("/v2/namespaces/{namespace}/repositories/{repo}/images/{digest}/tags",
+		func(res http.ResponseWriter, req *http.Request) {
+			vars := mux.Vars(req)
+			digest := vars["digest"]
+
+			res.Header().Set("Content-Type", "application/json")
+			res.WriteHeader(http.StatusOK)
+
+			if tagSlice, ok := te.ImageTagMap[digest]; ok {
+				var tags []TagWithStatus
+				for enum, tag := range tagSlice {
+					tags = append(tags, TagWithStatus{Tag: tag, IsCurrent: enum == 0})
+				}
+				results := CheckImageResp{Results: tags}
+				data, _ := json.Marshal(results)
+				res.Write(data)
+			} else {
+				results := CheckImageResp{Results: nil}
+				data, _ := json.Marshal(results)
+				res.Write(data)
+			}
+		}).Methods("GET")
+
+	router.HandleFunc("/v2/namespaces/namespace/repositories/testrepo/images",
+		func(res http.ResponseWriter, req *http.Request) {
+			var resSlice []GetTagDigestResult
+			for image, tagSlice := range te.ImageTagMap {
+				var tags []TagWithStatus
+				for enum, tag := range tagSlice {
+					tags = append(tags, TagWithStatus{Tag: tag, IsCurrent: enum == 0})
+				}
+				resSlice = append(resSlice, GetTagDigestResult{Digest: image, Tags: tags})
+			}
+			res.Header().Set("Content-Type", "application/json")
+			res.WriteHeader(http.StatusOK)
+			results := GetTagDigestResp{Results: resSlice}
+			data, _ := json.Marshal(results)
+			res.Write(data)
+		})
+
+	ts := httptest.NewServer(router)
+	te.Ts = ts
+
+	dockerhubApi := DockerhubApi{
+		url:       ts.URL,
+		namespace: "namespace",
+		username:  "username",
+		secret:    "secret",
+		token:     "fake token",
+	}
+
+	te.Clients.DockerhubApi = &dockerhubApi
 	return &te
 }
 
@@ -124,6 +189,7 @@ func (te *testEngine) TearDown() {
 			log.LogAppErr("Couldn't remove container", err)
 		}
 	}
+	te.Ts.Close()
 }
 
 // named tag is what it will appear as in the docker registryDomain
@@ -134,7 +200,9 @@ func (te *testEngine) PushNewTag(namedTag, actualTag string) {
 	publicImageName := fmt.Sprintf("%s:%s", te.Conf.GetString("base_public_image"), actualTag)
 	err := te.helper.AddImageToRegistry(publicImageName, mockImageName)
 
-	// te.MockServer.ImageTag[actualTag] = namedTag
+	originalTags := te.ImageTagMap[actualTag]
+	te.ImageTagMap[actualTag] = append([]string{namedTag}, originalTags...)
+
 	if err != nil {
 		panic(fmt.Errorf("couldn't add image to registry: %v", err))
 	}
diff --git a/testutils/mock_server.go b/testutils/mock_server.go
deleted file mode 100644
index de67796..0000000
--- a/testutils/mock_server.go
+++ /dev/null
@@ -1,59 +0,0 @@
-package testutils
-
-import (
-	"fmt"
-	"github.com/gorilla/mux"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-)
-
-type MockDockerhubServer struct {
-	Ts *httptest.Server
-	ImageTag map[string]string
-}
-
-func SetUpMockDockerhubServer(imageTag map[string]string) *MockDockerhubServer {
-	router := mux.NewRouter()
-
-	router.HandleFunc("/v2/users/login", func(res http.ResponseWriter, req *http.Request) {
-		res.Write([]byte(`{"token": "test token"}`))
-	}).Methods("GET")
-
-	router.HandleFunc("/v2/namespaces/{namespace}/repositories/{repo}/images/{digest}/tags",
-		func(res http.ResponseWriter, req *http.Request) {
-			vars := mux.Vars(req)
-			digest := vars["digest"]
-
-			if tag, ok := imageTag[digest]; ok {
-				res.Write([]byte(fmt.Sprintf(`{"results": [{"tag": "%s", "is_current": true}]}`, tag)))
-			} else {
-				res.Write([]byte(`{"results": nil}`))
-			}
-		}).Methods("GET")
-
-	router.HandleFunc("/v2/namespaces/{namespace}/repositories/{repo}/images",
-		func(res http.ResponseWriter, req *http.Request) {
-			var resSlice []string
-			for image, tag := range imageTag {
-				imageStr := fmt.Sprintf(`{"digest": "%s", "tags": ["tag": "%s", "is_current": true]}`, image, tag)
-				resSlice = append(resSlice, imageStr)
-			}
-			resString := strings.Join(resSlice, ",")
-			res.Write([]byte(fmt.Sprintf(`{"results": [%s]}`, resString)))
-		})
-
-	ts := httptest.NewServer(router)
-	defer ts.Close()
-
-	mds := MockDockerhubServer{
-		Ts: ts,
-		ImageTag: imageTag,
-	}
-
-	return &mds
-}
-
-func (mds *MockDockerhubServer) PushNewTag (tag string, image string) {
-	mds.ImageTag[image] = tag
-}

From fdc2b82de612f68d9f4341639f6725dbb1df2cf3 Mon Sep 17 00:00:00 2001
From: qbiqing <bq2134@columbia.edu>
Date: Wed, 28 Jul 2021 09:58:12 +0800
Subject: [PATCH 11/25] remove unused

---
 main.go | 61 ---------------------------------------------------------
 1 file changed, 61 deletions(-)

diff --git a/main.go b/main.go
index 824c824..6adc928 100644
--- a/main.go
+++ b/main.go
@@ -70,7 +70,6 @@ func SetUpRouter(conf *viper.Viper, clients *client.Clients) *gin.Engine {
 	r.GET("/tags/:repo_name", handler.GetTagHandler)
 	r.GET("/repos", handler.RepoSummaryHandler)
 	r.GET("/debug/caches", handler.CacheSummaryHandler)
-	r.POST("/tags/:repo_name/listener", handler.DockerhubWebhookListener)
 
 	return r
 }
@@ -109,14 +108,6 @@ type deployBody struct {
 	AutoDeploy *bool   `json:"auto_deploy",omitempty`
 }
 
-type webhookBody struct {
-	PushData *pushData `json:"push_data"`
-}
-
-type pushData struct {
-	Tag *string `json:"tag"`
-}
-
 func (h *Handler) ResetTagHandler(c *gin.Context) {
 
 	pinnedTag := ""
@@ -369,55 +360,3 @@ func (h *Handler) CacheSummaryHandler(c *gin.Context) {
 
 	c.JSON(200, rtn)
 }
-
-func (h *Handler) DockerhubWebhookListener(c *gin.Context) {
-	// check if repoName is valid
-	repoName := c.Param("repo_name")
-	validName := false
-	for _, repo := range h.conf.GetStringSlice("watched_repositories") {
-		if repoName == repo {
-			validName = true
-			break
-		}
-	}
-	if !validName {
-		_, registryDomain, _, _ := utils.ExtractRegistryInfo(h.conf, repoName)
-		c.JSON(400, gin.H{
-			"message": fmt.Sprintf("Error: The specified repo name %s is not inside the docker repository registry %s", repoName, registryDomain),
-		})
-		return
-	}
-
-	pinnedTag, err := h.clients.PostgresClient.GetPinnedTag(repoName)
-	if err != nil {
-		log.LogAppErr(fmt.Sprintf("Couldn't fetch pinned tag while checking whether to deploy for %s", repoName), err)
-		c.JSON(500, gin.H{
-			"message": fmt.Sprintf("Unable to fetch repo %s tag, err: %s", repoName, err),
-		})
-		return
-	}
-
-	var webhookBody webhookBody
-	err = c.BindJSON(&webhookBody)
-	if err != nil {
-		c.JSON(400, gin.H{
-			"message": fmt.Sprintf("Error: %s", err),
-		})
-		return
-	}
-
-	if *(webhookBody.PushData.Tag) == pinnedTag {
-		utils.PostSlackUpdate(h.conf, fmt.Sprintf("Update: the SHA of tag `%s` in `%s` changed. Auto deployment will happen shortly.", pinnedTag, repoName))
-		log.LogAppInfo(fmt.Sprintf("Auto deploying tag %s for repo %s", pinnedTag, repoName))
-
-		h.clients.DeployPinnedTag(h.conf, repoName)
-		h.clients.UpdateCaches(repoName)
-		c.JSON(200, gin.H{
-			"message": fmt.Sprintf("Deploying to %s", pinnedTag),
-		})
-	} else {
-		c.JSON(200, gin.H{
-			"message": fmt.Sprintf("Received push data, branch %s not in deployment", pinnedTag),
-		})
-	}
-}

From c7b8e344765497820aef27ddf8deb8796f5f1bcb Mon Sep 17 00:00:00 2001
From: qbiqing <bq2134@columbia.edu>
Date: Wed, 28 Jul 2021 12:05:06 +0800
Subject: [PATCH 12/25] integration tests run

---
 client/client.go               |  4 ----
 client/client_test.go          |  6 ++++--
 client/dockerhub_api_client.go |  7 +++++--
 client/setup.go                | 33 +++++++++++++++++++--------------
 testutils/testutils.go         | 11 +++++++++++
 5 files changed, 39 insertions(+), 22 deletions(-)

diff --git a/client/client.go b/client/client.go
index 0b80fd8..3bb667b 100644
--- a/client/client.go
+++ b/client/client.go
@@ -328,14 +328,10 @@ func (client *Clients) ShouldDeploy(repoName string) (bool, error) {
 	isDigestChanged, err := client.isTagDigestChanged(repoName)
 	if err != nil {
 		log.LogAppErr(fmt.Sprintf("Couldn't check tag digest changed while checking whether to deploy for %s", repoName), err)
-		fmt.Println("returning at isDigestChanged, ", err)
 		return false, err
 	}
 
 	if (pinnedTag == "" && client.isNewReleaseTagAvailable(repoName)) || isDigestChanged {
-		fmt.Println("returning 1")
-		fmt.Println("isTagDigestChanged", isDigestChanged)
-		fmt.Println("isNewReleaseTagAvailable", client.isNewReleaseTagAvailable(repoName))
 		return true, nil
 	}
 
diff --git a/client/client_test.go b/client/client_test.go
index ade770b..501f2b1 100644
--- a/client/client_test.go
+++ b/client/client_test.go
@@ -3,8 +3,9 @@
 package client
 
 import (
-	"github.com/stretchr/testify/assert"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 // test ShouldDeploy for a multitude of conditions
@@ -69,7 +70,8 @@ func TestAutoDeployLatestTag(t *testing.T) {
 	shouldDeploy, _ = te.Clients.ShouldDeploy(te.TestRepoName)
 	te.Clients.UpdateCaches(te.TestRepoName)
 	tagToDeploy, _ = te.Clients.GetFormattedPinnedTag(te.TestRepoName)
-	assert.False(t, shouldDeploy)
+	// should re-deploy after pinned tag is changed
+	assert.True(t, shouldDeploy)
 	assert.Equal(t, "test", tagToDeploy)
 
 	// "test" is now based on "alpine", rather than the original "latest"
diff --git a/client/dockerhub_api_client.go b/client/dockerhub_api_client.go
index 5290b0b..1c9422b 100644
--- a/client/dockerhub_api_client.go
+++ b/client/dockerhub_api_client.go
@@ -139,8 +139,11 @@ func (api *DockerhubApi) CheckImageIsCurrent(repository, digest string, checkTag
 		}
 	}
 	// image tag does not match the tag to be checked
-	// this throws an error as the stored digest should match the tag
-	return nil, errors.New(fmt.Sprintf("Digest %s does not have tag %s", digest, checkTag))
+	// this means the cached digest belongs to a previous tag
+	// return false (not current) because we want the cache to be updated
+	log.LogAppInfo(fmt.Sprintf(fmt.Sprintf("Digest %s does not have tag %s", digest, checkTag)))
+	isCurrent := false
+	return &isCurrent, nil
 }
 
 // We can't just use the registry GetTagDigest because the digests from the
diff --git a/client/setup.go b/client/setup.go
index 2866fe4..ac9c210 100644
--- a/client/setup.go
+++ b/client/setup.go
@@ -2,6 +2,7 @@ package client
 
 import (
 	"fmt"
+	"strconv"
 	"testing"
 
 	"github.com/dsaidgovsg/registrywatcher/config"
@@ -12,9 +13,10 @@ import (
 	"github.com/spf13/viper"
 
 	"encoding/json"
-	"github.com/gorilla/mux"
 	"net/http"
 	"net/http/httptest"
+
+	"github.com/gorilla/mux"
 )
 
 type testEngine struct {
@@ -22,7 +24,7 @@ type testEngine struct {
 	Conf         *viper.Viper
 	helper       *testutils.TestHelper
 	Clients      *Clients
-	ImageTagMap  map[string][]string
+	ImageTagMap  map[string][]string // digest: [tag, is_current]
 	Ts           *httptest.Server
 	TestRepoName string
 }
@@ -92,11 +94,9 @@ func SetUpClientTest(t *testing.T) *testEngine {
 			res.Header().Set("Content-Type", "application/json")
 			res.WriteHeader(http.StatusOK)
 
-			if tagSlice, ok := te.ImageTagMap[digest]; ok {
-				var tags []TagWithStatus
-				for enum, tag := range tagSlice {
-					tags = append(tags, TagWithStatus{Tag: tag, IsCurrent: enum == 0})
-				}
+			if tag, ok := te.ImageTagMap[digest]; ok {
+				isCurrent, _ := strconv.ParseBool(tag[1])
+				tags := []TagWithStatus{{Tag: tag[0], IsCurrent: isCurrent}}
 				results := CheckImageResp{Results: tags}
 				data, _ := json.Marshal(results)
 				res.Write(data)
@@ -110,11 +110,9 @@ func SetUpClientTest(t *testing.T) *testEngine {
 	router.HandleFunc("/v2/namespaces/namespace/repositories/testrepo/images",
 		func(res http.ResponseWriter, req *http.Request) {
 			var resSlice []GetTagDigestResult
-			for image, tagSlice := range te.ImageTagMap {
-				var tags []TagWithStatus
-				for enum, tag := range tagSlice {
-					tags = append(tags, TagWithStatus{Tag: tag, IsCurrent: enum == 0})
-				}
+			for image, tag := range te.ImageTagMap {
+				isCurrent, _ := strconv.ParseBool(tag[1])
+				tags := []TagWithStatus{{Tag: tag[0], IsCurrent: isCurrent}}
 				resSlice = append(resSlice, GetTagDigestResult{Digest: image, Tags: tags})
 			}
 			res.Header().Set("Content-Type", "application/json")
@@ -200,8 +198,15 @@ func (te *testEngine) PushNewTag(namedTag, actualTag string) {
 	publicImageName := fmt.Sprintf("%s:%s", te.Conf.GetString("base_public_image"), actualTag)
 	err := te.helper.AddImageToRegistry(publicImageName, mockImageName)
 
-	originalTags := te.ImageTagMap[actualTag]
-	te.ImageTagMap[actualTag] = append([]string{namedTag}, originalTags...)
+	imageDigest := testutils.RandSeq(10)
+	te.ImageTagMap[imageDigest] = []string{namedTag, "true"}
+
+	// make is_current false for images holding the same tag
+	for image, tag := range te.ImageTagMap {
+		if tag[0] == namedTag && image != imageDigest {
+			tag[1] = "false"
+		}
+	}
 
 	if err != nil {
 		panic(fmt.Errorf("couldn't add image to registry: %v", err))
diff --git a/testutils/testutils.go b/testutils/testutils.go
index b2ed1b5..c87235c 100644
--- a/testutils/testutils.go
+++ b/testutils/testutils.go
@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"math"
+	"math/rand"
 	"net"
 	"net/http"
 	"os"
@@ -356,3 +357,13 @@ func ConstructRegistryAuth(identity, secret string) (string, error) {
 
 	return base64.URLEncoding.EncodeToString(buf), nil
 }
+
+func RandSeq(n int) string {
+	var letters = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+	b := make([]byte, n)
+	rand.Seed(time.Now().UnixNano())
+	for i := range b {
+		b[i] = letters[rand.Intn(len(letters))]
+	}
+	return string(b)
+}

From f55ca56fb698a890a5dcc24281142e076a9c14bf Mon Sep 17 00:00:00 2001
From: qbiqing <bq2134@columbia.edu>
Date: Fri, 30 Jul 2021 11:30:34 +0800
Subject: [PATCH 13/25] .gitignore, add Makefile

---
 .gitignore                  |  2 ++
 Dockerfile-golang           |  2 +-
 Makefile                    |  9 +++++++++
 codefresh.yml               |  1 +
 test.sh                     |  3 ++-
 testutils/snakeoil/cert.pem | 19 -------------------
 testutils/snakeoil/key.pem  | 28 ----------------------------
 7 files changed, 15 insertions(+), 49 deletions(-)
 create mode 100644 Makefile
 delete mode 100644 testutils/snakeoil/cert.pem
 delete mode 100644 testutils/snakeoil/key.pem

diff --git a/.gitignore b/.gitignore
index 9a318ce..0cbdb96 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,8 @@ registrywatcher
 !test.toml
 !sample.toml
 
+testutils/snakeoil/*.pem
+
 # dependencies
 node_modules
 
diff --git a/Dockerfile-golang b/Dockerfile-golang
index 07b4b7c..ce89ac2 100644
--- a/Dockerfile-golang
+++ b/Dockerfile-golang
@@ -1,6 +1,6 @@
 FROM golang:1.12-alpine
 
 # Only install hard dependencies
-RUN apk add --no-cache gcc git musl-dev; \
+RUN apk add --no-cache gcc git musl-dev make; \
     go get -v -u gotest.tools/gotestsum; \
     :
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..9b96ce1
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,9 @@
+# Setup name variables for the package/tool
+NAME := registrywatcher
+PKG := github.com/dsaidgovsg/$(NAME)
+
+.PHONY: snakeoil
+snakeoil: ## Update snakeoil certs for testing.
+	go run $(GOROOT)/src/crypto/tls/generate_cert.go --host localhost,127.0.0.1 --ca
+	mv $(CURDIR)/key.pem $(CURDIR)/testutils/snakeoil/key.pem
+	mv $(CURDIR)/cert.pem $(CURDIR)/testutils/snakeoil/cert.pem
diff --git a/codefresh.yml b/codefresh.yml
index 8a9d14b..31bd6bb 100644
--- a/codefresh.yml
+++ b/codefresh.yml
@@ -34,6 +34,7 @@ steps:
     working_directory: ${{main_clone}}
     environment:
     - GOPATH=/codefresh/volume/.go
+    - GOROOT=/usr/local/go
     commands:
     - ./test.sh
     when:
diff --git a/test.sh b/test.sh
index 2439d35..5bb2012 100755
--- a/test.sh
+++ b/test.sh
@@ -19,5 +19,6 @@ assert_each_go_file gofmt -l
 echo "OK"
 
 echo "-- go unit tests --"
-gotestsum -- -tags=unit ./...
+make snakeoil
+gotestsum -- -tags="unit integration" ./...
 echo "OK"
diff --git a/testutils/snakeoil/cert.pem b/testutils/snakeoil/cert.pem
deleted file mode 100644
index 63a4ad9..0000000
--- a/testutils/snakeoil/cert.pem
+++ /dev/null
@@ -1,19 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDITCCAgmgAwIBAgIQeZpSa+H8ymWFJjpvPaNaUTANBgkqhkiG9w0BAQsFADAS
-MRAwDgYDVQQKEwdBY21lIENvMB4XDTIxMDcyNjA2MTI0MloXDTIyMDcyNjA2MTI0
-MlowEjEQMA4GA1UEChMHQWNtZSBDbzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
-AQoCggEBAMkqOYxszVgyXQs8LbGfoMPjFdMzmS/AZSxUjY1GUsBF2gKv2FY5CC7o
-SGbrZOpTieLxzmND7Fx/J7IcaablfVOnPymSbqrE/xzBjha3ivCu7fSMUz+yWSSs
-jID2jPd8OShGW+nimrj1r4SnfZZlB1Fch0F2jf1nU35Nlz+1ZhqGWPTZlQoo40qw
-YAAwzD/ypCOwFWtPLCwiht43cE2ymReD/J//ZhJM7wpVTj+8ZbW3wJK6N/Y9oRg/
-Tf9dA6LNkSD/OVFuxkgX6ucOM2mM8o8X6KAirT7ieFdUuVF8JS0dgEsEDdoDyrIL
-iAiJYhTKilv4aFTWUZkvOrYWVOHr2gMCAwEAAaNzMHEwDgYDVR0PAQH/BAQDAgKk
-MBMGA1UdJQQMMAoGCCsGAQUFBwMBMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYE
-FMmAx81t71R+M82Sb3rfc5+7jjjXMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAA
-ATANBgkqhkiG9w0BAQsFAAOCAQEAvzZmkRuUHLwqkfPyTVH99wZS3DWf4AgbJZP2
-PYJpIRJlEqqviUwUqoq5VE0H8VLV6k1nDi/Y3kKOQE0ZhvtP62cqp6Ej+C/fkWtH
-JQSdQudQd5CtAh3FoaHnJrwdNqvJ7lWPxXVZE0eCT6HXJyns2EzYHyXCDkrMi8jJ
-wEdt2W9yOkgtfnotHCV6uuLFab7wmvcnqt4x71YbtfN0Iq3vNx1d9A5i4Qhz4fHT
-xC3icbMBgdHqdKEbFBrPvtQfRVg4huftRNzgGDV+FU9uuM1CCtQ4WlrGpH86RsTW
-ekG9Nezx9v8nHZP9rC1YeruITRlgd4I4/BynhGEsuUD20929Tw==
------END CERTIFICATE-----
diff --git a/testutils/snakeoil/key.pem b/testutils/snakeoil/key.pem
deleted file mode 100644
index 0ec5368..0000000
--- a/testutils/snakeoil/key.pem
+++ /dev/null
@@ -1,28 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDJKjmMbM1YMl0L
-PC2xn6DD4xXTM5kvwGUsVI2NRlLARdoCr9hWOQgu6Ehm62TqU4ni8c5jQ+xcfyey
-HGmm5X1Tpz8pkm6qxP8cwY4Wt4rwru30jFM/slkkrIyA9oz3fDkoRlvp4pq49a+E
-p32WZQdRXIdBdo39Z1N+TZc/tWYahlj02ZUKKONKsGAAMMw/8qQjsBVrTywsIobe
-N3BNspkXg/yf/2YSTO8KVU4/vGW1t8CSujf2PaEYP03/XQOizZEg/zlRbsZIF+rn
-DjNpjPKPF+igIq0+4nhXVLlRfCUtHYBLBA3aA8qyC4gIiWIUyopb+GhU1lGZLzq2
-FlTh69oDAgMBAAECggEAc0sBCqt9ETRkQUEvSyMa58opww9Kl6SkF8EqE7MfUFV4
-epu/2ML9Hq1AecQl5WsVlv1RdXEDcYvq+1jvcJkMuDodqDzlsngPYrLINdriBcms
-VcFrIRGdp7hBGtSzvE/1DlEs8qJOXP1qqBO6lcy3YfkvB+Tjwj1LDBN6QSg9Kaua
-Y9vx5o41uNvlV9T5R5gwHEYcxtXYvH6bCT71BtQ3UbT8NQX/LIKWUKvnkVQtOqHR
-2FCXmYwwCucUYyB51FXKuEk4rNd32iU93BRTHEBhylcsyJBywX2/2JYv6VK09UQ3
-4KTVOz+dRfo7t+WN+QZemUvgafWUnaQzyb8ygl7uoQKBgQD91RRBJWU2IFk2fkC3
-6DvO64ukZ38WY1BR2vGpnc+ebuiffXTAjYPgPbiCm7sZPboQef7P4MeM1SInQe66
-KfrVjVQIvn3BTH99b+4DfEKlw0CbjkJJhwtsXLzTKipMgSAfTSPablOi2Re4IEp4
-HH11d6ckr2UOSiet/UbWIq2PEwKBgQDK4gF4U4fJOYO6U62lsdOcOyj+cgJ7f3L2
-aaZBb+Z7OK+pQ9BZIE/NkokRzjH8Q6dfQ05WYgBssZY26XQdV6XDXdN2mMnMQmUg
-bvdakW0ujPQywbQJ0i+EfVtaVBgHJAENLnPQRFz+xdMNPV7YiB8Cx0CoGiqFJUyG
-+grkIbm3UQKBgF4e3ZTOdfodApeXNw46ebrfUndSvSgjCUxg4A0h7DN2MqG5Fkqz
-7rt+h2U+s6DHZ6/mXJreYYASI8jkxM2Y9V6sOgXIBL9hS0DK+TmnoNrwTIuhs2iq
-P40d0hKG5wpZMv0IUIwVLsQKtaXbq2FgRxEAniSXOOSUkKQOb27Fiw+rAoGBAJUn
-ZLdGibxZKUvxuubm/lCMec3emY6bmzQnZdAT6VlLynDhZyovBBx5D4ZklgEE5+a0
-P8JoGDHQuUOzi+cKidZAg3dcSnd9lIUsr3MJGUCw5dLu72KNANwhoOiejG+re45q
-SUCjEc9INqo0JSeBo3LSc+liG0J5snJFH36qgLORAoGBAOq5NJczg1SOWORyG6L7
-JoQ3mpwtM2PazNUZD0UcDTFJvwkFS8KGXyBfk9si2vUVJOZAZRKBq8K5pIlPF3TD
-CLuoD3HMxwS4LGpvZa/kEZvk3nHROOQegArIigne3Bbo3qIt09jOeTZIBOM0FzeE
-UwiqkMMojrYsL1bHEJMRSMPd
------END PRIVATE KEY-----

From 5e13e36f9aa15f8123a77867c407e7546171d51a Mon Sep 17 00:00:00 2001
From: qbiqing <bq2134@columbia.edu>
Date: Fri, 30 Jul 2021 11:48:49 +0800
Subject: [PATCH 14/25] debug CI

---
 .gitignore | 2 +-
 Makefile   | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/.gitignore b/.gitignore
index 0cbdb96..4f4cde6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,7 +4,7 @@ registrywatcher
 !test.toml
 !sample.toml
 
-testutils/snakeoil/*.pem
+testutils/snakeoil
 
 # dependencies
 node_modules
diff --git a/Makefile b/Makefile
index 9b96ce1..6b39c9e 100644
--- a/Makefile
+++ b/Makefile
@@ -5,5 +5,6 @@ PKG := github.com/dsaidgovsg/$(NAME)
 .PHONY: snakeoil
 snakeoil: ## Update snakeoil certs for testing.
 	go run $(GOROOT)/src/crypto/tls/generate_cert.go --host localhost,127.0.0.1 --ca
-	mv $(CURDIR)/key.pem $(CURDIR)/testutils/snakeoil/key.pem
-	mv $(CURDIR)/cert.pem $(CURDIR)/testutils/snakeoil/cert.pem
+	mkdir -p testutils/snakeoil
+	mv cert.pem $(CURDIR)/testutils/snakeoil/cert.pem
+	mv key.pem $(CURDIR)/testutils/snakeoil/key.pem

From 6ad435a7c06fd0dffffc38f1524c9db6058a775e Mon Sep 17 00:00:00 2001
From: qbiqing <bq2134@columbia.edu>
Date: Mon, 2 Aug 2021 11:13:20 +0800
Subject: [PATCH 15/25] try using services

---
 codefresh.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/codefresh.yml b/codefresh.yml
index 31bd6bb..080e3af 100644
--- a/codefresh.yml
+++ b/codefresh.yml
@@ -37,6 +37,10 @@ steps:
     - GOROOT=/usr/local/go
     commands:
     - ./test.sh
+    services:
+      composition:
+        docker:
+          image: docker
     when:
       steps:
       - name: build_builder

From 225736700153d15dcc95eac803394c2ffd507a9f Mon Sep 17 00:00:00 2001
From: qbiqing <bq2134@columbia.edu>
Date: Mon, 2 Aug 2021 11:13:20 +0800
Subject: [PATCH 16/25] amend Makefile

---
 Makefile      |  2 ++
 README.md     | 12 ++----------
 codefresh.yml |  4 ----
 test.sh       |  3 +--
 4 files changed, 5 insertions(+), 16 deletions(-)

diff --git a/Makefile b/Makefile
index 6b39c9e..157b2b7 100644
--- a/Makefile
+++ b/Makefile
@@ -2,6 +2,8 @@
 NAME := registrywatcher
 PKG := github.com/dsaidgovsg/$(NAME)
 
+GOROOT := $(shell go env | grep GOROOT | cut -b 8-)
+
 .PHONY: snakeoil
 snakeoil: ## Update snakeoil certs for testing.
 	go run $(GOROOT)/src/crypto/tls/generate_cert.go --host localhost,127.0.0.1 --ca
diff --git a/README.md b/README.md
index d0ff0c9..2f0ce6f 100644
--- a/README.md
+++ b/README.md
@@ -115,18 +115,10 @@ You need to make one modification to your docker daemon config (usually at ~/.do
 
 Add the flag `--insecure-registry localhost:5000` to your docker daemon, documented [here](https://docs.docker.com/registry/insecure/) for testing against an insecure registry.
 
-You may need to update the certificates from time to time.
-
-Find `GOROOT` via
-
-```bash
-go env
-```
+You may need to update the certificates from time to time. To do so, run
 
 ```bash
-go run $GOROOT/src/crypto/tls/generate_cert.go --host localhost,127.0.0.1 --ca
-mv $(CURDIR)/key.pem $(CURDIR)/testutils/snakeoil/key.pem
-mv $(CURDIR)/cert.pem $(CURDIR)/testutils/snakeoil/cert.pem
+make snakeoil
 ```
 
 ## Tests
diff --git a/codefresh.yml b/codefresh.yml
index 080e3af..31bd6bb 100644
--- a/codefresh.yml
+++ b/codefresh.yml
@@ -37,10 +37,6 @@ steps:
     - GOROOT=/usr/local/go
     commands:
     - ./test.sh
-    services:
-      composition:
-        docker:
-          image: docker
     when:
       steps:
       - name: build_builder
diff --git a/test.sh b/test.sh
index 5bb2012..91c78e0 100755
--- a/test.sh
+++ b/test.sh
@@ -19,6 +19,5 @@ assert_each_go_file gofmt -l
 echo "OK"
 
 echo "-- go unit tests --"
-make snakeoil
-gotestsum -- -tags="unit integration" ./...
+gotestsum -- -tags="unit" ./...
 echo "OK"

From 0f5aff8fe9c7c4b733aca30bfada628b900c8a12 Mon Sep 17 00:00:00 2001
From: qbiqing <bq2134@columbia.edu>
Date: Mon, 2 Aug 2021 15:03:04 +0800
Subject: [PATCH 17/25] refactor UpdateCaches

---
 client/client.go         | 10 +++++++---
 client/client_test.go    |  5 -----
 main.go                  |  6 ------
 worker/watcher_worker.go |  3 ---
 4 files changed, 7 insertions(+), 17 deletions(-)

diff --git a/client/client.go b/client/client.go
index 3bb667b..4024c6b 100644
--- a/client/client.go
+++ b/client/client.go
@@ -152,6 +152,8 @@ func (client *Clients) DeployPinnedTag(conf *viper.Viper, repoName string) {
 	jobID := utils.GetRepoNomadJob(conf, repoName)
 	taskName := utils.GetRepoNomadTaskName(conf, repoName)
 	client.NomadClient.UpdateNomadJobTag(jobID, repoName, taskName, pinnedTag)
+	// update after deploying new sha, so it will not trigger autodeployment
+	client.updateCaches(repoName)
 }
 
 func (client *Clients) PopulateCaches(repoName string) {
@@ -271,7 +273,7 @@ func (client *Clients) isTagDigestChanged(repoName string) (bool, error) {
 	return !*digestIsCurrent, nil
 }
 
-func (client *Clients) UpdateCaches(repoName string) {
+func (client *Clients) updateCaches(repoName string) {
 	validTags, err := client.getSHATags(repoName)
 	if err != nil {
 		log.LogAppErr(fmt.Sprintf("Couldn't fetch tags from registry while updating cache for %s", repoName), err)
@@ -309,7 +311,7 @@ func (client *Clients) UpdateCaches(repoName string) {
 }
 
 // this function compares cached values with the actual values,
-// so only update the cache after calling, not before
+// so only update the cache before returning non-error cases
 func (client *Clients) ShouldDeploy(repoName string) (bool, error) {
 	autoDeploy, err := client.PostgresClient.GetAutoDeployFlag(repoName)
 	if err != nil {
@@ -317,6 +319,7 @@ func (client *Clients) ShouldDeploy(repoName string) (bool, error) {
 		return false, err
 	}
 	if !autoDeploy {
+		client.updateCaches(repoName)
 		return false, nil
 	}
 
@@ -332,8 +335,9 @@ func (client *Clients) ShouldDeploy(repoName string) (bool, error) {
 	}
 
 	if (pinnedTag == "" && client.isNewReleaseTagAvailable(repoName)) || isDigestChanged {
+		client.updateCaches(repoName)
 		return true, nil
 	}
-
+	client.updateCaches(repoName)
 	return false, nil
 }
diff --git a/client/client_test.go b/client/client_test.go
index 501f2b1..d9f61f3 100644
--- a/client/client_test.go
+++ b/client/client_test.go
@@ -31,7 +31,6 @@ func TestAutoDeployLatestTag(t *testing.T) {
 	te.PushNewTag(newTag, "latest")
 
 	shouldDeploy, _ := te.Clients.ShouldDeploy(te.TestRepoName)
-	te.Clients.UpdateCaches(te.TestRepoName)
 	tagToDeploy, _ := te.Clients.GetFormattedPinnedTag(te.TestRepoName)
 	// v0.0.2 is not a new release version
 	assert.False(t, shouldDeploy)
@@ -42,7 +41,6 @@ func TestAutoDeployLatestTag(t *testing.T) {
 	te.PushNewTag(newTag, "latest")
 
 	shouldDeploy, _ = te.Clients.ShouldDeploy(te.TestRepoName)
-	te.Clients.UpdateCaches(te.TestRepoName)
 	tagToDeploy, _ = te.Clients.GetFormattedPinnedTag(te.TestRepoName)
 	// v0.2.0 is a new release version,
 	assert.True(t, shouldDeploy)
@@ -53,7 +51,6 @@ func TestAutoDeployLatestTag(t *testing.T) {
 	te.PushNewTag(newTag, "latest")
 
 	shouldDeploy, _ = te.Clients.ShouldDeploy(te.TestRepoName)
-	te.Clients.UpdateCaches(te.TestRepoName)
 	tagToDeploy, _ = te.Clients.GetFormattedPinnedTag(te.TestRepoName)
 	// v0.0.9 is not a new release version
 	assert.False(t, shouldDeploy)
@@ -68,7 +65,6 @@ func TestAutoDeployLatestTag(t *testing.T) {
 	te.UpdatePinnedTag(newTag)
 
 	shouldDeploy, _ = te.Clients.ShouldDeploy(te.TestRepoName)
-	te.Clients.UpdateCaches(te.TestRepoName)
 	tagToDeploy, _ = te.Clients.GetFormattedPinnedTag(te.TestRepoName)
 	// should re-deploy after pinned tag is changed
 	assert.True(t, shouldDeploy)
@@ -78,7 +74,6 @@ func TestAutoDeployLatestTag(t *testing.T) {
 	te.PushNewTag(newTag, "alpine")
 
 	shouldDeploy, _ = te.Clients.ShouldDeploy(te.TestRepoName)
-	te.Clients.UpdateCaches(te.TestRepoName)
 	tagToDeploy, _ = te.Clients.GetFormattedPinnedTag(te.TestRepoName)
 	assert.True(t, shouldDeploy)
 	assert.Equal(t, "test", tagToDeploy)
diff --git a/main.go b/main.go
index 6adc928..fe0d71e 100644
--- a/main.go
+++ b/main.go
@@ -152,8 +152,6 @@ func (h *Handler) ResetTagHandler(c *gin.Context) {
 	} else {
 		log.LogAppInfo(fmt.Sprintf("Updated pinned_tag for repo %s from %s to %s succesfully, deployment of pinned_tag will happen shortly", repoName, originalTag, pinnedTag))
 		h.clients.DeployPinnedTag(h.conf, repoName)
-		// update after deploying new sha, so it will not trigger autodeployment
-		h.clients.UpdateCaches(repoName)
 		c.JSON(200, gin.H{
 			"message": fmt.Sprintf("Deploying to %s", pinnedTag),
 		})
@@ -236,8 +234,6 @@ func (h *Handler) DeployTagHandler(c *gin.Context) {
 	originalTag, err := h.clients.PostgresClient.GetPinnedTag(repoName)
 	if originalTag == pinnedTag {
 		h.clients.DeployPinnedTag(h.conf, repoName)
-		// update after deploying new sha, so it will not trigger autodeployment
-		h.clients.UpdateCaches(repoName)
 		c.JSON(200, gin.H{
 			"message": fmt.Sprintf("Deploying to %s", pinnedTag),
 		})
@@ -255,8 +251,6 @@ func (h *Handler) DeployTagHandler(c *gin.Context) {
 	} else {
 		log.LogAppInfo(fmt.Sprintf("Updated pinned_tag for repo %s from %s to %s succesfully, deployment of pinned_tag will happen shortly", repoName, originalTag, pinnedTag))
 		h.clients.DeployPinnedTag(h.conf, repoName)
-		// update after deploying new sha, so it will not trigger autodeployment
-		h.clients.UpdateCaches(repoName)
 		c.JSON(200, gin.H{
 			"message": fmt.Sprintf("Deploying to %s", pinnedTag),
 		})
diff --git a/worker/watcher_worker.go b/worker/watcher_worker.go
index e15a00c..db95142 100644
--- a/worker/watcher_worker.go
+++ b/worker/watcher_worker.go
@@ -44,12 +44,9 @@ func (ww *WatcherWorker) initialize() {
 func (ww *WatcherWorker) runOnce() {
 	shouldDeploy, err := ww.clients.ShouldDeploy(ww.repoName)
 	if err != nil {
-		ww.clients.UpdateCaches(ww.repoName)
 		return
 	}
 	originalTag, err := ww.clients.GetFormattedPinnedTag(ww.repoName)
-	// update AFTER checking for changes
-	ww.clients.UpdateCaches(ww.repoName)
 	if err != nil || !shouldDeploy {
 		return
 	}

From 9dd4251b111c01d6f7b1ae34b2d430ec4dda6552 Mon Sep 17 00:00:00 2001
From: qbiqing <bq2134@columbia.edu>
Date: Tue, 3 Aug 2021 09:54:56 +0800
Subject: [PATCH 18/25] fix comments

---
 Dockerfile-golang              |  5 ++---
 README.md                      |  7 +++++--
 client/client_test.go          | 10 +---------
 client/dockerhub_api_client.go |  4 ++--
 4 files changed, 10 insertions(+), 16 deletions(-)

diff --git a/Dockerfile-golang b/Dockerfile-golang
index ce89ac2..7195f3c 100644
--- a/Dockerfile-golang
+++ b/Dockerfile-golang
@@ -1,6 +1,5 @@
 FROM golang:1.12-alpine
 
 # Only install hard dependencies
-RUN apk add --no-cache gcc git musl-dev make; \
-    go get -v -u gotest.tools/gotestsum; \
-    :
+RUN apk add --no-cache gcc git musl-dev make && \
+    go get -v -u gotest.tools/gotestsum
diff --git a/README.md b/README.md
index 2f0ce6f..4de1bf6 100644
--- a/README.md
+++ b/README.md
@@ -128,6 +128,9 @@ Run locally:
 ```bash
 // run both integration and unit tests
 gotestsum -- -tags="integration unit" ./...
+
+// clean testcache
+go clean -testcache
 ```
 
 The tests don't test the nomad deployment capability of the service.
@@ -136,8 +139,6 @@ The tests don't test the nomad deployment capability of the service.
 
 A makefile to avoid having to change your local docker daemon config so as to run the tests as docker-in-docker for CI integration.
 
-A makefile step to regenerate testing certificates in `testutils/snakeoil`
-
 An interface to swap in the deployment agent (only Nomad supported for now).
 
 Tests take too long to run, this is mainly due each integration test case spinning up and down its own docker containers.
@@ -147,3 +148,5 @@ Nomad mock server cannot run jobs, which blocks writing of integration tests inv
 Buttons should implement debouncing
 
 Bug where only new tags/SHA changes AFTER you toggle back auto deployment will trigger an update, meaning any updates during the window where auto deployment was turned off are moot.
+
+Clean up unused registry methods.
diff --git a/client/client_test.go b/client/client_test.go
index d9f61f3..990b3aa 100644
--- a/client/client_test.go
+++ b/client/client_test.go
@@ -60,17 +60,9 @@ func TestAutoDeployLatestTag(t *testing.T) {
 		resolves correctly for autodeployment of custom tags
 	*/
 
-	// set to "test:latest"
+	// set to "test:alpine"
 	newTag = "test"
 	te.UpdatePinnedTag(newTag)
-
-	shouldDeploy, _ = te.Clients.ShouldDeploy(te.TestRepoName)
-	tagToDeploy, _ = te.Clients.GetFormattedPinnedTag(te.TestRepoName)
-	// should re-deploy after pinned tag is changed
-	assert.True(t, shouldDeploy)
-	assert.Equal(t, "test", tagToDeploy)
-
-	// "test" is now based on "alpine", rather than the original "latest"
 	te.PushNewTag(newTag, "alpine")
 
 	shouldDeploy, _ = te.Clients.ShouldDeploy(te.TestRepoName)
diff --git a/client/dockerhub_api_client.go b/client/dockerhub_api_client.go
index 1c9422b..c09d7ba 100644
--- a/client/dockerhub_api_client.go
+++ b/client/dockerhub_api_client.go
@@ -146,8 +146,8 @@ func (api *DockerhubApi) CheckImageIsCurrent(repository, digest string, checkTag
 	return &isCurrent, nil
 }
 
-// We can't just use the registry GetTagDigest because the digests from the
-// registry and from the dockerhub API do not match
+// Note that the digest returned from the Dockerhub API does not match
+// the digest from docker registry manifest V2 API
 func (api *DockerhubApi) GetTagDigestFromApi(repository string, checkTag string) (
 	*string, error) {
 	endpoint := fmt.Sprintf("/v2/namespaces/%s/repositories/%s/images?", api.namespace,

From c620b7dab2368e0e74c8a7a5e4429d12bfed4f5f Mon Sep 17 00:00:00 2001
From: qbiqing <bq2134@columbia.edu>
Date: Tue, 3 Aug 2021 10:22:41 +0800
Subject: [PATCH 19/25] minor edits

---
 client/setup.go | 14 +++++++-------
 codefresh.yml   |  1 -
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/client/setup.go b/client/setup.go
index ac9c210..1a82016 100644
--- a/client/setup.go
+++ b/client/setup.go
@@ -37,8 +37,8 @@ func (te *testEngine) printState() {
 	fmt.Println("cached tags", cachedTags)
 
 	pinnedTag, _ := te.Clients.GetFormattedPinnedTag(te.TestRepoName)
-	registryTagDigest, _ := te.Clients.DockerhubApi.GetTagDigestFromApi(te.TestRepoName, pinnedTag)
-	fmt.Println("registry tag digest", *registryTagDigest)
+	tagDigest, _ := te.Clients.DockerhubApi.GetTagDigestFromApi(te.TestRepoName, pinnedTag)
+	fmt.Println("registry tag digest", *tagDigest)
 
 	cachedTagDigest, _ := te.Clients.GetCachedTagDigest(te.TestRepoName)
 	fmt.Println("cached tag digest", cachedTagDigest)
@@ -70,16 +70,16 @@ func SetUpClientTest(t *testing.T) *testEngine {
 	te.containerIDs = append(te.containerIDs, regID)
 	te.containerIDs = append(te.containerIDs, pgID)
 
-	// initialize mock server
-	te.ImageTagMap = make(map[string][]string)
-
 	// initialize the clients
 	te.Clients = SetUpTestClients(t, conf)
 
 	// we use this so much might as well keep it in the struct
 	te.TestRepoName = te.Conf.GetStringSlice("watched_repositories")[0]
 
-	// set up mock Dockerhub API and imageTag store
+	// initialize mock imageTag store
+	te.ImageTagMap = make(map[string][]string)
+
+	// initialize mock Dockerhub server
 	router := mux.NewRouter()
 
 	router.HandleFunc("/v2/users/login", func(res http.ResponseWriter, req *http.Request) {
@@ -198,7 +198,7 @@ func (te *testEngine) PushNewTag(namedTag, actualTag string) {
 	publicImageName := fmt.Sprintf("%s:%s", te.Conf.GetString("base_public_image"), actualTag)
 	err := te.helper.AddImageToRegistry(publicImageName, mockImageName)
 
-	imageDigest := testutils.RandSeq(10)
+	imageDigest := testutils.RandSeq(20)
 	te.ImageTagMap[imageDigest] = []string{namedTag, "true"}
 
 	// make is_current false for images holding the same tag
diff --git a/codefresh.yml b/codefresh.yml
index 31bd6bb..8a9d14b 100644
--- a/codefresh.yml
+++ b/codefresh.yml
@@ -34,7 +34,6 @@ steps:
     working_directory: ${{main_clone}}
     environment:
     - GOPATH=/codefresh/volume/.go
-    - GOROOT=/usr/local/go
     commands:
     - ./test.sh
     when:

From f7c652a43ad3d1f05da50618d361393081254fc8 Mon Sep 17 00:00:00 2001
From: qbiqing <bq2134@columbia.edu>
Date: Tue, 3 Aug 2021 17:11:37 +0800
Subject: [PATCH 20/25] amend tests

---
 client/client_test.go  |  8 +++++++-
 client/setup.go        | 33 +++++++++++++++++----------------
 testutils/testutils.go | 11 -----------
 3 files changed, 24 insertions(+), 28 deletions(-)

diff --git a/client/client_test.go b/client/client_test.go
index 990b3aa..746067a 100644
--- a/client/client_test.go
+++ b/client/client_test.go
@@ -63,8 +63,14 @@ func TestAutoDeployLatestTag(t *testing.T) {
 	// set to "test:alpine"
 	newTag = "test"
 	te.UpdatePinnedTag(newTag)
-	te.PushNewTag(newTag, "alpine")
+	shouldDeploy, _ = te.Clients.ShouldDeploy(te.TestRepoName)
+	tagToDeploy, _ = te.Clients.GetFormattedPinnedTag(te.TestRepoName)
+	// should not deploy as the image is not changed, even though
+	// the pinned tag is changed
+	assert.False(t, shouldDeploy)
+	assert.Equal(t, "test", tagToDeploy)
 
+	te.PushNewTag(newTag, "alpine")
 	shouldDeploy, _ = te.Clients.ShouldDeploy(te.TestRepoName)
 	tagToDeploy, _ = te.Clients.GetFormattedPinnedTag(te.TestRepoName)
 	assert.True(t, shouldDeploy)
diff --git a/client/setup.go b/client/setup.go
index 1a82016..076b0b9 100644
--- a/client/setup.go
+++ b/client/setup.go
@@ -2,7 +2,6 @@ package client
 
 import (
 	"fmt"
-	"strconv"
 	"testing"
 
 	"github.com/dsaidgovsg/registrywatcher/config"
@@ -24,7 +23,7 @@ type testEngine struct {
 	Conf         *viper.Viper
 	helper       *testutils.TestHelper
 	Clients      *Clients
-	ImageTagMap  map[string][]string // digest: [tag, is_current]
+	ImageTagMap  map[string][]TagWithStatus // digest: [tag, is_current]
 	Ts           *httptest.Server
 	TestRepoName string
 }
@@ -38,7 +37,7 @@ func (te *testEngine) printState() {
 
 	pinnedTag, _ := te.Clients.GetFormattedPinnedTag(te.TestRepoName)
 	tagDigest, _ := te.Clients.DockerhubApi.GetTagDigestFromApi(te.TestRepoName, pinnedTag)
-	fmt.Println("registry tag digest", *tagDigest)
+	fmt.Println("new tag digest", *tagDigest)
 
 	cachedTagDigest, _ := te.Clients.GetCachedTagDigest(te.TestRepoName)
 	fmt.Println("cached tag digest", cachedTagDigest)
@@ -77,7 +76,7 @@ func SetUpClientTest(t *testing.T) *testEngine {
 	te.TestRepoName = te.Conf.GetStringSlice("watched_repositories")[0]
 
 	// initialize mock imageTag store
-	te.ImageTagMap = make(map[string][]string)
+	te.ImageTagMap = make(map[string][]TagWithStatus)
 
 	// initialize mock Dockerhub server
 	router := mux.NewRouter()
@@ -94,9 +93,7 @@ func SetUpClientTest(t *testing.T) *testEngine {
 			res.Header().Set("Content-Type", "application/json")
 			res.WriteHeader(http.StatusOK)
 
-			if tag, ok := te.ImageTagMap[digest]; ok {
-				isCurrent, _ := strconv.ParseBool(tag[1])
-				tags := []TagWithStatus{{Tag: tag[0], IsCurrent: isCurrent}}
+			if tags, ok := te.ImageTagMap[digest]; ok {
 				results := CheckImageResp{Results: tags}
 				data, _ := json.Marshal(results)
 				res.Write(data)
@@ -110,9 +107,7 @@ func SetUpClientTest(t *testing.T) *testEngine {
 	router.HandleFunc("/v2/namespaces/namespace/repositories/testrepo/images",
 		func(res http.ResponseWriter, req *http.Request) {
 			var resSlice []GetTagDigestResult
-			for image, tag := range te.ImageTagMap {
-				isCurrent, _ := strconv.ParseBool(tag[1])
-				tags := []TagWithStatus{{Tag: tag[0], IsCurrent: isCurrent}}
+			for image, tags := range te.ImageTagMap {
 				resSlice = append(resSlice, GetTagDigestResult{Digest: image, Tags: tags})
 			}
 			res.Header().Set("Content-Type", "application/json")
@@ -198,13 +193,19 @@ func (te *testEngine) PushNewTag(namedTag, actualTag string) {
 	publicImageName := fmt.Sprintf("%s:%s", te.Conf.GetString("base_public_image"), actualTag)
 	err := te.helper.AddImageToRegistry(publicImageName, mockImageName)
 
-	imageDigest := testutils.RandSeq(20)
-	te.ImageTagMap[imageDigest] = []string{namedTag, "true"}
+	imageDigest := actualTag
+	if _, ok := te.ImageTagMap[imageDigest]; ok {
+		te.ImageTagMap[imageDigest] = append(te.ImageTagMap[imageDigest], TagWithStatus{namedTag, true})
+	} else {
+		te.ImageTagMap[imageDigest] = []TagWithStatus{{namedTag, true}}
+	}
 
-	// make is_current false for images holding the same tag
-	for image, tag := range te.ImageTagMap {
-		if tag[0] == namedTag && image != imageDigest {
-			tag[1] = "false"
+	// make is_current false for older images holding the same tag
+	for image, tags := range te.ImageTagMap {
+		for i, tag := range tags {
+			if tag.Tag == namedTag && image != imageDigest {
+				tags[i].IsCurrent = false
+			}
 		}
 	}
 
diff --git a/testutils/testutils.go b/testutils/testutils.go
index c87235c..b2ed1b5 100644
--- a/testutils/testutils.go
+++ b/testutils/testutils.go
@@ -10,7 +10,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"math"
-	"math/rand"
 	"net"
 	"net/http"
 	"os"
@@ -357,13 +356,3 @@ func ConstructRegistryAuth(identity, secret string) (string, error) {
 
 	return base64.URLEncoding.EncodeToString(buf), nil
 }
-
-func RandSeq(n int) string {
-	var letters = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
-	b := make([]byte, n)
-	rand.Seed(time.Now().UnixNano())
-	for i := range b {
-		b[i] = letters[rand.Intn(len(letters))]
-	}
-	return string(b)
-}

From 0781859c4c29e97deee2a266b9b669ab6fc636d1 Mon Sep 17 00:00:00 2001
From: qbiqing <bq2134@columbia.edu>
Date: Tue, 3 Aug 2021 17:17:14 +0800
Subject: [PATCH 21/25] fix comments

---
 client/client_test.go | 5 ++++-
 test.sh               | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/client/client_test.go b/client/client_test.go
index 746067a..157adf4 100644
--- a/client/client_test.go
+++ b/client/client_test.go
@@ -60,9 +60,10 @@ func TestAutoDeployLatestTag(t *testing.T) {
 		resolves correctly for autodeployment of custom tags
 	*/
 
-	// set to "test:alpine"
+	// set to "test:latest"
 	newTag = "test"
 	te.UpdatePinnedTag(newTag)
+
 	shouldDeploy, _ = te.Clients.ShouldDeploy(te.TestRepoName)
 	tagToDeploy, _ = te.Clients.GetFormattedPinnedTag(te.TestRepoName)
 	// should not deploy as the image is not changed, even though
@@ -70,7 +71,9 @@ func TestAutoDeployLatestTag(t *testing.T) {
 	assert.False(t, shouldDeploy)
 	assert.Equal(t, "test", tagToDeploy)
 
+	// "test" is now based on "alpine", rather than the original "latest"
 	te.PushNewTag(newTag, "alpine")
+
 	shouldDeploy, _ = te.Clients.ShouldDeploy(te.TestRepoName)
 	tagToDeploy, _ = te.Clients.GetFormattedPinnedTag(te.TestRepoName)
 	assert.True(t, shouldDeploy)
diff --git a/test.sh b/test.sh
index 91c78e0..2439d35 100755
--- a/test.sh
+++ b/test.sh
@@ -19,5 +19,5 @@ assert_each_go_file gofmt -l
 echo "OK"
 
 echo "-- go unit tests --"
-gotestsum -- -tags="unit" ./...
+gotestsum -- -tags=unit ./...
 echo "OK"

From c80e7ee472b78c88b4c99cc198285deefe9649df Mon Sep 17 00:00:00 2001
From: qbiqing <bq2134@columbia.edu>
Date: Tue, 3 Aug 2021 18:20:56 +0800
Subject: [PATCH 22/25] add comment

---
 client/client_test.go | 6 ++++--
 client/setup.go       | 9 +++++++--
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/client/client_test.go b/client/client_test.go
index 157adf4..64dcdb8 100644
--- a/client/client_test.go
+++ b/client/client_test.go
@@ -8,7 +8,9 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-// test ShouldDeploy for a multitude of conditions
+/*
+	Test ShouldDeploy for a multitude of conditions.
+*/
 func TestAutoDeployLatestTag(t *testing.T) {
 	te := SetUpClientTest(t)
 	defer te.TearDown()
@@ -66,7 +68,7 @@ func TestAutoDeployLatestTag(t *testing.T) {
 
 	shouldDeploy, _ = te.Clients.ShouldDeploy(te.TestRepoName)
 	tagToDeploy, _ = te.Clients.GetFormattedPinnedTag(te.TestRepoName)
-	// should not deploy as the image is not changed, even though
+	// should not deploy as all tags are based on the digest "latest", even though
 	// the pinned tag is changed
 	assert.False(t, shouldDeploy)
 	assert.Equal(t, "test", tagToDeploy)
diff --git a/client/setup.go b/client/setup.go
index 076b0b9..22d5e5c 100644
--- a/client/setup.go
+++ b/client/setup.go
@@ -185,8 +185,13 @@ func (te *testEngine) TearDown() {
 	te.Ts.Close()
 }
 
-// named tag is what it will appear as in the docker registryDomain
-// actual tag is what the tag is based on (this is for testing purposes only)
+/*
+	For the following tests, note that the base image tags "latest" and "alpine"
+	are taken to be equivalent to image digest strings in the mock image-tag store.
+	The mock image tag store is used in turn because we are mocking the Dockerhub API server.
+	While in the real world, image digests are SHA hashes, this still allows us to test
+	the auto-deploy behaviour as we use the digest string to check if the underlying image is changed.
+*/
 func (te *testEngine) PushNewTag(namedTag, actualTag string) {
 	_, registryDomain, registryPrefix, _ := utils.ExtractRegistryInfo(te.Conf, te.TestRepoName)
 	mockImageName := utils.ConstructImageName(registryDomain, registryPrefix, te.TestRepoName, namedTag)

From 225bbe72dfe9dee700e9eb5d67f64e9e3cfb195b Mon Sep 17 00:00:00 2001
From: qbiqing <bq2134@columbia.edu>
Date: Tue, 3 Aug 2021 18:22:18 +0800
Subject: [PATCH 23/25] edit comment

---
 client/setup.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/client/setup.go b/client/setup.go
index 22d5e5c..ed11271 100644
--- a/client/setup.go
+++ b/client/setup.go
@@ -186,11 +186,11 @@ func (te *testEngine) TearDown() {
 }
 
 /*
-	For the following tests, note that the base image tags "latest" and "alpine"
-	are taken to be equivalent to image digest strings in the mock image-tag store.
-	The mock image tag store is used in turn because we are mocking the Dockerhub API server.
-	While in the real world, image digests are SHA hashes, this still allows us to test
-	the auto-deploy behaviour as we use the digest string to check if the underlying image is changed.
+	For tests, note that the base image tags "latest" and "alpine" are taken to be equivalent
+	to image digest strings in the mock image-tag store. The mock image tag store is used
+	because we are mocking the Dockerhub API server. While in the real world, image digests
+	are SHA hashes, this still allows us to test the auto-deploy behaviour as we use the
+	digest string to check if the underlying image is changed.
 */
 func (te *testEngine) PushNewTag(namedTag, actualTag string) {
 	_, registryDomain, registryPrefix, _ := utils.ExtractRegistryInfo(te.Conf, te.TestRepoName)

From df2557a56228495204bfda57eeb097ee4fed4bc4 Mon Sep 17 00:00:00 2001
From: qbiqing <bq2134@columbia.edu>
Date: Tue, 3 Aug 2021 18:24:45 +0800
Subject: [PATCH 24/25] remove outdated

---
 client/dockerhub_api_client.go | 1 -
 client/setup.go                | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/client/dockerhub_api_client.go b/client/dockerhub_api_client.go
index c09d7ba..7837aca 100644
--- a/client/dockerhub_api_client.go
+++ b/client/dockerhub_api_client.go
@@ -62,7 +62,6 @@ func InitializeDockerhubApi(conf *viper.Viper) (*DockerhubApi, error) {
 
 func (api *DockerhubApi) Authenticate() (*string, error) {
 	addr := fmt.Sprintf("%s%s", api.url, "/v2/users/login")
-	log.LogAppErr("in authenticate", nil)
 	log.LogAppInfo(fmt.Sprintf("dockerhub.users.login url=%s", addr))
 
 	data := map[string]string{"username": api.username, "password": api.secret}
diff --git a/client/setup.go b/client/setup.go
index ed11271..d27df88 100644
--- a/client/setup.go
+++ b/client/setup.go
@@ -23,7 +23,7 @@ type testEngine struct {
 	Conf         *viper.Viper
 	helper       *testutils.TestHelper
 	Clients      *Clients
-	ImageTagMap  map[string][]TagWithStatus // digest: [tag, is_current]
+	ImageTagMap  map[string][]TagWithStatus
 	Ts           *httptest.Server
 	TestRepoName string
 }

From 8bbd4e80bc166321b86255fe5e0688f6831d367e Mon Sep 17 00:00:00 2001
From: qbiqing <bq2134@columbia.edu>
Date: Tue, 3 Aug 2021 18:29:30 +0800
Subject: [PATCH 25/25] minor edit

---
 client/dockerhub_api_client.go | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/client/dockerhub_api_client.go b/client/dockerhub_api_client.go
index 7837aca..41bd7f0 100644
--- a/client/dockerhub_api_client.go
+++ b/client/dockerhub_api_client.go
@@ -195,10 +195,8 @@ func (api *DockerhubApi) GetTagDigestFromApi(repository string, checkTag string)
 
 	for _, item := range deserialized.Results {
 		for _, tag := range item.Tags {
-			if tag.Tag == checkTag {
-				if tag.IsCurrent {
-					return &item.Digest, nil
-				}
+			if tag.Tag == checkTag && tag.IsCurrent {
+				return &item.Digest, nil
 			}
 		}
 	}