feat: add usage collection

Dockerfile

@@ -12,10 +12,11 @@ FROM gcr.io/distroless/base AS runtime
 
 WORKDIR /pipeline-backend
 
+COPY --from=build /go/src/config ./config
+COPY --from=build /go/src/release-please ./release-please
 COPY --from=build /go/src/internal/db/migration ./internal/db/migration
-COPY --from=build /pipeline-backend-migrate ./
 
-COPY --from=build /go/src/config ./config
+COPY --from=build /pipeline-backend-migrate ./
 COPY --from=build /pipeline-backend ./
 
 ENTRYPOINT ["./pipeline-backend"]

Makefile

@@ -1,10 +1,10 @@
 .DEFAULT_GOAL:=help
 
-DEVELOP_SERVICES := pipeline_backend
+DEV := pipeline_backend
 DEP := mgmt_backend connector_backend model_backend triton_conda_env
-DB := pg_sql
+DB := pg_sql redis
 TRITON := triton_server
-TEMPORAL := temporal redis redoc_openapi
+TEMPORAL := temporal
 
 #============================================================================
 

cmd/main/main.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"syscall"
 
+	"github.com/go-redis/redis/v9"
 	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 	"github.com/rs/cors"
 	"golang.org/x/net/http2"
@@ -29,10 +30,13 @@ import (
 	"github.com/instill-ai/pipeline-backend/pkg/handler"
 	"github.com/instill-ai/pipeline-backend/pkg/repository"
 	"github.com/instill-ai/pipeline-backend/pkg/service"
+	"github.com/instill-ai/pipeline-backend/pkg/usage"
+	"github.com/instill-ai/x/repo"
 
-	cache "github.com/instill-ai/pipeline-backend/internal/cache"
 	database "github.com/instill-ai/pipeline-backend/internal/db"
 	pipelinePB "github.com/instill-ai/protogen-go/vdp/pipeline/v1alpha"
+	usagePB "github.com/instill-ai/protogen-go/vdp/usage/v1alpha"
+	usageclient "github.com/instill-ai/usage-client/usage"
 )
 
 func grpcHandlerFunc(grpcServer *grpc.Server, gwHandler http.Handler, CORSOrigins []string) http.Handler {
@@ -67,9 +71,6 @@ func main() {
 	db := database.GetConnection()
 	defer database.Close(db)
 
-	cache.Init()
-	defer cache.Close()
-
 	// Create tls based credential.
 	var creds credentials.TransportCredentials
 	var err error
@@ -113,16 +114,32 @@ func main() {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
+	userServiceClient, userServiceClientConn := external.InitUserServiceClient()
+	defer userServiceClientConn.Close()
+
+	connectorServiceClient, connectorServiceClientConn := external.InitConnectorServiceClient()
+	defer connectorServiceClientConn.Close()
+
+	modelServiceClient, modelServiceClientConn := external.InitModelServiceClient()
+	defer modelServiceClientConn.Close()
+
+	redisClient := redis.NewClient(&config.Config.Cache.Redis.RedisOptions)
+	defer redisClient.Close()
+
+	repository := repository.NewRepository(db)
+
+	service := service.NewService(
+		repository,
+		userServiceClient,
+		connectorServiceClient,
+		modelServiceClient,
+		redisClient,
+	)
+
 	grpcS := grpc.NewServer(grpcServerOpts...)
 	pipelinePB.RegisterPipelineServiceServer(
 		grpcS,
-		handler.NewHandler(
-			service.NewService(
-				repository.NewRepository(db),
-				external.InitUserServiceClient(),
-				external.InitConnectorServiceClient(),
-				external.InitModelServiceClient(),
-			)),
+		handler.NewHandler(service),
 	)
 
 	gwS := runtime.NewServeMux(
@@ -145,6 +162,30 @@ func main() {
 		panic(err)
 	}
 
+	// Start usage reporter
+	if !config.Config.Server.DisableUsage {
+		version, err := repo.ReadReleaseManifest("release-please/manifest.json")
+		if err != nil {
+			logger.Fatal(err.Error())
+		}
+
+		usageServiceClient, usageServiceClientConn := external.InitUsageServiceClient()
+		defer usageServiceClientConn.Close()
+
+		usg := usage.NewUsage(repository, userServiceClient, redisClient)
+		err = usageclient.StartReporter(
+			context.Background(),
+			usageServiceClient,
+			usagePB.Session_SERVICE_PIPELINE,
+			config.Config.Server.Edition,
+			version,
+			usg.RetrieveUsageData)
+		if err != nil {
+			logger.Error(fmt.Sprintf("Unable to start usage reporter: %v\n", err))
+		}
+	}
+
+	// Start gRPC server
 	var dialOpts []grpc.DialOption
 	if config.Config.Server.HTTPS.Cert != "" && config.Config.Server.HTTPS.Key != "" {
 		dialOpts = []grpc.DialOption{grpc.WithTransportCredentials(creds)}

config/config.go

@@ -6,7 +6,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/go-redis/redis/v8"
+	"github.com/go-redis/redis/v9"
 	"github.com/knadh/koanf"
 	"github.com/knadh/koanf/parsers/yaml"
 	"github.com/knadh/koanf/providers/env"
@@ -28,6 +28,7 @@ type AppConfig struct {
 	MgmtBackend      MgmtBackendConfig      `koanf:"mgmtbackend"`
 	ConnectorBackend ConnectorBackendConfig `koanf:"connectorbackend"`
 	ModelBackend     ModelBackendConfig     `koanf:"modelbackend"`
+	UsageBackend     UsageBackendConfig     `koanf:"usagebackend"`
 }
 
 // ServerConfig defines HTTP server configurations
@@ -37,10 +38,9 @@ type ServerConfig struct {
 		Cert string `koanf:"cert"`
 		Key  string `koanf:"key"`
 	}
-	CORSOrigins []string `koanf:"corsorigins"`
-	Paginate    struct {
-		Salt string `koanf:"salt"`
-	}
+	CORSOrigins  []string `koanf:"corsorigins"`
+	Edition      string   `koanf:"edition"`
+	DisableUsage bool     `koanf:"disableusage"`
 }
 
 // DatabaseConfig related to database
@@ -101,6 +101,16 @@ type ModelBackendConfig struct {
 	}
 }
 
+// UsageBackendConfig related to usage-backend
+type UsageBackendConfig struct {
+	Host  string `koanf:"host"`
+	Port  int    `koanf:"port"`
+	HTTPS struct {
+		Cert string `koanf:"cert"`
+		Key  string `koanf:"key"`
+	}
+}
+
 // Init - Assign global config to decoded config struct
 func Init() error {
 	logger, _ := logger.GetZapLogger()

config/config.yaml

@@ -7,6 +7,8 @@ server:
     - http://localhost
     - https://instill-inc.tech
     - https://instill.tech
+  edition: local-ce:dev
+  disableusage: false
 database:
   username: postgres
   password: password
@@ -45,3 +47,9 @@ modelbackend:
   https:
     cert:   # /ssl/tls.crt
     key:    # /ssl/tls.key
+usagebackend:
+  host: localhost
+  port: 8084
+  https:
+    cert:  # /ssl/tls.crt
+    key:  # /ssl/tls.crt

docker-compose.yml

@@ -15,8 +15,6 @@ volumes:
 services:
   mgmt_backend_migrate:
     container_name: mgmt-backend-migrate
-    build:
-      context: .
     image: instill/mgmt-backend:dev
     restart: on-failure
     environment:
@@ -31,8 +29,6 @@ services:
 
   mgmt_backend_init:
     container_name: mgmt-backend-init
-    build:
-      context: .
     image: instill/mgmt-backend:dev
     restart: on-failure
     environment:
@@ -46,8 +42,6 @@ services:
 
   mgmt_backend:
     container_name: mgmt-backend
-    build:
-      context: .
     image: instill/mgmt-backend:dev
     restart: unless-stopped
     environment:
@@ -56,6 +50,8 @@ services:
       CFG_DATABASE_PORT: 5432
       CFG_DATABASE_USERNAME: postgres
       CFG_DATABASE_PASSWORD: password
+      CFG_USAGEBACKEND_HOST: usage-backend-cloud
+      CFG_USAGEBACKEND_PORT: 8084
     ports:
       - 8080:8080
     depends_on:
@@ -97,6 +93,8 @@ services:
       CFG_CONNECTORBACKEND_PORT: 8082
       CFG_MODELBACKEND_HOST: model_backend
       CFG_MODELBACKEND_PORT: 8083
+      CFG_USAGEBACKEND_HOST: usage-backend-cloud
+      CFG_USAGEBACKEND_PORT: 8084
     ports:
       - 8081:8081
     volumes:

go.mod

@@ -3,16 +3,17 @@ module github.com/instill-ai/pipeline-backend
 go 1.18
 
 require (
-	github.com/go-redis/redis/v8 v8.11.4
+	github.com/go-redis/redis/v9 v9.0.0-beta.1
 	github.com/gofrs/uuid v4.0.0+incompatible
 	github.com/gogo/status v1.1.0
 	github.com/golang-migrate/migrate/v4 v4.15.1
 	github.com/golang/mock v1.6.0
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.3
 	github.com/iancoleman/strcase v0.2.0
-	github.com/instill-ai/protogen-go v0.1.5-alpha.0.20220530153944-e1c33d5f7d13
-	github.com/instill-ai/x v0.1.0-alpha.0.20220517204940-5a70916ce425
+	github.com/instill-ai/protogen-go v0.1.5-alpha.0.20220606124550-53f79f9c7d74
+	github.com/instill-ai/usage-client v0.0.0-20220606142220-c17424a565e4
+	github.com/instill-ai/x v0.1.0-alpha.0.20220604235252-39fcffc82edb
 	github.com/knadh/koanf v1.4.0
 	github.com/mennanov/fieldmask-utils v0.5.0
 	github.com/rs/cors v1.8.2
@@ -22,11 +23,12 @@ require (
 	golang.org/x/net v0.0.0-20220225172249-27dd8689420f
 	google.golang.org/grpc v1.45.0
 	google.golang.org/protobuf v1.27.1
-	gorm.io/driver/postgres v1.2.3
-	gorm.io/gorm v1.22.5
+	gorm.io/driver/postgres v1.3.6
+	gorm.io/gorm v1.23.5
 )
 
 require (
+	github.com/catalinc/hashcash v0.0.0-20161205220751-e6bc29ff4de9 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
@@ -39,13 +41,13 @@ require (
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.0 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
-	github.com/jackc/pgconn v1.10.1
+	github.com/jackc/pgconn v1.12.1
 	github.com/jackc/pgio v1.0.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
-	github.com/jackc/pgproto3/v2 v2.2.0 // indirect
+	github.com/jackc/pgproto3/v2 v2.3.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
-	github.com/jackc/pgtype v1.9.0 // indirect
-	github.com/jackc/pgx/v4 v4.14.0 // indirect
+	github.com/jackc/pgtype v1.11.0 // indirect
+	github.com/jackc/pgx/v4 v4.16.1 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.4 // indirect
 	github.com/lib/pq v1.10.2 // indirect