diff --git a/Dockerfile b/Dockerfile index 7225e98..20ed3b3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,7 @@ WORKDIR /build COPY kubelab-backend/go.mod kubelab-backend/go.sum kubelab-backend/main.go ./ COPY kubelab-backend/hooks ./hooks COPY kubelab-backend/pkg ./pkg +COPY kubelab-backend/vcluster-values.yaml ./vcluster-values.yaml RUN apk --no-cache add upx make git gcc libtool musl-dev ca-certificates dumb-init \ && go mod tidy \ && CGO_ENABLED=0 go build \ @@ -20,6 +21,7 @@ RUN npm run build FROM alpine as runtime WORKDIR /app/kubelab COPY --from=backend-builder /build/kubelab /app/kubelab/kubelab +COPY --from=backend-builder /build/vcluster-values.yaml /app/kubelab/vcluster-values.yaml COPY ./kubelab-backend/pb_migrations ./pb_migrations COPY --from=ui-builder /build/build /app/kubelab/pb_public EXPOSE 8090 diff --git a/kubelab-backend/.gitignore b/kubelab-backend/.gitignore index 6c2bf35..cec492b 100644 --- a/kubelab-backend/.gitignore +++ b/kubelab-backend/.gitignore @@ -4,5 +4,5 @@ /pb_data /tmp /bin -kubelab ./pocketbase +kubelab diff --git a/kubelab-backend/go.mod b/kubelab-backend/go.mod index 0f8073c..ed2a84f 100644 --- a/kubelab-backend/go.mod +++ b/kubelab-backend/go.mod @@ -5,8 +5,7 @@ go 1.20 require ( github.com/caarlos0/env/v8 v8.0.0 github.com/pocketbase/dbx v1.10.0 - github.com/pocketbase/pocketbase v0.17.4 - gopkg.in/yaml.v2 v2.4.0 + github.com/pocketbase/pocketbase v0.17.5 helm.sh/helm/v3 v3.11.2 k8s.io/api v0.27.2 k8s.io/apimachinery v0.27.2 @@ -101,6 +100,7 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230807174057-1744710a1577 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.26.3 // indirect k8s.io/apiserver v0.26.3 // indirect diff --git a/kubelab-backend/go.sum b/kubelab-backend/go.sum index 9f87273..1e428d1 100644 --- a/kubelab-backend/go.sum +++ b/kubelab-backend/go.sum @@ -600,8 +600,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pocketbase/dbx v1.10.0 h1:58VIT7r6T+BnVbYVosvGBsPjQEic3/VFRYGT823vWSQ= github.com/pocketbase/dbx v1.10.0/go.mod h1:xXRCIAKTHMgUCyCKZm55pUOdvFziJjQfXaWKhu2vhMs= -github.com/pocketbase/pocketbase v0.17.4 h1:PwT84MMoXPVdv/EC/dfanCsqfByY1xMNlaNWvlM5MfY= -github.com/pocketbase/pocketbase v0.17.4/go.mod h1:IqsgywiDX5ewaUdeG+0NNckBLWp5mFhJSca1PH9zSp4= +github.com/pocketbase/pocketbase v0.17.5 h1:3KwMrlSMt1rcwZK2Z1/wFa5p/rkDHpcyeCRvUDXMb+Y= +github.com/pocketbase/pocketbase v0.17.5/go.mod h1:IqsgywiDX5ewaUdeG+0NNckBLWp5mFhJSca1PH9zSp4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/poy/onpar v0.0.0-20200406201722-06f95a1c68e8/go.mod h1:nSbFQvMj97ZyhFRSJYtut+msi4sOY6zJDGCdSc+/rZU= diff --git a/kubelab-backend/main.go b/kubelab-backend/main.go index 4cfd14d..adc5f63 100644 --- a/kubelab-backend/main.go +++ b/kubelab-backend/main.go @@ -1,26 +1,20 @@ package main import ( - "io" "log" - "net/http" "os" "path/filepath" "strings" - "time" "github.com/natrontech/kubelab/hooks" + "github.com/natrontech/kubelab/pkg/controller" "github.com/natrontech/kubelab/pkg/env" - "github.com/natrontech/kubelab/pkg/helm" "github.com/natrontech/kubelab/pkg/k8s" "github.com/pocketbase/pocketbase" "github.com/pocketbase/pocketbase/apis" "github.com/pocketbase/pocketbase/core" - "github.com/pocketbase/pocketbase/models" "github.com/pocketbase/pocketbase/plugins/jsvm" "github.com/pocketbase/pocketbase/plugins/migratecmd" - "gopkg.in/yaml.v2" - v1 "k8s.io/api/core/v1" ) func defaultPublicDir() string { @@ -76,271 +70,12 @@ func main() { }) app.OnRecordBeforeUpdateRequest().Add(func(e *core.RecordUpdateEvent) error { - // check collection name - if e.Collection.Name == "lab_sessions" { - if e.Record.GetBool("clusterRunning") { - - // deploy a new vcluster - helmclient, err := helm.CreateHelmClient(e.Record.GetString("lab"), e.Record.GetString("user")) - if err != nil { - log.Println(err) - return err - } - - err = helm.AddHelmRepositoryToClient(helmclient, "loft-sh", "https://charts.loft.sh") - if err != nil { - log.Println(err) - return err - } - - // create yaml struct - yamlValues := struct { - Sync struct { - PersistentVolumes struct { - Enabled bool `yaml:"enabled"` - } `yaml:"persistentvolumes"` - StorageClasses struct { - Enabled bool `yaml:"enabled"` - } `yaml:"storageclasses"` - Ingresses struct { - Enabled bool `yaml:"enabled"` - } `yaml:"ingresses"` - HostStorageClasses struct { - Enabled bool `yaml:"enabled"` - } `yaml:"hoststorageclasses"` - } `yaml:"sync"` - Storage struct { - Persistence bool `yaml:"persistence"` - } `yaml:"storage"` - Isolation struct { - Enabled bool `yaml:"enabled"` - PodSecurityStandard string `yaml:"podSecurityStandard"` - NodeProxyPermission struct { - Enabled bool `yaml:"enabled"` - } `yaml:"nodeProxyPermission"` - ResourceQuota struct { - Enabled bool `yaml:"enabled"` - } `yaml:"resourceQuota"` - LimitRange struct { - Enabled bool `yaml:"enabled"` - Default map[string]string `yaml:"default"` - DefaultRequest map[string]string `yaml:"defaultRequest"` - } `yaml:"limitRange"` - NetworkPolicy struct { - Enabled bool `yaml:"enabled"` - OutgoingConnections struct { - IPBlock struct { - CIDR string `yaml:"cidr"` - Except []string `yaml:"except"` - } `yaml:"ipBlock"` - } `yaml:"outgoingConnections"` - } `yaml:"networkPolicy"` - } `yaml:"isolation"` - }{} - - // set values - yamlValues.Sync.PersistentVolumes.Enabled = true - yamlValues.Sync.StorageClasses.Enabled = false - yamlValues.Sync.Ingresses.Enabled = true - yamlValues.Sync.HostStorageClasses.Enabled = true - yamlValues.Storage.Persistence = false - yamlValues.Isolation.Enabled = true - yamlValues.Isolation.PodSecurityStandard = "baseline" - yamlValues.Isolation.NodeProxyPermission.Enabled = false - yamlValues.Isolation.ResourceQuota.Enabled = false - yamlValues.Isolation.LimitRange.Enabled = true - yamlValues.Isolation.LimitRange.Default = map[string]string{ - "ephemeral-storage": "8Gi", - "memory": "512Mi", - "cpu": "1", - } - yamlValues.Isolation.LimitRange.DefaultRequest = map[string]string{ - "ephemeral-storage": "3Gi", - "memory": "128Mi", - "cpu": "100m", - } - yamlValues.Isolation.NetworkPolicy.Enabled = true - yamlValues.Isolation.NetworkPolicy.OutgoingConnections.IPBlock.CIDR = "8.8.8.8/32" - - // convert to yaml - yamlValuesBytes, err := yaml.Marshal(yamlValues) - if err != nil { - log.Println(err) - return err - } - - _, err = helm.CreateOrUpdateHelmRelease( - helmclient, - "loft-sh/vcluster", - "vcluster", - helm.GetNamespaceName(e.Record.GetString("lab"), e.Record.GetString("user")), - "0.15.2", - string(string(yamlValuesBytes)), - ) - if err != nil { - log.Println(err) - return err - } - - err = k8s.CreateResourceQuota(helm.GetNamespaceName(e.Record.GetString("lab"), e.Record.GetString("user")), env.Config.ResourceName, env.Config.PodsLimit, env.Config.StorageLimit) - if err != nil { - log.Println(err) - return err - } - - time.Sleep(15 * time.Second) - - } else { - // delete the namespace - err := k8s.DeleteNamespace(helm.GetNamespaceName(e.Record.GetString("lab"), e.Record.GetString("user"))) - if err != nil { - log.Println(err) - } - - time.Sleep(15 * time.Second) - } - } - - if e.Collection.Name == "exercise_sessions" { - if e.Record.GetBool("agentRunning") { - var err error - var exercise *models.Record - // retrieve the exercise - exercise, err = app.Dao().FindRecordById("exercises", e.Record.GetString("exercise")) - if err != nil { - log.Println(err) - return err - } - - // check if the namespace exists - err = k8s.CreateNamespace(helm.GetNamespaceName(exercise.GetString("lab"), e.Record.GetString("user"))) - if err != nil { - log.Println(err) - } else { - return err - } - - // check if vcluster pod exists 'vcluster-0' - _, err = k8s.GetPodByName(helm.GetNamespaceName(exercise.GetString("lab"), e.Record.GetString("user")), "vcluster-0") - if err != nil { - log.Println(err) - return err - } - - // get kubeconfig secret called 'vc-vcluster' - var secret *v1.Secret - secret, err = k8s.GetSecretByName(helm.GetNamespaceName(exercise.GetString("lab"), e.Record.GetString("user")), "vc-vcluster") - if err != nil { - log.Println(err) - return err - } - - // get exercise.GetString("bootstrap") this is a url to a bootstrap script over https github raw - bootstrap, err := http.Get(exercise.GetString("bootstrap")) - if err != nil { - log.Println(err) - return err - } - - defer bootstrap.Body.Close() - - // read the body - bootstrapBody, err := io.ReadAll(bootstrap.Body) - if err != nil { - log.Println(err) - return err - } - - check, err := http.Get(exercise.GetString("check")) - if err != nil { - log.Println(err) - return err - } - - defer check.Body.Close() - - // read the body - checkBody, err := io.ReadAll(check.Body) - if err != nil { - log.Println(err) - return err - } - - // create a new deployment - _, err = k8s.CreateDeployment( - "kubelab-agent-"+exercise.Id, - helm.GetNamespaceName(exercise.GetString("lab"), e.Record.GetString("user")), - env.Config.KubelabImage, - 1, - string(secret.Data["config"]), - string(bootstrapBody), - string(checkBody), - env.Config.AllowedHosts, - ) - if err != nil { - log.Println(err) - } - - // create a new service - _, err = k8s.CreateService( - helm.GetNamespaceName(exercise.GetString("lab"), e.Record.GetString("user")), - "kubelab-agent-"+exercise.Id, - 8376, - ) - if err != nil { - log.Println(err) - } - - // create a new ingress - _, err = k8s.CreateIngress( - helm.GetNamespaceName(exercise.GetString("lab"), e.Record.GetString("user")), - "kubelab-"+exercise.GetString("lab")+"-"+exercise.Id+"-"+e.Record.GetString("user"), - env.Config.AllowedHosts, - "kubelab-agent-"+exercise.Id, - "kubelab-"+exercise.GetString("lab")+"-"+exercise.Id+"-"+e.Record.GetString("user"), - ) - - // check if deployment is ready - err = k8s.WaitForDeployment(helm.GetNamespaceName(exercise.GetString("lab"), e.Record.GetString("user")), "kubelab-agent-"+exercise.Id) - if err != nil { - log.Println(err) - } - - // sleep for 5 seconds - time.Sleep(5 * time.Second) - - } else { - var err error - var exercise *models.Record - // retrieve the exercise - exercise, err = app.Dao().FindRecordById("exercises", e.Record.GetString("exercise")) - if err != nil { - log.Println(err) - return err - } - // delete the deployment - err = k8s.DeleteDeployment(helm.GetNamespaceName(exercise.GetString("lab"), e.Record.GetString("user")), "kubelab-agent-"+exercise.Id) - if err != nil { - log.Println(err) - // return err - } - - // delete the service - err = k8s.DeleteService(helm.GetNamespaceName(exercise.GetString("lab"), e.Record.GetString("user")), "kubelab-agent-"+exercise.Id) - if err != nil { - log.Println(err) - // return err - } - - // delete the ingress - err = k8s.DeleteIngress(helm.GetNamespaceName(exercise.GetString("lab"), e.Record.GetString("user")), "kubelab-"+exercise.GetString("lab")+"-"+exercise.Id+"-"+e.Record.GetString("user")) - if err != nil { - log.Println(err) - // return err - } - } + switch e.Collection.Name { + case "lab_sessions": + return controller.HandleLabSessions(e, app) + case "exercise_sessions": + return controller.HandleExerciseSessions(e, app) } - return nil }) diff --git a/kubelab-backend/pkg/controller/exercise.go b/kubelab-backend/pkg/controller/exercise.go new file mode 100644 index 0000000..efaf7a5 --- /dev/null +++ b/kubelab-backend/pkg/controller/exercise.go @@ -0,0 +1,129 @@ +package controller + +import ( + "log" + "time" + + "github.com/natrontech/kubelab/pkg/env" + "github.com/natrontech/kubelab/pkg/helm" + "github.com/natrontech/kubelab/pkg/k8s" + "github.com/pocketbase/pocketbase" + "github.com/pocketbase/pocketbase/core" +) + +func setupExerciseResources(e *core.RecordUpdateEvent, app *pocketbase.PocketBase) error { + exercise, err := app.Dao().FindRecordById("exercises", e.Record.GetString("exercise")) + if err != nil { + return logAndReturnErr(err) + } + + user, err := app.Dao().FindRecordById("users", e.Record.GetString("user")) + if err != nil { + return err + } + + namespaceParams := k8s.NamespaceParams{ + Name: namespaceName(e, exercise.GetString("lab")), + UserRecord: user, + } + + if err = k8s.CreateNamespace(namespaceParams); err != nil { + log.Println(err) + } + + if _, err = k8s.GetPodByName(namespaceName(e, exercise.GetString("lab")), "vcluster-0"); err != nil { + return logAndReturnErr(err) + } + + secret, err := k8s.GetSecretByName(namespaceName(e, exercise.GetString("lab")), "vc-vcluster") + if err != nil { + return logAndReturnErr(err) + } + + bootstrapBody, err := fetchBodyFromURL(exercise.GetString("bootstrap")) + if err != nil { + return logAndReturnErr(err) + } + + checkBody, err := fetchBodyFromURL(exercise.GetString("check")) + if err != nil { + return logAndReturnErr(err) + } + + deploymentParams := k8s.DeploymentParams{ + Name: "kubelab-agent-" + exercise.Id, + Namespace: helm.GetNamespaceName(exercise.GetString("lab"), e.Record.GetString("user")), + Image: env.Config.KubelabImage, + Replicas: 1, + Kubeconfig: string(secret.Data["config"]), + Bootstrap: string(bootstrapBody), + Check: string(checkBody), + Host: env.Config.AllowedHosts, + UserRecord: user, + } + + // create a new deployment + _, err = k8s.CreateDeployment(deploymentParams) + if err != nil { + log.Println(err) + } + + serviceParams := k8s.ServiceParams{ + Name: "kubelab-agent-" + exercise.Id, + Namespace: helm.GetNamespaceName(exercise.GetString("lab"), e.Record.GetString("user")), + Port: 8376, + UserRecord: user, + } + + // create a new service + _, err = k8s.CreateService(serviceParams) + if err != nil { + log.Println(err) + } + + ingressParams := k8s.IngressParams{ + Name: "kubelab-agent-" + exercise.Id, + Namespace: helm.GetNamespaceName(exercise.GetString("lab"), e.Record.GetString("user")), + ServiceName: "kubelab-agent-" + exercise.Id, + Host: env.Config.AllowedHosts, + Path: "kubelab-" + exercise.GetString("lab") + "-" + exercise.Id + "-" + e.Record.GetString("user"), + UserRecord: user, + } + + // create a new ingress + _, err = k8s.CreateIngress(ingressParams) + if err != nil { + log.Println(err) + } + + // check if deployment is ready + err = k8s.WaitForDeployment(helm.GetNamespaceName(exercise.GetString("lab"), e.Record.GetString("user")), "kubelab-agent-"+exercise.Id) + if err != nil { + log.Println(err) + } + + // sleep for 5 seconds + time.Sleep(5 * time.Second) + return nil +} + +func deleteExerciseResources(e *core.RecordUpdateEvent, app *pocketbase.PocketBase) error { + exercise, err := app.Dao().FindRecordById("exercises", e.Record.GetString("exercise")) + if err != nil { + return logAndReturnErr(err) + } + + deleteFuncs := []func(string, string) error{ + k8s.DeleteDeployment, + k8s.DeleteService, + k8s.DeleteIngress, + } + + for _, fn := range deleteFuncs { + if err = fn(namespaceName(e, exercise.GetString("lab")), "kubelab-agent-"+exercise.Id); err != nil { + log.Println(err) + } + } + + return nil +} diff --git a/kubelab-backend/pkg/controller/handler.go b/kubelab-backend/pkg/controller/handler.go new file mode 100644 index 0000000..f5ab7b1 --- /dev/null +++ b/kubelab-backend/pkg/controller/handler.go @@ -0,0 +1,20 @@ +package controller + +import ( + "github.com/pocketbase/pocketbase" + "github.com/pocketbase/pocketbase/core" +) + +func HandleLabSessions(e *core.RecordUpdateEvent, app *pocketbase.PocketBase) error { + if e.Record.GetBool("clusterRunning") { + return deployVCluster(e, app) + } + return deleteClusterResources(e, app) +} + +func HandleExerciseSessions(e *core.RecordUpdateEvent, app *pocketbase.PocketBase) error { + if e.Record.GetBool("agentRunning") { + return setupExerciseResources(e, app) + } + return deleteExerciseResources(e, app) +} diff --git a/kubelab-backend/pkg/controller/lab.go b/kubelab-backend/pkg/controller/lab.go new file mode 100644 index 0000000..d5585d5 --- /dev/null +++ b/kubelab-backend/pkg/controller/lab.go @@ -0,0 +1,86 @@ +package controller + +import ( + "log" + "os" + "time" + + "github.com/natrontech/kubelab/pkg/env" + "github.com/natrontech/kubelab/pkg/helm" + "github.com/natrontech/kubelab/pkg/k8s" + "github.com/natrontech/kubelab/pkg/util" + "github.com/pocketbase/pocketbase" + "github.com/pocketbase/pocketbase/core" +) + +func deployVCluster(e *core.RecordUpdateEvent, app *pocketbase.PocketBase) error { + helmclient, err := helm.CreateHelmClient(e.Record.GetString("lab"), e.Record.GetString("user")) + if err != nil { + return logAndReturnErr(err) + } + + user, err := app.Dao().FindRecordById("users", e.Record.GetString("user")) + if err != nil { + return err + } + + if err = helm.AddHelmRepositoryToClient(helmclient, "loft-sh", "https://charts.loft.sh"); err != nil { + return logAndReturnErr(err) + } + + yamlValues, err := os.ReadFile(env.Config.VClusterValuesFilePath) + if err != nil { + return logAndReturnErr(err) + } + + labels := map[string]string{ + "kubelab.ch": e.Record.GetString("lab"), + "kubelab.ch/userId": e.Record.GetString("user"), + "kubelab.ch/username": user.GetString("username"), + "kubelab.ch/displayName": util.StringParser(user.GetString("name")), + } + + // add string at the end of yamlValues + yamlValues = append(yamlValues, []byte("\nlabels:\n")...) + for k, v := range labels { + yamlValues = append(yamlValues, []byte(" "+k+": "+v+"\n")...) + } + + // add string at the end of yamlValues + yamlValues = append(yamlValues, []byte("\npodLabels:\n")...) + for k, v := range labels { + yamlValues = append(yamlValues, []byte(" "+k+": "+v+"\n")...) + } + + // add string at the end of yamlValues + yamlValues = append(yamlValues, []byte("\ncoredns:\n podLabels:\n")...) + for k, v := range labels { + yamlValues = append(yamlValues, []byte(" "+k+": "+v+"\n")...) + } + + if _, err = helm.CreateOrUpdateHelmRelease( + helmclient, + "loft-sh/vcluster", + "vcluster", + namespaceName(e, e.Record.GetString("lab")), + env.Config.VClusterChartVersion, + string(yamlValues), + ); err != nil { + return logAndReturnErr(err) + } + + if err = k8s.CreateResourceQuota(namespaceName(e, e.Record.GetString("lab")), env.Config.ResourceName, env.Config.PodsLimit, env.Config.StorageLimit); err != nil { + return logAndReturnErr(err) + } + + time.Sleep(15 * time.Second) + return nil +} + +func deleteClusterResources(e *core.RecordUpdateEvent, app *pocketbase.PocketBase) error { + if err := k8s.DeleteNamespace(namespaceName(e, e.Record.GetString("lab"))); err != nil { + log.Println(err) + } + time.Sleep(15 * time.Second) + return nil +} diff --git a/kubelab-backend/pkg/controller/util.go b/kubelab-backend/pkg/controller/util.go new file mode 100644 index 0000000..24a53be --- /dev/null +++ b/kubelab-backend/pkg/controller/util.go @@ -0,0 +1,28 @@ +package controller + +import ( + "io" + "log" + "net/http" + + "github.com/natrontech/kubelab/pkg/helm" + "github.com/pocketbase/pocketbase/core" +) + +func namespaceName(e *core.RecordUpdateEvent, lab string) string { + return helm.GetNamespaceName(lab, e.Record.GetString("user")) +} + +func logAndReturnErr(err error) error { + log.Println(err) + return err +} + +func fetchBodyFromURL(url string) ([]byte, error) { + response, err := http.Get(url) + if err != nil { + return nil, logAndReturnErr(err) + } + defer response.Body.Close() + return io.ReadAll(response.Body) +} diff --git a/kubelab-backend/pkg/env/env.go b/kubelab-backend/pkg/env/env.go index 13d567d..2dd5485 100644 --- a/kubelab-backend/pkg/env/env.go +++ b/kubelab-backend/pkg/env/env.go @@ -7,13 +7,15 @@ import ( ) type config struct { - Local bool `env:"LOCAL"` - KubelabImage string `env:"KUBELAB_AGENT_IMAGE"` - AllowedHosts string `env:"ALLOWED_HOSTS"` - ResourceName string `env:"RESOURCE_NAME"` - IngressClass string `env:"AGENT_INGRESS_CLASS"` - PodsLimit string `env:"PODS_LIMIT"` - StorageLimit string `env:"STORAGE_LIMIT"` + Local bool `env:"LOCAL"` + KubelabImage string `env:"KUBELAB_AGENT_IMAGE" envDefault:"ghcr.io/natrontech/kubelab-agent:latest"` + AllowedHosts string `env:"ALLOWED_HOSTS" envDefault:"*"` + ResourceName string `env:"RESOURCE_NAME" envDefault:"kubelab"` + IngressClass string `env:"AGENT_INGRESS_CLASS" envDefault:"nginx"` + PodsLimit string `env:"PODS_LIMIT" envDefault:"70"` + StorageLimit string `env:"STORAGE_LIMIT" envDefault:"50Gi"` + VClusterChartVersion string `env:"VCLUSTER_CHART_VERSION" envDefault:"0.15.5"` + VClusterValuesFilePath string `env:"VCLUSTER_VALUES_FILE_PATH" envDefault:"./vcluster-values.yaml"` } var Config config @@ -26,29 +28,4 @@ func Init() { if Config.Local { log.Println("Running in local mode") } - - if Config.KubelabImage == "" { - Config.KubelabImage = "ghcr.io/natrontech/kubelab-agent:latest" - } - - if Config.AllowedHosts == "" { - Config.AllowedHosts = "*" - } - - if Config.ResourceName == "" { - Config.ResourceName = "kubelab" - } - - if Config.IngressClass == "" { - Config.IngressClass = "nginx" - } - - if Config.PodsLimit == "" { - Config.PodsLimit = "70" - } - - if Config.StorageLimit == "" { - Config.StorageLimit = "50Gi" - } - } diff --git a/kubelab-backend/pkg/helm/helm.go b/kubelab-backend/pkg/helm/helm.go index a010998..24f4a79 100644 --- a/kubelab-backend/pkg/helm/helm.go +++ b/kubelab-backend/pkg/helm/helm.go @@ -11,15 +11,13 @@ import ( "helm.sh/helm/v3/pkg/repo" ) -var ( - Prefix = "kubelab" -) +const Prefix = "kubelab" -func GetNamespaceName(labName string, username string) string { +func GetNamespaceName(labName, username string) string { return Prefix + "-" + util.StringParser(labName) + "-" + util.StringParser(username) } -func CreateHelmClient(labName string, username string) (helmclient.Client, error) { +func CreateHelmClient(labName, username string) (helmclient.Client, error) { opt := &helmclient.Options{ Namespace: GetNamespaceName(labName, username), Debug: true, @@ -27,29 +25,19 @@ func CreateHelmClient(labName string, username string) (helmclient.Client, error DebugLog: func(format string, v ...interface{}) {}, } - helmClient, err := helmclient.New(opt) - if err != nil { - return nil, err - } - - return helmClient, nil + return helmclient.New(opt) } -func AddHelmRepositoryToClient(helmClient helmclient.Client, repositoryName string, repositoryURL string) error { +func AddHelmRepositoryToClient(helmClient helmclient.Client, repositoryName, repositoryURL string) error { chartRepo := repo.Entry{ Name: strings.ToLower(repositoryName), URL: repositoryURL, } - if err := helmClient.AddOrUpdateChartRepo(chartRepo); err != nil { - return err - } - - return nil + return helmClient.AddOrUpdateChartRepo(chartRepo) } -func CreateOrUpdateHelmRelease(helmClient helmclient.Client, chartName string, releaseName string, namespace string, version string, valuesYaml string) (*release.Release, error) { - +func CreateOrUpdateHelmRelease(helmClient helmclient.Client, chartName, releaseName, namespace, version, valuesYaml string) (rel *release.Release, err error) { chartSpec := helmclient.ChartSpec{ ChartName: strings.ToLower(chartName), ReleaseName: strings.ToLower(releaseName), @@ -60,11 +48,8 @@ func CreateOrUpdateHelmRelease(helmClient helmclient.Client, chartName string, r ValuesYaml: valuesYaml, } - if release, err := helmClient.InstallOrUpgradeChart(context.Background(), &chartSpec, nil); err != nil { - return nil, err - } else { - return release, nil - } + rel, err = helmClient.InstallOrUpgradeChart(context.Background(), &chartSpec, nil) + return } func GetHelmRelease(helmClient helmclient.Client, releaseName string) (*release.Release, error) { diff --git a/kubelab-backend/pkg/k8s/deployment.go b/kubelab-backend/pkg/k8s/deployment.go index 91e73d6..86e0919 100644 --- a/kubelab-backend/pkg/k8s/deployment.go +++ b/kubelab-backend/pkg/k8s/deployment.go @@ -5,44 +5,55 @@ import ( "strings" "time" + "github.com/natrontech/kubelab/pkg/util" + "github.com/pocketbase/pocketbase/models" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func CreateDeployment(name string, namespace string, image string, replicas int32, kubeconfig string, bootstrap string, check string, host string) (*appsv1.Deployment, error) { +type DeploymentParams struct { + Name string + Namespace string + Image string + Replicas int32 + Kubeconfig string + Bootstrap string + Check string + Host string + UserRecord *models.Record +} - // search in string kubeconfig for 'localhost' and replace it with 'vcluster' - newKubeconfig := strings.Replace(kubeconfig, "localhost:8443", "vcluster:443", -1) +func CreateDeployment(params DeploymentParams) (*appsv1.Deployment, error) { + params.Kubeconfig = strings.Replace(params.Kubeconfig, "localhost:8443", "vcluster:443", -1) - configMap := &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kubeconfig", - }, - Data: map[string]string{ - "config": newKubeconfig, - }, - } - if _, err := Clientset.CoreV1().ConfigMaps(namespace).Create(Ctx, configMap, metav1.CreateOptions{}); err != nil { + createConfigMap(params.Namespace, "kubeconfig", map[string]string{"config": params.Kubeconfig}) + createConfigMap(params.Namespace, "scripts-"+params.Name, map[string]string{ + "check.sh": params.Check, + "bootstrap.sh": params.Bootstrap, + }) + + deployment := constructDeployment(params.Name, params.Namespace, params.Image, params.Replicas, params.Host, params.UserRecord) + deployed, err := Clientset.AppsV1().Deployments(params.Namespace).Create(Ctx, deployment, metav1.CreateOptions{}) + if err != nil { log.Println(err) } + return deployed, nil +} - // create config maps for scripts - scriptsConfigMap := &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "scripts-" + name, - }, - Data: map[string]string{ - "check.sh": check, - "bootstrap.sh": bootstrap, - }, +func createConfigMap(namespace, name string, data map[string]string) { + configMap := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Data: data, } - - if _, err := Clientset.CoreV1().ConfigMaps(namespace).Create(Ctx, scriptsConfigMap, metav1.CreateOptions{}); err != nil { + if _, err := Clientset.CoreV1().ConfigMaps(namespace).Create(Ctx, configMap, metav1.CreateOptions{}); err != nil { log.Println(err) } +} - deployment := &appsv1.Deployment{ +func constructDeployment(name, namespace, image string, replicas int32, host string, userRecord *models.Record) *appsv1.Deployment { + scriptVolumeName := "scripts-" + name + return &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: namespace, @@ -51,13 +62,19 @@ func CreateDeployment(name string, namespace string, image string, replicas int3 Replicas: &replicas, Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - "kubelab.natron.io": name, + "kubelab.ch": name, + "kubelab.ch/userId": userRecord.GetString("id"), + "kubelab.ch/username": userRecord.GetString("username"), + "kubelab.ch/displayName": util.StringParser(userRecord.GetString("name")), }, }, Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - "kubelab.natron.io": name, + "kubelab.ch": name, + "kubelab.ch/userId": userRecord.GetString("id"), + "kubelab.ch/username": userRecord.GetString("username"), + "kubelab.ch/displayName": util.StringParser(userRecord.GetString("name")), "vcluster.loft.sh/managed-by": "vcluster", }, }, @@ -77,11 +94,11 @@ func CreateDeployment(name string, namespace string, image string, replicas int3 MountPath: "/config-writable", }, { - Name: "scripts-" + name, + Name: scriptVolumeName, MountPath: "/scripts", }, { - Name: "scripts-" + name + "-writable", + Name: scriptVolumeName + "-writable", MountPath: "/scripts-writable", }, }, @@ -106,7 +123,7 @@ func CreateDeployment(name string, namespace string, image string, replicas int3 ReadOnly: true, }, { - Name: "scripts-" + name + "-writable", + Name: scriptVolumeName + "-writable", MountPath: "/home/kubelab-agent/.kubelab", ReadOnly: true, }, @@ -131,17 +148,17 @@ func CreateDeployment(name string, namespace string, image string, replicas int3 }, }, { - Name: "scripts-" + name, + Name: scriptVolumeName, VolumeSource: v1.VolumeSource{ ConfigMap: &v1.ConfigMapVolumeSource{ LocalObjectReference: v1.LocalObjectReference{ - Name: "scripts-" + name, + Name: scriptVolumeName, }, }, }, }, { - Name: "scripts-" + name + "-writable", + Name: scriptVolumeName + "-writable", VolumeSource: v1.VolumeSource{ EmptyDir: &v1.EmptyDirVolumeSource{}, }, @@ -151,40 +168,30 @@ func CreateDeployment(name string, namespace string, image string, replicas int3 }, }, } +} - deployment, err := Clientset.AppsV1().Deployments(namespace).Create(Ctx, deployment, metav1.CreateOptions{}) - if err != nil { - log.Println(err) - } +func DeleteDeployment(namespace, name string) error { + deleteConfigMap(namespace, "kubeconfig") + deleteConfigMap(namespace, "scripts-"+name) - return deployment, nil + return Clientset.AppsV1().Deployments(namespace).Delete(Ctx, name, metav1.DeleteOptions{}) } -func DeleteDeployment(namespace string, name string) error { - // Delete the configmap first - if err := Clientset.CoreV1().ConfigMaps(namespace).Delete(Ctx, "kubeconfig", metav1.DeleteOptions{}); err != nil { +func deleteConfigMap(namespace, name string) { + if err := Clientset.CoreV1().ConfigMaps(namespace).Delete(Ctx, name, metav1.DeleteOptions{}); err != nil { log.Println(err) } - - if err := Clientset.CoreV1().ConfigMaps(namespace).Delete(Ctx, "scripts-"+name, metav1.DeleteOptions{}); err != nil { - log.Println(err) - } - - return Clientset.AppsV1().Deployments(namespace).Delete(Ctx, name, metav1.DeleteOptions{}) } -func WaitForDeployment(namespace string, name string) error { +func WaitForDeployment(namespace, name string) error { for { deployment, err := Clientset.AppsV1().Deployments(namespace).Get(Ctx, name, metav1.GetOptions{}) if err != nil { return err } - - // check if all containers are ready if deployment.Status.ReadyReplicas == *deployment.Spec.Replicas { return nil } - time.Sleep(1 * time.Second) } } diff --git a/kubelab-backend/pkg/k8s/ingress.go b/kubelab-backend/pkg/k8s/ingress.go index 3866f80..b306df2 100644 --- a/kubelab-backend/pkg/k8s/ingress.go +++ b/kubelab-backend/pkg/k8s/ingress.go @@ -2,17 +2,28 @@ package k8s import ( "github.com/natrontech/kubelab/pkg/env" + "github.com/natrontech/kubelab/pkg/util" + "github.com/pocketbase/pocketbase/models" networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +type IngressParams struct { + Namespace string + Name string + Host string + ServiceName string + Path string + UserRecord *models.Record +} + // create a ingress with a hostpath with the namespace name pointed to a service -func CreateIngress(namespace string, name string, host string, serviceName string, path string) (*networkingv1.Ingress, error) { +func CreateIngress(params IngressParams) (*networkingv1.Ingress, error) { ingress := &networkingv1.Ingress{ ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, + Name: params.Name, + Namespace: params.Namespace, Annotations: map[string]string{ "nginx.ingress.kubernetes.io/affinity": "cookie", "nginx.ingress.kubernetes.io/proxy-connect-timeout": "3600", @@ -23,29 +34,35 @@ func CreateIngress(namespace string, name string, host string, serviceName strin "nginx.ingress.kubernetes.io/session-cookie-expires": "172800", "nginx.ingress.kubernetes.io/session-cookie-max-age": "172800", "nginx.ingress.kubernetes.io/session-cookie-name": "route", - "nginx.ingress.kubernetes.io/websocket-services": serviceName, - "nginx.org/websocket-services": serviceName, + "nginx.ingress.kubernetes.io/websocket-services": params.ServiceName, + "nginx.org/websocket-services": params.ServiceName, + }, + Labels: map[string]string{ + "kubelab.ch": params.Name, + "kubelab.ch/userId": params.UserRecord.GetString("id"), + "kubelab.ch/username": params.UserRecord.GetString("username"), + "kubelab.ch/displayName": util.StringParser(params.UserRecord.GetString("name")), }, }, Spec: networkingv1.IngressSpec{ IngressClassName: func() *string { s := env.Config.IngressClass; return &s }(), TLS: []networkingv1.IngressTLS{ { - Hosts: []string{host}, + Hosts: []string{params.Host}, }, }, Rules: []networkingv1.IngressRule{ { - Host: host, + Host: params.Host, IngressRuleValue: networkingv1.IngressRuleValue{ HTTP: &networkingv1.HTTPIngressRuleValue{ Paths: []networkingv1.HTTPIngressPath{ { - Path: "/" + path + "(/|$)(.*)", + Path: "/" + params.Path + "(/|$)(.*)", PathType: func() *networkingv1.PathType { p := networkingv1.PathTypeImplementationSpecific; return &p }(), Backend: networkingv1.IngressBackend{ Service: &networkingv1.IngressServiceBackend{ - Name: serviceName, + Name: params.ServiceName, Port: networkingv1.ServiceBackendPort{ Number: 8376, }, @@ -60,7 +77,7 @@ func CreateIngress(namespace string, name string, host string, serviceName strin }, } - return Clientset.NetworkingV1().Ingresses(namespace).Create(Ctx, ingress, metav1.CreateOptions{}) + return Clientset.NetworkingV1().Ingresses(params.Namespace).Create(Ctx, ingress, metav1.CreateOptions{}) } diff --git a/kubelab-backend/pkg/k8s/namespace.go b/kubelab-backend/pkg/k8s/namespace.go index 0e41e45..b69ca13 100644 --- a/kubelab-backend/pkg/k8s/namespace.go +++ b/kubelab-backend/pkg/k8s/namespace.go @@ -4,6 +4,8 @@ import ( "strconv" "strings" + "github.com/natrontech/kubelab/pkg/util" + "github.com/pocketbase/pocketbase/models" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -22,16 +24,30 @@ func GetTotalNamespaces() (string, error) { return "unknown", nil } -func CreateNamespace(namespace string) error { +type NamespaceParams struct { + Name string + UserRecord *models.Record +} + +func CreateNamespace(params NamespaceParams) error { ns := &v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: namespace, + Name: params.Name, Labels: map[string]string{ - "kubelab.natron.io/created-by": "kubelab", + "kubelab.ch": params.Name, + "kubelab.ch/userId": params.UserRecord.GetString("id"), + "kubelab.ch/username": params.UserRecord.GetString("username"), + "kubelab.ch/displayName": util.StringParser(params.UserRecord.GetString("name")), }, }, } _, err := Clientset.CoreV1().Namespaces().Create(Ctx, ns, metav1.CreateOptions{}) + + // if err already exists, update + if err != nil && strings.Contains(err.Error(), "already exists") { + _, err = Clientset.CoreV1().Namespaces().Update(Ctx, ns, metav1.UpdateOptions{}) + } + return err } diff --git a/kubelab-backend/pkg/k8s/service.go b/kubelab-backend/pkg/k8s/service.go index 810a0b5..c64b3b1 100644 --- a/kubelab-backend/pkg/k8s/service.go +++ b/kubelab-backend/pkg/k8s/service.go @@ -1,30 +1,45 @@ package k8s import ( + "github.com/natrontech/kubelab/pkg/util" + "github.com/pocketbase/pocketbase/models" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func CreateService(namespace string, name string, port int32) (*v1.Service, error) { +type ServiceParams struct { + Namespace string + Name string + Port int32 + UserRecord *models.Record +} + +func CreateService(params ServiceParams) (*v1.Service, error) { service := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, + Name: params.Name, + Namespace: params.Namespace, + Labels: map[string]string{ + "kubelab.ch": params.Name, + "kubelab.ch/userId": params.UserRecord.GetString("id"), + "kubelab.ch/username": params.UserRecord.GetString("username"), + "kubelab.ch/displayName": util.StringParser(params.UserRecord.GetString("name")), + }, }, Spec: v1.ServiceSpec{ Selector: map[string]string{ - "kubelab.natron.io": name, + "kubelab.ch": params.Name, }, Ports: []v1.ServicePort{ { - Port: port, + Port: params.Port, }, }, Type: v1.ServiceTypeClusterIP, }, } - return Clientset.CoreV1().Services(namespace).Create(Ctx, service, metav1.CreateOptions{}) + return Clientset.CoreV1().Services(params.Namespace).Create(Ctx, service, metav1.CreateOptions{}) } func DeleteService(namespace string, name string) error { diff --git a/kubelab-backend/vcluster-values.yaml b/kubelab-backend/vcluster-values.yaml new file mode 100644 index 0000000..b555466 --- /dev/null +++ b/kubelab-backend/vcluster-values.yaml @@ -0,0 +1,34 @@ +sync: + persistentvolumes: + enabled: true + storageclasses: + enabled: false + ingresses: + enabled: true + hoststorageclasses: + enabled: true +storage: + persistence: false +isolation: + enabled: true + podSecurityStandard: baseline + nodeProxyPermission: + enabled: false + resourceQuota: + enabled: false + limitRange: + enabled: true + default: + ephemeral-storage: 8Gi + memory: 512Mi + cpu: 1 + defaultRequest: + ephemeral-storage: 3Gi + memory: 128Mi + cpu: 100m + networkPolicy: + enabled: true + outgoingConnections: + ipBlock: + cidr: 8.8.8.8/32 + except: []