feat(k8s): local mode for container modules (#2949)

core/package.json

@@ -137,6 +137,7 @@
     "titleize": "^2.1.0",
     "tmp-promise": "^3.0.2",
     "toposort": "^2.0.2",
+    "touch": "^3.1.0",
     "tough-cookie": "^4.0.0",
     "ts-stream": "^3.0.0",
     "typeorm-with-better-sqlite3": "^0.2.27",
@@ -270,4 +271,4 @@
     ]
   },
   "gitHead": "b0647221a4d2ff06952bae58000b104215aed922"
-}
+}

core/src/actions.ts

@@ -615,7 +615,13 @@ export class ActionRouter implements TypeGuard {
     })
 
     const runtimeContext = emptyRuntimeContext
-    const status = await this.getServiceStatus({ ...params, runtimeContext, devMode: false, hotReload: false })
+    const status = await this.getServiceStatus({
+      ...params,
+      runtimeContext,
+      devMode: false,
+      hotReload: false,
+      localMode: false,
+    })
 
     if (status.state === "missing") {
       log.setSuccess({
@@ -804,6 +810,7 @@ export class ActionRouter implements TypeGuard {
           service,
           devModeServiceNames: [],
           hotReloadServiceNames: [],
+          localModeServiceNames: [],
         })
     )
     const results = await this.garden.processTasks(tasks, { throwOnError: true })
@@ -826,6 +833,7 @@ export class ActionRouter implements TypeGuard {
           fromWatch: false,
           devModeServiceNames: [],
           hotReloadServiceNames: [],
+          localModeServiceNames: [],
         })
     )
 

core/src/commands/call.ts

@@ -81,6 +81,7 @@ export class CallCommand extends Command<Args> {
       graph,
       devMode: false,
       hotReload: false,
+      localMode: false,
       runtimeContext,
     })
 

core/src/commands/deploy.ts

@@ -25,7 +25,7 @@ import { printHeader } from "../logger/util"
 import { BaseTask } from "../tasks/base"
 import {
   getDevModeModules,
-  getDevModeServiceNames,
+  getMatchingServiceNames,
   getHotReloadServiceNames,
   validateHotReloadServiceNames,
 } from "./helpers"
@@ -67,6 +67,17 @@ export const deployOpts = {
     `,
     alias: "hot",
   }),
+  "local-mode": new StringsParameter({
+    help: deline`[EXPERIMENTAL] The name(s) of the service(s) to be started locally with local mode enabled.
+    Use comma as a separator to specify multiple services. Use * to deploy all
+    services with local mode enabled. When this option is used,
+    the command is run in persistent mode.
+
+    This always takes the precedence over the dev mode if there are any conflicts,
+    i.e. if the same services are passed to both \`--dev\` and \`--local\` options.
+    `,
+    alias: "local",
+  }),
   "skip": new StringsParameter({
     help: "The name(s) of services you'd like to skip when deploying.",
   }),
@@ -110,6 +121,8 @@ export class DeployCommand extends Command<Args, Opts> {
         garden deploy --watch              # watch for changes to code
         garden deploy --dev=my-service     # deploys all services, with dev mode enabled for my-service
         garden deploy --dev                # deploys all compatible services with dev mode enabled
+        garden deploy --local=my-service   # deploys all services, with local mode enabled for my-service
+        garden deploy --local              # deploys all compatible services with local mode enabled
         garden deploy --env stage          # deploy your services to an environment called stage
         garden deploy --skip service-b     # deploy all services except service-b
   `
@@ -122,7 +135,7 @@ export class DeployCommand extends Command<Args, Opts> {
   outputsSchema = () => processCommandResultSchema()
 
   isPersistent({ opts }: PrepareParams<Args, Opts>) {
-    return !!opts.watch || !!opts["hot-reload"] || !!opts["dev-mode"] || !!opts.forward
+    return !!opts.watch || !!opts["hot-reload"] || !!opts["dev-mode"] || !!opts["local-mode"] || !!opts.forward
   }
 
   printHeader({ headerLog }) {
@@ -183,7 +196,10 @@ export class DeployCommand extends Command<Args, Opts> {
     }
 
     const modules = Array.from(new Set(services.map((s) => s.module)))
-    const devModeServiceNames = getDevModeServiceNames(opts["dev-mode"], initGraph)
+    const localModeServiceNames = getMatchingServiceNames(opts["local-mode"], initGraph)
+    const devModeServiceNames = getMatchingServiceNames(opts["dev-mode"], initGraph).filter(
+      (name) => !localModeServiceNames.includes(name)
+    )
     const hotReloadServiceNames = getHotReloadServiceNames(opts["hot-reload"], initGraph)
 
     let watch = opts.watch
@@ -218,6 +234,7 @@ export class DeployCommand extends Command<Args, Opts> {
           skipRuntimeDependencies,
           devModeServiceNames,
           hotReloadServiceNames,
+          localModeServiceNames,
         })
     )
 
@@ -239,6 +256,7 @@ export class DeployCommand extends Command<Args, Opts> {
           servicesWatched: services.map((s) => s.name),
           devModeServiceNames,
           hotReloadServiceNames,
+          localModeServiceNames,
         })
 
         return tasks

core/src/commands/dev.ts

@@ -7,30 +7,35 @@
  */
 
 import Bluebird from "bluebird"
-import deline = require("deline")
-import dedent = require("dedent")
 import chalk from "chalk"
 import { readFile } from "fs-extra"
 import { flatten } from "lodash"
-import moment = require("moment")
 import { join } from "path"
 
 import { getModuleWatchTasks } from "../tasks/helpers"
-import { Command, CommandResult, CommandParams, handleProcessResults, PrepareParams } from "./base"
+import { Command, CommandParams, CommandResult, handleProcessResults, PrepareParams } from "./base"
 import { STATIC_DIR } from "../constants"
 import { processModules } from "../process"
 import { GardenModule } from "../types/module"
 import { getTestTasks } from "../tasks/test"
 import { ConfigGraph } from "../config-graph"
-import { getDevModeModules, getHotReloadServiceNames, validateHotReloadServiceNames } from "./helpers"
+import {
+  getDevModeModules,
+  getHotReloadServiceNames,
+  getMatchingServiceNames,
+  validateHotReloadServiceNames,
+} from "./helpers"
 import { startServer } from "../server/server"
 import { BuildTask } from "../tasks/build"
 import { DeployTask } from "../tasks/deploy"
 import { Garden } from "../garden"
 import { LogEntry } from "../logger/log-entry"
-import { StringsParameter, BooleanParameter } from "../cli/params"
+import { BooleanParameter, StringsParameter } from "../cli/params"
 import { printHeader } from "../logger/util"
 import { GardenService } from "../types/service"
+import deline = require("deline")
+import dedent = require("dedent")
+import moment = require("moment")
 
 const ansiBannerPath = join(STATIC_DIR, "garden-banner-2.txt")
 
@@ -50,6 +55,17 @@ const devOpts = {
     `,
     alias: "hot",
   }),
+  "local-mode": new StringsParameter({
+    help: deline`[EXPERIMENTAL] The name(s) of the service(s) to be started locally with local mode enabled.
+    Use comma as a separator to specify multiple services. Use * to deploy all
+    services with local mode enabled. When this option is used,
+    the command is run in persistent mode.
+
+    This always takes the precedence over the dev mode if there are any conflicts,
+    i.e. if the same services are passed to both \`--dev\` and \`--local\` options.
+    `,
+    alias: "local",
+  }),
   "skip-tests": new BooleanParameter({
     help: "Disable running the tests.",
   }),
@@ -85,6 +101,8 @@ export class DevCommand extends Command<DevCommandArgs, DevCommandOpts> {
         garden dev
         garden dev --hot=foo-service,bar-service  # enable hot reloading for foo-service and bar-service
         garden dev --hot=*                        # enable hot reloading for all compatible services
+        garden dev --local=service-1,service-2    # enable local mode for service-1 and service-2
+        garden dev --local=*                      # enable local mode for all compatible services
         garden dev --skip-tests=                  # skip running any tests
         garden dev --force                        # force redeploy of services when the command starts
         garden dev --name integ                   # run all tests with the name 'integ' in the project
@@ -152,13 +170,15 @@ export class DevCommand extends Command<DevCommandArgs, DevCommandOpts> {
       }
     }
 
+    const localModeServiceNames = getMatchingServiceNames(opts["local-mode"], graph)
+
     const services = graph.getServices({ names: args.services })
 
     const devModeServiceNames = services
       .map((s) => s.name)
       // Since dev mode is implicit when using this command, we consider explicitly enabling hot reloading to
       // take precedence over dev mode.
-      .filter((name) => !hotReloadServiceNames.includes(name))
+      .filter((name) => !hotReloadServiceNames.includes(name) && !localModeServiceNames.includes(name))
 
     const initialTasks = await getDevCommandInitialTasks({
       garden,
@@ -168,6 +188,7 @@ export class DevCommand extends Command<DevCommandArgs, DevCommandOpts> {
       services,
       devModeServiceNames,
       hotReloadServiceNames,
+      localModeServiceNames,
       skipTests,
       forceDeploy: opts.force,
     })
@@ -190,6 +211,7 @@ export class DevCommand extends Command<DevCommandArgs, DevCommandOpts> {
           servicesWatched: devModeServiceNames,
           devModeServiceNames,
           hotReloadServiceNames,
+          localModeServiceNames,
           testNames: opts["test-names"],
           skipTests,
         })
@@ -208,6 +230,7 @@ export async function getDevCommandInitialTasks({
   services,
   devModeServiceNames,
   hotReloadServiceNames,
+  localModeServiceNames,
   skipTests,
   forceDeploy,
 }: {
@@ -218,6 +241,7 @@ export async function getDevCommandInitialTasks({
   services: GardenService[]
   devModeServiceNames: string[]
   hotReloadServiceNames: string[]
+  localModeServiceNames: string[]
   skipTests: boolean
   forceDeploy: boolean
 }) {
@@ -242,6 +266,7 @@ export async function getDevCommandInitialTasks({
             module,
             devModeServiceNames,
             hotReloadServiceNames,
+            localModeServiceNames,
             force: forceDeploy,
             forceBuild: false,
           })
@@ -264,6 +289,7 @@ export async function getDevCommandInitialTasks({
           fromWatch: false,
           devModeServiceNames,
           hotReloadServiceNames,
+          localModeServiceNames,
         })
     )
 
@@ -278,6 +304,7 @@ export async function getDevCommandWatchTasks({
   servicesWatched,
   devModeServiceNames,
   hotReloadServiceNames,
+  localModeServiceNames,
   testNames,
   skipTests,
 }: {
@@ -288,6 +315,7 @@ export async function getDevCommandWatchTasks({
   servicesWatched: string[]
   devModeServiceNames: string[]
   hotReloadServiceNames: string[]
+  localModeServiceNames: string[]
   testNames: string[] | undefined
   skipTests: boolean
 }) {
@@ -299,6 +327,7 @@ export async function getDevCommandWatchTasks({
     servicesWatched,
     devModeServiceNames,
     hotReloadServiceNames,
+    localModeServiceNames,
   })
 
   if (!skipTests) {
@@ -315,6 +344,7 @@ export async function getDevCommandWatchTasks({
             fromWatch: true,
             devModeServiceNames,
             hotReloadServiceNames,
+            localModeServiceNames,
           })
         )
       )

core/src/commands/helpers.ts

@@ -17,7 +17,7 @@ import { GardenTask } from "../types/task"
 import { GardenTest } from "../types/test"
 import { uniqByName } from "../util/util"
 
-export function getDevModeServiceNames(namesFromOpt: string[] | undefined, configGraph: ConfigGraph) {
+export function getMatchingServiceNames(namesFromOpt: string[] | undefined, configGraph: ConfigGraph) {
   const names = namesFromOpt || []
   if (names.includes("*") || (!!namesFromOpt && namesFromOpt.length === 0)) {
     return configGraph.getServices().map((s) => s.name)

core/src/commands/run/service.ts

@@ -96,6 +96,7 @@ export class RunServiceCommand extends Command<Args, Opts> {
       service,
       devModeServiceNames: [],
       hotReloadServiceNames: [],
+      localModeServiceNames: [],
     })
     const dependencyResults = await garden.processTasks(await deployTask.resolveDependencies())
 

core/src/commands/run/task.ts

@@ -103,6 +103,7 @@ export class RunTaskCommand extends Command<Args, Opts> {
       forceBuild: opts["force-build"],
       devModeServiceNames: [],
       hotReloadServiceNames: [],
+      localModeServiceNames: [],
     })
     const graphResults = await garden.processTasks([taskTask])
 

core/src/commands/run/test.ts

@@ -136,6 +136,7 @@ export class RunTestCommand extends Command<Args, Opts> {
       test,
       devModeServiceNames: [],
       hotReloadServiceNames: [],
+      localModeServiceNames: [],
     })
 
     const graphResults = await garden.processTasks([testTask])

core/src/commands/test.ts

@@ -174,6 +174,7 @@ export class TestCommand extends Command<Args, Opts> {
           forceBuild,
           devModeServiceNames: [],
           hotReloadServiceNames: [],
+          localModeServiceNames: [],
           skipRuntimeDependencies,
         })
       )
@@ -205,6 +206,7 @@ export class TestCommand extends Command<Args, Opts> {
               fromWatch: true,
               devModeServiceNames: [],
               hotReloadServiceNames: [],
+              localModeServiceNames: [],
             })
           )
         )

core/src/events.ts

@@ -276,6 +276,7 @@ export interface Events extends LoggerEvents {
     serviceName: string
     devMode: boolean
     hotReload: boolean
+    localMode: boolean
     force: boolean
     forceBuild: boolean
     skipDependencies: boolean

core/src/garden.ts

@@ -1240,7 +1240,7 @@ export const resolveGardenParams = profileAsync(async function _resolveGardenPar
   const vcsInfo = await gitHandler.getPathInfo(log, projectRoot)
 
   // Since we iterate/traverse them before fully validating them (which we do after resolving template strings), we
-  // validdate that `config.environments` and `config.providers` are both arrays.
+  // validate that `config.environments` and `config.providers` are both arrays.
   // This prevents cryptic type errors when the user mistakely writes down e.g. a map instead of an array.
   validateWithPath({
     config: config.environments,

core/src/outputs.ts

@@ -91,6 +91,7 @@ export async function resolveProjectOutputs(garden: Garden, log: LogEntry): Prom
           service,
           devModeServiceNames: [],
           hotReloadServiceNames: [],
+          localModeServiceNames: [],
         })
     ),
     ...tasks.map(
@@ -104,6 +105,7 @@ export async function resolveProjectOutputs(garden: Garden, log: LogEntry): Prom
           task,
           devModeServiceNames: [],
           hotReloadServiceNames: [],
+          localModeServiceNames: [],
         })
     ),
   ]