Adjust invoker playbook to pull runtime and blackbox docker images. (#…

…3680) Remove default runtime prefix and tag; the runtimes.json file should fully specify these. Remove docker.{registry,prefix,tax} entanglement with runtimes as these are used for the system images not the runtimes. Optionally specify a runtimes.registry to pull images from there instead of dockerhub.
apache · Jun 18, 2018 · 2047c0f · 2047c0f
1 parent 4181f7c
commit 2047c0f
Show file tree

Hide file tree

Showing 14 changed files with 114 additions and 106 deletions.
diff --git a/ansible/files/runtimes.json b/ansible/files/runtimes.json
@@ -4,15 +4,19 @@
             {
                 "kind": "nodejs",
                 "image": {
-                    "name": "nodejsaction"
+                    "prefix": "openwhisk",
+                    "name": "nodejsaction",
+                    "tag": "latest"
                 },
                 "deprecated": true
             },
             {
                 "kind": "nodejs:6",
                 "default": true,
                 "image": {
-                    "name": "nodejs6action"
+                    "prefix": "openwhisk",
+                    "name": "nodejs6action",
+                    "tag": "latest"
                 },
                 "deprecated": false,
                 "stemCells": [{
@@ -24,7 +28,9 @@
                 "kind": "nodejs:8",
                 "default": false,
                 "image": {
-                    "name": "action-nodejs-v8"
+                    "prefix": "openwhisk",
+                    "name": "action-nodejs-v8",
+                    "tag": "latest"
                 },
                 "deprecated": false
             }
@@ -33,22 +39,28 @@
             {
                 "kind": "python",
                 "image": {
-                    "name": "python2action"
+                    "prefix": "openwhisk",
+                    "name": "python2action",
+                    "tag": "latest"
                 },
                 "deprecated": false
             },
             {
                 "kind": "python:2",
                 "default": true,
                 "image": {
-                    "name": "python2action"
+                    "prefix": "openwhisk",
+                    "name": "python2action",
+                    "tag": "latest"
                 },
                 "deprecated": false
             },
             {
                 "kind": "python:3",
                 "image": {
-                    "name": "python3action"
+                    "prefix": "openwhisk",
+                    "name": "python3action",
+                    "tag": "latest"
                 },
                 "deprecated": false
             }
@@ -57,29 +69,37 @@
             {
                 "kind": "swift",
                 "image": {
-                    "name": "swiftaction"
+                    "prefix": "openwhisk",
+                    "name": "swiftaction",
+                    "tag": "latest"
                 },
                 "deprecated": true
             },
             {
                 "kind": "swift:3",
                 "image": {
-                    "name": "swift3action"
+                    "prefix": "openwhisk",
+                    "name": "swift3action",
+                    "tag": "latest"
                 },
                 "deprecated": true
             },
             {
                 "kind": "swift:3.1.1",
                 "image": {
-                    "name": "action-swift-v3.1.1"
+                    "prefix": "openwhisk",
+                    "name": "action-swift-v3.1.1",
+                    "tag": "latest"
                 },
                 "deprecated": false
             },
             {
                 "kind": "swift:4.1",
                 "default": true,
                 "image": {
-                    "name": "action-swift-v4.1"
+                    "prefix": "openwhisk",
+                    "name": "action-swift-v4.1",
+                    "tag": "latest"
                 },
                 "deprecated": false
             }
@@ -89,7 +109,9 @@
                 "kind": "java",
                 "default": true,
                 "image": {
-                    "name": "java8action"
+                    "prefix": "openwhisk",
+                    "name": "java8action",
+                    "tag": "latest"
                 },
                 "deprecated": false,
                 "attached": {
@@ -106,14 +128,18 @@
                 "default": true,
                 "deprecated": false,
                 "image": {
-                    "name": "action-php-v7.1"
+                    "prefix": "openwhisk",
+                    "name": "action-php-v7.1",
+                    "tag": "latest"
                 }
             }
         ]
     },
     "blackboxes": [
         {
-            "name": "dockerskeleton"
+            "prefix": "openwhisk",
+            "name": "dockerskeleton",
+            "tag": "latest"
         }
     ]
 }
diff --git a/ansible/group_vars/all b/ansible/group_vars/all
@@ -29,14 +29,17 @@ whisk:
     date: "{{ansible_date_time.iso8601}}"
 
 ##
-# list of supported runtimes (see whisk.core.entity.ExecManifest for schema).
-# briefly:
-#   defaultImagePrefix: the default image prefix when not given explicitly
-#   defaultImageTag: the default image tag
-#   runtimes: set of language runtime families grouped by language (e.g., nodejs, python)
-#   blackboxes: list of pre-populated docker action images as "name" with optional "prefix" and "tag"
-#   bypassPullForLocalImages: optional, if true, allow images with a prefix that matches {{ docker.image.prefix }}
-#                             to skip docker pull in invoker even if the image is not part of the blackbox set
+# configuration parameters related to support runtimes (see whisk.core.entity.ExecManifest for schema of the manifest).
+# briefly the parameters are:
+#
+#   runtimes_registry: optional registry (with trailing slack) where to pull docker images from for runtimes and backbox images
+#
+#   skip_pull_runtimes: this will skip pulling the images to the invoker (images must exist there somehow)
+#
+#   runtimes_manifest: set of language runtime families grouped by language (e.g., nodejs, python) and blackbox images to pre-pull
+#
+#   runtimes_bypass_pull_for_local_images: optional, if true, allow images with a prefix that matches
+#       {{ runtimes_local_image_prefix }} to skip docker pull in invoker even if the image is not part of the blackbox set
 #
 runtimesManifest: "{{ runtimes_manifest | default(lookup('file', openwhisk_home ~ '/ansible/files/runtimes.json') | from_json) }}"
 

diff --git a/ansible/roles/invoker/tasks/deploy.yml b/ansible/roles/invoker/tasks/deploy.yml
@@ -13,18 +13,28 @@
   retries: "{{ docker.pull.retries }}"
   delay: "{{ docker.pull.delay }}"
 
-- name: "pull runtime action images with tag {{docker.image.tag}}"
-  shell: "docker pull {{docker_registry}}{{docker.image.prefix}}/{{item}}:{{docker.image.tag}}"
-  with_items: "{{ runtimesManifest.runtimes.values() | sum(start=[]) | selectattr('deprecated', 'equalto',false)  | map(attribute='image.name') | list | unique }}"
-  when: docker_registry != ""
+###
+# This task assumes that the images are local to the invoker host already if there is no prefix or tag
+# which is usually the case for a local deployment. A distributed deployment will specify the prefix, or tag
+# to pull the images from the appropriate registry. If a runtimes_registry is optionally specified, pull images
+# from there; this permits a (private) registry to be used for caching the images. The registry if specified
+# must include a trailing '/'.
+#
+- name: "pull runtime action images per manifest"
+  shell: "docker pull {{runtimes_registry | default()}}{{item.prefix}}/{{item.name}}:{{item.tag | default()}}"
+  with_items: "{{ runtimesManifest.runtimes.values() | sum(start=[]) | selectattr('deprecated', 'equalto',false)  | map(attribute='image') | list | unique }}"
+  when: skip_pull_runtimes is not defined or skip_pull_runtimes == True
   retries: "{{ docker.pull.retries }}"
   delay: "{{ docker.pull.delay }}"
 
-- name: "pull blackboxes action images with tag {{docker.image.tag}}"
-  shell: "docker pull {{docker_registry}}{{docker.image.prefix}}/{{item.name}}:{{docker.image.tag}}"
+###
+# See comment above for pulling other runtime images.
+#
+- name: "pull blackboxes action images per manifest"
+  shell: "docker pull {{runtimes_registry | default()}}{{item.prefix}}/{{item.name}}:{{item.tag | default()}}"
   with_items:
     - "{{ runtimesManifest.blackboxes }}"
-  when: docker_registry != ""
+  when: skip_pull_runtimes is not defined or skip_pull_runtimes == True
   retries: "{{ docker.pull.retries }}"
   delay: "{{ docker.pull.delay }}"
 
@@ -179,14 +189,10 @@
       "WHISK_API_HOST_PROTO": "{{ whisk_api_host_proto | default('https') }}"
       "WHISK_API_HOST_PORT": "{{ whisk_api_host_port | default('443') }}"
       "WHISK_API_HOST_NAME": "{{ whisk_api_host_name | default(groups['edge'] | first) }}"
+      "RUNTIMES_REGISTRY": "{{ runtimes_registry | default('') }}"
       "RUNTIMES_MANIFEST": "{{ runtimesManifest | to_json }}"
-      "CONFIG_whisk_runtimes_defaultImagePrefix": "{{ runtimes_default_image_prefix | default() }}"
-      "CONFIG_whisk_runtimes_defaultImageTag": "{{ runtimes_default_image_tag | default() }}"
       "CONFIG_whisk_runtimes_bypassPullForLocalImages": "{{ runtimes_bypass_pull_for_local_images | default() }}"
       "CONFIG_whisk_runtimes_localImagePrefix": "{{ runtimes_local_image_prefix | default() }}"
-      "DOCKER_REGISTRY": "{{ docker_registry }}"
-      "DOCKER_IMAGE_PREFIX": "{{ docker.image.prefix }}"
-      "DOCKER_IMAGE_TAG": "{{ docker.image.tag }}"
       "CONFIG_whisk_containerFactory_containerArgs_network": "{{ invoker_container_network_name | default('bridge') }}"
       "INVOKER_CONTAINER_POLICY": "{{ invoker_container_policy_name | default()}}"
       "CONFIG_whisk_containerPool_numCore": "{{ invoker.numcore }}"

diff --git a/common/scala/src/main/resources/application.conf b/common/scala/src/main/resources/application.conf
@@ -146,8 +146,6 @@ whisk {
     }
     # action runtimes configuration
     runtimes {
-        default-image-prefix = "openwhisk"
-        default-image-tag = "latest"
         bypass-pull-for-local-images = false
         local-image-prefix = "whisk"
     }

diff --git a/common/scala/src/main/scala/whisk/core/WhiskConfig.scala b/common/scala/src/main/scala/whisk/core/WhiskConfig.scala
@@ -51,13 +51,9 @@ class WhiskConfig(requiredProperties: Map[String, String],
   }
 
   val servicePort = this(WhiskConfig.servicePort)
-  val dockerRegistry = this(WhiskConfig.dockerRegistry)
   val dockerEndpoint = this(WhiskConfig.dockerEndpoint)
   val dockerPort = this(WhiskConfig.dockerPort)
 
-  val dockerImagePrefix = this(WhiskConfig.dockerImagePrefix)
-  val dockerImageTag = this(WhiskConfig.dockerImageTag)
-
   val invokerName = this(WhiskConfig.invokerName)
 
   val wskApiHost = this(WhiskConfig.wskApiProtocol) + "://" + this(WhiskConfig.wskApiHostname) + ":" + this(
@@ -77,6 +73,7 @@ class WhiskConfig(requiredProperties: Map[String, String],
   val dbPrefix = this(WhiskConfig.dbPrefix)
   val mainDockerEndpoint = this(WhiskConfig.mainDockerEndpoint)
 
+  val runtimesRegistry = this(WhiskConfig.runtimesRegistry)
   val runtimesManifest = this(WhiskConfig.runtimesManifest)
   val actionInvokePerMinuteLimit = this(WhiskConfig.actionInvokePerMinuteLimit)
   val actionInvokeConcurrentLimit = this(WhiskConfig.actionInvokeConcurrentLimit)
@@ -149,7 +146,6 @@ object WhiskConfig {
   }
 
   val servicePort = "port"
-  val dockerRegistry = "docker.registry"
   val dockerPort = "docker.port"
 
   val dockerEndpoint = "main.docker.endpoint"
@@ -163,9 +159,6 @@ object WhiskConfig {
 
   val whiskVersion = Map(whiskVersionDate -> null, whiskVersionBuildno -> null)
 
-  val dockerImagePrefix = "docker.image.prefix"
-  val dockerImageTag = "docker.image.tag"
-
   val invokerName = "invoker.name"
 
   val wskApiProtocol = "whisk.api.host.proto"
@@ -194,6 +187,7 @@ object WhiskConfig {
   val kafkaHosts = Map(kafkaHostList -> null)
   val zookeeperHosts = Map(zookeeperHostList -> null)
 
+  val runtimesRegistry = "runtimes.registry"
   val runtimesManifest = "runtimes.manifest"
 
   val actionSequenceMaxLimit = "limits.actions.sequence.maxLength"

diff --git a/common/scala/src/main/scala/whisk/core/entity/ExecManifest.scala b/common/scala/src/main/scala/whisk/core/entity/ExecManifest.scala
@@ -72,24 +72,20 @@ protected[core] object ExecManifest {
    * @return Runtimes instance
    */
   protected[entity] def runtimes(config: JsObject, runtimeManifestConfig: RuntimeManifestConfig): Try[Runtimes] = Try {
-
-    val prefix = runtimeManifestConfig.defaultImagePrefix
-    val tag = runtimeManifestConfig.defaultImageTag
-
     val runtimes = config.fields
       .get("runtimes")
       .map(_.convertTo[Map[String, Set[RuntimeManifest]]].map {
         case (name, versions) =>
           RuntimeFamily(name, versions.map { mf =>
-            val img = ImageName(mf.image.name, mf.image.prefix.orElse(prefix), mf.image.tag.orElse(tag))
+            val img = ImageName(mf.image.name, mf.image.prefix, mf.image.tag)
             mf.copy(image = img)
           })
       }.toSet)
 
     val blackbox = config.fields
       .get("blackboxes")
       .map(_.convertTo[Set[ImageName]].map { image =>
-        ImageName(image.name, image.prefix.orElse(prefix), image.tag.orElse(tag))
+        ImageName(image.name, image.prefix, image.tag)
       })
 
     val bypassPullForLocalImages = runtimeManifestConfig.bypassPullForLocalImages
@@ -100,16 +96,14 @@ protected[core] object ExecManifest {
   }
 
   /**
-   * Misc options related to runtime manifests
-   * @param defaultImagePrefix the default image prefix when not given explicitly
-   * @param defaultImageTag the default image tag
+   * Misc options related to runtime manifests.
+   *
    * @param bypassPullForLocalImages if true, allow images with a prefix that matches localImagePrefix
-   *                                 to skip docker pull in invoker even if the image is not part of the blackbox set
+   *                                 to skip docker pull on invoker even if the image is not part of the blackbox set;
+   *                                 this is useful for testing with local images that aren't published to the runtimes registry
    * @param localImagePrefix image prefix for bypassPullForLocalImages
    */
-  protected[core] case class RuntimeManifestConfig(defaultImagePrefix: Option[String] = None,
-                                                   defaultImageTag: Option[String] = None,
-                                                   bypassPullForLocalImages: Option[Boolean] = None,
+  protected[core] case class RuntimeManifestConfig(bypassPullForLocalImages: Option[Boolean] = None,
                                                    localImagePrefix: Option[String] = None)
 
   /**
@@ -159,18 +153,18 @@ protected[core] object ExecManifest {
     }
 
     /**
-     * The internal name of the image for an action kind. It overrides
-     * the prefix with an internal name. Optionally overrides tag.
+     * The internal name of the image for an action kind relative to a registry.
      */
-    def localImageName(registry: String, prefix: String, tagOverride: Option[String] = None): String = {
+    def localImageName(registry: String): String = {
       val r = Option(registry)
         .filter(_.nonEmpty)
         .map { reg =>
           if (reg.endsWith("/")) reg else reg + "/"
         }
         .getOrElse("")
-      val p = Option(prefix).filter(_.nonEmpty).map(_ + "/").getOrElse("")
-      r + p + name + ":" + tagOverride.orElse(tag).getOrElse(ImageName.defaultImageTag)
+      val p = prefix.filter(_.nonEmpty).map(_ + "/").getOrElse("")
+      val t = tag.filter(_.nonEmpty).map(":" + _).getOrElse("")
+      r + p + name + t
     }
 
     /**
@@ -190,7 +184,7 @@ protected[core] object ExecManifest {
   }
 
   protected[core] object ImageName {
-    protected val defaultImageTag = "latest"
+    private val defaultImageTag = "latest"
     private val componentRegex = """([a-z0-9._-]+)""".r
     private val tagRegex = """([\w.-]{0,128})""".r
 

diff --git a/common/scala/src/main/scala/whisk/core/mesos/MesosContainerFactory.scala b/common/scala/src/main/scala/whisk/core/mesos/MesosContainerFactory.scala
@@ -114,7 +114,7 @@ class MesosContainerFactory(config: WhiskConfig,
     val image = if (userProvidedImage) {
       actionImage.publicImageName
     } else {
-      actionImage.localImageName(config.dockerRegistry, config.dockerImagePrefix, Some(config.dockerImageTag))
+      actionImage.localImageName(config.runtimesRegistry)
     }
     val constraintStrings = if (userProvidedImage) {
       mesosConfig.blackboxConstraints

diff --git a/core/invoker/src/main/scala/whisk/core/containerpool/docker/DockerContainerFactory.scala b/core/invoker/src/main/scala/whisk/core/containerpool/docker/DockerContainerFactory.scala
@@ -61,7 +61,7 @@ class DockerContainerFactory(instance: InstanceId,
     val image = if (userProvidedImage) {
       actionImage.publicImageName
     } else {
-      actionImage.localImageName(config.dockerRegistry, config.dockerImagePrefix, Some(config.dockerImageTag))
+      actionImage.localImageName(config.runtimesRegistry)
     }
 
     DockerContainer.create(

diff --git a/...voker/src/main/scala/whisk/core/containerpool/kubernetes/KubernetesContainerFactory.scala b/...voker/src/main/scala/whisk/core/containerpool/kubernetes/KubernetesContainerFactory.scala
@@ -69,7 +69,7 @@ class KubernetesContainerFactory(label: String, config: WhiskConfig)(implicit ac
     val image = if (userProvidedImage) {
       actionImage.publicImageName
     } else {
-      actionImage.localImageName(config.dockerRegistry, config.dockerImagePrefix, Some(config.dockerImageTag))
+      actionImage.localImageName(config.runtimesRegistry)
     }
 
     KubernetesContainer.create(

diff --git a/core/invoker/src/main/scala/whisk/core/invoker/Invoker.scala b/core/invoker/src/main/scala/whisk/core/invoker/Invoker.scala
@@ -52,12 +52,11 @@ object Invoker {
    * An object which records the environment variables required for this component to run.
    */
   def requiredProperties =
-    Map(servicePort -> 8080.toString(), dockerRegistry -> null, dockerImagePrefix -> null) ++
+    Map(servicePort -> 8080.toString, invokerName -> "", runtimesRegistry -> "") ++
       ExecManifest.requiredProperties ++
       kafkaHosts ++
       zookeeperHosts ++
-      wskApiHost ++ Map(dockerImageTag -> "latest") ++
-      Map(invokerName -> "")
+      wskApiHost
 
   def main(args: Array[String]): Unit = {
     Kamon.start()