feat(#89): add support for pulling the Locust image from private regi…

…stries

refactor(#89): rename resource attributes

docs/getting_started.md

@@ -67,7 +67,11 @@ spec:
 7. The amount of _worker_ nodes to spawn in the cluster.
 8. [Optional] Name of _configMap_ to mount into the pod
 
-Note that other options are available. In particular, you can add labels and annotations as well. For example:
+#### Other options
+
+##### Labels and annotations
+
+You can add labels and annotations to generated Pods. For example:
 
 ```yaml title="locusttest-cr.yaml"
 apiVersion: locust.io/v1
@@ -95,6 +99,25 @@ spec:
 
 Both labels and annotations can be added to the Prometheus configuration, so that metrics are associated with the appropriate information, such as the test and tenant ids. You can read more about this in the [Prometheus documentation](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) site.
 
+##### Image pull policy and secrets
+
+You can use a private image for master and worker images, and configure its pull policy and corresponding secret. For example:
+
+```yaml title="locusttest-cr.yaml"
+apiVersion: locust.io/v1
+...
+spec:
+  image: ghcr.io/mycompany/locust:latest #(1)!
+  imagePullPolicy: Always #(2)!
+  imagePullSecrets: #(3)!
+    - gcr-secret
+  ...
+```
+
+1. [Optional] Specify which Locust image to use for both master and worker containers.
+2. [Optional] Specify the pull policy to use for containers defined within master and worker containers.
+3. [Optional] Specify an existing pull secret to use for master and worker pods.
+
 ### Step 3: Deploy _Locust k8s Operator_ in the cluster.
 
 The recommended way to install the _Operator_ is by using the official HELM chart. Documentation on how to perform that

docs/helm_deploy.md

@@ -30,7 +30,7 @@ In order to deploy using helm, follow the below steps:
 
 [//]: # (Resources urls)
 
-[crd-locusttest.yaml]: https://github.com/AbdelrhmanHamouda/locust-k8s-operator/blob/master/charts/locust-k8s-operator/templates/crd-locusttest.yaml
+[crd-locusttest.yaml]: https://github.com/AbdelrhmanHamouda/locust-k8s-operator/blob/master/charts/locust-k8s-operator/crds/locust-test-crd.yaml
 
 [serviceaccount-and-roles.yaml]: https://github.com/AbdelrhmanHamouda/locust-k8s-operator/blob/master/charts/locust-k8s-operator/templates/serviceaccount-and-roles.yaml
 

kube/crd/locust-test-crd.yaml

@@ -109,6 +109,17 @@ spec:
                 image: # Child field 'image'
                   description: Locust image
                   type: string
+                imagePullPolicy:
+                  description: Image pull policy
+                  type: string
+                  enum:
+                    - "Always"
+                    - "IfNotPresent"
+                imagePullSecrets:
+                  description: Secrets for pulling images from private registries
+                  type: array
+                  items:
+                    type: string
                 configMap: # Child field 'configMap'
                   description: Configuration map name containing the test
                   type: string

kube/sample-cr/locust-test-cr.yaml

@@ -4,6 +4,10 @@ metadata:
   name: demo.test
 spec:
   image: locustio/locust:latest
+  # [Optional-Section] Image pull policy and secrets
+  imagePullPolicy: Always
+  imagePullSecrets:
+    - "my-private-registry-secret"
 
   # [Optional-Section] Labels
   labels:

src/main/java/com/locust/operator/controller/dto/LoadGenerationNode.java

@@ -25,6 +25,8 @@ public class LoadGenerationNode {
     private List<String> command;
     private OperationalMode operationalMode;
     private String image;
+    private String imagePullPolicy;
+    private List<String> imagePullSecrets;
     private Integer replicas;
     private List<Integer> ports;
     private String configMap;

src/main/java/com/locust/operator/controller/utils/LoadGenHelpers.java

@@ -65,6 +65,8 @@ public LoadGenerationNode generateLoadGenNodeObject(LocustTest resource, Operati
             constructNodeCommand(resource, mode),
             mode,
             getNodeImage(resource),
+            getPullPolicy(resource),
+            getPullSecrets(resource),
             getReplicaCount(resource, mode),
             getNodePorts(resource, mode),
             getConfigMap(resource));
@@ -89,6 +91,14 @@ private String getNodeImage(LocustTest resource) {
 
     }
 
+    private String getPullPolicy(LocustTest resource) {
+        return resource.getSpec().getImagePullPolicy();
+    }
+
+    private List<String> getPullSecrets(LocustTest resource) {
+        return resource.getSpec().getImagePullSecrets();
+    }
+
     public LocustTestAffinity getNodeAffinity(LocustTest resource) {
 
         return config.isAffinityCrInjectionEnabled() ? resource.getSpec().getAffinity() : null;

src/main/java/com/locust/operator/controller/utils/resource/manage/ResourceCreationHelpers.java

@@ -12,6 +12,8 @@
 import io.fabric8.kubernetes.api.model.ContainerPortBuilder;
 import io.fabric8.kubernetes.api.model.EnvVar;
 import io.fabric8.kubernetes.api.model.EnvVarBuilder;
+import io.fabric8.kubernetes.api.model.LocalObjectReference;
+import io.fabric8.kubernetes.api.model.LocalObjectReferenceBuilder;
 import io.fabric8.kubernetes.api.model.NodeAffinity;
 import io.fabric8.kubernetes.api.model.NodeAffinityBuilder;
 import io.fabric8.kubernetes.api.model.NodeSelector;
@@ -215,6 +217,8 @@ private ObjectMeta prepareTemplateMetadata(LoadGenerationNode nodeConfig, String
     private PodSpec prepareTemplateSpec(LoadGenerationNode nodeConfig) {
 
         PodSpec templateSpec = new PodSpecBuilder()
+            // images
+            .withImagePullSecrets(prepareImagePullSecrets(nodeConfig))
 
             // Containers
             .withContainers(prepareContainerList(nodeConfig))
@@ -230,6 +234,17 @@ private PodSpec prepareTemplateSpec(LoadGenerationNode nodeConfig) {
 
     }
 
+    private List<LocalObjectReference> prepareImagePullSecrets(LoadGenerationNode nodeConfig) {
+        final List<LocalObjectReference> references = nodeConfig.getImagePullSecrets()
+            .stream()
+            .map(secretName -> new LocalObjectReferenceBuilder().withName(secretName).build())
+            .collect(Collectors.toList());
+
+        log.debug("Prepared image pull secrets: {}", references);
+
+        return references;
+    }
+
     private List<Volume> prepareVolumesList(LoadGenerationNode nodeConfig) {
 
         List<Volume> volumeList = new ArrayList<>();
@@ -339,7 +354,7 @@ private List<Container> prepareContainerList(LoadGenerationNode nodeConfig) {
 
         // Inject metrics container only if `master`
         if (nodeConfig.getOperationalMode().equals(MASTER)) {
-            constantsList.add(prepareMetricsExporterContainer());
+            constantsList.add(prepareMetricsExporterContainer(nodeConfig.getImagePullPolicy()));
         }
 
         return constantsList;
@@ -351,9 +366,10 @@ private List<Container> prepareContainerList(LoadGenerationNode nodeConfig) {
      * <p>
      * Reference: <a href="https://github.com/ContainerSolutions/locust_exporter">locust exporter docs</a>
      *
+     * @param pullPolicy The image pull policy
      * @return Container
      */
-    private Container prepareMetricsExporterContainer() {
+    private Container prepareMetricsExporterContainer(final String pullPolicy) {
 
         HashMap<String, String> envMap = new HashMap<>();
 
@@ -367,6 +383,7 @@ private Container prepareMetricsExporterContainer() {
 
             // Image
             .withImage(EXPORTER_IMAGE)
+            .withImagePullPolicy(pullPolicy)
 
             // Ports
             .withPorts(new ContainerPortBuilder().withContainerPort(LOCUST_EXPORTER_PORT).build())
@@ -400,6 +417,7 @@ private Container prepareLoadGenContainer(LoadGenerationNode nodeConfig) {
 
             // Image
             .withImage(nodeConfig.getImage())
+            .withImagePullPolicy(nodeConfig.getImagePullPolicy())
 
             // Ports
             .withPorts(prepareContainerPorts(nodeConfig.getPorts()))

src/main/java/com/locust/operator/customresource/LocustTestSpec.java

@@ -36,5 +36,7 @@ public class LocustTestSpec implements KubernetesResource {
     private Integer workerReplicas;
     private String configMap;
     private String image;
+    private String imagePullPolicy;
+    private List<String> imagePullSecrets;
 
 }

src/test/java/com/locust/operator/controller/TestFixtures.java

@@ -16,6 +16,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -32,11 +33,13 @@
 @NoArgsConstructor(access = PRIVATE)
 public class TestFixtures {
 
-    public static final String CRD_FILE_PATH = "charts/locust-k8s-operator/templates/crd-locusttest.yaml";
+    public static final String CRD_FILE_PATH = "charts/locust-k8s-operator/crds/locust-test-crd.yaml";
     public static final String DEFAULT_API_VERSION = GROUP + "/" + VERSION;
     public static final String KIND = "LocustTest";
     public static final String DEFAULT_SEED_COMMAND = "--locustfile src/demo.py";
     public static final String DEFAULT_TEST_IMAGE = "xlocust:latest";
+    public static final String DEFAULT_IMAGE_PULL_POLICY = "IfNotPresent";
+    public static final List<String> DEFAULT_IMAGE_PULL_SECRETS = Collections.emptyList();
     public static final String DEFAULT_TEST_CONFIGMAP = "demo-test-configmap";
     public static final String DEFAULT_NAMESPACE = "default";
     public static final int REPLICAS = 50;
@@ -120,6 +123,8 @@ public static LocustTest prepareLocustTest(String resourceName, Integer replicas
         spec.setWorkerCommandSeed(DEFAULT_SEED_COMMAND);
         spec.setConfigMap(DEFAULT_TEST_CONFIGMAP);
         spec.setImage(DEFAULT_TEST_IMAGE);
+        spec.setImagePullPolicy(DEFAULT_IMAGE_PULL_POLICY);
+        spec.setImagePullSecrets(DEFAULT_IMAGE_PULL_SECRETS);
         spec.setWorkerReplicas(replicas);
 
         var labels = new HashMap<String, Map<String, String>>();

src/test/java/com/locust/operator/controller/utils/TestFixtures.java

@@ -8,6 +8,8 @@
 import com.locust.operator.customresource.internaldto.LocustTestNodeAffinity;
 import com.locust.operator.customresource.internaldto.LocustTestToleration;
 import io.fabric8.kubernetes.api.model.KubernetesResourceList;
+import io.fabric8.kubernetes.api.model.LocalObjectReference;
+import io.fabric8.kubernetes.api.model.PodList;
 import io.fabric8.kubernetes.api.model.batch.v1.JobList;
 import lombok.NoArgsConstructor;
 import lombok.SneakyThrows;
@@ -20,6 +22,8 @@
 import java.util.Map;
 
 import static com.github.stefanbirkner.systemlambda.SystemLambda.withEnvironmentVariable;
+import static com.locust.operator.controller.TestFixtures.DEFAULT_IMAGE_PULL_POLICY;
+import static com.locust.operator.controller.TestFixtures.DEFAULT_IMAGE_PULL_SECRETS;
 import static com.locust.operator.controller.TestFixtures.REPLICAS;
 import static com.locust.operator.controller.dto.OperationalMode.MASTER;
 import static com.locust.operator.controller.dto.OperatorType.EQUAL;
@@ -94,6 +98,8 @@ public static LoadGenerationNode prepareNodeConfig(String nodeName, OperationalM
             .command(List.of(DEFAULT_SEED_COMMAND.split(CONTAINER_ARGS_SEPARATOR)))
             .operationalMode(mode)
             .image(DEFAULT_TEST_IMAGE)
+            .imagePullPolicy(DEFAULT_IMAGE_PULL_POLICY)
+            .imagePullSecrets(DEFAULT_IMAGE_PULL_SECRETS)
             .replicas(mode.equals(MASTER) ? MASTER_REPLICA_COUNT : REPLICAS)
             .ports(mode.equals(MASTER) ? DEFAULT_MASTER_PORT_LIST : DEFAULT_WORKER_PORT_LIST)
             .build();
@@ -132,6 +138,17 @@ public static LoadGenerationNode prepareNodeConfigWithTolerations(String nodeNam
 
     }
 
+    public static LoadGenerationNode prepareNodeConfigWithPullPolicyAndSecrets(
+        String nodeName, OperationalMode mode, String pullPolicy, List<String> pullSecrets) {
+
+        val nodeConfig = prepareNodeConfig(nodeName, mode);
+        nodeConfig.setImagePullPolicy(pullPolicy);
+        nodeConfig.setImagePullSecrets(pullSecrets);
+
+        return nodeConfig;
+
+    }
+
     public static <T extends KubernetesResourceList<?>> void assertK8sResourceCreation(String nodeName, T resourceList) {
 
         assertSoftly(softly -> {
@@ -141,6 +158,30 @@ public static <T extends KubernetesResourceList<?>> void assertK8sResourceCreati
 
     }
 
+    public static void assertImagePullData(LoadGenerationNode nodeConfig, PodList podList) {
+
+        podList.getItems().forEach(pod -> {
+            final List<String> references = pod.getSpec()
+                .getImagePullSecrets()
+                .stream()
+                .map(LocalObjectReference::getName)
+                .toList();
+
+            assertSoftly(softly -> {
+                softly.assertThat(references).isEqualTo(nodeConfig.getImagePullSecrets());
+            });
+
+            pod.getSpec()
+                .getContainers()
+                .forEach(container -> {
+                    assertSoftly(softly -> {
+                        softly.assertThat(container.getImagePullPolicy()).isEqualTo(nodeConfig.getImagePullPolicy());
+                    });
+                });
+        });
+
+    }
+
     public static void assertK8sNodeAffinity(LoadGenerationNode nodeConfig, JobList jobList, String k8sNodeLabelKey) {
 
         jobList.getItems().forEach(job -> {

src/test/java/com/locust/operator/controller/utils/resource/manage/ResourceCreationManagerTests.java

@@ -14,14 +14,18 @@
 import org.mockito.Mock;
 import org.mockito.MockitoAnnotations;
 
+import java.util.List;
+
 import static com.locust.operator.controller.dto.OperationalMode.MASTER;
+import static com.locust.operator.controller.utils.TestFixtures.assertImagePullData;
 import static com.locust.operator.controller.utils.TestFixtures.assertK8sNodeAffinity;
 import static com.locust.operator.controller.utils.TestFixtures.assertK8sResourceCreation;
 import static com.locust.operator.controller.utils.TestFixtures.assertK8sTolerations;
 import static com.locust.operator.controller.utils.TestFixtures.containerEnvironmentMap;
 import static com.locust.operator.controller.utils.TestFixtures.executeWithK8sMockServer;
 import static com.locust.operator.controller.utils.TestFixtures.prepareNodeConfig;
 import static com.locust.operator.controller.utils.TestFixtures.prepareNodeConfigWithNodeAffinity;
+import static com.locust.operator.controller.utils.TestFixtures.prepareNodeConfigWithPullPolicyAndSecrets;
 import static com.locust.operator.controller.utils.TestFixtures.prepareNodeConfigWithTolerations;
 import static org.mockito.Mockito.when;
 
@@ -120,7 +124,7 @@ void createJobWithNodeAffinityTest() {
         assertK8sNodeAffinity(nodeConfig, jobList, k8sNodeLabelKey);
 
     }
-    
+
     @Test
     @DisplayName("Functional: Create a kubernetes Job with Tolerations and Toleration Operator set to Equal")
     void createJobWithTolerationsAndOperatorEqualTest() {
@@ -179,4 +183,29 @@ void createJobWithTolerationsAndOperatorExistsTest() {
         assertK8sTolerations(jobList, toleration);
 
     }
+
+    @Test
+    @DisplayName("Functional: Create a kubernetes Pod with image pull policy and secrets")
+    void createPodTest() {
+
+        // * Setup
+        val namespace = "default";
+        val nodeName = "mnt-demo-test";
+        val resourceName = "mnt.demo-test";
+        val nodeConfig = prepareNodeConfigWithPullPolicyAndSecrets(
+            nodeName, MASTER, "Always", List.of("my-private-registry-secret", "gcr-cred-secret")
+        );
+
+        // * Act
+        executeWithK8sMockServer(k8sServerUrl, () -> CreationManager.createJob(nodeConfig, namespace, resourceName));
+
+        // Get All Pods created by the method
+        val podList = testClient.pods().inNamespace(namespace).list();
+        log.debug("Acquired Pod list: {}", podList);
+
+        // * Assert
+        assertImagePullData(nodeConfig, podList);
+
+    }
+
 }