Adding new Windows agent and organize Mac agent configurations (#169)

* Adding new Windows agent and organize Mac agent configurations Signed-off-by: Peter Zhu <[email protected]> * Update ami number Signed-off-by: Peter Zhu <[email protected]> * Update ntp windows Signed-off-by: Peter Zhu <[email protected]> * Add mac launchtimeout Signed-off-by: Peter Zhu <[email protected]>
opensearch-project · Jul 25, 2022 · c2f5e81 · c2f5e81
1 parent 5e1d255
commit c2f5e81
Show file tree

Hide file tree

Showing 11 changed files with 146 additions and 38 deletions.
diff --git a/README.md b/README.md
@@ -153,6 +153,10 @@ Example:
 npm run cdk deploy OpenSearch-CI-Dev -- -c useSsl=false -c runWithOidc=false -c macAgent=true
 ```
 
+#### Windows agents
+###### Prerequisite
+Make sure there is an existing Windows AMI with necessary requirements, see [packer directory](./packer/README.md) for more information and AMI build commands.
+
 #### Runnning additional commands
 In cases where you need to run additional logic/commands, such as adding a cron to emit ssl cert expiry metric, you can pass the commands as a script using `additionalCommands` context parameter.
 Below sample will write the python script to $HOME/hello-world path on jenkins master node and then execute it once the jenkins master node has been brought up. 

diff --git a/lib/ci-stack.ts b/lib/ci-stack.ts
@@ -105,7 +105,8 @@ export class CIStack extends Stack {
     const listenerCertificate = ListenerCertificate.fromArn(certificateArn.secretValue.toString());
     const agentNode = new AgentNodes();
     const agentNodes: AgentNodeProps[] = [agentNode.AL2_X64, agentNode.AL2_X64_DOCKER_HOST, agentNode.AL2_X64_DOCKER_HOST_PERF_TEST,
-      agentNode.AL2_ARM64, agentNode.AL2_ARM64_DOCKER_HOST, agentNode.UBUNTU_X64, agentNode.UBUNTU_X64_DOCKER_BUILDER];
+      agentNode.AL2_ARM64, agentNode.AL2_ARM64_DOCKER_HOST, agentNode.UBUNTU2004_X64, agentNode.UBUNTU2004_X64_DOCKER_BUILDER,
+      agentNode.MACOS12_X64_MULTI_HOST, agentNode.WINDOWS2019_X64];
 
     const mainJenkinsNode = new JenkinsMainNode(this, {
       vpc,

diff --git a/lib/compute/agent-node-config.ts b/lib/compute/agent-node-config.ts
@@ -18,6 +18,7 @@ import { load } from 'js-yaml';
 import { JenkinsMainNode } from './jenkins-main-node';
 
 export interface AgentNodeProps {
+   agentType: string;
    amiId: string;
    instanceType: string;
    workerLabelString: string;
@@ -130,11 +131,14 @@ export class AgentNodeConfig {
      const configTemplates: any = [];
 
      templates.forEach((element) => {
-       configTemplates.push(this.getTemplate(stack, element, props));
+       if (element.agentType == 'unix') {
+         configTemplates.push(this.getUnixTemplate(stack, element, props));
+       } else if (element.agentType == 'mac' && macAgent == 'true') {
+         configTemplates.push(this.getMacTemplate(stack, element, props));
+       } else if (element.agentType == 'windows') {
+         configTemplates.push(this.getWindowsTemplate(stack, element, props));
+       }
      });
-     if (macAgent == 'true') {
-       configTemplates.push(this.getMacTemplate(stack, props));
-     }
 
      const agentNodeYamlConfig = [{
        amazonEC2: {
@@ -149,7 +153,7 @@ export class AgentNodeConfig {
      return jenkinsYaml;
    }
 
-   private getTemplate(stack: Stack, config: AgentNodeProps, props: AgentNodeNetworkProps): { [x: string]: any; } {
+   private getUnixTemplate(stack: Stack, config: AgentNodeProps, props: AgentNodeNetworkProps): { [x: string]: any; } {
      return {
        ami: config.amiId,
        amiType:
@@ -195,30 +199,32 @@ export class AgentNodeConfig {
      };
    }
 
-   private getMacTemplate(stack: Stack, props: AgentNodeNetworkProps): { [x: string]: any; } {
+   private getMacTemplate(stack: Stack, config: AgentNodeProps, props: AgentNodeNetworkProps): { [x: string]: any; } {
      return {
-       ami: 'ami-0379811a08268a97e',
+       ami: config.amiId,
        amiType:
         { macData: { sshPort: '22' } },
        associatePublicIp: false,
        connectBySSHProcess: false,
        connectionStrategy: 'PRIVATE_IP',
        customDeviceMapping: '/dev/sda1=:300:true:gp3::encrypted',
        deleteRootOnTermination: true,
-       description: 'jenkinsAgentNode-Jenkins-Agent-MacOS12-X64-Mac1Metal-Multi-Host',
+       description: `jenkinsAgentNode-${config.workerLabelString}`,
        ebsEncryptRootVolume: 'ENCRYPTED',
        ebsOptimized: true,
        hostKeyVerificationStrategy: 'OFF',
        iamInstanceProfile: this.AgentNodeInstanceProfileArn,
        idleTerminationMinutes: '720',
-       labelString: 'Jenkins-Agent-MacOS12-X64-Mac1Metal-Multi-Host',
-       maxTotalUses: -1,
+       labelString: config.workerLabelString,
+       launchTimeoutStr: '1000',
+       initScript: config.initScript,
+       maxTotalUses: config.maxTotalUses,
        minimumNumberOfInstances: 1,
        minimumNumberOfSpareInstances: 0,
        mode: 'EXCLUSIVE',
        monitoring: true,
-       numExecutors: '6',
-       remoteAdmin: 'ec2-user',
+       numExecutors: config.numExecutors,
+       remoteAdmin: config.remoteUser,
        remoteFS: '/var/jenkins',
        securityGroups: props.agentNodeSecurityGroup,
        stopOnTerminate: false,
@@ -227,15 +233,15 @@ export class AgentNodeConfig {
        tags: [
          {
            name: 'Name',
-           value: `${stack.stackName}/AgentNode/Jenkins-Agent-MacOS12-X64-Mac1Metal-Multi-Host`,
+           value: `${stack.stackName}/AgentNode/${config.workerLabelString}`,
          },
          {
            name: 'type',
-           value: 'jenkinsAgentNode-Jenkins-Agent-MacOS12-X64-Mac1Metal-Multi-Host',
+           value: `jenkinsAgentNode-${config.workerLabelString}`,
          },
        ],
        tenancy: 'Host',
-       type: 'Mac1Metal',
+       type: config.instanceType,
        nodeProperties: [
          {
            envVars: {
@@ -252,4 +258,54 @@ export class AgentNodeConfig {
        useEphemeralDevices: false,
      };
    }
+
+   private getWindowsTemplate(stack: Stack, config: AgentNodeProps, props: AgentNodeNetworkProps): { [x: string]: any; } {
+     return {
+       ami: config.amiId,
+       amiType:
+         {
+           windowsData: {
+             allowSelfSignedCertificate: false, bootDelay: '7.5', specifyPassword: false, useHTTPS: false,
+           },
+         },
+       associatePublicIp: false,
+       connectBySSHProcess: false,
+       connectionStrategy: 'PRIVATE_IP',
+       customDeviceMapping: '/dev/sda1=:300:true:::encrypted',
+       deleteRootOnTermination: true,
+       description: `jenkinsAgentNode-${config.workerLabelString}`,
+       ebsEncryptRootVolume: 'ENCRYPTED',
+       ebsOptimized: true,
+       hostKeyVerificationStrategy: 'OFF',
+       iamInstanceProfile: this.AgentNodeInstanceProfileArn,
+       idleTerminationMinutes: '120',
+       initScript: config.initScript,
+       labelString: config.workerLabelString,
+       launchTimeoutStr: '1000',
+       maxTotalUses: config.maxTotalUses,
+       minimumNumberOfInstances: 0,
+       minimumNumberOfSpareInstances: 1,
+       mode: 'EXCLUSIVE',
+       monitoring: true,
+       numExecutors: config.numExecutors,
+       remoteAdmin: config.remoteUser,
+       remoteFS: `C:\\Users\\${config.remoteUser}\\jenkins`,
+       securityGroups: props.agentNodeSecurityGroup,
+       stopOnTerminate: false,
+       subnetId: props.subnetId,
+       t2Unlimited: false,
+       tags: [{
+         name: 'Name',
+         value: `${stack.stackName}/AgentNode/${config.workerLabelString}`,
+       },
+       {
+         name: 'type',
+         value: `jenkinsAgentNode-${config.workerLabelString}`,
+       },
+       ],
+       tenancy: 'Default',
+       type: config.instanceType,
+       useEphemeralDevices: false,
+     };
+   }
 }
diff --git a/lib/compute/agent-nodes.ts b/lib/compute/agent-nodes.ts
@@ -20,82 +20,120 @@ export class AgentNodes {
 
     readonly AL2_ARM64_DOCKER_HOST: AgentNodeProps;
 
-    readonly UBUNTU_X64: AgentNodeProps;
+    readonly UBUNTU2004_X64: AgentNodeProps;
 
-    readonly UBUNTU_X64_DOCKER_BUILDER: AgentNodeProps;
+    readonly UBUNTU2004_X64_DOCKER_BUILDER: AgentNodeProps;
+
+    readonly MACOS12_X64_MULTI_HOST: AgentNodeProps;
+
+    readonly WINDOWS2019_X64: AgentNodeProps;
 
     constructor() {
       this.AL2_X64 = {
+        agentType: 'unix',
         workerLabelString: 'Jenkins-Agent-AL2-X64-C54xlarge-Single-Host',
         instanceType: 'C54xlarge',
         remoteUser: 'ec2-user',
         maxTotalUses: -1,
         numExecutors: 1,
         amiId: 'ami-00a07e55fcad01043',
         initScript: 'sudo yum clean all && sudo rm -rf /var/cache/yum /var/lib/yum/history && sudo yum repolist &&'
-        + ' sudo yum update --skip-broken --exclude=openssh* --exclude=docker* -y',
+        + ' sudo yum update --skip-broken --exclude=openssh* --exclude=docker* -y && sudo yum install -y ntp &&'
+        + ' sudo systemctl restart ntpd && sudo systemctl enable ntpd',
       };
       this.AL2_X64_DOCKER_HOST = {
+        agentType: 'unix',
         workerLabelString: 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host',
         instanceType: 'C54xlarge',
         remoteUser: 'ec2-user',
         maxTotalUses: -1,
         numExecutors: 8,
         amiId: 'ami-00a07e55fcad01043',
         initScript: 'sudo yum clean all && sudo rm -rf /var/cache/yum /var/lib/yum/history && sudo yum repolist &&'
-        + ' sudo yum update --skip-broken --exclude=openssh* --exclude=docker* -y',
+        + ' sudo yum update --skip-broken --exclude=openssh* --exclude=docker* -y && sudo yum install -y ntp &&'
+        + ' sudo systemctl restart ntpd && sudo systemctl enable ntpd',
       };
       this.AL2_X64_DOCKER_HOST_PERF_TEST = {
+        agentType: 'unix',
         workerLabelString: 'Jenkins-Agent-AL2-X64-M52xlarge-Docker-Host-Perf-Test',
         instanceType: 'M52xlarge',
         remoteUser: 'ec2-user',
         maxTotalUses: -1,
         numExecutors: 8,
         amiId: 'ami-00a07e55fcad01043',
         initScript: 'sudo yum clean all && sudo rm -rf /var/cache/yum /var/lib/yum/history && sudo yum repolist &&'
-        + ' sudo yum update --skip-broken --exclude=openssh* --exclude=docker* -y',
+        + ' sudo yum update --skip-broken --exclude=openssh* --exclude=docker* -y && sudo yum install -y ntp &&'
+        + ' sudo systemctl restart ntpd && sudo systemctl enable ntpd',
       };
       this.AL2_ARM64 = {
+        agentType: 'unix',
         workerLabelString: 'Jenkins-Agent-AL2-Arm64-C6g4xlarge-Single-Host',
         instanceType: 'C6g4xlarge',
         remoteUser: 'ec2-user',
         maxTotalUses: -1,
         numExecutors: 1,
         amiId: 'ami-020c52efb1a60f1ae',
         initScript: 'sudo yum clean all && sudo rm -rf /var/cache/yum /var/lib/yum/history && sudo yum repolist &&'
-        + ' sudo yum update --skip-broken --exclude=openssh* --exclude=docker* -y',
+        + ' sudo yum update --skip-broken --exclude=openssh* --exclude=docker* -y && sudo yum install -y ntp &&'
+        + ' sudo systemctl restart ntpd && sudo systemctl enable ntpd',
       };
       this.AL2_ARM64_DOCKER_HOST = {
+        agentType: 'unix',
         workerLabelString: 'Jenkins-Agent-AL2-Arm64-C6g4xlarge-Docker-Host',
         instanceType: 'C6g4xlarge',
         remoteUser: 'ec2-user',
         maxTotalUses: -1,
         numExecutors: 8,
         amiId: 'ami-020c52efb1a60f1ae',
         initScript: 'sudo yum clean all && sudo rm -rf /var/cache/yum /var/lib/yum/history && sudo yum repolist &&'
-        + ' sudo yum update --skip-broken --exclude=openssh* --exclude=docker* -y',
+        + ' sudo yum update --skip-broken --exclude=openssh* --exclude=docker* -y && sudo yum install -y ntp &&'
+        + ' sudo systemctl restart ntpd && sudo systemctl enable ntpd',
       };
-      this.UBUNTU_X64 = {
+      this.UBUNTU2004_X64 = {
+        agentType: 'unix',
         workerLabelString: 'Jenkins-Agent-Ubuntu2004-X64-C524xlarge-Single-Host',
         instanceType: 'C524xlarge',
         remoteUser: 'ubuntu',
         maxTotalUses: 1,
         numExecutors: 1,
         amiId: 'ami-0f6ceb3b3687a3fba',
         initScript: 'sudo apt-mark hold docker docker.io openssh-server && docker ps &&'
-        + ' (sudo killall -9 apt-get apt 2>&1 || echo) &&'
-        + ' sudo apt-get update -y && sudo apt-get upgrade -y && sudo apt-get install docker-compose -y',
+        + ' sudo apt-get update && (sudo killall -9 apt-get apt 2>&1 || echo) &&'
+        + ' sudo apt-get upgrade -y && sudo apt-get install -y ntp &&'
+        + ' sudo systemctl restart ntp && sudo systemctl enable ntp',
       };
-      this.UBUNTU_X64_DOCKER_BUILDER = {
+      this.UBUNTU2004_X64_DOCKER_BUILDER = {
+        agentType: 'unix',
         workerLabelString: 'Jenkins-Agent-Ubuntu2004-X64-M52xlarge-Docker-Builder',
         instanceType: 'M52xlarge',
         remoteUser: 'ubuntu',
         maxTotalUses: -1,
         numExecutors: 1,
         amiId: 'ami-0f6ceb3b3687a3fba',
         initScript: 'sudo apt-mark hold docker docker.io openssh-server && docker ps &&'
-        + ' (sudo killall -9 apt-get apt 2>&1 || echo) &&'
-        + ' sudo apt-get update -y && sudo apt-get upgrade -y',
+        + ' sudo apt-get update && (sudo killall -9 apt-get apt 2>&1 || echo) &&'
+        + ' sudo apt-get upgrade -y && sudo apt-get install -y ntp &&'
+        + ' sudo systemctl restart ntp && sudo systemctl enable ntp',
+      };
+      this.MACOS12_X64_MULTI_HOST = {
+        agentType: 'mac',
+        workerLabelString: 'Jenkins-Agent-MacOS12-X64-Mac1Metal-Multi-Host',
+        instanceType: 'Mac1Metal',
+        remoteUser: 'ec2-user',
+        maxTotalUses: -1,
+        numExecutors: 6,
+        amiId: 'ami-0379811a08268a97e',
+        initScript: 'echo',
+      };
+      this.WINDOWS2019_X64 = {
+        agentType: 'windows',
+        workerLabelString: 'Jenkins-Agent-Windows2019-X64-C54xlarge-Single-Host',
+        instanceType: 'C54xlarge',
+        remoteUser: 'Administrator',
+        maxTotalUses: -1,
+        numExecutors: 1,
+        amiId: 'ami-07591ca4ef792c2d4',
+        initScript: 'echo',
       };
     }
 }
diff --git a/lib/compute/jenkins-main-node.ts b/lib/compute/jenkins-main-node.ts
@@ -209,6 +209,7 @@ export class JenkinsMainNode {
             'elasticfilesystem:DescribeFileSystems',
             'elasticfilesystem:DescribeMountTargets',
             'ec2:DescribeAvailabilityZones',
+            'ec2:GetPasswordData',
           ],
           resources: ['*'],
           conditions: {

diff --git a/lib/security/ci-security-groups.ts b/lib/security/ci-security-groups.ts
@@ -40,6 +40,8 @@ export class JenkinsSecurityGroups {
       description: 'Agent Node of Jenkins',
     });
     this.agentNodeSG.addIngressRule(this.mainNodeSG, Port.tcp(22), 'Main node SSH Access into agent nodes');
+    this.agentNodeSG.addIngressRule(this.mainNodeSG, Port.tcp(445), 'Main node SMB Access into agent nodes for Windows');
+    this.agentNodeSG.addIngressRule(this.mainNodeSG, Port.tcp(5985), 'Main node WinRM HTTP Access into agent nodes for Windows');
 
     this.efsSG = new SecurityGroup(stack, 'efsSG', {
       vpc,

diff --git a/packer/README.md b/packer/README.md
@@ -7,9 +7,9 @@
 * **.json:** All templates are now in JSON format, we have not converted them into HCL2 yet.
 
 ### Templates
-* jenkins-agent-win2016-x64.json: Windows 2016 Server.
-* jenkins-agent-win2019-x64.json: Windows 2019 Server (Recommended).
-* jenkins-agent-win2019-x64-alpine-wsl.json: Windows 2019 Server with WSL enabled running Alpine 3.
+* jenkins-agent-win2016-x64.json: Windows 2016 x86_64 Server.
+* jenkins-agent-win2019-x64.json: Windows 2019 x86_64 Server (Recommended).
+* jenkins-agent-win2019-x64-alpine-wsl.json: Windows 2019 x86_64 Server with WSL enabled running Alpine 3.
 * jenkins-agent-macos12-x64.json: MacOS 12 with x86_64_mac os_architecture.
 
 ### Usages

diff --git a/packer/jenkins-agent-win2016-x64.json b/packer/jenkins-agent-win2016-x64.json
@@ -1,6 +1,7 @@
 {
   "variables":{
-    "name-base":"Jenkins-Agent-Windows2016",
+    "name-base":"Jenkins-Agent-Windows2016-x64",
+    "os-version": "Windows2016",
     "build-region":"us-east-1",
     "build-vpc":"vpc-<>",
     "build-subnet":"subnet-<>",
@@ -43,7 +44,7 @@
       "winrm_insecure":true,
       "tags":{
         "Name": "{{user `name-base`}}-{{user `build-time`}}",
-        "OS_Version":"{{user `name-base`}}",
+        "OS_Version":"{{user `os-version`}}",
         "User":"Packer",
         "Encrypted_AMI":"False",
         "Created":"{{user `build-time`}}"

diff --git a/packer/jenkins-agent-win2019-x64-alpine-wsl.json b/packer/jenkins-agent-win2019-x64-alpine-wsl.json
@@ -1,6 +1,7 @@
 {
   "variables": {
-    "name-base":"Jenkins-Agent-Windows2019",
+    "name-base":"Jenkins-Agent-Windows2019-x64",
+    "os-version": "Windows2019",
     "build-region":"us-east-1",
     "build-vpc":"vpc-<>",
     "build-subnet":"subnet-<>",
@@ -43,7 +44,7 @@
       "winrm_insecure":true,
       "tags": {
         "Name": "{{user `name-base`}}-{{user `build-time`}}",
-        "OS_Version":"{{user `name-base`}}",
+        "OS_Version":"{{user `os-version`}}",
         "User":"Packer",
         "Encrypted_AMI":"False",
         "Created":"{{user `build-time`}}"

diff --git a/packer/jenkins-agent-win2019-x64.json b/packer/jenkins-agent-win2019-x64.json
@@ -1,6 +1,7 @@
 {
   "variables": {
-    "name-base":"Jenkins-Agent-Windows2019",
+    "name-base":"Jenkins-Agent-Windows2019-x64",
+    "os-version": "Windows2019",
     "build-region":"us-east-1",
     "build-vpc":"vpc-<>",
     "build-subnet":"subnet-<>",
@@ -43,7 +44,7 @@
       "winrm_insecure":true,
       "tags": {
         "Name": "{{user `name-base`}}-{{user `build-time`}}",
-        "OS_Version":"{{user `name-base`}}",
+        "OS_Version":"{{user `os-version`}}",
         "User":"Packer",
         "Encrypted_AMI":"False",
         "Created":"{{user `build-time`}}"

diff --git a/packer/scripts/windows/userdata.ps1 b/packer/scripts/windows/userdata.ps1
@@ -19,5 +19,8 @@ cmd /c net start winrm
 echo "Create Jenkins Home"
 New-Item -Path 'C:\\Users\\Administrator\\jenkins' -ItemType Directory
 
+echo "NTP Time Syncing"
+cmd /c net time \\pool.ntp.org /set /y
+
 </powershell>