diff --git a/README.md b/README.md index b9018725..5260686e 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,10 @@ Example: npm run cdk deploy OpenSearch-CI-Dev -- -c useSsl=false -c runWithOidc=false -c macAgent=true ``` +#### Windows agents +###### Prerequisite +Make sure there is an existing Windows AMI with necessary requirements, see [packer directory](./packer/README.md) for more information and AMI build commands. + #### Runnning additional commands In cases where you need to run additional logic/commands, such as adding a cron to emit ssl cert expiry metric, you can pass the commands as a script using `additionalCommands` context parameter. Below sample will write the python script to $HOME/hello-world path on jenkins master node and then execute it once the jenkins master node has been brought up. diff --git a/lib/ci-stack.ts b/lib/ci-stack.ts index 41696e87..73bff3fd 100644 --- a/lib/ci-stack.ts +++ b/lib/ci-stack.ts @@ -105,7 +105,8 @@ export class CIStack extends Stack { const listenerCertificate = ListenerCertificate.fromArn(certificateArn.secretValue.toString()); const agentNode = new AgentNodes(); const agentNodes: AgentNodeProps[] = [agentNode.AL2_X64, agentNode.AL2_X64_DOCKER_HOST, agentNode.AL2_X64_DOCKER_HOST_PERF_TEST, - agentNode.AL2_ARM64, agentNode.AL2_ARM64_DOCKER_HOST, agentNode.UBUNTU_X64, agentNode.UBUNTU_X64_DOCKER_BUILDER]; + agentNode.AL2_ARM64, agentNode.AL2_ARM64_DOCKER_HOST, agentNode.UBUNTU2004_X64, agentNode.UBUNTU2004_X64_DOCKER_BUILDER, + agentNode.MACOS12_X64_MULTI_HOST, agentNode.WINDOWS2019_X64]; const mainJenkinsNode = new JenkinsMainNode(this, { vpc, diff --git a/lib/compute/agent-node-config.ts b/lib/compute/agent-node-config.ts index a6bc1992..de08cce5 100644 --- a/lib/compute/agent-node-config.ts +++ b/lib/compute/agent-node-config.ts @@ -18,6 +18,7 @@ import { load } from 'js-yaml'; import { JenkinsMainNode } from './jenkins-main-node'; export interface AgentNodeProps { + agentType: string; amiId: string; instanceType: string; workerLabelString: string; @@ -130,11 +131,14 @@ export class AgentNodeConfig { const configTemplates: any = []; templates.forEach((element) => { - configTemplates.push(this.getTemplate(stack, element, props)); + if (element.agentType == 'unix') { + configTemplates.push(this.getUnixTemplate(stack, element, props)); + } else if (element.agentType == 'mac' && macAgent == 'true') { + configTemplates.push(this.getMacTemplate(stack, element, props)); + } else if (element.agentType == 'windows') { + configTemplates.push(this.getWindowsTemplate(stack, element, props)); + } }); - if (macAgent == 'true') { - configTemplates.push(this.getMacTemplate(stack, props)); - } const agentNodeYamlConfig = [{ amazonEC2: { @@ -149,7 +153,7 @@ export class AgentNodeConfig { return jenkinsYaml; } - private getTemplate(stack: Stack, config: AgentNodeProps, props: AgentNodeNetworkProps): { [x: string]: any; } { + private getUnixTemplate(stack: Stack, config: AgentNodeProps, props: AgentNodeNetworkProps): { [x: string]: any; } { return { ami: config.amiId, amiType: @@ -195,9 +199,9 @@ export class AgentNodeConfig { }; } - private getMacTemplate(stack: Stack, props: AgentNodeNetworkProps): { [x: string]: any; } { + private getMacTemplate(stack: Stack, config: AgentNodeProps, props: AgentNodeNetworkProps): { [x: string]: any; } { return { - ami: 'ami-0379811a08268a97e', + ami: config.amiId, amiType: { macData: { sshPort: '22' } }, associatePublicIp: false, @@ -205,20 +209,22 @@ export class AgentNodeConfig { connectionStrategy: 'PRIVATE_IP', customDeviceMapping: '/dev/sda1=:300:true:gp3::encrypted', deleteRootOnTermination: true, - description: 'jenkinsAgentNode-Jenkins-Agent-MacOS12-X64-Mac1Metal-Multi-Host', + description: `jenkinsAgentNode-${config.workerLabelString}`, ebsEncryptRootVolume: 'ENCRYPTED', ebsOptimized: true, hostKeyVerificationStrategy: 'OFF', iamInstanceProfile: this.AgentNodeInstanceProfileArn, idleTerminationMinutes: '720', - labelString: 'Jenkins-Agent-MacOS12-X64-Mac1Metal-Multi-Host', - maxTotalUses: -1, + labelString: config.workerLabelString, + launchTimeoutStr: '1000', + initScript: config.initScript, + maxTotalUses: config.maxTotalUses, minimumNumberOfInstances: 1, minimumNumberOfSpareInstances: 0, mode: 'EXCLUSIVE', monitoring: true, - numExecutors: '6', - remoteAdmin: 'ec2-user', + numExecutors: config.numExecutors, + remoteAdmin: config.remoteUser, remoteFS: '/var/jenkins', securityGroups: props.agentNodeSecurityGroup, stopOnTerminate: false, @@ -227,15 +233,15 @@ export class AgentNodeConfig { tags: [ { name: 'Name', - value: `${stack.stackName}/AgentNode/Jenkins-Agent-MacOS12-X64-Mac1Metal-Multi-Host`, + value: `${stack.stackName}/AgentNode/${config.workerLabelString}`, }, { name: 'type', - value: 'jenkinsAgentNode-Jenkins-Agent-MacOS12-X64-Mac1Metal-Multi-Host', + value: `jenkinsAgentNode-${config.workerLabelString}`, }, ], tenancy: 'Host', - type: 'Mac1Metal', + type: config.instanceType, nodeProperties: [ { envVars: { @@ -252,4 +258,54 @@ export class AgentNodeConfig { useEphemeralDevices: false, }; } + + private getWindowsTemplate(stack: Stack, config: AgentNodeProps, props: AgentNodeNetworkProps): { [x: string]: any; } { + return { + ami: config.amiId, + amiType: + { + windowsData: { + allowSelfSignedCertificate: false, bootDelay: '7.5', specifyPassword: false, useHTTPS: false, + }, + }, + associatePublicIp: false, + connectBySSHProcess: false, + connectionStrategy: 'PRIVATE_IP', + customDeviceMapping: '/dev/sda1=:300:true:::encrypted', + deleteRootOnTermination: true, + description: `jenkinsAgentNode-${config.workerLabelString}`, + ebsEncryptRootVolume: 'ENCRYPTED', + ebsOptimized: true, + hostKeyVerificationStrategy: 'OFF', + iamInstanceProfile: this.AgentNodeInstanceProfileArn, + idleTerminationMinutes: '120', + initScript: config.initScript, + labelString: config.workerLabelString, + launchTimeoutStr: '1000', + maxTotalUses: config.maxTotalUses, + minimumNumberOfInstances: 0, + minimumNumberOfSpareInstances: 1, + mode: 'EXCLUSIVE', + monitoring: true, + numExecutors: config.numExecutors, + remoteAdmin: config.remoteUser, + remoteFS: `C:\\Users\\${config.remoteUser}\\jenkins`, + securityGroups: props.agentNodeSecurityGroup, + stopOnTerminate: false, + subnetId: props.subnetId, + t2Unlimited: false, + tags: [{ + name: 'Name', + value: `${stack.stackName}/AgentNode/${config.workerLabelString}`, + }, + { + name: 'type', + value: `jenkinsAgentNode-${config.workerLabelString}`, + }, + ], + tenancy: 'Default', + type: config.instanceType, + useEphemeralDevices: false, + }; + } } diff --git a/lib/compute/agent-nodes.ts b/lib/compute/agent-nodes.ts index c774863f..00508705 100644 --- a/lib/compute/agent-nodes.ts +++ b/lib/compute/agent-nodes.ts @@ -20,12 +20,17 @@ export class AgentNodes { readonly AL2_ARM64_DOCKER_HOST: AgentNodeProps; - readonly UBUNTU_X64: AgentNodeProps; + readonly UBUNTU2004_X64: AgentNodeProps; - readonly UBUNTU_X64_DOCKER_BUILDER: AgentNodeProps; + readonly UBUNTU2004_X64_DOCKER_BUILDER: AgentNodeProps; + + readonly MACOS12_X64_MULTI_HOST: AgentNodeProps; + + readonly WINDOWS2019_X64: AgentNodeProps; constructor() { this.AL2_X64 = { + agentType: 'unix', workerLabelString: 'Jenkins-Agent-AL2-X64-C54xlarge-Single-Host', instanceType: 'C54xlarge', remoteUser: 'ec2-user', @@ -33,9 +38,11 @@ export class AgentNodes { numExecutors: 1, amiId: 'ami-00a07e55fcad01043', initScript: 'sudo yum clean all && sudo rm -rf /var/cache/yum /var/lib/yum/history && sudo yum repolist &&' - + ' sudo yum update --skip-broken --exclude=openssh* --exclude=docker* -y', + + ' sudo yum update --skip-broken --exclude=openssh* --exclude=docker* -y && sudo yum install -y ntp &&' + + ' sudo systemctl restart ntpd && sudo systemctl enable ntpd', }; this.AL2_X64_DOCKER_HOST = { + agentType: 'unix', workerLabelString: 'Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host', instanceType: 'C54xlarge', remoteUser: 'ec2-user', @@ -43,9 +50,11 @@ export class AgentNodes { numExecutors: 8, amiId: 'ami-00a07e55fcad01043', initScript: 'sudo yum clean all && sudo rm -rf /var/cache/yum /var/lib/yum/history && sudo yum repolist &&' - + ' sudo yum update --skip-broken --exclude=openssh* --exclude=docker* -y', + + ' sudo yum update --skip-broken --exclude=openssh* --exclude=docker* -y && sudo yum install -y ntp &&' + + ' sudo systemctl restart ntpd && sudo systemctl enable ntpd', }; this.AL2_X64_DOCKER_HOST_PERF_TEST = { + agentType: 'unix', workerLabelString: 'Jenkins-Agent-AL2-X64-M52xlarge-Docker-Host-Perf-Test', instanceType: 'M52xlarge', remoteUser: 'ec2-user', @@ -53,9 +62,11 @@ export class AgentNodes { numExecutors: 8, amiId: 'ami-00a07e55fcad01043', initScript: 'sudo yum clean all && sudo rm -rf /var/cache/yum /var/lib/yum/history && sudo yum repolist &&' - + ' sudo yum update --skip-broken --exclude=openssh* --exclude=docker* -y', + + ' sudo yum update --skip-broken --exclude=openssh* --exclude=docker* -y && sudo yum install -y ntp &&' + + ' sudo systemctl restart ntpd && sudo systemctl enable ntpd', }; this.AL2_ARM64 = { + agentType: 'unix', workerLabelString: 'Jenkins-Agent-AL2-Arm64-C6g4xlarge-Single-Host', instanceType: 'C6g4xlarge', remoteUser: 'ec2-user', @@ -63,9 +74,11 @@ export class AgentNodes { numExecutors: 1, amiId: 'ami-020c52efb1a60f1ae', initScript: 'sudo yum clean all && sudo rm -rf /var/cache/yum /var/lib/yum/history && sudo yum repolist &&' - + ' sudo yum update --skip-broken --exclude=openssh* --exclude=docker* -y', + + ' sudo yum update --skip-broken --exclude=openssh* --exclude=docker* -y && sudo yum install -y ntp &&' + + ' sudo systemctl restart ntpd && sudo systemctl enable ntpd', }; this.AL2_ARM64_DOCKER_HOST = { + agentType: 'unix', workerLabelString: 'Jenkins-Agent-AL2-Arm64-C6g4xlarge-Docker-Host', instanceType: 'C6g4xlarge', remoteUser: 'ec2-user', @@ -73,9 +86,11 @@ export class AgentNodes { numExecutors: 8, amiId: 'ami-020c52efb1a60f1ae', initScript: 'sudo yum clean all && sudo rm -rf /var/cache/yum /var/lib/yum/history && sudo yum repolist &&' - + ' sudo yum update --skip-broken --exclude=openssh* --exclude=docker* -y', + + ' sudo yum update --skip-broken --exclude=openssh* --exclude=docker* -y && sudo yum install -y ntp &&' + + ' sudo systemctl restart ntpd && sudo systemctl enable ntpd', }; - this.UBUNTU_X64 = { + this.UBUNTU2004_X64 = { + agentType: 'unix', workerLabelString: 'Jenkins-Agent-Ubuntu2004-X64-C524xlarge-Single-Host', instanceType: 'C524xlarge', remoteUser: 'ubuntu', @@ -83,10 +98,12 @@ export class AgentNodes { numExecutors: 1, amiId: 'ami-0f6ceb3b3687a3fba', initScript: 'sudo apt-mark hold docker docker.io openssh-server && docker ps &&' - + ' (sudo killall -9 apt-get apt 2>&1 || echo) &&' - + ' sudo apt-get update -y && sudo apt-get upgrade -y && sudo apt-get install docker-compose -y', + + ' sudo apt-get update && (sudo killall -9 apt-get apt 2>&1 || echo) &&' + + ' sudo apt-get upgrade -y && sudo apt-get install -y ntp &&' + + ' sudo systemctl restart ntp && sudo systemctl enable ntp', }; - this.UBUNTU_X64_DOCKER_BUILDER = { + this.UBUNTU2004_X64_DOCKER_BUILDER = { + agentType: 'unix', workerLabelString: 'Jenkins-Agent-Ubuntu2004-X64-M52xlarge-Docker-Builder', instanceType: 'M52xlarge', remoteUser: 'ubuntu', @@ -94,8 +111,29 @@ export class AgentNodes { numExecutors: 1, amiId: 'ami-0f6ceb3b3687a3fba', initScript: 'sudo apt-mark hold docker docker.io openssh-server && docker ps &&' - + ' (sudo killall -9 apt-get apt 2>&1 || echo) &&' - + ' sudo apt-get update -y && sudo apt-get upgrade -y', + + ' sudo apt-get update && (sudo killall -9 apt-get apt 2>&1 || echo) &&' + + ' sudo apt-get upgrade -y && sudo apt-get install -y ntp &&' + + ' sudo systemctl restart ntp && sudo systemctl enable ntp', + }; + this.MACOS12_X64_MULTI_HOST = { + agentType: 'mac', + workerLabelString: 'Jenkins-Agent-MacOS12-X64-Mac1Metal-Multi-Host', + instanceType: 'Mac1Metal', + remoteUser: 'ec2-user', + maxTotalUses: -1, + numExecutors: 6, + amiId: 'ami-0379811a08268a97e', + initScript: 'echo', + }; + this.WINDOWS2019_X64 = { + agentType: 'windows', + workerLabelString: 'Jenkins-Agent-Windows2019-X64-C54xlarge-Single-Host', + instanceType: 'C54xlarge', + remoteUser: 'Administrator', + maxTotalUses: -1, + numExecutors: 1, + amiId: 'ami-07591ca4ef792c2d4', + initScript: 'echo', }; } } diff --git a/lib/compute/jenkins-main-node.ts b/lib/compute/jenkins-main-node.ts index 4baf0d59..6cf6be92 100644 --- a/lib/compute/jenkins-main-node.ts +++ b/lib/compute/jenkins-main-node.ts @@ -209,6 +209,7 @@ export class JenkinsMainNode { 'elasticfilesystem:DescribeFileSystems', 'elasticfilesystem:DescribeMountTargets', 'ec2:DescribeAvailabilityZones', + 'ec2:GetPasswordData', ], resources: ['*'], conditions: { diff --git a/lib/security/ci-security-groups.ts b/lib/security/ci-security-groups.ts index 6c0983dc..e0580dcf 100644 --- a/lib/security/ci-security-groups.ts +++ b/lib/security/ci-security-groups.ts @@ -40,6 +40,8 @@ export class JenkinsSecurityGroups { description: 'Agent Node of Jenkins', }); this.agentNodeSG.addIngressRule(this.mainNodeSG, Port.tcp(22), 'Main node SSH Access into agent nodes'); + this.agentNodeSG.addIngressRule(this.mainNodeSG, Port.tcp(445), 'Main node SMB Access into agent nodes for Windows'); + this.agentNodeSG.addIngressRule(this.mainNodeSG, Port.tcp(5985), 'Main node WinRM HTTP Access into agent nodes for Windows'); this.efsSG = new SecurityGroup(stack, 'efsSG', { vpc, diff --git a/packer/README.md b/packer/README.md index 76c6d984..361e08fc 100644 --- a/packer/README.md +++ b/packer/README.md @@ -7,9 +7,9 @@ * **.json:** All templates are now in JSON format, we have not converted them into HCL2 yet. ### Templates -* jenkins-agent-win2016-x64.json: Windows 2016 Server. -* jenkins-agent-win2019-x64.json: Windows 2019 Server (Recommended). -* jenkins-agent-win2019-x64-alpine-wsl.json: Windows 2019 Server with WSL enabled running Alpine 3. +* jenkins-agent-win2016-x64.json: Windows 2016 x86_64 Server. +* jenkins-agent-win2019-x64.json: Windows 2019 x86_64 Server (Recommended). +* jenkins-agent-win2019-x64-alpine-wsl.json: Windows 2019 x86_64 Server with WSL enabled running Alpine 3. * jenkins-agent-macos12-x64.json: MacOS 12 with x86_64_mac os_architecture. ### Usages diff --git a/packer/jenkins-agent-win2016-x64.json b/packer/jenkins-agent-win2016-x64.json index 5755a106..c2d194a1 100644 --- a/packer/jenkins-agent-win2016-x64.json +++ b/packer/jenkins-agent-win2016-x64.json @@ -1,6 +1,7 @@ { "variables":{ - "name-base":"Jenkins-Agent-Windows2016", + "name-base":"Jenkins-Agent-Windows2016-x64", + "os-version": "Windows2016", "build-region":"us-east-1", "build-vpc":"vpc-<>", "build-subnet":"subnet-<>", @@ -43,7 +44,7 @@ "winrm_insecure":true, "tags":{ "Name": "{{user `name-base`}}-{{user `build-time`}}", - "OS_Version":"{{user `name-base`}}", + "OS_Version":"{{user `os-version`}}", "User":"Packer", "Encrypted_AMI":"False", "Created":"{{user `build-time`}}" diff --git a/packer/jenkins-agent-win2019-x64-alpine-wsl.json b/packer/jenkins-agent-win2019-x64-alpine-wsl.json index 91885ef2..7704b5e4 100644 --- a/packer/jenkins-agent-win2019-x64-alpine-wsl.json +++ b/packer/jenkins-agent-win2019-x64-alpine-wsl.json @@ -1,6 +1,7 @@ { "variables": { - "name-base":"Jenkins-Agent-Windows2019", + "name-base":"Jenkins-Agent-Windows2019-x64", + "os-version": "Windows2019", "build-region":"us-east-1", "build-vpc":"vpc-<>", "build-subnet":"subnet-<>", @@ -43,7 +44,7 @@ "winrm_insecure":true, "tags": { "Name": "{{user `name-base`}}-{{user `build-time`}}", - "OS_Version":"{{user `name-base`}}", + "OS_Version":"{{user `os-version`}}", "User":"Packer", "Encrypted_AMI":"False", "Created":"{{user `build-time`}}" diff --git a/packer/jenkins-agent-win2019-x64.json b/packer/jenkins-agent-win2019-x64.json index 265bf565..658c331f 100644 --- a/packer/jenkins-agent-win2019-x64.json +++ b/packer/jenkins-agent-win2019-x64.json @@ -1,6 +1,7 @@ { "variables": { - "name-base":"Jenkins-Agent-Windows2019", + "name-base":"Jenkins-Agent-Windows2019-x64", + "os-version": "Windows2019", "build-region":"us-east-1", "build-vpc":"vpc-<>", "build-subnet":"subnet-<>", @@ -43,7 +44,7 @@ "winrm_insecure":true, "tags": { "Name": "{{user `name-base`}}-{{user `build-time`}}", - "OS_Version":"{{user `name-base`}}", + "OS_Version":"{{user `os-version`}}", "User":"Packer", "Encrypted_AMI":"False", "Created":"{{user `build-time`}}" diff --git a/packer/scripts/windows/userdata.ps1 b/packer/scripts/windows/userdata.ps1 index 601e4ebd..1809eb90 100644 --- a/packer/scripts/windows/userdata.ps1 +++ b/packer/scripts/windows/userdata.ps1 @@ -19,5 +19,8 @@ cmd /c net start winrm echo "Create Jenkins Home" New-Item -Path 'C:\\Users\\Administrator\\jenkins' -ItemType Directory +echo "NTP Time Syncing" +cmd /c net time \\pool.ntp.org /set /y +