From b49b81b1c3eebb6a9fd39d2304aa6507938b0877 Mon Sep 17 00:00:00 2001 From: Peter Darton Date: Wed, 28 Jun 2017 16:45:58 +0100 Subject: [PATCH 1/3] Change EOL codes from CRLF to LF. When CloudProvisioningAlgorithm, CloudProvisioningRecord and CloudProvisioningState were added, they were mistakenly added with CRLF end of line codes, which makes future "diff" operations difficult (as they often show as "entire file contents removed and replaced with new file contents"). This commit changes the end-of-line codes from CRLF to LF - no other changes made. --- .../tools/CloudProvisioningAlgorithm.java | 276 +++--- .../tools/CloudProvisioningRecord.java | 384 ++++---- .../vsphere/tools/CloudProvisioningState.java | 572 ++++++------ .../tools/CloudProvisioningAlgorithmTest.java | 578 ++++++------ .../tools/CloudProvisioningStateTest.java | 822 +++++++++--------- 5 files changed, 1316 insertions(+), 1316 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithm.java b/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithm.java index 30d489ad..323a1ad0 100644 --- a/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithm.java +++ b/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithm.java @@ -1,139 +1,139 @@ -package org.jenkinsci.plugins.vsphere.tools; - -import java.math.BigInteger; -import java.util.Collection; -import java.util.Iterator; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.UUID; - -import org.jenkinsci.plugins.vSphereCloudSlaveTemplate; -import org.kohsuke.accmod.Restricted; -import org.kohsuke.accmod.restrictions.NoExternalUse; - -/** - * How we decide what template to create the next slave on. - */ -public final class CloudProvisioningAlgorithm { - private CloudProvisioningAlgorithm() { - } - - /** - * Given a bunch of templates to choose from, works out which one we should - * use next. - * - * @param provisionables - * Template records to decide between. - * @return The record with the most free capacity, or null if there are none - * with any capacity. - */ - public static CloudProvisioningRecord findTemplateWithMostFreeCapacity( - Collection provisionables) { - final SortedSet sortedSet = new TreeSet( - CloudProvisioningRecord.leastUsedFirst); - sortedSet.addAll(provisionables); - final Iterator iterator = sortedSet.iterator(); - if (iterator.hasNext()) { - final CloudProvisioningRecord bestOption = iterator.next(); - if (bestOption.hasCapacityForMore()) { - return bestOption; - } - } - return null; - } - - /** - * Chooses a name for a new node. - *
    - *
  • If the template has a limited number of instances available then the - * name will be of the form "prefix_number" where "number" is a number that - * should be between 1 and the number of permitted instances.
    • - *
  • If the template has an unlimited number of instances available then - * the name will be of the form "prefix_random" where "random" is a random - * UUID's 32-byte number (rendered using a high radix to keep the string - * short).
    • - *
- * - * @param record - * Our record regarding the template the slave will be created - * from. - * @return A name for the new node. This will start with the - * {@link vSphereCloudSlaveTemplate#getCloneNamePrefix()}. - */ - public static String findUnusedName(CloudProvisioningRecord record) { - final vSphereCloudSlaveTemplate template = record.getTemplate(); - final String cloneNamePrefix = template.getCloneNamePrefix(); - final Set existingNames = new TreeSet(); - existingNames.addAll(record.getCurrentlyPlanned()); - existingNames.addAll(record.getCurrentlyProvisioned()); - final int templateInstanceCap = template.getTemplateInstanceCap(); - final boolean hasCap = templateInstanceCap > 0 && templateInstanceCap < Integer.MAX_VALUE; - final int maxAttempts = hasCap ? (templateInstanceCap + 1) : 100; - for (int attempt = 0; attempt < maxAttempts; attempt++) { - final String suffix = hasCap ? calcSequentialSuffix(attempt) : calcRandomSuffix(attempt); - final String nodeName = cloneNamePrefix + "_" + suffix; - if (!existingNames.contains(nodeName)) { - return nodeName; - } - } - throw new IllegalStateException("Unable to find unused name for slave for record " + record.toString() - + ", even after " + maxAttempts + " attempts."); - } - - private static String calcSequentialSuffix(final int attempt) { - final int slaveNumber = attempt + 1; - final String suffix = Integer.toString(slaveNumber); - return suffix; - } - - private static String calcRandomSuffix(int attempt) { - // get "unique" UUID - final UUID uuid = UUID.randomUUID(); - // put both "long"s into a BigInteger. - final long lsb = uuid.getLeastSignificantBits(); - final long msb = uuid.getMostSignificantBits(); - final BigInteger bigNumber = toBigInteger(msb, lsb); - // turn into a string - final String suffix = bigNumber.toString(Character.MAX_RADIX); - return suffix; - } - - /** - * Turns two 64-bit long numbers into a positive 128-bit {@link BigInteger} - * that's in the range 0 to 2128-1. - *

- * Note: This is only package-level access for unit-testing. - *

- * - * @param msb - * The most-significant 64 bits. - * @param lsb - * The least-significant 64 bits. - * @return A {@link BigInteger}. - */ - @Restricted(NoExternalUse.class) - static BigInteger toBigInteger(final long msb, final long lsb) { - final byte[] bytes = new byte[17]; - int b = 0; - bytes[b++] = (byte) 0; // ensure we're all positive - bytes[b++] = (byte) (msb >> 56); - bytes[b++] = (byte) (msb >> 48); - bytes[b++] = (byte) (msb >> 40); - bytes[b++] = (byte) (msb >> 32); - bytes[b++] = (byte) (msb >> 24); - bytes[b++] = (byte) (msb >> 16); - bytes[b++] = (byte) (msb >> 8); - bytes[b++] = (byte) (msb); - bytes[b++] = (byte) (lsb >> 56); - bytes[b++] = (byte) (lsb >> 48); - bytes[b++] = (byte) (lsb >> 40); - bytes[b++] = (byte) (lsb >> 32); - bytes[b++] = (byte) (lsb >> 24); - bytes[b++] = (byte) (lsb >> 16); - bytes[b++] = (byte) (lsb >> 8); - bytes[b++] = (byte) (lsb); - final BigInteger bigNumber = new BigInteger(bytes); - return bigNumber; - } +package org.jenkinsci.plugins.vsphere.tools; + +import java.math.BigInteger; +import java.util.Collection; +import java.util.Iterator; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.UUID; + +import org.jenkinsci.plugins.vSphereCloudSlaveTemplate; +import org.kohsuke.accmod.Restricted; +import org.kohsuke.accmod.restrictions.NoExternalUse; + +/** + * How we decide what template to create the next slave on. + */ +public final class CloudProvisioningAlgorithm { + private CloudProvisioningAlgorithm() { + } + + /** + * Given a bunch of templates to choose from, works out which one we should + * use next. + * + * @param provisionables + * Template records to decide between. + * @return The record with the most free capacity, or null if there are none + * with any capacity. + */ + public static CloudProvisioningRecord findTemplateWithMostFreeCapacity( + Collection provisionables) { + final SortedSet sortedSet = new TreeSet( + CloudProvisioningRecord.leastUsedFirst); + sortedSet.addAll(provisionables); + final Iterator iterator = sortedSet.iterator(); + if (iterator.hasNext()) { + final CloudProvisioningRecord bestOption = iterator.next(); + if (bestOption.hasCapacityForMore()) { + return bestOption; + } + } + return null; + } + + /** + * Chooses a name for a new node. + *
    + *
  • If the template has a limited number of instances available then the + * name will be of the form "prefix_number" where "number" is a number that + * should be between 1 and the number of permitted instances.
    • + *
  • If the template has an unlimited number of instances available then + * the name will be of the form "prefix_random" where "random" is a random + * UUID's 32-byte number (rendered using a high radix to keep the string + * short).
    • + *
+ * + * @param record + * Our record regarding the template the slave will be created + * from. + * @return A name for the new node. This will start with the + * {@link vSphereCloudSlaveTemplate#getCloneNamePrefix()}. + */ + public static String findUnusedName(CloudProvisioningRecord record) { + final vSphereCloudSlaveTemplate template = record.getTemplate(); + final String cloneNamePrefix = template.getCloneNamePrefix(); + final Set existingNames = new TreeSet(); + existingNames.addAll(record.getCurrentlyPlanned()); + existingNames.addAll(record.getCurrentlyProvisioned()); + final int templateInstanceCap = template.getTemplateInstanceCap(); + final boolean hasCap = templateInstanceCap > 0 && templateInstanceCap < Integer.MAX_VALUE; + final int maxAttempts = hasCap ? (templateInstanceCap + 1) : 100; + for (int attempt = 0; attempt < maxAttempts; attempt++) { + final String suffix = hasCap ? calcSequentialSuffix(attempt) : calcRandomSuffix(attempt); + final String nodeName = cloneNamePrefix + "_" + suffix; + if (!existingNames.contains(nodeName)) { + return nodeName; + } + } + throw new IllegalStateException("Unable to find unused name for slave for record " + record.toString() + + ", even after " + maxAttempts + " attempts."); + } + + private static String calcSequentialSuffix(final int attempt) { + final int slaveNumber = attempt + 1; + final String suffix = Integer.toString(slaveNumber); + return suffix; + } + + private static String calcRandomSuffix(int attempt) { + // get "unique" UUID + final UUID uuid = UUID.randomUUID(); + // put both "long"s into a BigInteger. + final long lsb = uuid.getLeastSignificantBits(); + final long msb = uuid.getMostSignificantBits(); + final BigInteger bigNumber = toBigInteger(msb, lsb); + // turn into a string + final String suffix = bigNumber.toString(Character.MAX_RADIX); + return suffix; + } + + /** + * Turns two 64-bit long numbers into a positive 128-bit {@link BigInteger} + * that's in the range 0 to 2128-1. + *

+ * Note: This is only package-level access for unit-testing. + *

+ * + * @param msb + * The most-significant 64 bits. + * @param lsb + * The least-significant 64 bits. + * @return A {@link BigInteger}. + */ + @Restricted(NoExternalUse.class) + static BigInteger toBigInteger(final long msb, final long lsb) { + final byte[] bytes = new byte[17]; + int b = 0; + bytes[b++] = (byte) 0; // ensure we're all positive + bytes[b++] = (byte) (msb >> 56); + bytes[b++] = (byte) (msb >> 48); + bytes[b++] = (byte) (msb >> 40); + bytes[b++] = (byte) (msb >> 32); + bytes[b++] = (byte) (msb >> 24); + bytes[b++] = (byte) (msb >> 16); + bytes[b++] = (byte) (msb >> 8); + bytes[b++] = (byte) (msb); + bytes[b++] = (byte) (lsb >> 56); + bytes[b++] = (byte) (lsb >> 48); + bytes[b++] = (byte) (lsb >> 40); + bytes[b++] = (byte) (lsb >> 32); + bytes[b++] = (byte) (lsb >> 24); + bytes[b++] = (byte) (lsb >> 16); + bytes[b++] = (byte) (lsb >> 8); + bytes[b++] = (byte) (lsb); + final BigInteger bigNumber = new BigInteger(bytes); + return bigNumber; + } } \ No newline at end of file diff --git a/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningRecord.java b/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningRecord.java index 727539ab..a82e31bb 100644 --- a/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningRecord.java +++ b/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningRecord.java @@ -1,193 +1,193 @@ -package org.jenkinsci.plugins.vsphere.tools; - -import java.util.Comparator; -import java.util.Set; -import java.util.TreeSet; - -import org.jenkinsci.plugins.vSphereCloudSlaveTemplate; - -/** - * There's a delay between when we give a bunch of slave nodes to Jenkins (when - * it asks us to provision some) and when those nodes appear in vSphere and in - * Jenkins, so we need to keep a record of what's in progress so we don't - * over-commit. - */ -public final class CloudProvisioningRecord { - private final vSphereCloudSlaveTemplate template; - private Set currentlyProvisioned; - private Set currentlyPlanned; - - CloudProvisioningRecord(vSphereCloudSlaveTemplate template) { - this.template = template; - this.currentlyProvisioned = new TreeSet(); - this.currentlyPlanned = new TreeSet(); - } - - public vSphereCloudSlaveTemplate getTemplate() { - return template; - } - - @Override - public String toString() { - return String.format("Template[prefix=%s, provisioned=%s, planned=%s, max=%d, fullness=%.3f%%]", getTemplate() - .getCloneNamePrefix(), getCurrentlyProvisioned(), getCurrentlyPlanned(), calcMaxToProvision(), - calcFullness() * 100.0); - } - - Set getCurrentlyProvisioned() { - return currentlyProvisioned; - } - - boolean addCurrentlyActive(String nodeName) { - return currentlyProvisioned.add(nodeName); - } - - boolean removeCurrentlyActive(String nodeName) { - return currentlyProvisioned.remove(nodeName); - } - - Set getCurrentlyPlanned() { - return currentlyPlanned; - } - - boolean addCurrentlyPlanned(String nodeName) { - return currentlyPlanned.add(nodeName); - } - - boolean removeCurrentlyPlanned(String nodeName) { - return currentlyPlanned.remove(nodeName); - } - - private int calcMaxToProvision() { - final int templateInstanceCap = template.getTemplateInstanceCap(); - final int maxToProvision = templateInstanceCap == 0 ? Integer.MAX_VALUE : templateInstanceCap; - return maxToProvision; - } - - private boolean hasFiniteCapacity() { - final int templateInstanceCap = template.getTemplateInstanceCap(); - final int maxToProvision = templateInstanceCap == 0 ? Integer.MAX_VALUE : templateInstanceCap; - return maxToProvision != Integer.MAX_VALUE; - } - - private double calcFullness() { - final int maxToProvision = calcMaxToProvision(); - return ((double) calcCurrentCommitment()) / (double) maxToProvision; - } - - boolean hasCapacityForMore() { - final int totalCommitment = calcCurrentCommitment(); - final int maxToProvision = calcMaxToProvision(); - return maxToProvision > totalCommitment; - } - - private int calcCurrentCommitment() { - return currentlyProvisioned.size() + currentlyPlanned.size(); - } - - /** - * Sorts {@link CloudProvisioningRecord}s, putting the ones with most free - * capacity first. - *

- * When comparing two records with finite capacity then their usage:limit - * ratios are compared, otherwise current usage levels are compared. - */ - static final Comparator leastUsedFirst = new Comparator() { - private static final int theyAreTheSame = 0; - private static final int bShouldComeLast = -1; - private static final int aShouldComeLast = 1; - - @Override - public int compare(CloudProvisioningRecord a, CloudProvisioningRecord b) { - if (b == a) { - return theyAreTheSame; - } - final int compareByCapacity; - if (a.hasFiniteCapacity() && b.hasFiniteCapacity()) { - compareByCapacity = compareByUsageRatio(a, b); - } else { - compareByCapacity = compareByUsage(a, b); - } - if (compareByCapacity != theyAreTheSame) { - return compareByCapacity; - } - final int compareByMaxCapacity = compareByMaxCapacity(a, b); - if (compareByMaxCapacity != theyAreTheSame) { - return compareByMaxCapacity; - } - return tieBreak(a, b); - } - - /** if both have instance caps, we rank by utilization:capacity ratio */ - private int compareByUsageRatio(CloudProvisioningRecord a, CloudProvisioningRecord b) { - // sort by utilization:capacity ratio - lowest usage comes first - final double aFullness = a.calcFullness(); - final double bFullness = b.calcFullness(); - if (aFullness > bFullness) { - return aShouldComeLast; - } - if (aFullness < bFullness) { - return bShouldComeLast; - } - return theyAreTheSame; - } - - /** - * if either has no instance cap, we rank by least usage UNLESS one of - * them is full - */ - private int compareByUsage(CloudProvisioningRecord a, CloudProvisioningRecord b) { - // sort by "is full" - ones that are full come last - final boolean aFull = !a.hasCapacityForMore(); - final boolean bFull = !b.hasCapacityForMore(); - if (aFull != bFull) { - if (aFull) { - return aShouldComeLast; - } else { - return bShouldComeLast; - } - } - // sort by utilization - lowest usage comes first - final double aUsage = a.calcCurrentCommitment(); - final double bUsage = b.calcCurrentCommitment(); - if (aUsage > bUsage) { - return aShouldComeLast; - } - if (aUsage < bUsage) { - return bShouldComeLast; - } - return theyAreTheSame; - } - - /** Try rank by capacity */ - private int compareByMaxCapacity(CloudProvisioningRecord a, CloudProvisioningRecord b) { - // by absolute capacity - highest comes first - final int aCapacity = a.calcMaxToProvision(); - final int bCapacity = b.calcMaxToProvision(); - if (bCapacity > aCapacity) { - return aShouldComeLast; - } - if (bCapacity < aCapacity) { - return bShouldComeLast; - } - return theyAreTheSame; - } - - /** - * if all else is equal we prefer the one with fewer VMs being started - * up - */ - private int tieBreak(CloudProvisioningRecord a, CloudProvisioningRecord b) { - // then by number of VMs being started - lowest comes first - final int aCurrentlyPlanned = a.currentlyPlanned.size(); - final int bCurrentlyPlanned = b.currentlyPlanned.size(); - if (aCurrentlyPlanned > bCurrentlyPlanned) { - return aShouldComeLast; - } - if (aCurrentlyPlanned < bCurrentlyPlanned) { - return bShouldComeLast; - } - return theyAreTheSame; - } - }; +package org.jenkinsci.plugins.vsphere.tools; + +import java.util.Comparator; +import java.util.Set; +import java.util.TreeSet; + +import org.jenkinsci.plugins.vSphereCloudSlaveTemplate; + +/** + * There's a delay between when we give a bunch of slave nodes to Jenkins (when + * it asks us to provision some) and when those nodes appear in vSphere and in + * Jenkins, so we need to keep a record of what's in progress so we don't + * over-commit. + */ +public final class CloudProvisioningRecord { + private final vSphereCloudSlaveTemplate template; + private Set currentlyProvisioned; + private Set currentlyPlanned; + + CloudProvisioningRecord(vSphereCloudSlaveTemplate template) { + this.template = template; + this.currentlyProvisioned = new TreeSet(); + this.currentlyPlanned = new TreeSet(); + } + + public vSphereCloudSlaveTemplate getTemplate() { + return template; + } + + @Override + public String toString() { + return String.format("Template[prefix=%s, provisioned=%s, planned=%s, max=%d, fullness=%.3f%%]", getTemplate() + .getCloneNamePrefix(), getCurrentlyProvisioned(), getCurrentlyPlanned(), calcMaxToProvision(), + calcFullness() * 100.0); + } + + Set getCurrentlyProvisioned() { + return currentlyProvisioned; + } + + boolean addCurrentlyActive(String nodeName) { + return currentlyProvisioned.add(nodeName); + } + + boolean removeCurrentlyActive(String nodeName) { + return currentlyProvisioned.remove(nodeName); + } + + Set getCurrentlyPlanned() { + return currentlyPlanned; + } + + boolean addCurrentlyPlanned(String nodeName) { + return currentlyPlanned.add(nodeName); + } + + boolean removeCurrentlyPlanned(String nodeName) { + return currentlyPlanned.remove(nodeName); + } + + private int calcMaxToProvision() { + final int templateInstanceCap = template.getTemplateInstanceCap(); + final int maxToProvision = templateInstanceCap == 0 ? Integer.MAX_VALUE : templateInstanceCap; + return maxToProvision; + } + + private boolean hasFiniteCapacity() { + final int templateInstanceCap = template.getTemplateInstanceCap(); + final int maxToProvision = templateInstanceCap == 0 ? Integer.MAX_VALUE : templateInstanceCap; + return maxToProvision != Integer.MAX_VALUE; + } + + private double calcFullness() { + final int maxToProvision = calcMaxToProvision(); + return ((double) calcCurrentCommitment()) / (double) maxToProvision; + } + + boolean hasCapacityForMore() { + final int totalCommitment = calcCurrentCommitment(); + final int maxToProvision = calcMaxToProvision(); + return maxToProvision > totalCommitment; + } + + private int calcCurrentCommitment() { + return currentlyProvisioned.size() + currentlyPlanned.size(); + } + + /** + * Sorts {@link CloudProvisioningRecord}s, putting the ones with most free + * capacity first. + *

+ * When comparing two records with finite capacity then their usage:limit + * ratios are compared, otherwise current usage levels are compared. + */ + static final Comparator leastUsedFirst = new Comparator() { + private static final int theyAreTheSame = 0; + private static final int bShouldComeLast = -1; + private static final int aShouldComeLast = 1; + + @Override + public int compare(CloudProvisioningRecord a, CloudProvisioningRecord b) { + if (b == a) { + return theyAreTheSame; + } + final int compareByCapacity; + if (a.hasFiniteCapacity() && b.hasFiniteCapacity()) { + compareByCapacity = compareByUsageRatio(a, b); + } else { + compareByCapacity = compareByUsage(a, b); + } + if (compareByCapacity != theyAreTheSame) { + return compareByCapacity; + } + final int compareByMaxCapacity = compareByMaxCapacity(a, b); + if (compareByMaxCapacity != theyAreTheSame) { + return compareByMaxCapacity; + } + return tieBreak(a, b); + } + + /** if both have instance caps, we rank by utilization:capacity ratio */ + private int compareByUsageRatio(CloudProvisioningRecord a, CloudProvisioningRecord b) { + // sort by utilization:capacity ratio - lowest usage comes first + final double aFullness = a.calcFullness(); + final double bFullness = b.calcFullness(); + if (aFullness > bFullness) { + return aShouldComeLast; + } + if (aFullness < bFullness) { + return bShouldComeLast; + } + return theyAreTheSame; + } + + /** + * if either has no instance cap, we rank by least usage UNLESS one of + * them is full + */ + private int compareByUsage(CloudProvisioningRecord a, CloudProvisioningRecord b) { + // sort by "is full" - ones that are full come last + final boolean aFull = !a.hasCapacityForMore(); + final boolean bFull = !b.hasCapacityForMore(); + if (aFull != bFull) { + if (aFull) { + return aShouldComeLast; + } else { + return bShouldComeLast; + } + } + // sort by utilization - lowest usage comes first + final double aUsage = a.calcCurrentCommitment(); + final double bUsage = b.calcCurrentCommitment(); + if (aUsage > bUsage) { + return aShouldComeLast; + } + if (aUsage < bUsage) { + return bShouldComeLast; + } + return theyAreTheSame; + } + + /** Try rank by capacity */ + private int compareByMaxCapacity(CloudProvisioningRecord a, CloudProvisioningRecord b) { + // by absolute capacity - highest comes first + final int aCapacity = a.calcMaxToProvision(); + final int bCapacity = b.calcMaxToProvision(); + if (bCapacity > aCapacity) { + return aShouldComeLast; + } + if (bCapacity < aCapacity) { + return bShouldComeLast; + } + return theyAreTheSame; + } + + /** + * if all else is equal we prefer the one with fewer VMs being started + * up + */ + private int tieBreak(CloudProvisioningRecord a, CloudProvisioningRecord b) { + // then by number of VMs being started - lowest comes first + final int aCurrentlyPlanned = a.currentlyPlanned.size(); + final int bCurrentlyPlanned = b.currentlyPlanned.size(); + if (aCurrentlyPlanned > bCurrentlyPlanned) { + return aShouldComeLast; + } + if (aCurrentlyPlanned < bCurrentlyPlanned) { + return bShouldComeLast; + } + return theyAreTheSame; + } + }; } \ No newline at end of file diff --git a/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningState.java b/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningState.java index be8f1183..a9d6b0ab 100644 --- a/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningState.java +++ b/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningState.java @@ -1,286 +1,286 @@ -package org.jenkinsci.plugins.vsphere.tools; - -import java.util.ArrayList; -import java.util.IdentityHashMap; -import java.util.List; -import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; - -import org.jenkinsci.plugins.vSphereCloud; -import org.jenkinsci.plugins.vSphereCloudSlaveTemplate; - -/** - * Utility class that works out what slaves we should start up in response to - * Jenkins asking us to start things. - *

- * We do this by keeping a record of every slave we start, and every slave we - * have active. That way, we can avoid over-provisioning. - *

- *

- * The idea is that we are told what slaves that the cloud is going to create, - * when the cloud has created them (or failed to) and when those slaves have - * died. This way we can keep track of everything, in order to allow the cloud - * to make accurate decisions regarding what to create next. - *

- * Note: This is not thread-safe. Callers must do their own synchronization. - */ -public class CloudProvisioningState { - private static final Logger LOGGER = Logger.getLogger(CloudProvisioningState.class.getName()); - /** - * Record of slaves we've told Jenkins to start up, which have yet to start. - */ - private final Map records = new IdentityHashMap(); - /** - * Our parent, so we can check what templates still exist (as the user may - * have added/removed some). - */ - private final vSphereCloud parent; - /** - * Where we log to. This is only instance-based for test-purposes, and - * transient to stop serialization problems. - */ - private transient final Logger logger; - - public CloudProvisioningState(vSphereCloud parent) { - this(parent, LOGGER); - } - - CloudProvisioningState(vSphereCloud parent, Logger logger) { - this.parent = parent; - this.logger = logger; - this.logger.log(Level.FINE, "Created for parent {0}", parent.toString()); - } - - /** - * To be called when we've decided to create a new node. Callers MUST ensure - * that {@link #provisionedSlaveNowActive(CloudProvisioningRecord, String)} - * or {@link #provisioningEndedInError(CloudProvisioningRecord, String)} - * gets called later. - * - * @param provisionable - * Our record for the template for the named node. - * @param nodeName - * The name of the VM. - */ - public void provisioningStarted(CloudProvisioningRecord provisionable, String nodeName) { - final boolean wasPreviouslyUnknownToPlanning = provisionable.addCurrentlyPlanned(nodeName); - final boolean wasAlreadyActive = provisionable.removeCurrentlyActive(nodeName); - logStateChange(Level.FINE, "Intending to create {0}", "wasPreviouslyUnknownToPlanning", - wasPreviouslyUnknownToPlanning, true, "wasAlreadyActive", wasAlreadyActive, false, nodeName); - } - - /** - * To be called when a newly created node (previously promised to - * {@link #provisioningStarted(CloudProvisioningRecord, String)}) comes up. - * Callers MUST ensure that - * {@link #provisionedSlaveNowTerminated(String)} - * gets called later. - * - * @param provisionable - * Our record for the template for the named node. - * @param nodeName - * The name of the VM. - */ - public void provisionedSlaveNowActive(CloudProvisioningRecord provisionable, String nodeName) { - final boolean wasNotPreviouslyActive = provisionable.addCurrentlyActive(nodeName); - final boolean wasPreviouslyPlanned = provisionable.removeCurrentlyPlanned(nodeName); - logStateChange(Level.FINE, "Marking {0} as active", "wasNotPreviouslyActive", wasNotPreviouslyActive, true, - "wasPreviouslyPlanned", wasPreviouslyPlanned, true, nodeName); - } - - /** - * To be called when a node we created (previously told to - * {@link #provisionedSlaveNowActive(CloudProvisioningRecord, String)}) has - * died. - * - * @param nodeName - * The name of the VM. - */ - public void provisionedSlaveNowTerminated(String nodeName) { - final Map.Entry entry = findEntryForVM(nodeName); - if (entry != null) { - final CloudProvisioningRecord provisionable = entry.getValue(); - final boolean wasPreviouslyPlanned = provisionable.removeCurrentlyPlanned(nodeName); - final boolean wasPreviouslyActive = provisionable.removeCurrentlyActive(nodeName); - if (recordIsPrunable(provisionable)) { - removeExistingRecord(provisionable); - } - logStateChange(Level.FINE, "Marking {0} as terminated", "wasPreviouslyPlanned", wasPreviouslyPlanned, - false, "wasPreviouslyActive", wasPreviouslyActive, true, nodeName); - } else { - logger.log(Level.WARNING, "Asked to mark {0} as terminated, but we have no record of it.", nodeName); - } - } - - /** - * To be called when a node that we previously promised to create (by - * calling {@link #provisioningStarted(CloudProvisioningRecord, String)}) - * failed to start. - * - * @param provisionable - * Our record for the template for the named node. - * @param nodeName - * The name of the VM. - */ - public void provisioningEndedInError(CloudProvisioningRecord provisionable, String nodeName) { - final boolean wasPreviouslyPlanned = provisionable.removeCurrentlyPlanned(nodeName); - final boolean wasPreviouslyActive = provisionable.removeCurrentlyActive(nodeName); - if (recordIsPrunable(provisionable)) { - removeExistingRecord(provisionable); - } - logStateChange(Level.INFO, "Marking {0} as failed", "wasPreviouslyPlanned", wasPreviouslyPlanned, true, - "wasPreviouslyActive", wasPreviouslyActive, false, nodeName); - } - - /** - * To be called every now and again to ensure that we're not caching records - * that will never be valid again. - */ - public void pruneUnwantedRecords() { - final List toBeRemoved = new ArrayList(records.size()); - for (final Map.Entry entry : records.entrySet()) { - final CloudProvisioningRecord record = entry.getValue(); - if (recordIsPrunable(record)) { - toBeRemoved.add(record); - } - } - for (final CloudProvisioningRecord record : toBeRemoved) { - removeExistingRecord(record); - } - } - - /** - * Given a set of templates, returns the equivalent records. - * - * @param templates - * The templates we are interested in. - * @return A list of {@link CloudProvisioningRecord}. - */ - public List calculateProvisionableTemplates(Iterable templates) { - final List result = new ArrayList(); - for (final vSphereCloudSlaveTemplate template : templates) { - final CloudProvisioningRecord provisionable = getOrCreateRecord(template); - result.add(provisionable); - } - return result; - } - - /** - * Counts all the known nodes, both active and in-progress, across all - * templates. - * - * @return The number of nodes that are active or soon-to-be-active. - */ - public int countNodes() { - int result = 0; - for (final CloudProvisioningRecord record : records.values()) { - result += record.getCurrentlyPlanned().size(); - result += record.getCurrentlyProvisioned().size(); - } - return result; - } - - /** - * Gets the record for the given template. If we didn't have one before, we - * create one. - * - * @param template - * The template in question. - * @return The one-and-only record for this template. - */ - public CloudProvisioningRecord getOrCreateRecord(final vSphereCloudSlaveTemplate template) { - final CloudProvisioningRecord existingRecord = getExistingRecord(template); - if (existingRecord != null) { - return existingRecord; - } - final CloudProvisioningRecord newRecord = new CloudProvisioningRecord(template); - logger.log(Level.FINE, "Creating new record for template {0} ({1})", - new Object[] { template.getCloneNamePrefix(), template.toString() }); - records.put(template, newRecord); - return newRecord; - } - - private CloudProvisioningRecord getExistingRecord(final vSphereCloudSlaveTemplate template) { - return records.get(template); - } - - private void removeExistingRecord(CloudProvisioningRecord existingRecord) { - final vSphereCloudSlaveTemplate template = existingRecord.getTemplate(); - logger.log(Level.FINE, "Disposing of record for template {0} ({1})", - new Object[] { template.getCloneNamePrefix(), template.toString() }); - records.remove(template); - } - - private boolean recordIsPrunable(CloudProvisioningRecord record) { - final boolean isEmpty = record.getCurrentlyProvisioned().isEmpty() && record.getCurrentlyPlanned().isEmpty(); - if (!isEmpty) { - return false; - } - final vSphereCloudSlaveTemplate template = record.getTemplate(); - final List knownTemplates = parent.getTemplates(); - final boolean isKnownToParent = knownTemplates.contains(template); - return !isKnownToParent; - } - - private Map.Entry findEntryForVM(String nodeName) { - for (final Map.Entry entry : records.entrySet()) { - final CloudProvisioningRecord record = entry.getValue(); - if (record.getCurrentlyProvisioned().contains(nodeName)) { - return entry; - } - if (record.getCurrentlyPlanned().contains(nodeName)) { - return entry; - } - } - return null; - } - - /** - * Logs a state change. If the state change isn't valid, it's logged as a - * warning. - * - * @param logLevel - * The level to log the message at, if the boolean arguments are - * as their expected values. - * @param logMsg - * The message to log. - * @param firstArgName - * What actualFirstArgValue represents - used when complaining - * about its value. - * @param actualFirstArgValue - * A state-change variable. - * @param expectedFirstArgValue - * The expected value of actualFirstArgValue. If that's not the - * case, we'll complain. - * @param secondArgName - * What actualSecondArgValue represents - used when complaining - * about its value. - * @param actualSecondArgValue - * A state-change variable. - * @param expectedSecondArgValue - * The expected value of actualSecondArgValue. If that's not the - * case, we'll complain. - * @param args - * The arguments for logMsg. Used if logMsg contains {0}, {1} - * etc. - */ - private void logStateChange(Level logLevel, String logMsg, String firstArgName, boolean actualFirstArgValue, - boolean expectedFirstArgValue, String secondArgName, boolean actualSecondArgValue, - boolean expectedSecondArgValue, Object... args) { - final boolean firstValid = actualFirstArgValue == expectedFirstArgValue; - final boolean secondValid = actualSecondArgValue == expectedSecondArgValue; - Level actualLevel = logLevel; - String actualMsg = logMsg; - if (!firstValid) { - actualMsg += " : " + firstArgName + "!=" + expectedFirstArgValue; - actualLevel = Level.WARNING; - } - if (!secondValid) { - actualMsg += " : " + secondArgName + "!=" + expectedSecondArgValue; - actualLevel = Level.WARNING; - } - final Logger loggerToUse = logger != null ? logger : LOGGER; - loggerToUse.log(actualLevel, actualMsg, args); - } -} +package org.jenkinsci.plugins.vsphere.tools; + +import java.util.ArrayList; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.jenkinsci.plugins.vSphereCloud; +import org.jenkinsci.plugins.vSphereCloudSlaveTemplate; + +/** + * Utility class that works out what slaves we should start up in response to + * Jenkins asking us to start things. + *

+ * We do this by keeping a record of every slave we start, and every slave we + * have active. That way, we can avoid over-provisioning. + *

+ *

+ * The idea is that we are told what slaves that the cloud is going to create, + * when the cloud has created them (or failed to) and when those slaves have + * died. This way we can keep track of everything, in order to allow the cloud + * to make accurate decisions regarding what to create next. + *

+ * Note: This is not thread-safe. Callers must do their own synchronization. + */ +public class CloudProvisioningState { + private static final Logger LOGGER = Logger.getLogger(CloudProvisioningState.class.getName()); + /** + * Record of slaves we've told Jenkins to start up, which have yet to start. + */ + private final Map records = new IdentityHashMap(); + /** + * Our parent, so we can check what templates still exist (as the user may + * have added/removed some). + */ + private final vSphereCloud parent; + /** + * Where we log to. This is only instance-based for test-purposes, and + * transient to stop serialization problems. + */ + private transient final Logger logger; + + public CloudProvisioningState(vSphereCloud parent) { + this(parent, LOGGER); + } + + CloudProvisioningState(vSphereCloud parent, Logger logger) { + this.parent = parent; + this.logger = logger; + this.logger.log(Level.FINE, "Created for parent {0}", parent.toString()); + } + + /** + * To be called when we've decided to create a new node. Callers MUST ensure + * that {@link #provisionedSlaveNowActive(CloudProvisioningRecord, String)} + * or {@link #provisioningEndedInError(CloudProvisioningRecord, String)} + * gets called later. + * + * @param provisionable + * Our record for the template for the named node. + * @param nodeName + * The name of the VM. + */ + public void provisioningStarted(CloudProvisioningRecord provisionable, String nodeName) { + final boolean wasPreviouslyUnknownToPlanning = provisionable.addCurrentlyPlanned(nodeName); + final boolean wasAlreadyActive = provisionable.removeCurrentlyActive(nodeName); + logStateChange(Level.FINE, "Intending to create {0}", "wasPreviouslyUnknownToPlanning", + wasPreviouslyUnknownToPlanning, true, "wasAlreadyActive", wasAlreadyActive, false, nodeName); + } + + /** + * To be called when a newly created node (previously promised to + * {@link #provisioningStarted(CloudProvisioningRecord, String)}) comes up. + * Callers MUST ensure that + * {@link #provisionedSlaveNowTerminated(String)} + * gets called later. + * + * @param provisionable + * Our record for the template for the named node. + * @param nodeName + * The name of the VM. + */ + public void provisionedSlaveNowActive(CloudProvisioningRecord provisionable, String nodeName) { + final boolean wasNotPreviouslyActive = provisionable.addCurrentlyActive(nodeName); + final boolean wasPreviouslyPlanned = provisionable.removeCurrentlyPlanned(nodeName); + logStateChange(Level.FINE, "Marking {0} as active", "wasNotPreviouslyActive", wasNotPreviouslyActive, true, + "wasPreviouslyPlanned", wasPreviouslyPlanned, true, nodeName); + } + + /** + * To be called when a node we created (previously told to + * {@link #provisionedSlaveNowActive(CloudProvisioningRecord, String)}) has + * died. + * + * @param nodeName + * The name of the VM. + */ + public void provisionedSlaveNowTerminated(String nodeName) { + final Map.Entry entry = findEntryForVM(nodeName); + if (entry != null) { + final CloudProvisioningRecord provisionable = entry.getValue(); + final boolean wasPreviouslyPlanned = provisionable.removeCurrentlyPlanned(nodeName); + final boolean wasPreviouslyActive = provisionable.removeCurrentlyActive(nodeName); + if (recordIsPrunable(provisionable)) { + removeExistingRecord(provisionable); + } + logStateChange(Level.FINE, "Marking {0} as terminated", "wasPreviouslyPlanned", wasPreviouslyPlanned, + false, "wasPreviouslyActive", wasPreviouslyActive, true, nodeName); + } else { + logger.log(Level.WARNING, "Asked to mark {0} as terminated, but we have no record of it.", nodeName); + } + } + + /** + * To be called when a node that we previously promised to create (by + * calling {@link #provisioningStarted(CloudProvisioningRecord, String)}) + * failed to start. + * + * @param provisionable + * Our record for the template for the named node. + * @param nodeName + * The name of the VM. + */ + public void provisioningEndedInError(CloudProvisioningRecord provisionable, String nodeName) { + final boolean wasPreviouslyPlanned = provisionable.removeCurrentlyPlanned(nodeName); + final boolean wasPreviouslyActive = provisionable.removeCurrentlyActive(nodeName); + if (recordIsPrunable(provisionable)) { + removeExistingRecord(provisionable); + } + logStateChange(Level.INFO, "Marking {0} as failed", "wasPreviouslyPlanned", wasPreviouslyPlanned, true, + "wasPreviouslyActive", wasPreviouslyActive, false, nodeName); + } + + /** + * To be called every now and again to ensure that we're not caching records + * that will never be valid again. + */ + public void pruneUnwantedRecords() { + final List toBeRemoved = new ArrayList(records.size()); + for (final Map.Entry entry : records.entrySet()) { + final CloudProvisioningRecord record = entry.getValue(); + if (recordIsPrunable(record)) { + toBeRemoved.add(record); + } + } + for (final CloudProvisioningRecord record : toBeRemoved) { + removeExistingRecord(record); + } + } + + /** + * Given a set of templates, returns the equivalent records. + * + * @param templates + * The templates we are interested in. + * @return A list of {@link CloudProvisioningRecord}. + */ + public List calculateProvisionableTemplates(Iterable templates) { + final List result = new ArrayList(); + for (final vSphereCloudSlaveTemplate template : templates) { + final CloudProvisioningRecord provisionable = getOrCreateRecord(template); + result.add(provisionable); + } + return result; + } + + /** + * Counts all the known nodes, both active and in-progress, across all + * templates. + * + * @return The number of nodes that are active or soon-to-be-active. + */ + public int countNodes() { + int result = 0; + for (final CloudProvisioningRecord record : records.values()) { + result += record.getCurrentlyPlanned().size(); + result += record.getCurrentlyProvisioned().size(); + } + return result; + } + + /** + * Gets the record for the given template. If we didn't have one before, we + * create one. + * + * @param template + * The template in question. + * @return The one-and-only record for this template. + */ + public CloudProvisioningRecord getOrCreateRecord(final vSphereCloudSlaveTemplate template) { + final CloudProvisioningRecord existingRecord = getExistingRecord(template); + if (existingRecord != null) { + return existingRecord; + } + final CloudProvisioningRecord newRecord = new CloudProvisioningRecord(template); + logger.log(Level.FINE, "Creating new record for template {0} ({1})", + new Object[] { template.getCloneNamePrefix(), template.toString() }); + records.put(template, newRecord); + return newRecord; + } + + private CloudProvisioningRecord getExistingRecord(final vSphereCloudSlaveTemplate template) { + return records.get(template); + } + + private void removeExistingRecord(CloudProvisioningRecord existingRecord) { + final vSphereCloudSlaveTemplate template = existingRecord.getTemplate(); + logger.log(Level.FINE, "Disposing of record for template {0} ({1})", + new Object[] { template.getCloneNamePrefix(), template.toString() }); + records.remove(template); + } + + private boolean recordIsPrunable(CloudProvisioningRecord record) { + final boolean isEmpty = record.getCurrentlyProvisioned().isEmpty() && record.getCurrentlyPlanned().isEmpty(); + if (!isEmpty) { + return false; + } + final vSphereCloudSlaveTemplate template = record.getTemplate(); + final List knownTemplates = parent.getTemplates(); + final boolean isKnownToParent = knownTemplates.contains(template); + return !isKnownToParent; + } + + private Map.Entry findEntryForVM(String nodeName) { + for (final Map.Entry entry : records.entrySet()) { + final CloudProvisioningRecord record = entry.getValue(); + if (record.getCurrentlyProvisioned().contains(nodeName)) { + return entry; + } + if (record.getCurrentlyPlanned().contains(nodeName)) { + return entry; + } + } + return null; + } + + /** + * Logs a state change. If the state change isn't valid, it's logged as a + * warning. + * + * @param logLevel + * The level to log the message at, if the boolean arguments are + * as their expected values. + * @param logMsg + * The message to log. + * @param firstArgName + * What actualFirstArgValue represents - used when complaining + * about its value. + * @param actualFirstArgValue + * A state-change variable. + * @param expectedFirstArgValue + * The expected value of actualFirstArgValue. If that's not the + * case, we'll complain. + * @param secondArgName + * What actualSecondArgValue represents - used when complaining + * about its value. + * @param actualSecondArgValue + * A state-change variable. + * @param expectedSecondArgValue + * The expected value of actualSecondArgValue. If that's not the + * case, we'll complain. + * @param args + * The arguments for logMsg. Used if logMsg contains {0}, {1} + * etc. + */ + private void logStateChange(Level logLevel, String logMsg, String firstArgName, boolean actualFirstArgValue, + boolean expectedFirstArgValue, String secondArgName, boolean actualSecondArgValue, + boolean expectedSecondArgValue, Object... args) { + final boolean firstValid = actualFirstArgValue == expectedFirstArgValue; + final boolean secondValid = actualSecondArgValue == expectedSecondArgValue; + Level actualLevel = logLevel; + String actualMsg = logMsg; + if (!firstValid) { + actualMsg += " : " + firstArgName + "!=" + expectedFirstArgValue; + actualLevel = Level.WARNING; + } + if (!secondValid) { + actualMsg += " : " + secondArgName + "!=" + expectedSecondArgValue; + actualLevel = Level.WARNING; + } + final Logger loggerToUse = logger != null ? logger : LOGGER; + loggerToUse.log(actualLevel, actualMsg, args); + } +} diff --git a/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithmTest.java b/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithmTest.java index 0ea08052..55114326 100644 --- a/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithmTest.java +++ b/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithmTest.java @@ -1,289 +1,289 @@ -package org.jenkinsci.plugins.vsphere.tools; - -import static org.junit.Assert.*; -import static org.hamcrest.CoreMatchers.*; -import hudson.slaves.JNLPLauncher; -import hudson.slaves.RetentionStrategy; - -import java.math.BigInteger; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.List; - -import org.jenkinsci.plugins.vSphereCloudSlaveTemplate; -import org.jenkinsci.plugins.vsphere.tools.CloudProvisioningRecord; -import org.junit.Before; -import org.junit.Test; - -public class CloudProvisioningAlgorithmTest { - - /** Used when faking up test data */ - private int instanceNumber; - - @Before - public void setup() { - instanceNumber = 0; - } - - @Test - public void findTemplateWithMostFreeCapacityGivenNoOptionsThenReturnsNull() { - // Given - final List emptyList = Collections.emptyList(); - - // When - final CloudProvisioningRecord actual = CloudProvisioningAlgorithm.findTemplateWithMostFreeCapacity(emptyList); - - // Then - assertThat(actual, nullValue()); - } - - @Test - public void findTemplateWithMostFreeCapacityGivenSomeActiveAndOneUnusedThenPrefersUnusedTemplate() { - // Given - final CloudProvisioningRecord zeroOfTwo = createInstance(2, 0, 0); - final CloudProvisioningRecord onePlannedOfTwo = createInstance(2, 0, 1); - final CloudProvisioningRecord oneExistsOfTwo = createInstance(2, 1, 0); - final List forwards = Arrays.asList(zeroOfTwo, onePlannedOfTwo, oneExistsOfTwo); - final List reverse = Arrays.asList(oneExistsOfTwo, onePlannedOfTwo, zeroOfTwo); - - // When - final CloudProvisioningRecord actual1 = CloudProvisioningAlgorithm.findTemplateWithMostFreeCapacity(forwards); - final CloudProvisioningRecord actual2 = CloudProvisioningAlgorithm.findTemplateWithMostFreeCapacity(reverse); - - // Then - assertThat(actual1, sameInstance(zeroOfTwo)); - assertThat(actual2, sameInstance(zeroOfTwo)); - } - - @Test - public void findTemplateWithMostFreeCapacityGivenEqualCapsThenDistributesTheLoadEvenly() { - // Given - final CloudProvisioningRecord a = createInstance(2, 0, 0); - final CloudProvisioningRecord b = createInstance(2, 0, 0); - final CloudProvisioningRecord c = createInstance(2, 0, 0); - final List records = Arrays.asList(a, b, c); - - // When/Then - testScenario(records, a, b, c, a, b, c, null); - } - - @Test - public void findTemplateWithMostFreeCapacityGivenEqualCapsButExistingUsageThenDistributesTheLoadEvenly() { - // Given - final CloudProvisioningRecord a = createInstance(2, 2, 0); - final CloudProvisioningRecord b = createInstance(2, 1, 0); - final CloudProvisioningRecord c = createInstance(2, 0, 0); - final List records = Arrays.asList(a, b, c); - - // When/Then - testScenario(records, c, b, c, null); - } - - @Test - public void findTemplateWithMostFreeCapacityGivenNoCapsThenDistributesTheLoadEvenly() { - // Given - final CloudProvisioningRecord a = createInstance(0, 0, 0); - final CloudProvisioningRecord b = createInstance(0, 0, 0); - final CloudProvisioningRecord c = createInstance(0, 0, 0); - final List records = Arrays.asList(a, b, c); - - // When/Then - testScenario(records, a, b, c, a, b, c, a, b, c, a); - } - - @Test - public void findTemplateWithMostFreeCapacityGivenUnequalCapsThenDistributesTheLoadFairly() { - findTemplateWithMostFreeCapacityGivenUnequalCapsThenDistributesTheLoadFairly(true); - } - - @Test - public void findTemplateWithMostFreeCapacityGivenUnequalCapsThenDistributesTheLoadFairly2() { - findTemplateWithMostFreeCapacityGivenUnequalCapsThenDistributesTheLoadFairly(false); - } - - private void findTemplateWithMostFreeCapacityGivenUnequalCapsThenDistributesTheLoadFairly(boolean forwards) { - // Given - final CloudProvisioningRecord capOf2 = createInstance(2, 0, 0); - final CloudProvisioningRecord capOf5 = createInstance(5, 0, 0); - final List records = forwards ? Arrays.asList(capOf2, capOf5) : Arrays.asList(capOf5, - capOf2); - - // When/Then - testScenario(records, capOf5, capOf2, capOf5, capOf5, capOf2, capOf5, capOf5, null); - } - - @Test - public void findTemplateWithMostFreeCapacityGivenOneCappedAndOneUncappedThenDistributesTheLoadEventlyUntilCapReached() { - findTemplateWithMostFreeCapacityGivenDifferentCapnessThenDistributesTheLoadEventlyUntilCapReached(true); - } - - @Test - public void findTemplateWithMostFreeCapacityGivenOneUncappedAndOneCappedThenDistributesTheLoadEventlyUntilCapReached() { - findTemplateWithMostFreeCapacityGivenDifferentCapnessThenDistributesTheLoadEventlyUntilCapReached(false); - } - - private void findTemplateWithMostFreeCapacityGivenDifferentCapnessThenDistributesTheLoadEventlyUntilCapReached( - boolean forwards) { - // Given - final CloudProvisioningRecord capOf2 = createInstance(2, 0, 0); - final CloudProvisioningRecord uncapped = createInstance(0, 0, 0); - final List records = forwards ? Arrays.asList(capOf2, uncapped) : Arrays.asList( - uncapped, capOf2); - - // When/Then - testScenario(records, uncapped, capOf2, uncapped, capOf2, uncapped, uncapped, uncapped); - } - - private static void testScenario(List records, CloudProvisioningRecord... expectedRecords) { - // Given records and expected return values - int i = 0; - for (final CloudProvisioningRecord expected : expectedRecords) { - final CloudProvisioningRecord actual = CloudProvisioningAlgorithm.findTemplateWithMostFreeCapacity(records); - i++; - assertThat("findTemplateWithMostFreeCapacity(" + records + ")#" + i, actual, sameInstance(expected)); - final String nodeName = "PlannedInStep" + i; - if (actual != null) { - actual.addCurrentlyPlanned(nodeName); - } - } - } - - @Test - public void toBigIntegerGivenTwoPow128MinusOneThenReturnsTwoPow128MinusOne() { - testToBigInteger(-1, -1, "340282366920938463463374607431768211455"); - } - - @Test - public void toBigIntegerGivenTwoPow64PlusOneThenReturnsTwoPow64PlusOne() { - testToBigInteger(1, 1, "18446744073709551617"); - } - - @Test - public void toBigIntegerGivenZeroThenReturnsZero() { - testToBigInteger(0, 0, "0"); - } - - @Test - public void toBigIntegerGivenPowersOfTwoThenReturnsPowersOfTwo() { - long lsb = 1; - long msb = 0; - BigInteger big = new BigInteger("1"); - for (int bit = 0; bit < 64; bit++) { - testToBigInteger(msb, lsb, big.toString()); - lsb <<= 1; - big = big.add(big); - } - msb = 1; - for (int bit = 0; bit < 64; bit++) { - testToBigInteger(msb, lsb, big.toString()); - msb <<= 1; - big = big.add(big); - } - } - - private void testToBigInteger(long msb, long lsb, String expected) { - // Given - final BigInteger expectedValue = new BigInteger(expected); - final byte[] expectedBytes = expectedValue.toByteArray(); - // When - final BigInteger actual = CloudProvisioningAlgorithm.toBigInteger(msb, lsb); - final byte[] actualBytes = actual.toByteArray(); - final String scenario = "toBigInteger(" + msb + "," + lsb + ") == " + expected + "\ni.e. " - + toHexString(actualBytes) + " ~= " + toHexString(expectedBytes); - // Then - assertThat(scenario, actual, equalTo(expectedValue)); - } - - @Test - public void findUnusedNameGivenZeroOfTwoExistsThenReturnsOneThenTwo() { - // Given - final CloudProvisioningRecord record = createInstance(2, 0, 0); - final String prefix = record.getTemplate().getCloneNamePrefix(); - final String expected1 = prefix + "_1"; - final String expected2 = prefix + "_2"; - - // When - final String actual1 = CloudProvisioningAlgorithm.findUnusedName(record); - record.addCurrentlyActive(actual1); - final String actual2 = CloudProvisioningAlgorithm.findUnusedName(record); - record.addCurrentlyActive(actual2); - - // Then - assertThat(actual1, equalTo(expected1)); - assertThat(actual2, equalTo(expected2)); - } - - @Test - public void findUnusedNameGivenOneOfTwoHasEndedThenReturnsOne() { - // Given - final CloudProvisioningRecord record = createInstance(2, 0, 0); - final String prefix = record.getTemplate().getCloneNamePrefix(); - final String expected = prefix + "_1"; - - // When - final String actual1 = CloudProvisioningAlgorithm.findUnusedName(record); - record.addCurrentlyActive(actual1); - final String actual2 = CloudProvisioningAlgorithm.findUnusedName(record); - record.addCurrentlyActive(actual2); - record.removeCurrentlyActive(actual1); - final String actual = CloudProvisioningAlgorithm.findUnusedName(record); - record.addCurrentlyActive(actual); - - // Then - assertThat(actual, equalTo(expected)); - } - - @Test - public void findUnusedNameGivenUncappedInstancesThenReturnsUniqueNames() { - // Given - final CloudProvisioningRecord record = createInstance(0, 5, 6); - final String prefix = record.getTemplate().getCloneNamePrefix(); - final List actuals = new ArrayList(); - - // When - for (int i = 0; i < 100; i++) { - final String actual = CloudProvisioningAlgorithm.findUnusedName(record); - record.addCurrentlyActive(actual); - actuals.add(actual); - } - - // Then - final List uniques = new ArrayList(new LinkedHashSet(actuals)); - assertThat(actuals, equalTo(uniques)); - assertThat(actuals, everyItem(startsWith(prefix + "_"))); - } - - private CloudProvisioningRecord createInstance(int capacity, int provisioned, int planned) { - final int iNum = ++instanceNumber; - final vSphereCloudSlaveTemplate template = stubTemplate(iNum + "cap" + capacity, capacity); - final CloudProvisioningRecord instance = new CloudProvisioningRecord(template); - for (int i = 0; i < provisioned; i++) { - final String nodeName = iNum + "provisioned#" + i; - instance.addCurrentlyPlanned(nodeName); - } - for (int i = 0; i < planned; i++) { - final String nodeName = iNum + "planned#" + i; - instance.addCurrentlyPlanned(nodeName); - } - return instance; - } - - private static vSphereCloudSlaveTemplate stubTemplate(String prefix, int templateInstanceCap) { - return new vSphereCloudSlaveTemplate(prefix, "", null, null, false, null, null, null, null, null, null, templateInstanceCap, 1, - null, null, null, false, false, 0, 0, false, null, null, null, new JNLPLauncher(), - RetentionStrategy.NOOP, null, null); - } - - private static String toHexString(byte[] bytes) { - final StringBuilder s = new StringBuilder("0x"); - for (final byte b : bytes) { - final int highDigit = (((int) b) >> 8) & 15; - final int lowDigit = ((int) b) & 15; - s.append(Integer.toString(highDigit, 16)); - s.append(Integer.toString(lowDigit, 16)); - } - return s.toString(); - } -} +package org.jenkinsci.plugins.vsphere.tools; + +import static org.junit.Assert.*; +import static org.hamcrest.CoreMatchers.*; +import hudson.slaves.JNLPLauncher; +import hudson.slaves.RetentionStrategy; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; + +import org.jenkinsci.plugins.vSphereCloudSlaveTemplate; +import org.jenkinsci.plugins.vsphere.tools.CloudProvisioningRecord; +import org.junit.Before; +import org.junit.Test; + +public class CloudProvisioningAlgorithmTest { + + /** Used when faking up test data */ + private int instanceNumber; + + @Before + public void setup() { + instanceNumber = 0; + } + + @Test + public void findTemplateWithMostFreeCapacityGivenNoOptionsThenReturnsNull() { + // Given + final List emptyList = Collections.emptyList(); + + // When + final CloudProvisioningRecord actual = CloudProvisioningAlgorithm.findTemplateWithMostFreeCapacity(emptyList); + + // Then + assertThat(actual, nullValue()); + } + + @Test + public void findTemplateWithMostFreeCapacityGivenSomeActiveAndOneUnusedThenPrefersUnusedTemplate() { + // Given + final CloudProvisioningRecord zeroOfTwo = createInstance(2, 0, 0); + final CloudProvisioningRecord onePlannedOfTwo = createInstance(2, 0, 1); + final CloudProvisioningRecord oneExistsOfTwo = createInstance(2, 1, 0); + final List forwards = Arrays.asList(zeroOfTwo, onePlannedOfTwo, oneExistsOfTwo); + final List reverse = Arrays.asList(oneExistsOfTwo, onePlannedOfTwo, zeroOfTwo); + + // When + final CloudProvisioningRecord actual1 = CloudProvisioningAlgorithm.findTemplateWithMostFreeCapacity(forwards); + final CloudProvisioningRecord actual2 = CloudProvisioningAlgorithm.findTemplateWithMostFreeCapacity(reverse); + + // Then + assertThat(actual1, sameInstance(zeroOfTwo)); + assertThat(actual2, sameInstance(zeroOfTwo)); + } + + @Test + public void findTemplateWithMostFreeCapacityGivenEqualCapsThenDistributesTheLoadEvenly() { + // Given + final CloudProvisioningRecord a = createInstance(2, 0, 0); + final CloudProvisioningRecord b = createInstance(2, 0, 0); + final CloudProvisioningRecord c = createInstance(2, 0, 0); + final List records = Arrays.asList(a, b, c); + + // When/Then + testScenario(records, a, b, c, a, b, c, null); + } + + @Test + public void findTemplateWithMostFreeCapacityGivenEqualCapsButExistingUsageThenDistributesTheLoadEvenly() { + // Given + final CloudProvisioningRecord a = createInstance(2, 2, 0); + final CloudProvisioningRecord b = createInstance(2, 1, 0); + final CloudProvisioningRecord c = createInstance(2, 0, 0); + final List records = Arrays.asList(a, b, c); + + // When/Then + testScenario(records, c, b, c, null); + } + + @Test + public void findTemplateWithMostFreeCapacityGivenNoCapsThenDistributesTheLoadEvenly() { + // Given + final CloudProvisioningRecord a = createInstance(0, 0, 0); + final CloudProvisioningRecord b = createInstance(0, 0, 0); + final CloudProvisioningRecord c = createInstance(0, 0, 0); + final List records = Arrays.asList(a, b, c); + + // When/Then + testScenario(records, a, b, c, a, b, c, a, b, c, a); + } + + @Test + public void findTemplateWithMostFreeCapacityGivenUnequalCapsThenDistributesTheLoadFairly() { + findTemplateWithMostFreeCapacityGivenUnequalCapsThenDistributesTheLoadFairly(true); + } + + @Test + public void findTemplateWithMostFreeCapacityGivenUnequalCapsThenDistributesTheLoadFairly2() { + findTemplateWithMostFreeCapacityGivenUnequalCapsThenDistributesTheLoadFairly(false); + } + + private void findTemplateWithMostFreeCapacityGivenUnequalCapsThenDistributesTheLoadFairly(boolean forwards) { + // Given + final CloudProvisioningRecord capOf2 = createInstance(2, 0, 0); + final CloudProvisioningRecord capOf5 = createInstance(5, 0, 0); + final List records = forwards ? Arrays.asList(capOf2, capOf5) : Arrays.asList(capOf5, + capOf2); + + // When/Then + testScenario(records, capOf5, capOf2, capOf5, capOf5, capOf2, capOf5, capOf5, null); + } + + @Test + public void findTemplateWithMostFreeCapacityGivenOneCappedAndOneUncappedThenDistributesTheLoadEventlyUntilCapReached() { + findTemplateWithMostFreeCapacityGivenDifferentCapnessThenDistributesTheLoadEventlyUntilCapReached(true); + } + + @Test + public void findTemplateWithMostFreeCapacityGivenOneUncappedAndOneCappedThenDistributesTheLoadEventlyUntilCapReached() { + findTemplateWithMostFreeCapacityGivenDifferentCapnessThenDistributesTheLoadEventlyUntilCapReached(false); + } + + private void findTemplateWithMostFreeCapacityGivenDifferentCapnessThenDistributesTheLoadEventlyUntilCapReached( + boolean forwards) { + // Given + final CloudProvisioningRecord capOf2 = createInstance(2, 0, 0); + final CloudProvisioningRecord uncapped = createInstance(0, 0, 0); + final List records = forwards ? Arrays.asList(capOf2, uncapped) : Arrays.asList( + uncapped, capOf2); + + // When/Then + testScenario(records, uncapped, capOf2, uncapped, capOf2, uncapped, uncapped, uncapped); + } + + private static void testScenario(List records, CloudProvisioningRecord... expectedRecords) { + // Given records and expected return values + int i = 0; + for (final CloudProvisioningRecord expected : expectedRecords) { + final CloudProvisioningRecord actual = CloudProvisioningAlgorithm.findTemplateWithMostFreeCapacity(records); + i++; + assertThat("findTemplateWithMostFreeCapacity(" + records + ")#" + i, actual, sameInstance(expected)); + final String nodeName = "PlannedInStep" + i; + if (actual != null) { + actual.addCurrentlyPlanned(nodeName); + } + } + } + + @Test + public void toBigIntegerGivenTwoPow128MinusOneThenReturnsTwoPow128MinusOne() { + testToBigInteger(-1, -1, "340282366920938463463374607431768211455"); + } + + @Test + public void toBigIntegerGivenTwoPow64PlusOneThenReturnsTwoPow64PlusOne() { + testToBigInteger(1, 1, "18446744073709551617"); + } + + @Test + public void toBigIntegerGivenZeroThenReturnsZero() { + testToBigInteger(0, 0, "0"); + } + + @Test + public void toBigIntegerGivenPowersOfTwoThenReturnsPowersOfTwo() { + long lsb = 1; + long msb = 0; + BigInteger big = new BigInteger("1"); + for (int bit = 0; bit < 64; bit++) { + testToBigInteger(msb, lsb, big.toString()); + lsb <<= 1; + big = big.add(big); + } + msb = 1; + for (int bit = 0; bit < 64; bit++) { + testToBigInteger(msb, lsb, big.toString()); + msb <<= 1; + big = big.add(big); + } + } + + private void testToBigInteger(long msb, long lsb, String expected) { + // Given + final BigInteger expectedValue = new BigInteger(expected); + final byte[] expectedBytes = expectedValue.toByteArray(); + // When + final BigInteger actual = CloudProvisioningAlgorithm.toBigInteger(msb, lsb); + final byte[] actualBytes = actual.toByteArray(); + final String scenario = "toBigInteger(" + msb + "," + lsb + ") == " + expected + "\ni.e. " + + toHexString(actualBytes) + " ~= " + toHexString(expectedBytes); + // Then + assertThat(scenario, actual, equalTo(expectedValue)); + } + + @Test + public void findUnusedNameGivenZeroOfTwoExistsThenReturnsOneThenTwo() { + // Given + final CloudProvisioningRecord record = createInstance(2, 0, 0); + final String prefix = record.getTemplate().getCloneNamePrefix(); + final String expected1 = prefix + "_1"; + final String expected2 = prefix + "_2"; + + // When + final String actual1 = CloudProvisioningAlgorithm.findUnusedName(record); + record.addCurrentlyActive(actual1); + final String actual2 = CloudProvisioningAlgorithm.findUnusedName(record); + record.addCurrentlyActive(actual2); + + // Then + assertThat(actual1, equalTo(expected1)); + assertThat(actual2, equalTo(expected2)); + } + + @Test + public void findUnusedNameGivenOneOfTwoHasEndedThenReturnsOne() { + // Given + final CloudProvisioningRecord record = createInstance(2, 0, 0); + final String prefix = record.getTemplate().getCloneNamePrefix(); + final String expected = prefix + "_1"; + + // When + final String actual1 = CloudProvisioningAlgorithm.findUnusedName(record); + record.addCurrentlyActive(actual1); + final String actual2 = CloudProvisioningAlgorithm.findUnusedName(record); + record.addCurrentlyActive(actual2); + record.removeCurrentlyActive(actual1); + final String actual = CloudProvisioningAlgorithm.findUnusedName(record); + record.addCurrentlyActive(actual); + + // Then + assertThat(actual, equalTo(expected)); + } + + @Test + public void findUnusedNameGivenUncappedInstancesThenReturnsUniqueNames() { + // Given + final CloudProvisioningRecord record = createInstance(0, 5, 6); + final String prefix = record.getTemplate().getCloneNamePrefix(); + final List actuals = new ArrayList(); + + // When + for (int i = 0; i < 100; i++) { + final String actual = CloudProvisioningAlgorithm.findUnusedName(record); + record.addCurrentlyActive(actual); + actuals.add(actual); + } + + // Then + final List uniques = new ArrayList(new LinkedHashSet(actuals)); + assertThat(actuals, equalTo(uniques)); + assertThat(actuals, everyItem(startsWith(prefix + "_"))); + } + + private CloudProvisioningRecord createInstance(int capacity, int provisioned, int planned) { + final int iNum = ++instanceNumber; + final vSphereCloudSlaveTemplate template = stubTemplate(iNum + "cap" + capacity, capacity); + final CloudProvisioningRecord instance = new CloudProvisioningRecord(template); + for (int i = 0; i < provisioned; i++) { + final String nodeName = iNum + "provisioned#" + i; + instance.addCurrentlyPlanned(nodeName); + } + for (int i = 0; i < planned; i++) { + final String nodeName = iNum + "planned#" + i; + instance.addCurrentlyPlanned(nodeName); + } + return instance; + } + + private static vSphereCloudSlaveTemplate stubTemplate(String prefix, int templateInstanceCap) { + return new vSphereCloudSlaveTemplate(prefix, "", null, null, false, null, null, null, null, null, null, templateInstanceCap, 1, + null, null, null, false, false, 0, 0, false, null, null, null, new JNLPLauncher(), + RetentionStrategy.NOOP, null, null); + } + + private static String toHexString(byte[] bytes) { + final StringBuilder s = new StringBuilder("0x"); + for (final byte b : bytes) { + final int highDigit = (((int) b) >> 8) & 15; + final int lowDigit = ((int) b) & 15; + s.append(Integer.toString(highDigit, 16)); + s.append(Integer.toString(lowDigit, 16)); + } + return s.toString(); + } +} diff --git a/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningStateTest.java b/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningStateTest.java index 5ba51592..30ac9361 100644 --- a/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningStateTest.java +++ b/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningStateTest.java @@ -1,411 +1,411 @@ -package org.jenkinsci.plugins.vsphere.tools; - -import static org.hamcrest.CoreMatchers.allOf; -import static org.hamcrest.CoreMatchers.any; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.everyItem; -import static org.hamcrest.collection.IsArrayContainingInOrder.arrayContaining; -import static org.hamcrest.collection.IsIterableContainingInOrder.contains; -import static org.junit.Assert.assertThat; -import hudson.model.Node.Mode; -import hudson.slaves.NodeProperty; -import hudson.slaves.JNLPLauncher; -import hudson.slaves.RetentionStrategy; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.logging.Handler; -import java.util.logging.Level; -import java.util.logging.LogRecord; -import java.util.logging.Logger; - -import org.hamcrest.Description; -import org.hamcrest.Matcher; -import org.hamcrest.TypeSafeMatcher; -import org.hamcrest.collection.IsIterableWithSize; -import org.jenkinsci.plugins.vSphereCloud; -import org.jenkinsci.plugins.vSphereCloudSlaveTemplate; -import org.jenkinsci.plugins.vsphere.VSphereConnectionConfig; -import org.jenkinsci.plugins.vsphere.VSphereGuestInfoProperty; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -public class CloudProvisioningStateTest { - private static List stubVSphereCloudTemplates; - private static vSphereCloud stubVSphereCloud; - private int recordNumber; - private int nodeNumber; - private Logger testLogger; - private List loggedMessages; - - @BeforeClass - public static void setupClass() { - stubVSphereCloudTemplates = new ArrayList(); - final VSphereConnectionConfig vsConnectionConfig = new VSphereConnectionConfig("vsHost", "credentialsId"); - stubVSphereCloud = new vSphereCloud(vsConnectionConfig, "vsDescription", 0, 0, stubVSphereCloudTemplates); - } - - @Before - public void setup() { - stubVSphereCloudTemplates.clear(); - recordNumber = 0; - nodeNumber = 0; - loggedMessages = new ArrayList(); - Logger logger = Logger.getLogger("CloudProvisioningStateTest"); - logger.setLevel(Level.ALL); - final Handler[] handlers = logger.getHandlers(); - for (final Handler handler : handlers) { - logger.removeHandler(handler); - } - final Handler testHandler = new Handler() { - @Override - public void publish(LogRecord record) { - loggedMessages.add(record); - } - - @Override - public void flush() { - } - - @Override - public void close() { - } - }; - logger.addHandler(testHandler); - testLogger = logger; - } - - @Test - public void constructorGivenCalledThenLogsConstructions() { - // Given - final Object[] expectedArgs = { stubVSphereCloud.toString() }; - - // When - createInstance(); - - // Then - assertThat(loggedMessages, contains(logMessage(Level.FINE, expectedArgs))); - } - - @Test - public void provisioningStartedGivenNoPreviousStateThenLogs() { - // Given - final String nodeName = createNodeName(); - final Object[] expectedArgs = { nodeName }; - final CloudProvisioningState instance = createInstance(); - final CloudProvisioningRecord provisionable = createRecord(instance); - wipeLog(); - - // When - instance.provisioningStarted(provisionable, nodeName); - - // Then - assertThat(loggedMessages, contains(logMessage(Level.FINE, expectedArgs))); - } - - @Test - public void provisioningStartedGivenPreviouslyStartedThenWarns() { - // Given - final String nodeName = createNodeName(); - final Object[] expectedArgs = { nodeName }; - final CloudProvisioningState instance = createInstance(); - final CloudProvisioningRecord provisionable = createRecord(instance); - instance.provisioningStarted(provisionable, nodeName); - wipeLog(); - - // When - instance.provisioningStarted(provisionable, nodeName); - - // Then - assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); - } - - @Test - public void normalLifecycleGivenNoErrorsThenLogs() { - // Given - final String nodeName = createNodeName(); - final Object[] expectedArgs = { nodeName }; - final CloudProvisioningState instance = createInstance(); - final CloudProvisioningRecord provisionable = createRecord(instance); - wipeLog(); - - // When - instance.provisioningStarted(provisionable, nodeName); - instance.provisionedSlaveNowActive(provisionable, nodeName); - instance.provisionedSlaveNowTerminated(nodeName); - - // Then - assertThat(loggedMessages, everyItem(logMessage(Level.FINE, expectedArgs))); - assertThat(loggedMessages, IsIterableWithSize. iterableWithSize(3)); - } - - @SuppressWarnings("unchecked") - @Test - public void failedToProvisionGivenNothingOutOfSequenceThenLogs() { - // Given - final String nodeName = createNodeName(); - final Object[] expectedArgs = { nodeName }; - final CloudProvisioningState instance = createInstance(); - final CloudProvisioningRecord provisionable = createRecord(instance); - wipeLog(); - - // When - instance.provisioningStarted(provisionable, nodeName); - instance.provisioningEndedInError(provisionable, nodeName); - - // Then - assertThat( - loggedMessages, - contains(logMessage(Level.FINE, expectedArgs), - logMessage(containsString("failed"), Level.INFO, expectedArgs))); - assertThat(loggedMessages, IsIterableWithSize. iterableWithSize(2)); - } - - @Test - public void provisionGivenOutOfOrderSequenceThenComplains() { - // Given - final String nodeName = createNodeName(); - final Object[] expectedArgs = { nodeName }; - final CloudProvisioningState instance = createInstance(); - final CloudProvisioningRecord provisionable = createRecord(instance); - wipeLog(); - - // When/Then - wipeLog(); - instance.provisionedSlaveNowTerminated(nodeName); - assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); - - wipeLog(); - instance.provisioningStarted(provisionable, nodeName); - assertThat(loggedMessages, contains(logMessage(Level.FINE, expectedArgs))); - - wipeLog(); - instance.provisioningStarted(provisionable, nodeName); - assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); - - wipeLog(); - instance.provisioningEndedInError(provisionable, nodeName); - assertThat(loggedMessages, contains(logMessage(Level.INFO, expectedArgs))); - - wipeLog(); - instance.provisioningEndedInError(provisionable, nodeName); - assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); - - wipeLog(); - instance.provisionedSlaveNowActive(provisionable, nodeName); - assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); - - wipeLog(); - instance.provisionedSlaveNowActive(provisionable, nodeName); - assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); - - wipeLog(); - instance.provisioningStarted(provisionable, nodeName); - assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); - - wipeLog(); - instance.provisioningStarted(provisionable, nodeName); - assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); - - wipeLog(); - instance.provisionedSlaveNowTerminated(nodeName); - assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); - - wipeLog(); - instance.provisionedSlaveNowTerminated(nodeName); - assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); - } - - @Test - public void pruneUnwantedRecordsGivenUnknownTemplatesThenRemovesRecordsForEmptyDeletedTemplates() { - // Given - final String deletedAndInactiveNodeName = createNodeName(); - final String deletedButActiveNodeName = createNodeName(); - final String livedAndDiedNodeName = createNodeName(); - final CloudProvisioningState instance = createInstance(); - // A template which the user deleted but still has an active slave - final CloudProvisioningRecord deletedButActiveRecord = createRecord(instance); - instance.provisioningStarted(deletedButActiveRecord, deletedButActiveNodeName); - instance.provisionedSlaveNowActive(deletedButActiveRecord, deletedButActiveNodeName); - // A template which the user deleted and is no longer needed - final CloudProvisioningRecord deletedAndInactiveRecord = createRecord(instance); - instance.provisioningStarted(deletedAndInactiveRecord, deletedAndInactiveNodeName); - instance.provisionedSlaveNowActive(deletedAndInactiveRecord, deletedAndInactiveNodeName); - final vSphereCloudSlaveTemplate deletedAndInactiveTemplate = deletedAndInactiveRecord.getTemplate(); - instance.provisionedSlaveNowTerminated(deletedAndInactiveNodeName); - // A template which is current but has no active slaves right now - final CloudProvisioningRecord activeRecord = createRecord(instance); - instance.provisioningStarted(activeRecord, livedAndDiedNodeName); - instance.provisionedSlaveNowActive(activeRecord, livedAndDiedNodeName); - instance.provisionedSlaveNowTerminated(livedAndDiedNodeName); - - // When - userHasDeletedSlaveTemplate(deletedButActiveRecord); - userHasDeletedSlaveTemplate(deletedAndInactiveRecord); - wipeLog(); - instance.pruneUnwantedRecords(); - - // Then - assertThat( - loggedMessages, - contains(logMessage(containsString("Disposing"), Level.FINE, deletedAndInactiveTemplate.getCloneNamePrefix(), deletedAndInactiveTemplate.toString()))); - } - - @Test - public void countNodesGivenNoTemplatesOrSlavesThenReturnsZero() { - // Given - final CloudProvisioningState instance = createInstance(); - - // When - final int actual = instance.countNodes(); - - // Then - assertThat(actual, equalTo(0)); - } - - @Test - public void countNodesGivenNoSlavesInAnyTemplatesThenReturnsZero() { - // Given - final CloudProvisioningState instance = createInstance(); - createRecord(instance); - final String node = createNodeName(); - final CloudProvisioningRecord previouslyActiveRecord = createRecord(instance); - previouslyActiveRecord.addCurrentlyActive(node); - previouslyActiveRecord.removeCurrentlyActive(node); - createRecord(instance); - - // When - final int actual = instance.countNodes(); - - // Then - assertThat(actual, equalTo(0)); - } - - @Test - public void countNodesGiven2ActiveSlavesThenReturns2() { - // Given - final CloudProvisioningState instance = createInstance(); - final CloudProvisioningRecord activeRecord = createRecord(instance); - activeRecord.addCurrentlyActive(createNodeName()); - activeRecord.addCurrentlyActive(createNodeName()); - - // When - final int actual = instance.countNodes(); - - // Then - assertThat(actual, equalTo(2)); - } - - @Test - public void countNodesGiven3ActiveAnd4PendingSlavesThenReturns7() { - // Given - final CloudProvisioningState instance = createInstance(); - final CloudProvisioningRecord recordWith2Active = createRecord(instance); - recordWith2Active.addCurrentlyActive(createNodeName()); - recordWith2Active.addCurrentlyActive(createNodeName()); - final CloudProvisioningRecord recordWith1Active4Planned = createRecord(instance); - recordWith1Active4Planned.addCurrentlyActive(createNodeName()); - recordWith1Active4Planned.addCurrentlyPlanned(createNodeName()); - recordWith1Active4Planned.addCurrentlyPlanned(createNodeName()); - recordWith1Active4Planned.addCurrentlyPlanned(createNodeName()); - recordWith1Active4Planned.addCurrentlyPlanned(createNodeName()); - - // When - final int actual = instance.countNodes(); - - // Then - assertThat(actual, equalTo(7)); - } - - private void wipeLog() { - loggedMessages.clear(); - } - - private CloudProvisioningState createInstance() { - return new CloudProvisioningState(stubVSphereCloud, testLogger); - } - - private CloudProvisioningRecord createRecord(CloudProvisioningState instance) { - recordNumber++; - final String cloneNamePrefix = "prefix" + recordNumber; - final vSphereCloudSlaveTemplate template = new vSphereCloudSlaveTemplate(cloneNamePrefix, "masterImageName", - null, "snapshotName", false, "cluster", "resourcePool", "datastore", "folder", "customizationSpec", "templateDescription", 0, 1, "remoteFS", - "", Mode.NORMAL, false, false, 0, 0, false, "targetResourcePool", "targetHost", null, - new JNLPLauncher(), RetentionStrategy.NOOP, Collections.> emptyList(), - Collections. emptyList()); - stubVSphereCloudTemplates.add(template); - final List templates = new ArrayList(); - templates.add(template); - final List records = instance.calculateProvisionableTemplates(templates); - assertThat(records, IsIterableWithSize. iterableWithSize(1)); - final CloudProvisioningRecord record = records.get(0); - return record; - } - - private String createNodeName() { - nodeNumber++; - final String nodeName = "N#" + nodeNumber; - return nodeName; - } - - private void userHasDeletedSlaveTemplate(CloudProvisioningRecord record) { - stubVSphereCloudTemplates.remove(record.getTemplate()); - } - - private static Matcher logMessage(final Level expectedLevel, final Object... expectedArgs) { - final List> messageMatchers = new ArrayList>( - expectedArgs.length); - for (int i = 0; i < expectedArgs.length; i++) { - final String expectedString = "{" + i + "}"; - messageMatchers.add(containsString(expectedString)); - } - final Matcher messageMatcher; - if (messageMatchers.isEmpty()) { - messageMatcher = any(String.class); - } else { - messageMatcher = allOf(messageMatchers); - } - return logMessage(messageMatcher, expectedLevel, expectedArgs); - } - - private static Matcher logMessage(final Matcher messageMatcher, final Level expectedLevel, - final Object... expectedArgs) { - final Matcher levelMatcher = equalTo(expectedLevel); - final Matcher parametersMatcher = arrayContaining(expectedArgs); - final Matcher itemMatcher = new TypeSafeMatcher(LogRecord.class) { - @Override - public boolean matchesSafely(LogRecord actual) { - final String actualMessage = actual.getMessage(); - final Level actualLevel = actual.getLevel(); - final Object[] actualParameters = actual.getParameters(); - return messageMatcher.matches(actualMessage) && levelMatcher.matches(actualLevel) - && parametersMatcher.matches(actualParameters); - } - - @Override - public void describeTo(Description description) { - description.appendText("LogRecord("); - description.appendText("message ").appendDescriptionOf(messageMatcher); - description.appendText(" && level ").appendDescriptionOf(levelMatcher); - description.appendText(" && parameters ").appendDescriptionOf(parametersMatcher); - description.appendText(")"); - } - - @Override - protected void describeMismatchSafely(LogRecord actual, Description description) { - final String actualMessage = actual.getMessage(); - final Level actualLevel = actual.getLevel(); - final Object[] actualParameters = actual.getParameters(); - description.appendText("was LogRecord("); - description.appendText("message=\"").appendValue(actualMessage); - description.appendText("\", level ").appendValue(actualLevel); - description.appendText(", parameters ").appendValueList("[", ",", "]", actualParameters); - description.appendText(")"); - } - }; - return itemMatcher; - } -} +package org.jenkinsci.plugins.vsphere.tools; + +import static org.hamcrest.CoreMatchers.allOf; +import static org.hamcrest.CoreMatchers.any; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.everyItem; +import static org.hamcrest.collection.IsArrayContainingInOrder.arrayContaining; +import static org.hamcrest.collection.IsIterableContainingInOrder.contains; +import static org.junit.Assert.assertThat; +import hudson.model.Node.Mode; +import hudson.slaves.NodeProperty; +import hudson.slaves.JNLPLauncher; +import hudson.slaves.RetentionStrategy; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; + +import org.hamcrest.Description; +import org.hamcrest.Matcher; +import org.hamcrest.TypeSafeMatcher; +import org.hamcrest.collection.IsIterableWithSize; +import org.jenkinsci.plugins.vSphereCloud; +import org.jenkinsci.plugins.vSphereCloudSlaveTemplate; +import org.jenkinsci.plugins.vsphere.VSphereConnectionConfig; +import org.jenkinsci.plugins.vsphere.VSphereGuestInfoProperty; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class CloudProvisioningStateTest { + private static List stubVSphereCloudTemplates; + private static vSphereCloud stubVSphereCloud; + private int recordNumber; + private int nodeNumber; + private Logger testLogger; + private List loggedMessages; + + @BeforeClass + public static void setupClass() { + stubVSphereCloudTemplates = new ArrayList(); + final VSphereConnectionConfig vsConnectionConfig = new VSphereConnectionConfig("vsHost", "credentialsId"); + stubVSphereCloud = new vSphereCloud(vsConnectionConfig, "vsDescription", 0, 0, stubVSphereCloudTemplates); + } + + @Before + public void setup() { + stubVSphereCloudTemplates.clear(); + recordNumber = 0; + nodeNumber = 0; + loggedMessages = new ArrayList(); + Logger logger = Logger.getLogger("CloudProvisioningStateTest"); + logger.setLevel(Level.ALL); + final Handler[] handlers = logger.getHandlers(); + for (final Handler handler : handlers) { + logger.removeHandler(handler); + } + final Handler testHandler = new Handler() { + @Override + public void publish(LogRecord record) { + loggedMessages.add(record); + } + + @Override + public void flush() { + } + + @Override + public void close() { + } + }; + logger.addHandler(testHandler); + testLogger = logger; + } + + @Test + public void constructorGivenCalledThenLogsConstructions() { + // Given + final Object[] expectedArgs = { stubVSphereCloud.toString() }; + + // When + createInstance(); + + // Then + assertThat(loggedMessages, contains(logMessage(Level.FINE, expectedArgs))); + } + + @Test + public void provisioningStartedGivenNoPreviousStateThenLogs() { + // Given + final String nodeName = createNodeName(); + final Object[] expectedArgs = { nodeName }; + final CloudProvisioningState instance = createInstance(); + final CloudProvisioningRecord provisionable = createRecord(instance); + wipeLog(); + + // When + instance.provisioningStarted(provisionable, nodeName); + + // Then + assertThat(loggedMessages, contains(logMessage(Level.FINE, expectedArgs))); + } + + @Test + public void provisioningStartedGivenPreviouslyStartedThenWarns() { + // Given + final String nodeName = createNodeName(); + final Object[] expectedArgs = { nodeName }; + final CloudProvisioningState instance = createInstance(); + final CloudProvisioningRecord provisionable = createRecord(instance); + instance.provisioningStarted(provisionable, nodeName); + wipeLog(); + + // When + instance.provisioningStarted(provisionable, nodeName); + + // Then + assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); + } + + @Test + public void normalLifecycleGivenNoErrorsThenLogs() { + // Given + final String nodeName = createNodeName(); + final Object[] expectedArgs = { nodeName }; + final CloudProvisioningState instance = createInstance(); + final CloudProvisioningRecord provisionable = createRecord(instance); + wipeLog(); + + // When + instance.provisioningStarted(provisionable, nodeName); + instance.provisionedSlaveNowActive(provisionable, nodeName); + instance.provisionedSlaveNowTerminated(nodeName); + + // Then + assertThat(loggedMessages, everyItem(logMessage(Level.FINE, expectedArgs))); + assertThat(loggedMessages, IsIterableWithSize. iterableWithSize(3)); + } + + @SuppressWarnings("unchecked") + @Test + public void failedToProvisionGivenNothingOutOfSequenceThenLogs() { + // Given + final String nodeName = createNodeName(); + final Object[] expectedArgs = { nodeName }; + final CloudProvisioningState instance = createInstance(); + final CloudProvisioningRecord provisionable = createRecord(instance); + wipeLog(); + + // When + instance.provisioningStarted(provisionable, nodeName); + instance.provisioningEndedInError(provisionable, nodeName); + + // Then + assertThat( + loggedMessages, + contains(logMessage(Level.FINE, expectedArgs), + logMessage(containsString("failed"), Level.INFO, expectedArgs))); + assertThat(loggedMessages, IsIterableWithSize. iterableWithSize(2)); + } + + @Test + public void provisionGivenOutOfOrderSequenceThenComplains() { + // Given + final String nodeName = createNodeName(); + final Object[] expectedArgs = { nodeName }; + final CloudProvisioningState instance = createInstance(); + final CloudProvisioningRecord provisionable = createRecord(instance); + wipeLog(); + + // When/Then + wipeLog(); + instance.provisionedSlaveNowTerminated(nodeName); + assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); + + wipeLog(); + instance.provisioningStarted(provisionable, nodeName); + assertThat(loggedMessages, contains(logMessage(Level.FINE, expectedArgs))); + + wipeLog(); + instance.provisioningStarted(provisionable, nodeName); + assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); + + wipeLog(); + instance.provisioningEndedInError(provisionable, nodeName); + assertThat(loggedMessages, contains(logMessage(Level.INFO, expectedArgs))); + + wipeLog(); + instance.provisioningEndedInError(provisionable, nodeName); + assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); + + wipeLog(); + instance.provisionedSlaveNowActive(provisionable, nodeName); + assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); + + wipeLog(); + instance.provisionedSlaveNowActive(provisionable, nodeName); + assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); + + wipeLog(); + instance.provisioningStarted(provisionable, nodeName); + assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); + + wipeLog(); + instance.provisioningStarted(provisionable, nodeName); + assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); + + wipeLog(); + instance.provisionedSlaveNowTerminated(nodeName); + assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); + + wipeLog(); + instance.provisionedSlaveNowTerminated(nodeName); + assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); + } + + @Test + public void pruneUnwantedRecordsGivenUnknownTemplatesThenRemovesRecordsForEmptyDeletedTemplates() { + // Given + final String deletedAndInactiveNodeName = createNodeName(); + final String deletedButActiveNodeName = createNodeName(); + final String livedAndDiedNodeName = createNodeName(); + final CloudProvisioningState instance = createInstance(); + // A template which the user deleted but still has an active slave + final CloudProvisioningRecord deletedButActiveRecord = createRecord(instance); + instance.provisioningStarted(deletedButActiveRecord, deletedButActiveNodeName); + instance.provisionedSlaveNowActive(deletedButActiveRecord, deletedButActiveNodeName); + // A template which the user deleted and is no longer needed + final CloudProvisioningRecord deletedAndInactiveRecord = createRecord(instance); + instance.provisioningStarted(deletedAndInactiveRecord, deletedAndInactiveNodeName); + instance.provisionedSlaveNowActive(deletedAndInactiveRecord, deletedAndInactiveNodeName); + final vSphereCloudSlaveTemplate deletedAndInactiveTemplate = deletedAndInactiveRecord.getTemplate(); + instance.provisionedSlaveNowTerminated(deletedAndInactiveNodeName); + // A template which is current but has no active slaves right now + final CloudProvisioningRecord activeRecord = createRecord(instance); + instance.provisioningStarted(activeRecord, livedAndDiedNodeName); + instance.provisionedSlaveNowActive(activeRecord, livedAndDiedNodeName); + instance.provisionedSlaveNowTerminated(livedAndDiedNodeName); + + // When + userHasDeletedSlaveTemplate(deletedButActiveRecord); + userHasDeletedSlaveTemplate(deletedAndInactiveRecord); + wipeLog(); + instance.pruneUnwantedRecords(); + + // Then + assertThat( + loggedMessages, + contains(logMessage(containsString("Disposing"), Level.FINE, deletedAndInactiveTemplate.getCloneNamePrefix(), deletedAndInactiveTemplate.toString()))); + } + + @Test + public void countNodesGivenNoTemplatesOrSlavesThenReturnsZero() { + // Given + final CloudProvisioningState instance = createInstance(); + + // When + final int actual = instance.countNodes(); + + // Then + assertThat(actual, equalTo(0)); + } + + @Test + public void countNodesGivenNoSlavesInAnyTemplatesThenReturnsZero() { + // Given + final CloudProvisioningState instance = createInstance(); + createRecord(instance); + final String node = createNodeName(); + final CloudProvisioningRecord previouslyActiveRecord = createRecord(instance); + previouslyActiveRecord.addCurrentlyActive(node); + previouslyActiveRecord.removeCurrentlyActive(node); + createRecord(instance); + + // When + final int actual = instance.countNodes(); + + // Then + assertThat(actual, equalTo(0)); + } + + @Test + public void countNodesGiven2ActiveSlavesThenReturns2() { + // Given + final CloudProvisioningState instance = createInstance(); + final CloudProvisioningRecord activeRecord = createRecord(instance); + activeRecord.addCurrentlyActive(createNodeName()); + activeRecord.addCurrentlyActive(createNodeName()); + + // When + final int actual = instance.countNodes(); + + // Then + assertThat(actual, equalTo(2)); + } + + @Test + public void countNodesGiven3ActiveAnd4PendingSlavesThenReturns7() { + // Given + final CloudProvisioningState instance = createInstance(); + final CloudProvisioningRecord recordWith2Active = createRecord(instance); + recordWith2Active.addCurrentlyActive(createNodeName()); + recordWith2Active.addCurrentlyActive(createNodeName()); + final CloudProvisioningRecord recordWith1Active4Planned = createRecord(instance); + recordWith1Active4Planned.addCurrentlyActive(createNodeName()); + recordWith1Active4Planned.addCurrentlyPlanned(createNodeName()); + recordWith1Active4Planned.addCurrentlyPlanned(createNodeName()); + recordWith1Active4Planned.addCurrentlyPlanned(createNodeName()); + recordWith1Active4Planned.addCurrentlyPlanned(createNodeName()); + + // When + final int actual = instance.countNodes(); + + // Then + assertThat(actual, equalTo(7)); + } + + private void wipeLog() { + loggedMessages.clear(); + } + + private CloudProvisioningState createInstance() { + return new CloudProvisioningState(stubVSphereCloud, testLogger); + } + + private CloudProvisioningRecord createRecord(CloudProvisioningState instance) { + recordNumber++; + final String cloneNamePrefix = "prefix" + recordNumber; + final vSphereCloudSlaveTemplate template = new vSphereCloudSlaveTemplate(cloneNamePrefix, "masterImageName", + null, "snapshotName", false, "cluster", "resourcePool", "datastore", "folder", "customizationSpec", "templateDescription", 0, 1, "remoteFS", + "", Mode.NORMAL, false, false, 0, 0, false, "targetResourcePool", "targetHost", null, + new JNLPLauncher(), RetentionStrategy.NOOP, Collections.> emptyList(), + Collections. emptyList()); + stubVSphereCloudTemplates.add(template); + final List templates = new ArrayList(); + templates.add(template); + final List records = instance.calculateProvisionableTemplates(templates); + assertThat(records, IsIterableWithSize. iterableWithSize(1)); + final CloudProvisioningRecord record = records.get(0); + return record; + } + + private String createNodeName() { + nodeNumber++; + final String nodeName = "N#" + nodeNumber; + return nodeName; + } + + private void userHasDeletedSlaveTemplate(CloudProvisioningRecord record) { + stubVSphereCloudTemplates.remove(record.getTemplate()); + } + + private static Matcher logMessage(final Level expectedLevel, final Object... expectedArgs) { + final List> messageMatchers = new ArrayList>( + expectedArgs.length); + for (int i = 0; i < expectedArgs.length; i++) { + final String expectedString = "{" + i + "}"; + messageMatchers.add(containsString(expectedString)); + } + final Matcher messageMatcher; + if (messageMatchers.isEmpty()) { + messageMatcher = any(String.class); + } else { + messageMatcher = allOf(messageMatchers); + } + return logMessage(messageMatcher, expectedLevel, expectedArgs); + } + + private static Matcher logMessage(final Matcher messageMatcher, final Level expectedLevel, + final Object... expectedArgs) { + final Matcher levelMatcher = equalTo(expectedLevel); + final Matcher parametersMatcher = arrayContaining(expectedArgs); + final Matcher itemMatcher = new TypeSafeMatcher(LogRecord.class) { + @Override + public boolean matchesSafely(LogRecord actual) { + final String actualMessage = actual.getMessage(); + final Level actualLevel = actual.getLevel(); + final Object[] actualParameters = actual.getParameters(); + return messageMatcher.matches(actualMessage) && levelMatcher.matches(actualLevel) + && parametersMatcher.matches(actualParameters); + } + + @Override + public void describeTo(Description description) { + description.appendText("LogRecord("); + description.appendText("message ").appendDescriptionOf(messageMatcher); + description.appendText(" && level ").appendDescriptionOf(levelMatcher); + description.appendText(" && parameters ").appendDescriptionOf(parametersMatcher); + description.appendText(")"); + } + + @Override + protected void describeMismatchSafely(LogRecord actual, Description description) { + final String actualMessage = actual.getMessage(); + final Level actualLevel = actual.getLevel(); + final Object[] actualParameters = actual.getParameters(); + description.appendText("was LogRecord("); + description.appendText("message=\"").appendValue(actualMessage); + description.appendText("\", level ").appendValue(actualLevel); + description.appendText(", parameters ").appendValueList("[", ",", "]", actualParameters); + description.appendText(")"); + } + }; + return itemMatcher; + } +} From 35c3feb51caddf0be8e030cbba972c99507ccb17 Mon Sep 17 00:00:00 2001 From: Peter Darton Date: Wed, 28 Jun 2017 17:08:59 +0100 Subject: [PATCH 2/3] Retry deletion of unwanted VMs Not every attempt to communicate with vSphere succeeds. Sometimes we fail to create a VM, sometimes we fail to destroy a VM. If we fail to create one, the existing logic will try again, as Jenkins will keep asking for new VMs. If we failed to delete one, we used to merely log the failure exception and then forget about it, which could result in unused VMs running in vSphere. This code change adds extra in-memory state so that we remember a list of all the VMs that we want to delete until we are sure that we have deleted them, and we keep trying to delete unwanted VMs until we've got rid of them. --- .../org/jenkinsci/plugins/vSphereCloud.java | 79 +++++- .../tools/CloudProvisioningAlgorithm.java | 6 +- .../tools/CloudProvisioningRecord.java | 61 ++++- .../vsphere/tools/CloudProvisioningState.java | 233 ++++++++++++++++-- .../tools/CloudProvisioningAlgorithmTest.java | 47 +++- .../tools/CloudProvisioningStateTest.java | 170 ++++++++++++- 6 files changed, 542 insertions(+), 54 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/vSphereCloud.java b/src/main/java/org/jenkinsci/plugins/vSphereCloud.java index a1ba4122..43ed1023 100644 --- a/src/main/java/org/jenkinsci/plugins/vSphereCloud.java +++ b/src/main/java/org/jenkinsci/plugins/vSphereCloud.java @@ -187,11 +187,10 @@ private void ensureLists() { for (final vSphereCloudProvisionedSlave n : NodeIterator.nodes(vSphereCloudProvisionedSlave.class)) { final String nodeName = n.getNodeName(); final vSphereCloudSlaveTemplate template = getTemplateForVM(nodeName); - if (template != null) { - final CloudProvisioningRecord provisionable = templateState.getOrCreateRecord(template); - templateState.provisioningStarted(provisionable, nodeName); - templateState.provisionedSlaveNowActive(provisionable, nodeName); - } + if (template == null) continue; + final CloudProvisioningRecord provisionable = templateState.getOrCreateRecord(template); + templateState.provisioningStarted(provisionable, nodeName); + templateState.provisionedSlaveNowActive(provisionable, nodeName); } } } @@ -333,6 +332,7 @@ public Collection provision(final Label label, int excessWorkload) synchronized (this) { ensureLists(); } + retryVMdeletionIfNecessary(Math.max(excessWorkload, 2)); final List plannedNodes = new ArrayList(); synchronized (templateState) { templateState.pruneUnwantedRecords(); @@ -371,6 +371,48 @@ public Collection provision(final Label label, int excessWorkload) } } + /** + * Has another go at deleting VMs we failed to delete earlier. It's possible + * that we were unable to talk to vSphere (or some other failure happened) + * when we decided to delete some VMs. We remember this sort of thing so we + * can retry later - this is where we use this information. + * + * @param maxToRetryDeletionOn + * The maximum number of VMs to try to remove this time around. + * Can be {@link Integer#MAX_VALUE} for unlimited. + */ + private void retryVMdeletionIfNecessary(final int maxToRetryDeletionOn) { + if (templateState == null) { + VSLOG.log(Level.INFO, "retryVMdeletionIfNecessary({0}): templateState==null", maxToRetryDeletionOn); + return; + } + // find all candidates and trim down the list + final List unwantedVMsThatNeedDeleting = templateState.getUnwantedVMsThatNeedDeleting(); + final int numberToAttemptToRetryThisTime = Math.min(maxToRetryDeletionOn, unwantedVMsThatNeedDeleting.size()); + final List nodeNamesToRetryDeletion = unwantedVMsThatNeedDeleting.subList(0, + numberToAttemptToRetryThisTime); + // now queue their deletion + synchronized (templateState) { + for (final String nodeName : nodeNamesToRetryDeletion) { + final Boolean isOkToDelete = templateState.isOkToDeleteUnwantedVM(nodeName); + if (isOkToDelete == Boolean.TRUE) { + final Runnable task = new Runnable() { + @Override + public void run() { + attemptDeletionOfSlave("retryVMdeletionIfNecessary(" + nodeName + ")", nodeName); + } + }; + VSLOG.log(Level.INFO, "retryVMdeletionIfNecessary({0}): scheduling deletion of {1}", new Object[] { maxToRetryDeletionOn, nodeName }); + Computer.threadPoolForRemoting.submit(task); + } else { + VSLOG.log(Level.FINER, + "retryVMdeletionIfNecessary({0}): not going to try deleting {1} as isOkToDeleteUnwantedVM({1})=={2}", + new Object[]{ maxToRetryDeletionOn, nodeName, isOkToDelete }); + } + } + } + } + /** * This is called by {@link vSphereCloudProvisionedSlave} instances once * they terminate, so we can take note of their passing and then destroy the @@ -383,19 +425,36 @@ void provisionedSlaveHasTerminated(final String cloneName) { ensureLists(); } VSLOG.log(Level.FINER, "provisionedSlaveHasTerminated({0}): recording in our runtime state...", cloneName); - // once we're done, remove our cached record. synchronized (templateState) { - templateState.provisionedSlaveNowTerminated(cloneName); + templateState.provisionedSlaveNowUnwanted(cloneName, true); } - VSLOG.log(Level.FINER, "provisionedSlaveHasTerminated({0}): destroying VM...", cloneName); + attemptDeletionOfSlave("provisionedSlaveHasTerminated(" + cloneName + ")", cloneName); + // We should also take this opportunity to see if we've got any other + // slaves that need deleting. + retryVMdeletionIfNecessary(1); + } + + private void attemptDeletionOfSlave(final String why, final String cloneName) { + VSLOG.log(Level.FINER, "{0}: destroying VM {1}...", new Object[]{ why, cloneName }); VSphere vSphere = null; + boolean successfullyDeleted = false; try { vSphere = vSphereInstance(); vSphere.destroyVm(cloneName, false); - VSLOG.log(Level.FINER, "provisionedSlaveHasTerminated({0}): VM destroyed.", cloneName); + successfullyDeleted = true; + VSLOG.log(Level.FINER, "{0}: VM {1} destroyed.", new Object[]{ why, cloneName }); + vSphere.disconnect(); + vSphere = null; } catch (VSphereException ex) { - VSLOG.log(Level.SEVERE, "provisionedSlaveHasTerminated(" + cloneName + "): Exception while trying to destroy VM", ex); + VSLOG.log(Level.SEVERE, why + ": Exception while trying to destroy VM " + cloneName, ex); } finally { + synchronized (templateState) { + if (successfullyDeleted) { + templateState.unwantedSlaveNowDeleted(cloneName); + } else { + templateState.unwantedSlaveNotDeleted(cloneName); + } + } if (vSphere != null) { vSphere.disconnect(); } diff --git a/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithm.java b/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithm.java index 323a1ad0..20f37720 100644 --- a/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithm.java +++ b/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithm.java @@ -64,12 +64,10 @@ public static CloudProvisioningRecord findTemplateWithMostFreeCapacity( public static String findUnusedName(CloudProvisioningRecord record) { final vSphereCloudSlaveTemplate template = record.getTemplate(); final String cloneNamePrefix = template.getCloneNamePrefix(); - final Set existingNames = new TreeSet(); - existingNames.addAll(record.getCurrentlyPlanned()); - existingNames.addAll(record.getCurrentlyProvisioned()); + final Set existingNames = record.getCurrentNames(); final int templateInstanceCap = template.getTemplateInstanceCap(); final boolean hasCap = templateInstanceCap > 0 && templateInstanceCap < Integer.MAX_VALUE; - final int maxAttempts = hasCap ? (templateInstanceCap + 1) : 100; + final int maxAttempts = hasCap ? (templateInstanceCap) : 100; for (int attempt = 0; attempt < maxAttempts; attempt++) { final String suffix = hasCap ? calcSequentialSuffix(attempt) : calcRandomSuffix(attempt); final String nodeName = cloneNamePrefix + "_" + suffix; diff --git a/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningRecord.java b/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningRecord.java index a82e31bb..8ea98493 100644 --- a/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningRecord.java +++ b/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningRecord.java @@ -1,6 +1,8 @@ package org.jenkinsci.plugins.vsphere.tools; import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.Set; import java.util.TreeSet; @@ -11,16 +13,21 @@ * it asks us to provision some) and when those nodes appear in vSphere and in * Jenkins, so we need to keep a record of what's in progress so we don't * over-commit. + * Similarly there's also a delay between when we decide to delete some and + * when we successfully delete the VMs, and so we need to know what VMs we need + * to delete, and also what ones we're in the process of deleting. */ public final class CloudProvisioningRecord { private final vSphereCloudSlaveTemplate template; - private Set currentlyProvisioned; - private Set currentlyPlanned; + private final Set currentlyProvisioned; + private final Set currentlyPlanned; + private final Map currentlyUnwanted; CloudProvisioningRecord(vSphereCloudSlaveTemplate template) { this.template = template; this.currentlyProvisioned = new TreeSet(); this.currentlyPlanned = new TreeSet(); + this.currentlyUnwanted = new LinkedHashMap(); } public vSphereCloudSlaveTemplate getTemplate() { @@ -29,8 +36,12 @@ public vSphereCloudSlaveTemplate getTemplate() { @Override public String toString() { - return String.format("Template[prefix=%s, provisioned=%s, planned=%s, max=%d, fullness=%.3f%%]", getTemplate() - .getCloneNamePrefix(), getCurrentlyProvisioned(), getCurrentlyPlanned(), calcMaxToProvision(), + return String.format("Template[prefix=%s, provisioned=%s, planned=%s, unwanted=%s, max=%d, fullness=%.3f%%]", + getTemplate().getCloneNamePrefix(), + getCurrentlyProvisioned(), + getCurrentlyPlanned(), + getCurrentlyUnwanted(), + calcMaxToProvision(), calcFullness() * 100.0); } @@ -58,6 +69,46 @@ boolean removeCurrentlyPlanned(String nodeName) { return currentlyPlanned.remove(nodeName); } + Map getCurrentlyUnwanted() { + return currentlyUnwanted; + } + + Boolean isCurrentlyUnwanted(String nodeName) { + return currentlyUnwanted.get(nodeName); + } + + Boolean setCurrentlyUnwanted(String nodeName, boolean beingDeleted) { + // ensure this node gets pushed to the end of the map by doing a remove then a put. + final Boolean oldValue = currentlyUnwanted.remove(nodeName); + currentlyUnwanted.put(nodeName, Boolean.valueOf(beingDeleted)); + return oldValue; + } + + boolean removeCurrentlyUnwanted(String nodeName) { + return currentlyUnwanted.remove(nodeName) != null; + } + + Set getCurrentNames() { + final Set existingNames = new TreeSet(); + existingNames.addAll(getCurrentlyPlanned()); + existingNames.addAll(getCurrentlyProvisioned()); + existingNames.addAll(getCurrentlyUnwanted().keySet()); + return existingNames; + } + + boolean contains(String nodeName) { + return currentlyProvisioned.contains(nodeName) || currentlyPlanned.contains(nodeName) + || currentlyUnwanted.containsKey(nodeName); + } + + int size() { + return currentlyProvisioned.size() + currentlyPlanned.size() + currentlyUnwanted.size(); + } + + boolean isEmpty() { + return currentlyProvisioned.isEmpty() && currentlyPlanned.isEmpty() && currentlyUnwanted.isEmpty(); + } + private int calcMaxToProvision() { final int templateInstanceCap = template.getTemplateInstanceCap(); final int maxToProvision = templateInstanceCap == 0 ? Integer.MAX_VALUE : templateInstanceCap; @@ -82,7 +133,7 @@ boolean hasCapacityForMore() { } private int calcCurrentCommitment() { - return currentlyProvisioned.size() + currentlyPlanned.size(); + return currentlyProvisioned.size() + currentlyPlanned.size() + currentlyUnwanted.size(); } /** diff --git a/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningState.java b/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningState.java index a9d6b0ab..507088d9 100644 --- a/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningState.java +++ b/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningState.java @@ -2,6 +2,8 @@ import java.util.ArrayList; import java.util.IdentityHashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.logging.Level; @@ -66,16 +68,20 @@ public CloudProvisioningState(vSphereCloud parent) { public void provisioningStarted(CloudProvisioningRecord provisionable, String nodeName) { final boolean wasPreviouslyUnknownToPlanning = provisionable.addCurrentlyPlanned(nodeName); final boolean wasAlreadyActive = provisionable.removeCurrentlyActive(nodeName); - logStateChange(Level.FINE, "Intending to create {0}", "wasPreviouslyUnknownToPlanning", - wasPreviouslyUnknownToPlanning, true, "wasAlreadyActive", wasAlreadyActive, false, nodeName); + final boolean wasPreviouslyUnwanted = provisionable.removeCurrentlyUnwanted(nodeName); + logStateChange(Level.FINE, "Intending to create {0}", + "wasPreviouslyUnknownToPlanning", wasPreviouslyUnknownToPlanning, true, + "wasAlreadyActive", wasAlreadyActive, false, + "wasPreviouslyUnwanted", wasPreviouslyUnwanted, false, + nodeName); } /** * To be called when a newly created node (previously promised to * {@link #provisioningStarted(CloudProvisioningRecord, String)}) comes up. * Callers MUST ensure that - * {@link #provisionedSlaveNowTerminated(String)} - * gets called later. + * {@link #provisionedSlaveNowUnwanted(String, boolean)} gets called later + * when we do not want it anymore. * * @param provisionable * Our record for the template for the named node. @@ -85,34 +91,155 @@ public void provisioningStarted(CloudProvisioningRecord provisionable, String no public void provisionedSlaveNowActive(CloudProvisioningRecord provisionable, String nodeName) { final boolean wasNotPreviouslyActive = provisionable.addCurrentlyActive(nodeName); final boolean wasPreviouslyPlanned = provisionable.removeCurrentlyPlanned(nodeName); - logStateChange(Level.FINE, "Marking {0} as active", "wasNotPreviouslyActive", wasNotPreviouslyActive, true, - "wasPreviouslyPlanned", wasPreviouslyPlanned, true, nodeName); + final boolean wasPreviouslyUnwanted = provisionable.removeCurrentlyUnwanted(nodeName); + logStateChange(Level.FINE, "Marking {0} as active", + "wasNotPreviouslyActive", wasNotPreviouslyActive, true, + "wasPreviouslyPlanned", wasPreviouslyPlanned, true, + "wasPreviouslyUnwanted", wasPreviouslyUnwanted, false, + nodeName); } /** * To be called when a node we created (previously told to - * {@link #provisionedSlaveNowActive(CloudProvisioningRecord, String)}) has - * died. + * {@link #provisionedSlaveNowActive(CloudProvisioningRecord, String)}) is + * no longer wanted and should be deleted. * * @param nodeName * The name of the VM. + * @param willAttemptImmediateDeletion + * If true then the caller must attempt to delete the slave and + * guarantee that they will call + * {@link #unwantedSlaveNotDeleted(String)} or + * {@link #unwantedSlaveNowDeleted(String)} as appropriate (just + * as if they'd called {@link #isOkToDeleteUnwantedVM(String)} + * and been told True). If false then the caller is under no such + * obligation. + */ + public void provisionedSlaveNowUnwanted(String nodeName, boolean willAttemptImmediateDeletion) { + final Map.Entry entry = findEntryForVM(nodeName); + if (entry != null) { + final CloudProvisioningRecord provisionable = entry.getValue(); + final boolean wasPreviouslyPlanned = provisionable.removeCurrentlyPlanned(nodeName); + final boolean wasPreviouslyActive = provisionable.removeCurrentlyActive(nodeName); + final boolean wasNotPreviouslyUnwanted = provisionable.setCurrentlyUnwanted(nodeName, willAttemptImmediateDeletion)==null; + logStateChange(Level.FINE, "Marking {0} for termination", + "wasPreviouslyPlanned", wasPreviouslyPlanned, false, + "wasPreviouslyActive", wasPreviouslyActive, true, + "wasNotPreviouslyUnwanted", wasNotPreviouslyUnwanted, true, + nodeName); + } else { + logger.log(Level.WARNING, "Asked to mark {0} for termination, but we have no record of it.", nodeName); + } + } + + /** + * To be called before commencing the deletion of a VM. + * + * @param nodeName + * The name of the VM being deleted. + * @return null if the VM is not unwanted (it may have recently been + * deleted). false if another thread is currently trying to delete + * it. true if deletion should be attempted, in which case the + * caller MUST later call {@link #unwantedSlaveNowDeleted(String)} + * or {@link #unwantedSlaveNotDeleted(String)}. + */ + public Boolean isOkToDeleteUnwantedVM(String nodeName) { + final Map.Entry entry = findEntryForVM(nodeName); + if (entry == null) { + return null; + } + final CloudProvisioningRecord record = entry.getValue(); + final Boolean thisNode = record.isCurrentlyUnwanted(nodeName); + if (thisNode == null) { + return null; + } + boolean someoneElseIsDeletingThis = thisNode.booleanValue(); + boolean isOkForUsToDeleteIt = !someoneElseIsDeletingThis; + if (isOkForUsToDeleteIt) { + record.setCurrentlyUnwanted(nodeName, true); + } + return Boolean.valueOf(isOkForUsToDeleteIt); + } + + /** + * MUST be called when a node previously declared to be unwanted (previously + * told to {@link #provisionedSlaveNowUnwanted(String, boolean)}) and that + * we were given clearance to delete + * ({@link #isOkToDeleteUnwantedVM(String)} returned true) has been + * successfully removed. + * + * @param nodeName + * The name of the VM that was successfully deleted. */ - public void provisionedSlaveNowTerminated(String nodeName) { + public void unwantedSlaveNowDeleted(String nodeName) { final Map.Entry entry = findEntryForVM(nodeName); if (entry != null) { final CloudProvisioningRecord provisionable = entry.getValue(); final boolean wasPreviouslyPlanned = provisionable.removeCurrentlyPlanned(nodeName); final boolean wasPreviouslyActive = provisionable.removeCurrentlyActive(nodeName); + final boolean wasPreviouslyUnwanted = provisionable.removeCurrentlyUnwanted(nodeName); if (recordIsPrunable(provisionable)) { removeExistingRecord(provisionable); } - logStateChange(Level.FINE, "Marking {0} as terminated", "wasPreviouslyPlanned", wasPreviouslyPlanned, - false, "wasPreviouslyActive", wasPreviouslyActive, true, nodeName); + logStateChange(Level.FINE, "Marking {0} as successfully terminated", + "wasPreviouslyPlanned", wasPreviouslyPlanned, false, + "wasPreviouslyActive", wasPreviouslyActive, false, + "wasPreviouslyUnwanted", wasPreviouslyUnwanted, true, + nodeName); } else { - logger.log(Level.WARNING, "Asked to mark {0} as terminated, but we have no record of it.", nodeName); + logger.log(Level.WARNING, "Asked to mark {0} as terminated, but we had no record of it.", nodeName); } } + /** + * MUST be called when a node previously declared to be unwanted (previously + * told to {@link #provisionedSlaveNowUnwanted(String, boolean)}) and that + * we were given clearance to delete + * ({@link #isOkToDeleteUnwantedVM(String)} returned true) failed to be + * removed. + * + * @param nodeName + * The name of the VM that failed to delete + */ + public void unwantedSlaveNotDeleted(String nodeName) { + final Map.Entry entry = findEntryForVM(nodeName); + if (entry != null) { + final CloudProvisioningRecord provisionable = entry.getValue(); + final boolean isPlanned = provisionable.getCurrentlyPlanned().contains(nodeName); + final boolean isActive = provisionable.getCurrentlyProvisioned().contains(nodeName); + final boolean isUnwanted = provisionable.setCurrentlyUnwanted(nodeName, false) != null; + logStateChange(Level.INFO, "Marking {0} as unsuccessfully terminated - we'll have to try again later", + "isPlanned", isPlanned, false, + "isActive", isActive, false, + "isUnwanted", isUnwanted, true, + nodeName); + } else { + logger.log(Level.WARNING, "Asked to mark {0} as unsuccessfully terminated, but we had no record of it.", + nodeName); + } + } + + /** + * To be called if we become aware that there is a VM that exist in vSphere + * (that we created) which we don't want anymore. + * + * @param template + * The template to which the node belonged. + * @param nodeName + * The name of the node that exists (despite our wishes). + */ + public void recordExistingUnwantedVM(final vSphereCloudSlaveTemplate template, String nodeName) { + final CloudProvisioningRecord record = getOrCreateRecord(template); + final boolean wasPreviouslyPlanned = record.removeCurrentlyPlanned(nodeName); + final boolean wasPreviouslyActive = record.removeCurrentlyActive(nodeName); + final boolean wasAlreadyUnwanted = record.setCurrentlyUnwanted(nodeName, false) != null; + logStateChange(Level.INFO, "Marking {0} as found in vSphere but unwanted", + "wasPreviouslyPlanned", wasPreviouslyPlanned, false, + "wasPreviouslyActive", wasPreviouslyActive, false, + "wasAlreadyUnwanted", wasAlreadyUnwanted, false, + nodeName); + } + /** * To be called when a node that we previously promised to create (by * calling {@link #provisioningStarted(CloudProvisioningRecord, String)}) @@ -126,11 +253,15 @@ public void provisionedSlaveNowTerminated(String nodeName) { public void provisioningEndedInError(CloudProvisioningRecord provisionable, String nodeName) { final boolean wasPreviouslyPlanned = provisionable.removeCurrentlyPlanned(nodeName); final boolean wasPreviouslyActive = provisionable.removeCurrentlyActive(nodeName); + final boolean wasPreviouslyUnwanted = provisionable.removeCurrentlyUnwanted(nodeName); if (recordIsPrunable(provisionable)) { removeExistingRecord(provisionable); } - logStateChange(Level.INFO, "Marking {0} as failed", "wasPreviouslyPlanned", wasPreviouslyPlanned, true, - "wasPreviouslyActive", wasPreviouslyActive, false, nodeName); + logStateChange(Level.INFO, "Marking {0} as failed", + "wasPreviouslyPlanned", wasPreviouslyPlanned, true, + "wasPreviouslyActive", wasPreviouslyActive, false, + "wasPreviouslyUnwanted", wasPreviouslyUnwanted, false, + nodeName); } /** @@ -167,16 +298,15 @@ public List calculateProvisionableTemplates(Iterable getUnwantedVMsThatNeedDeleting() { + // find out who needs what deleted + int count = 0; + final Map> allUnwantedVmsByRecord = new LinkedHashMap<>(); + for (final CloudProvisioningRecord record : records.values()) { + final Map currentlyUnwanted = record.getCurrentlyUnwanted(); + final List vmsInNeedOfDeletionForThisRecord = new ArrayList(currentlyUnwanted.size()); + for (Map.Entry entry : currentlyUnwanted.entrySet()) { + if (entry.getValue() == Boolean.FALSE) { + vmsInNeedOfDeletionForThisRecord.add(entry.getKey()); + } + } + count += vmsInNeedOfDeletionForThisRecord.size(); + allUnwantedVmsByRecord.put(record, vmsInNeedOfDeletionForThisRecord.iterator()); + } + // arrange the list in a round-robin order, taking the first from each + // template, followed by the second from each etc. + final List vmsInNeedOfDeletion = new ArrayList(count); + while (vmsInNeedOfDeletion.size() < count) { + for (Iterator i : allUnwantedVmsByRecord.values()) { + if (i.hasNext()) { + final String nodeName = i.next(); + vmsInNeedOfDeletion.add(nodeName); + } + } + } + return vmsInNeedOfDeletion; + } + private CloudProvisioningRecord getExistingRecord(final vSphereCloudSlaveTemplate template) { return records.get(template); } @@ -213,7 +380,7 @@ private void removeExistingRecord(CloudProvisioningRecord existingRecord) { } private boolean recordIsPrunable(CloudProvisioningRecord record) { - final boolean isEmpty = record.getCurrentlyProvisioned().isEmpty() && record.getCurrentlyPlanned().isEmpty(); + final boolean isEmpty = record.isEmpty(); if (!isEmpty) { return false; } @@ -226,10 +393,7 @@ private boolean recordIsPrunable(CloudProvisioningRecord record) { private Map.Entry findEntryForVM(String nodeName) { for (final Map.Entry entry : records.entrySet()) { final CloudProvisioningRecord record = entry.getValue(); - if (record.getCurrentlyProvisioned().contains(nodeName)) { - return entry; - } - if (record.getCurrentlyPlanned().contains(nodeName)) { + if (record.contains(nodeName)) { return entry; } } @@ -261,15 +425,26 @@ private Map.Entry findEntryF * @param expectedSecondArgValue * The expected value of actualSecondArgValue. If that's not the * case, we'll complain. + * @param thirdArgName + * What actualThirdArgValue represents - used when complaining + * about its value. + * @param actualThirdArgValue + * A state-change variable. + * @param expectedThirdArgValue + * The expected value of actualThirdArgValue. If that's not the + * case, we'll complain. * @param args * The arguments for logMsg. Used if logMsg contains {0}, {1} * etc. */ - private void logStateChange(Level logLevel, String logMsg, String firstArgName, boolean actualFirstArgValue, - boolean expectedFirstArgValue, String secondArgName, boolean actualSecondArgValue, - boolean expectedSecondArgValue, Object... args) { + private void logStateChange(Level logLevel, String logMsg, + String firstArgName, boolean actualFirstArgValue, boolean expectedFirstArgValue, + String secondArgName, boolean actualSecondArgValue, boolean expectedSecondArgValue, + String thirdArgName, boolean actualThirdArgValue, boolean expectedThirdArgValue, + Object... args) { final boolean firstValid = actualFirstArgValue == expectedFirstArgValue; final boolean secondValid = actualSecondArgValue == expectedSecondArgValue; + final boolean thirdValid = actualThirdArgValue == expectedThirdArgValue; Level actualLevel = logLevel; String actualMsg = logMsg; if (!firstValid) { @@ -280,6 +455,10 @@ private void logStateChange(Level logLevel, String logMsg, String firstArgName, actualMsg += " : " + secondArgName + "!=" + expectedSecondArgValue; actualLevel = Level.WARNING; } + if (!thirdValid) { + actualMsg += " : " + thirdArgName + "!=" + expectedThirdArgValue; + actualLevel = Level.WARNING; + } final Logger loggerToUse = logger != null ? logger : LOGGER; loggerToUse.log(actualLevel, actualMsg, args); } diff --git a/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithmTest.java b/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithmTest.java index 55114326..a6a0e717 100644 --- a/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithmTest.java +++ b/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithmTest.java @@ -215,6 +215,51 @@ public void findUnusedNameGivenZeroOfTwoExistsThenReturnsOneThenTwo() { assertThat(actual2, equalTo(expected2)); } + @Test + public void findUnusedNameGivenMiddleOfThreeStillExistsThenReturnsOneThenThree() { + // Given + final CloudProvisioningRecord record = createInstance(3, 0, 0); + final String prefix = record.getTemplate().getCloneNamePrefix(); + final String expected1 = prefix + "_1"; + final String unwanted = prefix + "_2"; + record.setCurrentlyUnwanted(unwanted, false); + final String expected2 = prefix + "_3"; + + // When + final String actual1 = CloudProvisioningAlgorithm.findUnusedName(record); + record.addCurrentlyActive(actual1); + final String actual2 = CloudProvisioningAlgorithm.findUnusedName(record); + record.addCurrentlyActive(actual2); + + // Then + assertThat(actual1, equalTo(expected1)); + assertThat(actual2, equalTo(expected2)); + } + + @Test + public void findUnusedNameGivenNoSpaceThenThrowsIllegalStateException() { + // Given + final CloudProvisioningRecord record = createInstance(3, 0, 0); + final String prefix = record.getTemplate().getCloneNamePrefix(); + final String unwanted = prefix + "_1"; + final String active = prefix + "_2"; + final String planned = prefix + "_3"; + record.setCurrentlyUnwanted(unwanted, false); + record.addCurrentlyActive(active); + record.addCurrentlyPlanned(planned); + final List records = Arrays.asList(record); + final CloudProvisioningRecord shouldBeNull = CloudProvisioningAlgorithm.findTemplateWithMostFreeCapacity(records); + assertThat(shouldBeNull, nullValue()); + + // When + try { + final String unexpected = CloudProvisioningAlgorithm.findUnusedName(record); + fail("Expected IllegalStateException, got '" + unexpected + "'."); + } catch (IllegalStateException expected) { + // Then passed. + } + } + @Test public void findUnusedNameGivenOneOfTwoHasEndedThenReturnsOne() { // Given @@ -261,7 +306,7 @@ private CloudProvisioningRecord createInstance(int capacity, int provisioned, in final CloudProvisioningRecord instance = new CloudProvisioningRecord(template); for (int i = 0; i < provisioned; i++) { final String nodeName = iNum + "provisioned#" + i; - instance.addCurrentlyPlanned(nodeName); + instance.addCurrentlyActive(nodeName); } for (int i = 0; i < planned; i++) { final String nodeName = iNum + "planned#" + i; diff --git a/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningStateTest.java b/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningStateTest.java index 30ac9361..8fa88ffa 100644 --- a/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningStateTest.java +++ b/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningStateTest.java @@ -1,6 +1,7 @@ package org.jenkinsci.plugins.vsphere.tools; import static org.hamcrest.CoreMatchers.allOf; +import static org.hamcrest.CoreMatchers.either; import static org.hamcrest.CoreMatchers.any; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; @@ -14,6 +15,7 @@ import hudson.slaves.RetentionStrategy; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.logging.Handler; @@ -135,11 +137,12 @@ public void normalLifecycleGivenNoErrorsThenLogs() { // When instance.provisioningStarted(provisionable, nodeName); instance.provisionedSlaveNowActive(provisionable, nodeName); - instance.provisionedSlaveNowTerminated(nodeName); + instance.provisionedSlaveNowUnwanted(nodeName, true); + instance.unwantedSlaveNowDeleted(nodeName); // Then assertThat(loggedMessages, everyItem(logMessage(Level.FINE, expectedArgs))); - assertThat(loggedMessages, IsIterableWithSize. iterableWithSize(3)); + assertThat(loggedMessages, IsIterableWithSize. iterableWithSize(4)); } @SuppressWarnings("unchecked") @@ -175,7 +178,11 @@ public void provisionGivenOutOfOrderSequenceThenComplains() { // When/Then wipeLog(); - instance.provisionedSlaveNowTerminated(nodeName); + instance.unwantedSlaveNowDeleted(nodeName); + assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); + + wipeLog(); + instance.provisionedSlaveNowUnwanted(nodeName, false); assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); wipeLog(); @@ -211,11 +218,11 @@ public void provisionGivenOutOfOrderSequenceThenComplains() { assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); wipeLog(); - instance.provisionedSlaveNowTerminated(nodeName); + instance.provisionedSlaveNowUnwanted(nodeName, false); assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); wipeLog(); - instance.provisionedSlaveNowTerminated(nodeName); + instance.provisionedSlaveNowUnwanted(nodeName, true); assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); } @@ -235,12 +242,14 @@ public void pruneUnwantedRecordsGivenUnknownTemplatesThenRemovesRecordsForEmptyD instance.provisioningStarted(deletedAndInactiveRecord, deletedAndInactiveNodeName); instance.provisionedSlaveNowActive(deletedAndInactiveRecord, deletedAndInactiveNodeName); final vSphereCloudSlaveTemplate deletedAndInactiveTemplate = deletedAndInactiveRecord.getTemplate(); - instance.provisionedSlaveNowTerminated(deletedAndInactiveNodeName); + instance.provisionedSlaveNowUnwanted(deletedAndInactiveNodeName, true); + instance.unwantedSlaveNowDeleted(deletedAndInactiveNodeName); // A template which is current but has no active slaves right now final CloudProvisioningRecord activeRecord = createRecord(instance); instance.provisioningStarted(activeRecord, livedAndDiedNodeName); instance.provisionedSlaveNowActive(activeRecord, livedAndDiedNodeName); - instance.provisionedSlaveNowTerminated(livedAndDiedNodeName); + instance.provisionedSlaveNowUnwanted(livedAndDiedNodeName, true); + instance.unwantedSlaveNowDeleted(livedAndDiedNodeName); // When userHasDeletedSlaveTemplate(deletedButActiveRecord); @@ -299,6 +308,21 @@ public void countNodesGiven2ActiveSlavesThenReturns2() { assertThat(actual, equalTo(2)); } + @Test + public void countNodesGiven2UnwantedSlavesThenReturns2() { + // Given + final CloudProvisioningState instance = createInstance(); + final CloudProvisioningRecord activeRecord = createRecord(instance); + activeRecord.setCurrentlyUnwanted(createNodeName(), false); + activeRecord.setCurrentlyUnwanted(createNodeName(), false); + + // When + final int actual = instance.countNodes(); + + // Then + assertThat(actual, equalTo(2)); + } + @Test public void countNodesGiven3ActiveAnd4PendingSlavesThenReturns7() { // Given @@ -320,6 +344,138 @@ public void countNodesGiven3ActiveAnd4PendingSlavesThenReturns7() { assertThat(actual, equalTo(7)); } + @Test + public void getUnwantedVMsThatNeedDeletingGivenNothingNeedsDeletingThenReturnsNothing() { + // Given + final CloudProvisioningState instance = createInstance(); + final CloudProvisioningRecord recordWith2Active = createRecord(instance); + recordWith2Active.addCurrentlyActive(createNodeName()); + recordWith2Active.addCurrentlyActive(createNodeName()); + final CloudProvisioningRecord recordWith1Active1Planned = createRecord(instance); + recordWith1Active1Planned.addCurrentlyActive(createNodeName()); + recordWith1Active1Planned.addCurrentlyPlanned(createNodeName()); + final List expected = Arrays.asList(); + + // When + final List actual = instance.getUnwantedVMsThatNeedDeleting(); + // Then + assertThat(actual, equalTo(expected)); + } + + @Test + public void getUnwantedVMsThatNeedDeletingGivenSeveralUnwantedThenReturnsDeletableVMsInCorrectOrder() { + // Given + final CloudProvisioningState instance = createInstance(); + final CloudProvisioningRecord r1 = createRecord(instance); + final String r1unwanted1 = createNodeName(); + final String r1unwanted2alreadyBeingDeleted = createNodeName(); + final String r1unwanted3 = createNodeName(); + r1.setCurrentlyUnwanted(r1unwanted1, false); + r1.setCurrentlyUnwanted(r1unwanted2alreadyBeingDeleted, true); + r1.setCurrentlyUnwanted(r1unwanted3, false); + final CloudProvisioningRecord r2 = createRecord(instance); + final String r2unwanted1alreadyBeingDeleted = createNodeName(); + final String r2unwanted2 = createNodeName(); + r2.setCurrentlyUnwanted(r2unwanted1alreadyBeingDeleted, true); + r2.setCurrentlyUnwanted(r2unwanted2, false); + // Expect to get the first not-being-deleted record from both before the + // second not-being-deleted record. However we don't mind the order that + // the records are processed, so there are two correct answers. + final List expectedA = Arrays.asList(r1unwanted1, r2unwanted2, r1unwanted3); + final List expectedB = Arrays.asList(r2unwanted2, r1unwanted1, r1unwanted3); + + // When + final List actual = instance.getUnwantedVMsThatNeedDeleting(); + // Then + assertThat(actual, either(equalTo(expectedA)).or(equalTo(expectedB))); + } + + @Test + public void unwantedSlaveNowDeletedGivenNothingElseToDeleteThenNothingRemains() { + // Given + final CloudProvisioningState instance = createInstance(); + final CloudProvisioningRecord r1 = createRecord(instance); + final String r1unwanted1 = createNodeName(); + final vSphereCloudSlaveTemplate r1t = r1.getTemplate(); + instance.recordExistingUnwantedVM(r1t, r1unwanted1); + final int numberOfNodesBeforeDeletion = instance.countNodes(); + assertThat(numberOfNodesBeforeDeletion, equalTo(1)); + final List noNodes = Arrays.asList(); + + // When + final Boolean actualIsOkToDelete = instance.isOkToDeleteUnwantedVM(r1unwanted1); + instance.unwantedSlaveNowDeleted(r1unwanted1); + final int actualNodesAfterDeletion = instance.countNodes(); + final List actualNodesToBeDeletedNext = instance.getUnwantedVMsThatNeedDeleting(); + + // Then + assertThat(actualIsOkToDelete, equalTo(Boolean.TRUE)); + assertThat(actualNodesAfterDeletion, equalTo(0)); + assertThat(actualNodesToBeDeletedNext, equalTo(noNodes)); + } + + @Test + public void unwantedSlaveNotDeletedGivenNothingElseToDeleteThenFailedDeletionRemains() { + // Given + final CloudProvisioningState instance = createInstance(); + final CloudProvisioningRecord r1 = createRecord(instance); + final String r1unwanted1 = createNodeName(); + final vSphereCloudSlaveTemplate r1t = r1.getTemplate(); + instance.recordExistingUnwantedVM(r1t, r1unwanted1); + final int numberOfNodesBeforeDeletion = instance.countNodes(); + assertThat(numberOfNodesBeforeDeletion, equalTo(1)); + final List sameNodeRemains = Arrays.asList(r1unwanted1); + + // When + final Boolean actualIsOkToDelete = instance.isOkToDeleteUnwantedVM(r1unwanted1); + instance.unwantedSlaveNotDeleted(r1unwanted1); + final int actualNodesAfterDeletion = instance.countNodes(); + final List actualNodesToBeDeletedNext = instance.getUnwantedVMsThatNeedDeleting(); + + // Then + assertThat(actualIsOkToDelete, equalTo(Boolean.TRUE)); + assertThat(actualNodesAfterDeletion, equalTo(1)); + assertThat(actualNodesToBeDeletedNext, equalTo(sameNodeRemains)); + } + + @Test + public void isOkToDeleteUnwantedVMGivenNobodyElseDeletingVMThenReturnsTrueOnceAndFalseThereafter() { + // Given + final CloudProvisioningState instance = createInstance(); + final CloudProvisioningRecord r1 = createRecord(instance); + final String r1unwanted1 = createNodeName(); + final vSphereCloudSlaveTemplate r1t = r1.getTemplate(); + instance.recordExistingUnwantedVM(r1t, r1unwanted1); + + // When + final Boolean actual1 = instance.isOkToDeleteUnwantedVM(r1unwanted1); + final Boolean actual2 = instance.isOkToDeleteUnwantedVM(r1unwanted1); + final Boolean actual3 = instance.isOkToDeleteUnwantedVM(r1unwanted1); + + // Then + assertThat(actual1, equalTo(Boolean.TRUE)); + assertThat(actual2, equalTo(Boolean.FALSE)); + assertThat(actual3, equalTo(Boolean.FALSE)); + } + + @Test + public void isOkToDeleteUnwantedVMGivenDeletionInProgressVMThenReturnsFalseUntilDeletionFails() { + // Given + final CloudProvisioningState instance = createInstance(); + final CloudProvisioningRecord r1 = createRecord(instance); + final String r1unwanted1 = createNodeName(); + r1.setCurrentlyUnwanted(r1unwanted1, true); + + // When + final Boolean actualBefore = instance.isOkToDeleteUnwantedVM(r1unwanted1); + instance.unwantedSlaveNotDeleted(r1unwanted1); + final Boolean actualAfter = instance.isOkToDeleteUnwantedVM(r1unwanted1); + + // Then + assertThat(actualBefore, equalTo(Boolean.FALSE)); + assertThat(actualAfter, equalTo(Boolean.TRUE)); + } + private void wipeLog() { loggedMessages.clear(); } From b9594afef5b74a99a65d0b9f426854e835346af8 Mon Sep 17 00:00:00 2001 From: Peter Darton Date: Thu, 6 Jul 2017 12:40:22 +0100 Subject: [PATCH 3/3] Slave termination now deletes VMs asynchronously. JENKINS-42187 applies to us too; same cause, same fix. So we avoid trying to delete VMs in-line with the slave's terminate method and instead schedule deletion for later. --- .../org/jenkinsci/plugins/vSphereCloud.java | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jenkinsci/plugins/vSphereCloud.java b/src/main/java/org/jenkinsci/plugins/vSphereCloud.java index 43ed1023..dad6e1b3 100644 --- a/src/main/java/org/jenkinsci/plugins/vSphereCloud.java +++ b/src/main/java/org/jenkinsci/plugins/vSphereCloud.java @@ -428,9 +428,21 @@ void provisionedSlaveHasTerminated(final String cloneName) { synchronized (templateState) { templateState.provisionedSlaveNowUnwanted(cloneName, true); } - attemptDeletionOfSlave("provisionedSlaveHasTerminated(" + cloneName + ")", cloneName); - // We should also take this opportunity to see if we've got any other - // slaves that need deleting. + // Deletion can take a long time, so we run it asynchronously because, + // at the point where we're called here, we've locked the remoting queue + // so Jenkins is largely crippled until we return. + // JENKINS-42187 describes the problem (for docker). + final Runnable task = new Runnable() { + @Override + public void run() { + attemptDeletionOfSlave("provisionedSlaveHasTerminated(" + cloneName + ")", cloneName); + } + }; + VSLOG.log(Level.INFO, "provisionedSlaveHasTerminated({0}): scheduling deletion of {0}", cloneName); + Computer.threadPoolForRemoting.submit(task); + // We also take this opportunity to see if we've got any other slaves + // that need deleting, and deal with at most one of those + // (asynchronously) as well. retryVMdeletionIfNecessary(1); } @@ -440,6 +452,10 @@ private void attemptDeletionOfSlave(final String why, final String cloneName) { boolean successfullyDeleted = false; try { vSphere = vSphereInstance(); + // Note: This can block indefinitely - it only completes when + // vSphere tells us the deletion has completed, and if vSphere has + // issues (e.g. a node failure) during that process then the + // deletion task can hang for ages. vSphere.destroyVm(cloneName, false); successfullyDeleted = true; VSLOG.log(Level.FINER, "{0}: VM {1} destroyed.", new Object[]{ why, cloneName });