diff --git a/src/main/java/org/jenkinsci/plugins/vSphereCloud.java b/src/main/java/org/jenkinsci/plugins/vSphereCloud.java index a1ba4122..dad6e1b3 100644 --- a/src/main/java/org/jenkinsci/plugins/vSphereCloud.java +++ b/src/main/java/org/jenkinsci/plugins/vSphereCloud.java @@ -187,11 +187,10 @@ private void ensureLists() { for (final vSphereCloudProvisionedSlave n : NodeIterator.nodes(vSphereCloudProvisionedSlave.class)) { final String nodeName = n.getNodeName(); final vSphereCloudSlaveTemplate template = getTemplateForVM(nodeName); - if (template != null) { - final CloudProvisioningRecord provisionable = templateState.getOrCreateRecord(template); - templateState.provisioningStarted(provisionable, nodeName); - templateState.provisionedSlaveNowActive(provisionable, nodeName); - } + if (template == null) continue; + final CloudProvisioningRecord provisionable = templateState.getOrCreateRecord(template); + templateState.provisioningStarted(provisionable, nodeName); + templateState.provisionedSlaveNowActive(provisionable, nodeName); } } } @@ -333,6 +332,7 @@ public Collection provision(final Label label, int excessWorkload) synchronized (this) { ensureLists(); } + retryVMdeletionIfNecessary(Math.max(excessWorkload, 2)); final List plannedNodes = new ArrayList(); synchronized (templateState) { templateState.pruneUnwantedRecords(); @@ -371,6 +371,48 @@ public Collection provision(final Label label, int excessWorkload) } } + /** + * Has another go at deleting VMs we failed to delete earlier. It's possible + * that we were unable to talk to vSphere (or some other failure happened) + * when we decided to delete some VMs. We remember this sort of thing so we + * can retry later - this is where we use this information. + * + * @param maxToRetryDeletionOn + * The maximum number of VMs to try to remove this time around. + * Can be {@link Integer#MAX_VALUE} for unlimited. + */ + private void retryVMdeletionIfNecessary(final int maxToRetryDeletionOn) { + if (templateState == null) { + VSLOG.log(Level.INFO, "retryVMdeletionIfNecessary({0}): templateState==null", maxToRetryDeletionOn); + return; + } + // find all candidates and trim down the list + final List unwantedVMsThatNeedDeleting = templateState.getUnwantedVMsThatNeedDeleting(); + final int numberToAttemptToRetryThisTime = Math.min(maxToRetryDeletionOn, unwantedVMsThatNeedDeleting.size()); + final List nodeNamesToRetryDeletion = unwantedVMsThatNeedDeleting.subList(0, + numberToAttemptToRetryThisTime); + // now queue their deletion + synchronized (templateState) { + for (final String nodeName : nodeNamesToRetryDeletion) { + final Boolean isOkToDelete = templateState.isOkToDeleteUnwantedVM(nodeName); + if (isOkToDelete == Boolean.TRUE) { + final Runnable task = new Runnable() { + @Override + public void run() { + attemptDeletionOfSlave("retryVMdeletionIfNecessary(" + nodeName + ")", nodeName); + } + }; + VSLOG.log(Level.INFO, "retryVMdeletionIfNecessary({0}): scheduling deletion of {1}", new Object[] { maxToRetryDeletionOn, nodeName }); + Computer.threadPoolForRemoting.submit(task); + } else { + VSLOG.log(Level.FINER, + "retryVMdeletionIfNecessary({0}): not going to try deleting {1} as isOkToDeleteUnwantedVM({1})=={2}", + new Object[]{ maxToRetryDeletionOn, nodeName, isOkToDelete }); + } + } + } + } + /** * This is called by {@link vSphereCloudProvisionedSlave} instances once * they terminate, so we can take note of their passing and then destroy the @@ -383,19 +425,52 @@ void provisionedSlaveHasTerminated(final String cloneName) { ensureLists(); } VSLOG.log(Level.FINER, "provisionedSlaveHasTerminated({0}): recording in our runtime state...", cloneName); - // once we're done, remove our cached record. synchronized (templateState) { - templateState.provisionedSlaveNowTerminated(cloneName); + templateState.provisionedSlaveNowUnwanted(cloneName, true); } - VSLOG.log(Level.FINER, "provisionedSlaveHasTerminated({0}): destroying VM...", cloneName); + // Deletion can take a long time, so we run it asynchronously because, + // at the point where we're called here, we've locked the remoting queue + // so Jenkins is largely crippled until we return. + // JENKINS-42187 describes the problem (for docker). + final Runnable task = new Runnable() { + @Override + public void run() { + attemptDeletionOfSlave("provisionedSlaveHasTerminated(" + cloneName + ")", cloneName); + } + }; + VSLOG.log(Level.INFO, "provisionedSlaveHasTerminated({0}): scheduling deletion of {0}", cloneName); + Computer.threadPoolForRemoting.submit(task); + // We also take this opportunity to see if we've got any other slaves + // that need deleting, and deal with at most one of those + // (asynchronously) as well. + retryVMdeletionIfNecessary(1); + } + + private void attemptDeletionOfSlave(final String why, final String cloneName) { + VSLOG.log(Level.FINER, "{0}: destroying VM {1}...", new Object[]{ why, cloneName }); VSphere vSphere = null; + boolean successfullyDeleted = false; try { vSphere = vSphereInstance(); + // Note: This can block indefinitely - it only completes when + // vSphere tells us the deletion has completed, and if vSphere has + // issues (e.g. a node failure) during that process then the + // deletion task can hang for ages. vSphere.destroyVm(cloneName, false); - VSLOG.log(Level.FINER, "provisionedSlaveHasTerminated({0}): VM destroyed.", cloneName); + successfullyDeleted = true; + VSLOG.log(Level.FINER, "{0}: VM {1} destroyed.", new Object[]{ why, cloneName }); + vSphere.disconnect(); + vSphere = null; } catch (VSphereException ex) { - VSLOG.log(Level.SEVERE, "provisionedSlaveHasTerminated(" + cloneName + "): Exception while trying to destroy VM", ex); + VSLOG.log(Level.SEVERE, why + ": Exception while trying to destroy VM " + cloneName, ex); } finally { + synchronized (templateState) { + if (successfullyDeleted) { + templateState.unwantedSlaveNowDeleted(cloneName); + } else { + templateState.unwantedSlaveNotDeleted(cloneName); + } + } if (vSphere != null) { vSphere.disconnect(); } diff --git a/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithm.java b/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithm.java index 30d489ad..20f37720 100644 --- a/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithm.java +++ b/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithm.java @@ -1,139 +1,137 @@ -package org.jenkinsci.plugins.vsphere.tools; - -import java.math.BigInteger; -import java.util.Collection; -import java.util.Iterator; -import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.UUID; - -import org.jenkinsci.plugins.vSphereCloudSlaveTemplate; -import org.kohsuke.accmod.Restricted; -import org.kohsuke.accmod.restrictions.NoExternalUse; - -/** - * How we decide what template to create the next slave on. - */ -public final class CloudProvisioningAlgorithm { - private CloudProvisioningAlgorithm() { - } - - /** - * Given a bunch of templates to choose from, works out which one we should - * use next. - * - * @param provisionables - * Template records to decide between. - * @return The record with the most free capacity, or null if there are none - * with any capacity. - */ - public static CloudProvisioningRecord findTemplateWithMostFreeCapacity( - Collection provisionables) { - final SortedSet sortedSet = new TreeSet( - CloudProvisioningRecord.leastUsedFirst); - sortedSet.addAll(provisionables); - final Iterator iterator = sortedSet.iterator(); - if (iterator.hasNext()) { - final CloudProvisioningRecord bestOption = iterator.next(); - if (bestOption.hasCapacityForMore()) { - return bestOption; - } - } - return null; - } - - /** - * Chooses a name for a new node. - *
    - *
  • If the template has a limited number of instances available then the - * name will be of the form "prefix_number" where "number" is a number that - * should be between 1 and the number of permitted instances.
    • - *
  • If the template has an unlimited number of instances available then - * the name will be of the form "prefix_random" where "random" is a random - * UUID's 32-byte number (rendered using a high radix to keep the string - * short).
    • - *
- * - * @param record - * Our record regarding the template the slave will be created - * from. - * @return A name for the new node. This will start with the - * {@link vSphereCloudSlaveTemplate#getCloneNamePrefix()}. - */ - public static String findUnusedName(CloudProvisioningRecord record) { - final vSphereCloudSlaveTemplate template = record.getTemplate(); - final String cloneNamePrefix = template.getCloneNamePrefix(); - final Set existingNames = new TreeSet(); - existingNames.addAll(record.getCurrentlyPlanned()); - existingNames.addAll(record.getCurrentlyProvisioned()); - final int templateInstanceCap = template.getTemplateInstanceCap(); - final boolean hasCap = templateInstanceCap > 0 && templateInstanceCap < Integer.MAX_VALUE; - final int maxAttempts = hasCap ? (templateInstanceCap + 1) : 100; - for (int attempt = 0; attempt < maxAttempts; attempt++) { - final String suffix = hasCap ? calcSequentialSuffix(attempt) : calcRandomSuffix(attempt); - final String nodeName = cloneNamePrefix + "_" + suffix; - if (!existingNames.contains(nodeName)) { - return nodeName; - } - } - throw new IllegalStateException("Unable to find unused name for slave for record " + record.toString() - + ", even after " + maxAttempts + " attempts."); - } - - private static String calcSequentialSuffix(final int attempt) { - final int slaveNumber = attempt + 1; - final String suffix = Integer.toString(slaveNumber); - return suffix; - } - - private static String calcRandomSuffix(int attempt) { - // get "unique" UUID - final UUID uuid = UUID.randomUUID(); - // put both "long"s into a BigInteger. - final long lsb = uuid.getLeastSignificantBits(); - final long msb = uuid.getMostSignificantBits(); - final BigInteger bigNumber = toBigInteger(msb, lsb); - // turn into a string - final String suffix = bigNumber.toString(Character.MAX_RADIX); - return suffix; - } - - /** - * Turns two 64-bit long numbers into a positive 128-bit {@link BigInteger} - * that's in the range 0 to 2128-1. - *

- * Note: This is only package-level access for unit-testing. - *

- * - * @param msb - * The most-significant 64 bits. - * @param lsb - * The least-significant 64 bits. - * @return A {@link BigInteger}. - */ - @Restricted(NoExternalUse.class) - static BigInteger toBigInteger(final long msb, final long lsb) { - final byte[] bytes = new byte[17]; - int b = 0; - bytes[b++] = (byte) 0; // ensure we're all positive - bytes[b++] = (byte) (msb >> 56); - bytes[b++] = (byte) (msb >> 48); - bytes[b++] = (byte) (msb >> 40); - bytes[b++] = (byte) (msb >> 32); - bytes[b++] = (byte) (msb >> 24); - bytes[b++] = (byte) (msb >> 16); - bytes[b++] = (byte) (msb >> 8); - bytes[b++] = (byte) (msb); - bytes[b++] = (byte) (lsb >> 56); - bytes[b++] = (byte) (lsb >> 48); - bytes[b++] = (byte) (lsb >> 40); - bytes[b++] = (byte) (lsb >> 32); - bytes[b++] = (byte) (lsb >> 24); - bytes[b++] = (byte) (lsb >> 16); - bytes[b++] = (byte) (lsb >> 8); - bytes[b++] = (byte) (lsb); - final BigInteger bigNumber = new BigInteger(bytes); - return bigNumber; - } +package org.jenkinsci.plugins.vsphere.tools; + +import java.math.BigInteger; +import java.util.Collection; +import java.util.Iterator; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.UUID; + +import org.jenkinsci.plugins.vSphereCloudSlaveTemplate; +import org.kohsuke.accmod.Restricted; +import org.kohsuke.accmod.restrictions.NoExternalUse; + +/** + * How we decide what template to create the next slave on. + */ +public final class CloudProvisioningAlgorithm { + private CloudProvisioningAlgorithm() { + } + + /** + * Given a bunch of templates to choose from, works out which one we should + * use next. + * + * @param provisionables + * Template records to decide between. + * @return The record with the most free capacity, or null if there are none + * with any capacity. + */ + public static CloudProvisioningRecord findTemplateWithMostFreeCapacity( + Collection provisionables) { + final SortedSet sortedSet = new TreeSet( + CloudProvisioningRecord.leastUsedFirst); + sortedSet.addAll(provisionables); + final Iterator iterator = sortedSet.iterator(); + if (iterator.hasNext()) { + final CloudProvisioningRecord bestOption = iterator.next(); + if (bestOption.hasCapacityForMore()) { + return bestOption; + } + } + return null; + } + + /** + * Chooses a name for a new node. + *
    + *
  • If the template has a limited number of instances available then the + * name will be of the form "prefix_number" where "number" is a number that + * should be between 1 and the number of permitted instances.
    • + *
  • If the template has an unlimited number of instances available then + * the name will be of the form "prefix_random" where "random" is a random + * UUID's 32-byte number (rendered using a high radix to keep the string + * short).
    • + *
+ * + * @param record + * Our record regarding the template the slave will be created + * from. + * @return A name for the new node. This will start with the + * {@link vSphereCloudSlaveTemplate#getCloneNamePrefix()}. + */ + public static String findUnusedName(CloudProvisioningRecord record) { + final vSphereCloudSlaveTemplate template = record.getTemplate(); + final String cloneNamePrefix = template.getCloneNamePrefix(); + final Set existingNames = record.getCurrentNames(); + final int templateInstanceCap = template.getTemplateInstanceCap(); + final boolean hasCap = templateInstanceCap > 0 && templateInstanceCap < Integer.MAX_VALUE; + final int maxAttempts = hasCap ? (templateInstanceCap) : 100; + for (int attempt = 0; attempt < maxAttempts; attempt++) { + final String suffix = hasCap ? calcSequentialSuffix(attempt) : calcRandomSuffix(attempt); + final String nodeName = cloneNamePrefix + "_" + suffix; + if (!existingNames.contains(nodeName)) { + return nodeName; + } + } + throw new IllegalStateException("Unable to find unused name for slave for record " + record.toString() + + ", even after " + maxAttempts + " attempts."); + } + + private static String calcSequentialSuffix(final int attempt) { + final int slaveNumber = attempt + 1; + final String suffix = Integer.toString(slaveNumber); + return suffix; + } + + private static String calcRandomSuffix(int attempt) { + // get "unique" UUID + final UUID uuid = UUID.randomUUID(); + // put both "long"s into a BigInteger. + final long lsb = uuid.getLeastSignificantBits(); + final long msb = uuid.getMostSignificantBits(); + final BigInteger bigNumber = toBigInteger(msb, lsb); + // turn into a string + final String suffix = bigNumber.toString(Character.MAX_RADIX); + return suffix; + } + + /** + * Turns two 64-bit long numbers into a positive 128-bit {@link BigInteger} + * that's in the range 0 to 2128-1. + *

+ * Note: This is only package-level access for unit-testing. + *

+ * + * @param msb + * The most-significant 64 bits. + * @param lsb + * The least-significant 64 bits. + * @return A {@link BigInteger}. + */ + @Restricted(NoExternalUse.class) + static BigInteger toBigInteger(final long msb, final long lsb) { + final byte[] bytes = new byte[17]; + int b = 0; + bytes[b++] = (byte) 0; // ensure we're all positive + bytes[b++] = (byte) (msb >> 56); + bytes[b++] = (byte) (msb >> 48); + bytes[b++] = (byte) (msb >> 40); + bytes[b++] = (byte) (msb >> 32); + bytes[b++] = (byte) (msb >> 24); + bytes[b++] = (byte) (msb >> 16); + bytes[b++] = (byte) (msb >> 8); + bytes[b++] = (byte) (msb); + bytes[b++] = (byte) (lsb >> 56); + bytes[b++] = (byte) (lsb >> 48); + bytes[b++] = (byte) (lsb >> 40); + bytes[b++] = (byte) (lsb >> 32); + bytes[b++] = (byte) (lsb >> 24); + bytes[b++] = (byte) (lsb >> 16); + bytes[b++] = (byte) (lsb >> 8); + bytes[b++] = (byte) (lsb); + final BigInteger bigNumber = new BigInteger(bytes); + return bigNumber; + } } \ No newline at end of file diff --git a/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningRecord.java b/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningRecord.java index 727539ab..8ea98493 100644 --- a/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningRecord.java +++ b/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningRecord.java @@ -1,193 +1,244 @@ -package org.jenkinsci.plugins.vsphere.tools; - -import java.util.Comparator; -import java.util.Set; -import java.util.TreeSet; - -import org.jenkinsci.plugins.vSphereCloudSlaveTemplate; - -/** - * There's a delay between when we give a bunch of slave nodes to Jenkins (when - * it asks us to provision some) and when those nodes appear in vSphere and in - * Jenkins, so we need to keep a record of what's in progress so we don't - * over-commit. - */ -public final class CloudProvisioningRecord { - private final vSphereCloudSlaveTemplate template; - private Set currentlyProvisioned; - private Set currentlyPlanned; - - CloudProvisioningRecord(vSphereCloudSlaveTemplate template) { - this.template = template; - this.currentlyProvisioned = new TreeSet(); - this.currentlyPlanned = new TreeSet(); - } - - public vSphereCloudSlaveTemplate getTemplate() { - return template; - } - - @Override - public String toString() { - return String.format("Template[prefix=%s, provisioned=%s, planned=%s, max=%d, fullness=%.3f%%]", getTemplate() - .getCloneNamePrefix(), getCurrentlyProvisioned(), getCurrentlyPlanned(), calcMaxToProvision(), - calcFullness() * 100.0); - } - - Set getCurrentlyProvisioned() { - return currentlyProvisioned; - } - - boolean addCurrentlyActive(String nodeName) { - return currentlyProvisioned.add(nodeName); - } - - boolean removeCurrentlyActive(String nodeName) { - return currentlyProvisioned.remove(nodeName); - } - - Set getCurrentlyPlanned() { - return currentlyPlanned; - } - - boolean addCurrentlyPlanned(String nodeName) { - return currentlyPlanned.add(nodeName); - } - - boolean removeCurrentlyPlanned(String nodeName) { - return currentlyPlanned.remove(nodeName); - } - - private int calcMaxToProvision() { - final int templateInstanceCap = template.getTemplateInstanceCap(); - final int maxToProvision = templateInstanceCap == 0 ? Integer.MAX_VALUE : templateInstanceCap; - return maxToProvision; - } - - private boolean hasFiniteCapacity() { - final int templateInstanceCap = template.getTemplateInstanceCap(); - final int maxToProvision = templateInstanceCap == 0 ? Integer.MAX_VALUE : templateInstanceCap; - return maxToProvision != Integer.MAX_VALUE; - } - - private double calcFullness() { - final int maxToProvision = calcMaxToProvision(); - return ((double) calcCurrentCommitment()) / (double) maxToProvision; - } - - boolean hasCapacityForMore() { - final int totalCommitment = calcCurrentCommitment(); - final int maxToProvision = calcMaxToProvision(); - return maxToProvision > totalCommitment; - } - - private int calcCurrentCommitment() { - return currentlyProvisioned.size() + currentlyPlanned.size(); - } - - /** - * Sorts {@link CloudProvisioningRecord}s, putting the ones with most free - * capacity first. - *

- * When comparing two records with finite capacity then their usage:limit - * ratios are compared, otherwise current usage levels are compared. - */ - static final Comparator leastUsedFirst = new Comparator() { - private static final int theyAreTheSame = 0; - private static final int bShouldComeLast = -1; - private static final int aShouldComeLast = 1; - - @Override - public int compare(CloudProvisioningRecord a, CloudProvisioningRecord b) { - if (b == a) { - return theyAreTheSame; - } - final int compareByCapacity; - if (a.hasFiniteCapacity() && b.hasFiniteCapacity()) { - compareByCapacity = compareByUsageRatio(a, b); - } else { - compareByCapacity = compareByUsage(a, b); - } - if (compareByCapacity != theyAreTheSame) { - return compareByCapacity; - } - final int compareByMaxCapacity = compareByMaxCapacity(a, b); - if (compareByMaxCapacity != theyAreTheSame) { - return compareByMaxCapacity; - } - return tieBreak(a, b); - } - - /** if both have instance caps, we rank by utilization:capacity ratio */ - private int compareByUsageRatio(CloudProvisioningRecord a, CloudProvisioningRecord b) { - // sort by utilization:capacity ratio - lowest usage comes first - final double aFullness = a.calcFullness(); - final double bFullness = b.calcFullness(); - if (aFullness > bFullness) { - return aShouldComeLast; - } - if (aFullness < bFullness) { - return bShouldComeLast; - } - return theyAreTheSame; - } - - /** - * if either has no instance cap, we rank by least usage UNLESS one of - * them is full - */ - private int compareByUsage(CloudProvisioningRecord a, CloudProvisioningRecord b) { - // sort by "is full" - ones that are full come last - final boolean aFull = !a.hasCapacityForMore(); - final boolean bFull = !b.hasCapacityForMore(); - if (aFull != bFull) { - if (aFull) { - return aShouldComeLast; - } else { - return bShouldComeLast; - } - } - // sort by utilization - lowest usage comes first - final double aUsage = a.calcCurrentCommitment(); - final double bUsage = b.calcCurrentCommitment(); - if (aUsage > bUsage) { - return aShouldComeLast; - } - if (aUsage < bUsage) { - return bShouldComeLast; - } - return theyAreTheSame; - } - - /** Try rank by capacity */ - private int compareByMaxCapacity(CloudProvisioningRecord a, CloudProvisioningRecord b) { - // by absolute capacity - highest comes first - final int aCapacity = a.calcMaxToProvision(); - final int bCapacity = b.calcMaxToProvision(); - if (bCapacity > aCapacity) { - return aShouldComeLast; - } - if (bCapacity < aCapacity) { - return bShouldComeLast; - } - return theyAreTheSame; - } - - /** - * if all else is equal we prefer the one with fewer VMs being started - * up - */ - private int tieBreak(CloudProvisioningRecord a, CloudProvisioningRecord b) { - // then by number of VMs being started - lowest comes first - final int aCurrentlyPlanned = a.currentlyPlanned.size(); - final int bCurrentlyPlanned = b.currentlyPlanned.size(); - if (aCurrentlyPlanned > bCurrentlyPlanned) { - return aShouldComeLast; - } - if (aCurrentlyPlanned < bCurrentlyPlanned) { - return bShouldComeLast; - } - return theyAreTheSame; - } - }; +package org.jenkinsci.plugins.vsphere.tools; + +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +import org.jenkinsci.plugins.vSphereCloudSlaveTemplate; + +/** + * There's a delay between when we give a bunch of slave nodes to Jenkins (when + * it asks us to provision some) and when those nodes appear in vSphere and in + * Jenkins, so we need to keep a record of what's in progress so we don't + * over-commit. + * Similarly there's also a delay between when we decide to delete some and + * when we successfully delete the VMs, and so we need to know what VMs we need + * to delete, and also what ones we're in the process of deleting. + */ +public final class CloudProvisioningRecord { + private final vSphereCloudSlaveTemplate template; + private final Set currentlyProvisioned; + private final Set currentlyPlanned; + private final Map currentlyUnwanted; + + CloudProvisioningRecord(vSphereCloudSlaveTemplate template) { + this.template = template; + this.currentlyProvisioned = new TreeSet(); + this.currentlyPlanned = new TreeSet(); + this.currentlyUnwanted = new LinkedHashMap(); + } + + public vSphereCloudSlaveTemplate getTemplate() { + return template; + } + + @Override + public String toString() { + return String.format("Template[prefix=%s, provisioned=%s, planned=%s, unwanted=%s, max=%d, fullness=%.3f%%]", + getTemplate().getCloneNamePrefix(), + getCurrentlyProvisioned(), + getCurrentlyPlanned(), + getCurrentlyUnwanted(), + calcMaxToProvision(), + calcFullness() * 100.0); + } + + Set getCurrentlyProvisioned() { + return currentlyProvisioned; + } + + boolean addCurrentlyActive(String nodeName) { + return currentlyProvisioned.add(nodeName); + } + + boolean removeCurrentlyActive(String nodeName) { + return currentlyProvisioned.remove(nodeName); + } + + Set getCurrentlyPlanned() { + return currentlyPlanned; + } + + boolean addCurrentlyPlanned(String nodeName) { + return currentlyPlanned.add(nodeName); + } + + boolean removeCurrentlyPlanned(String nodeName) { + return currentlyPlanned.remove(nodeName); + } + + Map getCurrentlyUnwanted() { + return currentlyUnwanted; + } + + Boolean isCurrentlyUnwanted(String nodeName) { + return currentlyUnwanted.get(nodeName); + } + + Boolean setCurrentlyUnwanted(String nodeName, boolean beingDeleted) { + // ensure this node gets pushed to the end of the map by doing a remove then a put. + final Boolean oldValue = currentlyUnwanted.remove(nodeName); + currentlyUnwanted.put(nodeName, Boolean.valueOf(beingDeleted)); + return oldValue; + } + + boolean removeCurrentlyUnwanted(String nodeName) { + return currentlyUnwanted.remove(nodeName) != null; + } + + Set getCurrentNames() { + final Set existingNames = new TreeSet(); + existingNames.addAll(getCurrentlyPlanned()); + existingNames.addAll(getCurrentlyProvisioned()); + existingNames.addAll(getCurrentlyUnwanted().keySet()); + return existingNames; + } + + boolean contains(String nodeName) { + return currentlyProvisioned.contains(nodeName) || currentlyPlanned.contains(nodeName) + || currentlyUnwanted.containsKey(nodeName); + } + + int size() { + return currentlyProvisioned.size() + currentlyPlanned.size() + currentlyUnwanted.size(); + } + + boolean isEmpty() { + return currentlyProvisioned.isEmpty() && currentlyPlanned.isEmpty() && currentlyUnwanted.isEmpty(); + } + + private int calcMaxToProvision() { + final int templateInstanceCap = template.getTemplateInstanceCap(); + final int maxToProvision = templateInstanceCap == 0 ? Integer.MAX_VALUE : templateInstanceCap; + return maxToProvision; + } + + private boolean hasFiniteCapacity() { + final int templateInstanceCap = template.getTemplateInstanceCap(); + final int maxToProvision = templateInstanceCap == 0 ? Integer.MAX_VALUE : templateInstanceCap; + return maxToProvision != Integer.MAX_VALUE; + } + + private double calcFullness() { + final int maxToProvision = calcMaxToProvision(); + return ((double) calcCurrentCommitment()) / (double) maxToProvision; + } + + boolean hasCapacityForMore() { + final int totalCommitment = calcCurrentCommitment(); + final int maxToProvision = calcMaxToProvision(); + return maxToProvision > totalCommitment; + } + + private int calcCurrentCommitment() { + return currentlyProvisioned.size() + currentlyPlanned.size() + currentlyUnwanted.size(); + } + + /** + * Sorts {@link CloudProvisioningRecord}s, putting the ones with most free + * capacity first. + *

+ * When comparing two records with finite capacity then their usage:limit + * ratios are compared, otherwise current usage levels are compared. + */ + static final Comparator leastUsedFirst = new Comparator() { + private static final int theyAreTheSame = 0; + private static final int bShouldComeLast = -1; + private static final int aShouldComeLast = 1; + + @Override + public int compare(CloudProvisioningRecord a, CloudProvisioningRecord b) { + if (b == a) { + return theyAreTheSame; + } + final int compareByCapacity; + if (a.hasFiniteCapacity() && b.hasFiniteCapacity()) { + compareByCapacity = compareByUsageRatio(a, b); + } else { + compareByCapacity = compareByUsage(a, b); + } + if (compareByCapacity != theyAreTheSame) { + return compareByCapacity; + } + final int compareByMaxCapacity = compareByMaxCapacity(a, b); + if (compareByMaxCapacity != theyAreTheSame) { + return compareByMaxCapacity; + } + return tieBreak(a, b); + } + + /** if both have instance caps, we rank by utilization:capacity ratio */ + private int compareByUsageRatio(CloudProvisioningRecord a, CloudProvisioningRecord b) { + // sort by utilization:capacity ratio - lowest usage comes first + final double aFullness = a.calcFullness(); + final double bFullness = b.calcFullness(); + if (aFullness > bFullness) { + return aShouldComeLast; + } + if (aFullness < bFullness) { + return bShouldComeLast; + } + return theyAreTheSame; + } + + /** + * if either has no instance cap, we rank by least usage UNLESS one of + * them is full + */ + private int compareByUsage(CloudProvisioningRecord a, CloudProvisioningRecord b) { + // sort by "is full" - ones that are full come last + final boolean aFull = !a.hasCapacityForMore(); + final boolean bFull = !b.hasCapacityForMore(); + if (aFull != bFull) { + if (aFull) { + return aShouldComeLast; + } else { + return bShouldComeLast; + } + } + // sort by utilization - lowest usage comes first + final double aUsage = a.calcCurrentCommitment(); + final double bUsage = b.calcCurrentCommitment(); + if (aUsage > bUsage) { + return aShouldComeLast; + } + if (aUsage < bUsage) { + return bShouldComeLast; + } + return theyAreTheSame; + } + + /** Try rank by capacity */ + private int compareByMaxCapacity(CloudProvisioningRecord a, CloudProvisioningRecord b) { + // by absolute capacity - highest comes first + final int aCapacity = a.calcMaxToProvision(); + final int bCapacity = b.calcMaxToProvision(); + if (bCapacity > aCapacity) { + return aShouldComeLast; + } + if (bCapacity < aCapacity) { + return bShouldComeLast; + } + return theyAreTheSame; + } + + /** + * if all else is equal we prefer the one with fewer VMs being started + * up + */ + private int tieBreak(CloudProvisioningRecord a, CloudProvisioningRecord b) { + // then by number of VMs being started - lowest comes first + final int aCurrentlyPlanned = a.currentlyPlanned.size(); + final int bCurrentlyPlanned = b.currentlyPlanned.size(); + if (aCurrentlyPlanned > bCurrentlyPlanned) { + return aShouldComeLast; + } + if (aCurrentlyPlanned < bCurrentlyPlanned) { + return bShouldComeLast; + } + return theyAreTheSame; + } + }; } \ No newline at end of file diff --git a/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningState.java b/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningState.java index be8f1183..507088d9 100644 --- a/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningState.java +++ b/src/main/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningState.java @@ -1,286 +1,465 @@ -package org.jenkinsci.plugins.vsphere.tools; - -import java.util.ArrayList; -import java.util.IdentityHashMap; -import java.util.List; -import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; - -import org.jenkinsci.plugins.vSphereCloud; -import org.jenkinsci.plugins.vSphereCloudSlaveTemplate; - -/** - * Utility class that works out what slaves we should start up in response to - * Jenkins asking us to start things. - *

- * We do this by keeping a record of every slave we start, and every slave we - * have active. That way, we can avoid over-provisioning. - *

- *

- * The idea is that we are told what slaves that the cloud is going to create, - * when the cloud has created them (or failed to) and when those slaves have - * died. This way we can keep track of everything, in order to allow the cloud - * to make accurate decisions regarding what to create next. - *

- * Note: This is not thread-safe. Callers must do their own synchronization. - */ -public class CloudProvisioningState { - private static final Logger LOGGER = Logger.getLogger(CloudProvisioningState.class.getName()); - /** - * Record of slaves we've told Jenkins to start up, which have yet to start. - */ - private final Map records = new IdentityHashMap(); - /** - * Our parent, so we can check what templates still exist (as the user may - * have added/removed some). - */ - private final vSphereCloud parent; - /** - * Where we log to. This is only instance-based for test-purposes, and - * transient to stop serialization problems. - */ - private transient final Logger logger; - - public CloudProvisioningState(vSphereCloud parent) { - this(parent, LOGGER); - } - - CloudProvisioningState(vSphereCloud parent, Logger logger) { - this.parent = parent; - this.logger = logger; - this.logger.log(Level.FINE, "Created for parent {0}", parent.toString()); - } - - /** - * To be called when we've decided to create a new node. Callers MUST ensure - * that {@link #provisionedSlaveNowActive(CloudProvisioningRecord, String)} - * or {@link #provisioningEndedInError(CloudProvisioningRecord, String)} - * gets called later. - * - * @param provisionable - * Our record for the template for the named node. - * @param nodeName - * The name of the VM. - */ - public void provisioningStarted(CloudProvisioningRecord provisionable, String nodeName) { - final boolean wasPreviouslyUnknownToPlanning = provisionable.addCurrentlyPlanned(nodeName); - final boolean wasAlreadyActive = provisionable.removeCurrentlyActive(nodeName); - logStateChange(Level.FINE, "Intending to create {0}", "wasPreviouslyUnknownToPlanning", - wasPreviouslyUnknownToPlanning, true, "wasAlreadyActive", wasAlreadyActive, false, nodeName); - } - - /** - * To be called when a newly created node (previously promised to - * {@link #provisioningStarted(CloudProvisioningRecord, String)}) comes up. - * Callers MUST ensure that - * {@link #provisionedSlaveNowTerminated(String)} - * gets called later. - * - * @param provisionable - * Our record for the template for the named node. - * @param nodeName - * The name of the VM. - */ - public void provisionedSlaveNowActive(CloudProvisioningRecord provisionable, String nodeName) { - final boolean wasNotPreviouslyActive = provisionable.addCurrentlyActive(nodeName); - final boolean wasPreviouslyPlanned = provisionable.removeCurrentlyPlanned(nodeName); - logStateChange(Level.FINE, "Marking {0} as active", "wasNotPreviouslyActive", wasNotPreviouslyActive, true, - "wasPreviouslyPlanned", wasPreviouslyPlanned, true, nodeName); - } - - /** - * To be called when a node we created (previously told to - * {@link #provisionedSlaveNowActive(CloudProvisioningRecord, String)}) has - * died. - * - * @param nodeName - * The name of the VM. - */ - public void provisionedSlaveNowTerminated(String nodeName) { - final Map.Entry entry = findEntryForVM(nodeName); - if (entry != null) { - final CloudProvisioningRecord provisionable = entry.getValue(); - final boolean wasPreviouslyPlanned = provisionable.removeCurrentlyPlanned(nodeName); - final boolean wasPreviouslyActive = provisionable.removeCurrentlyActive(nodeName); - if (recordIsPrunable(provisionable)) { - removeExistingRecord(provisionable); - } - logStateChange(Level.FINE, "Marking {0} as terminated", "wasPreviouslyPlanned", wasPreviouslyPlanned, - false, "wasPreviouslyActive", wasPreviouslyActive, true, nodeName); - } else { - logger.log(Level.WARNING, "Asked to mark {0} as terminated, but we have no record of it.", nodeName); - } - } - - /** - * To be called when a node that we previously promised to create (by - * calling {@link #provisioningStarted(CloudProvisioningRecord, String)}) - * failed to start. - * - * @param provisionable - * Our record for the template for the named node. - * @param nodeName - * The name of the VM. - */ - public void provisioningEndedInError(CloudProvisioningRecord provisionable, String nodeName) { - final boolean wasPreviouslyPlanned = provisionable.removeCurrentlyPlanned(nodeName); - final boolean wasPreviouslyActive = provisionable.removeCurrentlyActive(nodeName); - if (recordIsPrunable(provisionable)) { - removeExistingRecord(provisionable); - } - logStateChange(Level.INFO, "Marking {0} as failed", "wasPreviouslyPlanned", wasPreviouslyPlanned, true, - "wasPreviouslyActive", wasPreviouslyActive, false, nodeName); - } - - /** - * To be called every now and again to ensure that we're not caching records - * that will never be valid again. - */ - public void pruneUnwantedRecords() { - final List toBeRemoved = new ArrayList(records.size()); - for (final Map.Entry entry : records.entrySet()) { - final CloudProvisioningRecord record = entry.getValue(); - if (recordIsPrunable(record)) { - toBeRemoved.add(record); - } - } - for (final CloudProvisioningRecord record : toBeRemoved) { - removeExistingRecord(record); - } - } - - /** - * Given a set of templates, returns the equivalent records. - * - * @param templates - * The templates we are interested in. - * @return A list of {@link CloudProvisioningRecord}. - */ - public List calculateProvisionableTemplates(Iterable templates) { - final List result = new ArrayList(); - for (final vSphereCloudSlaveTemplate template : templates) { - final CloudProvisioningRecord provisionable = getOrCreateRecord(template); - result.add(provisionable); - } - return result; - } - - /** - * Counts all the known nodes, both active and in-progress, across all - * templates. - * - * @return The number of nodes that are active or soon-to-be-active. - */ - public int countNodes() { - int result = 0; - for (final CloudProvisioningRecord record : records.values()) { - result += record.getCurrentlyPlanned().size(); - result += record.getCurrentlyProvisioned().size(); - } - return result; - } - - /** - * Gets the record for the given template. If we didn't have one before, we - * create one. - * - * @param template - * The template in question. - * @return The one-and-only record for this template. - */ - public CloudProvisioningRecord getOrCreateRecord(final vSphereCloudSlaveTemplate template) { - final CloudProvisioningRecord existingRecord = getExistingRecord(template); - if (existingRecord != null) { - return existingRecord; - } - final CloudProvisioningRecord newRecord = new CloudProvisioningRecord(template); - logger.log(Level.FINE, "Creating new record for template {0} ({1})", - new Object[] { template.getCloneNamePrefix(), template.toString() }); - records.put(template, newRecord); - return newRecord; - } - - private CloudProvisioningRecord getExistingRecord(final vSphereCloudSlaveTemplate template) { - return records.get(template); - } - - private void removeExistingRecord(CloudProvisioningRecord existingRecord) { - final vSphereCloudSlaveTemplate template = existingRecord.getTemplate(); - logger.log(Level.FINE, "Disposing of record for template {0} ({1})", - new Object[] { template.getCloneNamePrefix(), template.toString() }); - records.remove(template); - } - - private boolean recordIsPrunable(CloudProvisioningRecord record) { - final boolean isEmpty = record.getCurrentlyProvisioned().isEmpty() && record.getCurrentlyPlanned().isEmpty(); - if (!isEmpty) { - return false; - } - final vSphereCloudSlaveTemplate template = record.getTemplate(); - final List knownTemplates = parent.getTemplates(); - final boolean isKnownToParent = knownTemplates.contains(template); - return !isKnownToParent; - } - - private Map.Entry findEntryForVM(String nodeName) { - for (final Map.Entry entry : records.entrySet()) { - final CloudProvisioningRecord record = entry.getValue(); - if (record.getCurrentlyProvisioned().contains(nodeName)) { - return entry; - } - if (record.getCurrentlyPlanned().contains(nodeName)) { - return entry; - } - } - return null; - } - - /** - * Logs a state change. If the state change isn't valid, it's logged as a - * warning. - * - * @param logLevel - * The level to log the message at, if the boolean arguments are - * as their expected values. - * @param logMsg - * The message to log. - * @param firstArgName - * What actualFirstArgValue represents - used when complaining - * about its value. - * @param actualFirstArgValue - * A state-change variable. - * @param expectedFirstArgValue - * The expected value of actualFirstArgValue. If that's not the - * case, we'll complain. - * @param secondArgName - * What actualSecondArgValue represents - used when complaining - * about its value. - * @param actualSecondArgValue - * A state-change variable. - * @param expectedSecondArgValue - * The expected value of actualSecondArgValue. If that's not the - * case, we'll complain. - * @param args - * The arguments for logMsg. Used if logMsg contains {0}, {1} - * etc. - */ - private void logStateChange(Level logLevel, String logMsg, String firstArgName, boolean actualFirstArgValue, - boolean expectedFirstArgValue, String secondArgName, boolean actualSecondArgValue, - boolean expectedSecondArgValue, Object... args) { - final boolean firstValid = actualFirstArgValue == expectedFirstArgValue; - final boolean secondValid = actualSecondArgValue == expectedSecondArgValue; - Level actualLevel = logLevel; - String actualMsg = logMsg; - if (!firstValid) { - actualMsg += " : " + firstArgName + "!=" + expectedFirstArgValue; - actualLevel = Level.WARNING; - } - if (!secondValid) { - actualMsg += " : " + secondArgName + "!=" + expectedSecondArgValue; - actualLevel = Level.WARNING; - } - final Logger loggerToUse = logger != null ? logger : LOGGER; - loggerToUse.log(actualLevel, actualMsg, args); - } -} +package org.jenkinsci.plugins.vsphere.tools; + +import java.util.ArrayList; +import java.util.IdentityHashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.jenkinsci.plugins.vSphereCloud; +import org.jenkinsci.plugins.vSphereCloudSlaveTemplate; + +/** + * Utility class that works out what slaves we should start up in response to + * Jenkins asking us to start things. + *

+ * We do this by keeping a record of every slave we start, and every slave we + * have active. That way, we can avoid over-provisioning. + *

+ *

+ * The idea is that we are told what slaves that the cloud is going to create, + * when the cloud has created them (or failed to) and when those slaves have + * died. This way we can keep track of everything, in order to allow the cloud + * to make accurate decisions regarding what to create next. + *

+ * Note: This is not thread-safe. Callers must do their own synchronization. + */ +public class CloudProvisioningState { + private static final Logger LOGGER = Logger.getLogger(CloudProvisioningState.class.getName()); + /** + * Record of slaves we've told Jenkins to start up, which have yet to start. + */ + private final Map records = new IdentityHashMap(); + /** + * Our parent, so we can check what templates still exist (as the user may + * have added/removed some). + */ + private final vSphereCloud parent; + /** + * Where we log to. This is only instance-based for test-purposes, and + * transient to stop serialization problems. + */ + private transient final Logger logger; + + public CloudProvisioningState(vSphereCloud parent) { + this(parent, LOGGER); + } + + CloudProvisioningState(vSphereCloud parent, Logger logger) { + this.parent = parent; + this.logger = logger; + this.logger.log(Level.FINE, "Created for parent {0}", parent.toString()); + } + + /** + * To be called when we've decided to create a new node. Callers MUST ensure + * that {@link #provisionedSlaveNowActive(CloudProvisioningRecord, String)} + * or {@link #provisioningEndedInError(CloudProvisioningRecord, String)} + * gets called later. + * + * @param provisionable + * Our record for the template for the named node. + * @param nodeName + * The name of the VM. + */ + public void provisioningStarted(CloudProvisioningRecord provisionable, String nodeName) { + final boolean wasPreviouslyUnknownToPlanning = provisionable.addCurrentlyPlanned(nodeName); + final boolean wasAlreadyActive = provisionable.removeCurrentlyActive(nodeName); + final boolean wasPreviouslyUnwanted = provisionable.removeCurrentlyUnwanted(nodeName); + logStateChange(Level.FINE, "Intending to create {0}", + "wasPreviouslyUnknownToPlanning", wasPreviouslyUnknownToPlanning, true, + "wasAlreadyActive", wasAlreadyActive, false, + "wasPreviouslyUnwanted", wasPreviouslyUnwanted, false, + nodeName); + } + + /** + * To be called when a newly created node (previously promised to + * {@link #provisioningStarted(CloudProvisioningRecord, String)}) comes up. + * Callers MUST ensure that + * {@link #provisionedSlaveNowUnwanted(String, boolean)} gets called later + * when we do not want it anymore. + * + * @param provisionable + * Our record for the template for the named node. + * @param nodeName + * The name of the VM. + */ + public void provisionedSlaveNowActive(CloudProvisioningRecord provisionable, String nodeName) { + final boolean wasNotPreviouslyActive = provisionable.addCurrentlyActive(nodeName); + final boolean wasPreviouslyPlanned = provisionable.removeCurrentlyPlanned(nodeName); + final boolean wasPreviouslyUnwanted = provisionable.removeCurrentlyUnwanted(nodeName); + logStateChange(Level.FINE, "Marking {0} as active", + "wasNotPreviouslyActive", wasNotPreviouslyActive, true, + "wasPreviouslyPlanned", wasPreviouslyPlanned, true, + "wasPreviouslyUnwanted", wasPreviouslyUnwanted, false, + nodeName); + } + + /** + * To be called when a node we created (previously told to + * {@link #provisionedSlaveNowActive(CloudProvisioningRecord, String)}) is + * no longer wanted and should be deleted. + * + * @param nodeName + * The name of the VM. + * @param willAttemptImmediateDeletion + * If true then the caller must attempt to delete the slave and + * guarantee that they will call + * {@link #unwantedSlaveNotDeleted(String)} or + * {@link #unwantedSlaveNowDeleted(String)} as appropriate (just + * as if they'd called {@link #isOkToDeleteUnwantedVM(String)} + * and been told True). If false then the caller is under no such + * obligation. + */ + public void provisionedSlaveNowUnwanted(String nodeName, boolean willAttemptImmediateDeletion) { + final Map.Entry entry = findEntryForVM(nodeName); + if (entry != null) { + final CloudProvisioningRecord provisionable = entry.getValue(); + final boolean wasPreviouslyPlanned = provisionable.removeCurrentlyPlanned(nodeName); + final boolean wasPreviouslyActive = provisionable.removeCurrentlyActive(nodeName); + final boolean wasNotPreviouslyUnwanted = provisionable.setCurrentlyUnwanted(nodeName, willAttemptImmediateDeletion)==null; + logStateChange(Level.FINE, "Marking {0} for termination", + "wasPreviouslyPlanned", wasPreviouslyPlanned, false, + "wasPreviouslyActive", wasPreviouslyActive, true, + "wasNotPreviouslyUnwanted", wasNotPreviouslyUnwanted, true, + nodeName); + } else { + logger.log(Level.WARNING, "Asked to mark {0} for termination, but we have no record of it.", nodeName); + } + } + + /** + * To be called before commencing the deletion of a VM. + * + * @param nodeName + * The name of the VM being deleted. + * @return null if the VM is not unwanted (it may have recently been + * deleted). false if another thread is currently trying to delete + * it. true if deletion should be attempted, in which case the + * caller MUST later call {@link #unwantedSlaveNowDeleted(String)} + * or {@link #unwantedSlaveNotDeleted(String)}. + */ + public Boolean isOkToDeleteUnwantedVM(String nodeName) { + final Map.Entry entry = findEntryForVM(nodeName); + if (entry == null) { + return null; + } + final CloudProvisioningRecord record = entry.getValue(); + final Boolean thisNode = record.isCurrentlyUnwanted(nodeName); + if (thisNode == null) { + return null; + } + boolean someoneElseIsDeletingThis = thisNode.booleanValue(); + boolean isOkForUsToDeleteIt = !someoneElseIsDeletingThis; + if (isOkForUsToDeleteIt) { + record.setCurrentlyUnwanted(nodeName, true); + } + return Boolean.valueOf(isOkForUsToDeleteIt); + } + + /** + * MUST be called when a node previously declared to be unwanted (previously + * told to {@link #provisionedSlaveNowUnwanted(String, boolean)}) and that + * we were given clearance to delete + * ({@link #isOkToDeleteUnwantedVM(String)} returned true) has been + * successfully removed. + * + * @param nodeName + * The name of the VM that was successfully deleted. + */ + public void unwantedSlaveNowDeleted(String nodeName) { + final Map.Entry entry = findEntryForVM(nodeName); + if (entry != null) { + final CloudProvisioningRecord provisionable = entry.getValue(); + final boolean wasPreviouslyPlanned = provisionable.removeCurrentlyPlanned(nodeName); + final boolean wasPreviouslyActive = provisionable.removeCurrentlyActive(nodeName); + final boolean wasPreviouslyUnwanted = provisionable.removeCurrentlyUnwanted(nodeName); + if (recordIsPrunable(provisionable)) { + removeExistingRecord(provisionable); + } + logStateChange(Level.FINE, "Marking {0} as successfully terminated", + "wasPreviouslyPlanned", wasPreviouslyPlanned, false, + "wasPreviouslyActive", wasPreviouslyActive, false, + "wasPreviouslyUnwanted", wasPreviouslyUnwanted, true, + nodeName); + } else { + logger.log(Level.WARNING, "Asked to mark {0} as terminated, but we had no record of it.", nodeName); + } + } + + /** + * MUST be called when a node previously declared to be unwanted (previously + * told to {@link #provisionedSlaveNowUnwanted(String, boolean)}) and that + * we were given clearance to delete + * ({@link #isOkToDeleteUnwantedVM(String)} returned true) failed to be + * removed. + * + * @param nodeName + * The name of the VM that failed to delete + */ + public void unwantedSlaveNotDeleted(String nodeName) { + final Map.Entry entry = findEntryForVM(nodeName); + if (entry != null) { + final CloudProvisioningRecord provisionable = entry.getValue(); + final boolean isPlanned = provisionable.getCurrentlyPlanned().contains(nodeName); + final boolean isActive = provisionable.getCurrentlyProvisioned().contains(nodeName); + final boolean isUnwanted = provisionable.setCurrentlyUnwanted(nodeName, false) != null; + logStateChange(Level.INFO, "Marking {0} as unsuccessfully terminated - we'll have to try again later", + "isPlanned", isPlanned, false, + "isActive", isActive, false, + "isUnwanted", isUnwanted, true, + nodeName); + } else { + logger.log(Level.WARNING, "Asked to mark {0} as unsuccessfully terminated, but we had no record of it.", + nodeName); + } + } + + /** + * To be called if we become aware that there is a VM that exist in vSphere + * (that we created) which we don't want anymore. + * + * @param template + * The template to which the node belonged. + * @param nodeName + * The name of the node that exists (despite our wishes). + */ + public void recordExistingUnwantedVM(final vSphereCloudSlaveTemplate template, String nodeName) { + final CloudProvisioningRecord record = getOrCreateRecord(template); + final boolean wasPreviouslyPlanned = record.removeCurrentlyPlanned(nodeName); + final boolean wasPreviouslyActive = record.removeCurrentlyActive(nodeName); + final boolean wasAlreadyUnwanted = record.setCurrentlyUnwanted(nodeName, false) != null; + logStateChange(Level.INFO, "Marking {0} as found in vSphere but unwanted", + "wasPreviouslyPlanned", wasPreviouslyPlanned, false, + "wasPreviouslyActive", wasPreviouslyActive, false, + "wasAlreadyUnwanted", wasAlreadyUnwanted, false, + nodeName); + } + + /** + * To be called when a node that we previously promised to create (by + * calling {@link #provisioningStarted(CloudProvisioningRecord, String)}) + * failed to start. + * + * @param provisionable + * Our record for the template for the named node. + * @param nodeName + * The name of the VM. + */ + public void provisioningEndedInError(CloudProvisioningRecord provisionable, String nodeName) { + final boolean wasPreviouslyPlanned = provisionable.removeCurrentlyPlanned(nodeName); + final boolean wasPreviouslyActive = provisionable.removeCurrentlyActive(nodeName); + final boolean wasPreviouslyUnwanted = provisionable.removeCurrentlyUnwanted(nodeName); + if (recordIsPrunable(provisionable)) { + removeExistingRecord(provisionable); + } + logStateChange(Level.INFO, "Marking {0} as failed", + "wasPreviouslyPlanned", wasPreviouslyPlanned, true, + "wasPreviouslyActive", wasPreviouslyActive, false, + "wasPreviouslyUnwanted", wasPreviouslyUnwanted, false, + nodeName); + } + + /** + * To be called every now and again to ensure that we're not caching records + * that will never be valid again. + */ + public void pruneUnwantedRecords() { + final List toBeRemoved = new ArrayList(records.size()); + for (final Map.Entry entry : records.entrySet()) { + final CloudProvisioningRecord record = entry.getValue(); + if (recordIsPrunable(record)) { + toBeRemoved.add(record); + } + } + for (final CloudProvisioningRecord record : toBeRemoved) { + removeExistingRecord(record); + } + } + + /** + * Given a set of templates, returns the equivalent records. + * + * @param templates + * The templates we are interested in. + * @return A list of {@link CloudProvisioningRecord}. + */ + public List calculateProvisionableTemplates(Iterable templates) { + final List result = new ArrayList(); + for (final vSphereCloudSlaveTemplate template : templates) { + final CloudProvisioningRecord provisionable = getOrCreateRecord(template); + result.add(provisionable); + } + return result; + } + + /** + * Counts all the known nodes, active, in-progress and being-deleted, across + * all templates. + * + * @return The number of nodes that exist (or will do). + */ + public int countNodes() { + int result = 0; + for (final CloudProvisioningRecord record : records.values()) { + result += record.size(); + } + return result; + } + + /** + * Gets the record for the given template. If we didn't have one before, we + * create one. + * + * @param template + * The template in question. + * @return The one-and-only record for this template. + */ + public CloudProvisioningRecord getOrCreateRecord(final vSphereCloudSlaveTemplate template) { + final CloudProvisioningRecord existingRecord = getExistingRecord(template); + if (existingRecord != null) { + return existingRecord; + } + final CloudProvisioningRecord newRecord = new CloudProvisioningRecord(template); + logger.log(Level.FINE, "Creating new record for template {0} ({1})", + new Object[] { template.getCloneNamePrefix(), template.toString() }); + records.put(template, newRecord); + return newRecord; + } + + /** + * Calculates the current list of "existing but unwanted" VMs, in priority + * order. Note: The returned data is not "live", it's a copy, so callers are + * free to edit the {@link List} they are given. + * + * @return A copy of the list of VMs that we know exist but no longer want, + * and which aren't in the process of being deleted by anyone. + */ + public List getUnwantedVMsThatNeedDeleting() { + // find out who needs what deleted + int count = 0; + final Map> allUnwantedVmsByRecord = new LinkedHashMap<>(); + for (final CloudProvisioningRecord record : records.values()) { + final Map currentlyUnwanted = record.getCurrentlyUnwanted(); + final List vmsInNeedOfDeletionForThisRecord = new ArrayList(currentlyUnwanted.size()); + for (Map.Entry entry : currentlyUnwanted.entrySet()) { + if (entry.getValue() == Boolean.FALSE) { + vmsInNeedOfDeletionForThisRecord.add(entry.getKey()); + } + } + count += vmsInNeedOfDeletionForThisRecord.size(); + allUnwantedVmsByRecord.put(record, vmsInNeedOfDeletionForThisRecord.iterator()); + } + // arrange the list in a round-robin order, taking the first from each + // template, followed by the second from each etc. + final List vmsInNeedOfDeletion = new ArrayList(count); + while (vmsInNeedOfDeletion.size() < count) { + for (Iterator i : allUnwantedVmsByRecord.values()) { + if (i.hasNext()) { + final String nodeName = i.next(); + vmsInNeedOfDeletion.add(nodeName); + } + } + } + return vmsInNeedOfDeletion; + } + + private CloudProvisioningRecord getExistingRecord(final vSphereCloudSlaveTemplate template) { + return records.get(template); + } + + private void removeExistingRecord(CloudProvisioningRecord existingRecord) { + final vSphereCloudSlaveTemplate template = existingRecord.getTemplate(); + logger.log(Level.FINE, "Disposing of record for template {0} ({1})", + new Object[] { template.getCloneNamePrefix(), template.toString() }); + records.remove(template); + } + + private boolean recordIsPrunable(CloudProvisioningRecord record) { + final boolean isEmpty = record.isEmpty(); + if (!isEmpty) { + return false; + } + final vSphereCloudSlaveTemplate template = record.getTemplate(); + final List knownTemplates = parent.getTemplates(); + final boolean isKnownToParent = knownTemplates.contains(template); + return !isKnownToParent; + } + + private Map.Entry findEntryForVM(String nodeName) { + for (final Map.Entry entry : records.entrySet()) { + final CloudProvisioningRecord record = entry.getValue(); + if (record.contains(nodeName)) { + return entry; + } + } + return null; + } + + /** + * Logs a state change. If the state change isn't valid, it's logged as a + * warning. + * + * @param logLevel + * The level to log the message at, if the boolean arguments are + * as their expected values. + * @param logMsg + * The message to log. + * @param firstArgName + * What actualFirstArgValue represents - used when complaining + * about its value. + * @param actualFirstArgValue + * A state-change variable. + * @param expectedFirstArgValue + * The expected value of actualFirstArgValue. If that's not the + * case, we'll complain. + * @param secondArgName + * What actualSecondArgValue represents - used when complaining + * about its value. + * @param actualSecondArgValue + * A state-change variable. + * @param expectedSecondArgValue + * The expected value of actualSecondArgValue. If that's not the + * case, we'll complain. + * @param thirdArgName + * What actualThirdArgValue represents - used when complaining + * about its value. + * @param actualThirdArgValue + * A state-change variable. + * @param expectedThirdArgValue + * The expected value of actualThirdArgValue. If that's not the + * case, we'll complain. + * @param args + * The arguments for logMsg. Used if logMsg contains {0}, {1} + * etc. + */ + private void logStateChange(Level logLevel, String logMsg, + String firstArgName, boolean actualFirstArgValue, boolean expectedFirstArgValue, + String secondArgName, boolean actualSecondArgValue, boolean expectedSecondArgValue, + String thirdArgName, boolean actualThirdArgValue, boolean expectedThirdArgValue, + Object... args) { + final boolean firstValid = actualFirstArgValue == expectedFirstArgValue; + final boolean secondValid = actualSecondArgValue == expectedSecondArgValue; + final boolean thirdValid = actualThirdArgValue == expectedThirdArgValue; + Level actualLevel = logLevel; + String actualMsg = logMsg; + if (!firstValid) { + actualMsg += " : " + firstArgName + "!=" + expectedFirstArgValue; + actualLevel = Level.WARNING; + } + if (!secondValid) { + actualMsg += " : " + secondArgName + "!=" + expectedSecondArgValue; + actualLevel = Level.WARNING; + } + if (!thirdValid) { + actualMsg += " : " + thirdArgName + "!=" + expectedThirdArgValue; + actualLevel = Level.WARNING; + } + final Logger loggerToUse = logger != null ? logger : LOGGER; + loggerToUse.log(actualLevel, actualMsg, args); + } +} diff --git a/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithmTest.java b/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithmTest.java index 0ea08052..a6a0e717 100644 --- a/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithmTest.java +++ b/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningAlgorithmTest.java @@ -1,289 +1,334 @@ -package org.jenkinsci.plugins.vsphere.tools; - -import static org.junit.Assert.*; -import static org.hamcrest.CoreMatchers.*; -import hudson.slaves.JNLPLauncher; -import hudson.slaves.RetentionStrategy; - -import java.math.BigInteger; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.List; - -import org.jenkinsci.plugins.vSphereCloudSlaveTemplate; -import org.jenkinsci.plugins.vsphere.tools.CloudProvisioningRecord; -import org.junit.Before; -import org.junit.Test; - -public class CloudProvisioningAlgorithmTest { - - /** Used when faking up test data */ - private int instanceNumber; - - @Before - public void setup() { - instanceNumber = 0; - } - - @Test - public void findTemplateWithMostFreeCapacityGivenNoOptionsThenReturnsNull() { - // Given - final List emptyList = Collections.emptyList(); - - // When - final CloudProvisioningRecord actual = CloudProvisioningAlgorithm.findTemplateWithMostFreeCapacity(emptyList); - - // Then - assertThat(actual, nullValue()); - } - - @Test - public void findTemplateWithMostFreeCapacityGivenSomeActiveAndOneUnusedThenPrefersUnusedTemplate() { - // Given - final CloudProvisioningRecord zeroOfTwo = createInstance(2, 0, 0); - final CloudProvisioningRecord onePlannedOfTwo = createInstance(2, 0, 1); - final CloudProvisioningRecord oneExistsOfTwo = createInstance(2, 1, 0); - final List forwards = Arrays.asList(zeroOfTwo, onePlannedOfTwo, oneExistsOfTwo); - final List reverse = Arrays.asList(oneExistsOfTwo, onePlannedOfTwo, zeroOfTwo); - - // When - final CloudProvisioningRecord actual1 = CloudProvisioningAlgorithm.findTemplateWithMostFreeCapacity(forwards); - final CloudProvisioningRecord actual2 = CloudProvisioningAlgorithm.findTemplateWithMostFreeCapacity(reverse); - - // Then - assertThat(actual1, sameInstance(zeroOfTwo)); - assertThat(actual2, sameInstance(zeroOfTwo)); - } - - @Test - public void findTemplateWithMostFreeCapacityGivenEqualCapsThenDistributesTheLoadEvenly() { - // Given - final CloudProvisioningRecord a = createInstance(2, 0, 0); - final CloudProvisioningRecord b = createInstance(2, 0, 0); - final CloudProvisioningRecord c = createInstance(2, 0, 0); - final List records = Arrays.asList(a, b, c); - - // When/Then - testScenario(records, a, b, c, a, b, c, null); - } - - @Test - public void findTemplateWithMostFreeCapacityGivenEqualCapsButExistingUsageThenDistributesTheLoadEvenly() { - // Given - final CloudProvisioningRecord a = createInstance(2, 2, 0); - final CloudProvisioningRecord b = createInstance(2, 1, 0); - final CloudProvisioningRecord c = createInstance(2, 0, 0); - final List records = Arrays.asList(a, b, c); - - // When/Then - testScenario(records, c, b, c, null); - } - - @Test - public void findTemplateWithMostFreeCapacityGivenNoCapsThenDistributesTheLoadEvenly() { - // Given - final CloudProvisioningRecord a = createInstance(0, 0, 0); - final CloudProvisioningRecord b = createInstance(0, 0, 0); - final CloudProvisioningRecord c = createInstance(0, 0, 0); - final List records = Arrays.asList(a, b, c); - - // When/Then - testScenario(records, a, b, c, a, b, c, a, b, c, a); - } - - @Test - public void findTemplateWithMostFreeCapacityGivenUnequalCapsThenDistributesTheLoadFairly() { - findTemplateWithMostFreeCapacityGivenUnequalCapsThenDistributesTheLoadFairly(true); - } - - @Test - public void findTemplateWithMostFreeCapacityGivenUnequalCapsThenDistributesTheLoadFairly2() { - findTemplateWithMostFreeCapacityGivenUnequalCapsThenDistributesTheLoadFairly(false); - } - - private void findTemplateWithMostFreeCapacityGivenUnequalCapsThenDistributesTheLoadFairly(boolean forwards) { - // Given - final CloudProvisioningRecord capOf2 = createInstance(2, 0, 0); - final CloudProvisioningRecord capOf5 = createInstance(5, 0, 0); - final List records = forwards ? Arrays.asList(capOf2, capOf5) : Arrays.asList(capOf5, - capOf2); - - // When/Then - testScenario(records, capOf5, capOf2, capOf5, capOf5, capOf2, capOf5, capOf5, null); - } - - @Test - public void findTemplateWithMostFreeCapacityGivenOneCappedAndOneUncappedThenDistributesTheLoadEventlyUntilCapReached() { - findTemplateWithMostFreeCapacityGivenDifferentCapnessThenDistributesTheLoadEventlyUntilCapReached(true); - } - - @Test - public void findTemplateWithMostFreeCapacityGivenOneUncappedAndOneCappedThenDistributesTheLoadEventlyUntilCapReached() { - findTemplateWithMostFreeCapacityGivenDifferentCapnessThenDistributesTheLoadEventlyUntilCapReached(false); - } - - private void findTemplateWithMostFreeCapacityGivenDifferentCapnessThenDistributesTheLoadEventlyUntilCapReached( - boolean forwards) { - // Given - final CloudProvisioningRecord capOf2 = createInstance(2, 0, 0); - final CloudProvisioningRecord uncapped = createInstance(0, 0, 0); - final List records = forwards ? Arrays.asList(capOf2, uncapped) : Arrays.asList( - uncapped, capOf2); - - // When/Then - testScenario(records, uncapped, capOf2, uncapped, capOf2, uncapped, uncapped, uncapped); - } - - private static void testScenario(List records, CloudProvisioningRecord... expectedRecords) { - // Given records and expected return values - int i = 0; - for (final CloudProvisioningRecord expected : expectedRecords) { - final CloudProvisioningRecord actual = CloudProvisioningAlgorithm.findTemplateWithMostFreeCapacity(records); - i++; - assertThat("findTemplateWithMostFreeCapacity(" + records + ")#" + i, actual, sameInstance(expected)); - final String nodeName = "PlannedInStep" + i; - if (actual != null) { - actual.addCurrentlyPlanned(nodeName); - } - } - } - - @Test - public void toBigIntegerGivenTwoPow128MinusOneThenReturnsTwoPow128MinusOne() { - testToBigInteger(-1, -1, "340282366920938463463374607431768211455"); - } - - @Test - public void toBigIntegerGivenTwoPow64PlusOneThenReturnsTwoPow64PlusOne() { - testToBigInteger(1, 1, "18446744073709551617"); - } - - @Test - public void toBigIntegerGivenZeroThenReturnsZero() { - testToBigInteger(0, 0, "0"); - } - - @Test - public void toBigIntegerGivenPowersOfTwoThenReturnsPowersOfTwo() { - long lsb = 1; - long msb = 0; - BigInteger big = new BigInteger("1"); - for (int bit = 0; bit < 64; bit++) { - testToBigInteger(msb, lsb, big.toString()); - lsb <<= 1; - big = big.add(big); - } - msb = 1; - for (int bit = 0; bit < 64; bit++) { - testToBigInteger(msb, lsb, big.toString()); - msb <<= 1; - big = big.add(big); - } - } - - private void testToBigInteger(long msb, long lsb, String expected) { - // Given - final BigInteger expectedValue = new BigInteger(expected); - final byte[] expectedBytes = expectedValue.toByteArray(); - // When - final BigInteger actual = CloudProvisioningAlgorithm.toBigInteger(msb, lsb); - final byte[] actualBytes = actual.toByteArray(); - final String scenario = "toBigInteger(" + msb + "," + lsb + ") == " + expected + "\ni.e. " - + toHexString(actualBytes) + " ~= " + toHexString(expectedBytes); - // Then - assertThat(scenario, actual, equalTo(expectedValue)); - } - - @Test - public void findUnusedNameGivenZeroOfTwoExistsThenReturnsOneThenTwo() { - // Given - final CloudProvisioningRecord record = createInstance(2, 0, 0); - final String prefix = record.getTemplate().getCloneNamePrefix(); - final String expected1 = prefix + "_1"; - final String expected2 = prefix + "_2"; - - // When - final String actual1 = CloudProvisioningAlgorithm.findUnusedName(record); - record.addCurrentlyActive(actual1); - final String actual2 = CloudProvisioningAlgorithm.findUnusedName(record); - record.addCurrentlyActive(actual2); - - // Then - assertThat(actual1, equalTo(expected1)); - assertThat(actual2, equalTo(expected2)); - } - - @Test - public void findUnusedNameGivenOneOfTwoHasEndedThenReturnsOne() { - // Given - final CloudProvisioningRecord record = createInstance(2, 0, 0); - final String prefix = record.getTemplate().getCloneNamePrefix(); - final String expected = prefix + "_1"; - - // When - final String actual1 = CloudProvisioningAlgorithm.findUnusedName(record); - record.addCurrentlyActive(actual1); - final String actual2 = CloudProvisioningAlgorithm.findUnusedName(record); - record.addCurrentlyActive(actual2); - record.removeCurrentlyActive(actual1); - final String actual = CloudProvisioningAlgorithm.findUnusedName(record); - record.addCurrentlyActive(actual); - - // Then - assertThat(actual, equalTo(expected)); - } - - @Test - public void findUnusedNameGivenUncappedInstancesThenReturnsUniqueNames() { - // Given - final CloudProvisioningRecord record = createInstance(0, 5, 6); - final String prefix = record.getTemplate().getCloneNamePrefix(); - final List actuals = new ArrayList(); - - // When - for (int i = 0; i < 100; i++) { - final String actual = CloudProvisioningAlgorithm.findUnusedName(record); - record.addCurrentlyActive(actual); - actuals.add(actual); - } - - // Then - final List uniques = new ArrayList(new LinkedHashSet(actuals)); - assertThat(actuals, equalTo(uniques)); - assertThat(actuals, everyItem(startsWith(prefix + "_"))); - } - - private CloudProvisioningRecord createInstance(int capacity, int provisioned, int planned) { - final int iNum = ++instanceNumber; - final vSphereCloudSlaveTemplate template = stubTemplate(iNum + "cap" + capacity, capacity); - final CloudProvisioningRecord instance = new CloudProvisioningRecord(template); - for (int i = 0; i < provisioned; i++) { - final String nodeName = iNum + "provisioned#" + i; - instance.addCurrentlyPlanned(nodeName); - } - for (int i = 0; i < planned; i++) { - final String nodeName = iNum + "planned#" + i; - instance.addCurrentlyPlanned(nodeName); - } - return instance; - } - - private static vSphereCloudSlaveTemplate stubTemplate(String prefix, int templateInstanceCap) { - return new vSphereCloudSlaveTemplate(prefix, "", null, null, false, null, null, null, null, null, null, templateInstanceCap, 1, - null, null, null, false, false, 0, 0, false, null, null, null, new JNLPLauncher(), - RetentionStrategy.NOOP, null, null); - } - - private static String toHexString(byte[] bytes) { - final StringBuilder s = new StringBuilder("0x"); - for (final byte b : bytes) { - final int highDigit = (((int) b) >> 8) & 15; - final int lowDigit = ((int) b) & 15; - s.append(Integer.toString(highDigit, 16)); - s.append(Integer.toString(lowDigit, 16)); - } - return s.toString(); - } -} +package org.jenkinsci.plugins.vsphere.tools; + +import static org.junit.Assert.*; +import static org.hamcrest.CoreMatchers.*; +import hudson.slaves.JNLPLauncher; +import hudson.slaves.RetentionStrategy; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; + +import org.jenkinsci.plugins.vSphereCloudSlaveTemplate; +import org.jenkinsci.plugins.vsphere.tools.CloudProvisioningRecord; +import org.junit.Before; +import org.junit.Test; + +public class CloudProvisioningAlgorithmTest { + + /** Used when faking up test data */ + private int instanceNumber; + + @Before + public void setup() { + instanceNumber = 0; + } + + @Test + public void findTemplateWithMostFreeCapacityGivenNoOptionsThenReturnsNull() { + // Given + final List emptyList = Collections.emptyList(); + + // When + final CloudProvisioningRecord actual = CloudProvisioningAlgorithm.findTemplateWithMostFreeCapacity(emptyList); + + // Then + assertThat(actual, nullValue()); + } + + @Test + public void findTemplateWithMostFreeCapacityGivenSomeActiveAndOneUnusedThenPrefersUnusedTemplate() { + // Given + final CloudProvisioningRecord zeroOfTwo = createInstance(2, 0, 0); + final CloudProvisioningRecord onePlannedOfTwo = createInstance(2, 0, 1); + final CloudProvisioningRecord oneExistsOfTwo = createInstance(2, 1, 0); + final List forwards = Arrays.asList(zeroOfTwo, onePlannedOfTwo, oneExistsOfTwo); + final List reverse = Arrays.asList(oneExistsOfTwo, onePlannedOfTwo, zeroOfTwo); + + // When + final CloudProvisioningRecord actual1 = CloudProvisioningAlgorithm.findTemplateWithMostFreeCapacity(forwards); + final CloudProvisioningRecord actual2 = CloudProvisioningAlgorithm.findTemplateWithMostFreeCapacity(reverse); + + // Then + assertThat(actual1, sameInstance(zeroOfTwo)); + assertThat(actual2, sameInstance(zeroOfTwo)); + } + + @Test + public void findTemplateWithMostFreeCapacityGivenEqualCapsThenDistributesTheLoadEvenly() { + // Given + final CloudProvisioningRecord a = createInstance(2, 0, 0); + final CloudProvisioningRecord b = createInstance(2, 0, 0); + final CloudProvisioningRecord c = createInstance(2, 0, 0); + final List records = Arrays.asList(a, b, c); + + // When/Then + testScenario(records, a, b, c, a, b, c, null); + } + + @Test + public void findTemplateWithMostFreeCapacityGivenEqualCapsButExistingUsageThenDistributesTheLoadEvenly() { + // Given + final CloudProvisioningRecord a = createInstance(2, 2, 0); + final CloudProvisioningRecord b = createInstance(2, 1, 0); + final CloudProvisioningRecord c = createInstance(2, 0, 0); + final List records = Arrays.asList(a, b, c); + + // When/Then + testScenario(records, c, b, c, null); + } + + @Test + public void findTemplateWithMostFreeCapacityGivenNoCapsThenDistributesTheLoadEvenly() { + // Given + final CloudProvisioningRecord a = createInstance(0, 0, 0); + final CloudProvisioningRecord b = createInstance(0, 0, 0); + final CloudProvisioningRecord c = createInstance(0, 0, 0); + final List records = Arrays.asList(a, b, c); + + // When/Then + testScenario(records, a, b, c, a, b, c, a, b, c, a); + } + + @Test + public void findTemplateWithMostFreeCapacityGivenUnequalCapsThenDistributesTheLoadFairly() { + findTemplateWithMostFreeCapacityGivenUnequalCapsThenDistributesTheLoadFairly(true); + } + + @Test + public void findTemplateWithMostFreeCapacityGivenUnequalCapsThenDistributesTheLoadFairly2() { + findTemplateWithMostFreeCapacityGivenUnequalCapsThenDistributesTheLoadFairly(false); + } + + private void findTemplateWithMostFreeCapacityGivenUnequalCapsThenDistributesTheLoadFairly(boolean forwards) { + // Given + final CloudProvisioningRecord capOf2 = createInstance(2, 0, 0); + final CloudProvisioningRecord capOf5 = createInstance(5, 0, 0); + final List records = forwards ? Arrays.asList(capOf2, capOf5) : Arrays.asList(capOf5, + capOf2); + + // When/Then + testScenario(records, capOf5, capOf2, capOf5, capOf5, capOf2, capOf5, capOf5, null); + } + + @Test + public void findTemplateWithMostFreeCapacityGivenOneCappedAndOneUncappedThenDistributesTheLoadEventlyUntilCapReached() { + findTemplateWithMostFreeCapacityGivenDifferentCapnessThenDistributesTheLoadEventlyUntilCapReached(true); + } + + @Test + public void findTemplateWithMostFreeCapacityGivenOneUncappedAndOneCappedThenDistributesTheLoadEventlyUntilCapReached() { + findTemplateWithMostFreeCapacityGivenDifferentCapnessThenDistributesTheLoadEventlyUntilCapReached(false); + } + + private void findTemplateWithMostFreeCapacityGivenDifferentCapnessThenDistributesTheLoadEventlyUntilCapReached( + boolean forwards) { + // Given + final CloudProvisioningRecord capOf2 = createInstance(2, 0, 0); + final CloudProvisioningRecord uncapped = createInstance(0, 0, 0); + final List records = forwards ? Arrays.asList(capOf2, uncapped) : Arrays.asList( + uncapped, capOf2); + + // When/Then + testScenario(records, uncapped, capOf2, uncapped, capOf2, uncapped, uncapped, uncapped); + } + + private static void testScenario(List records, CloudProvisioningRecord... expectedRecords) { + // Given records and expected return values + int i = 0; + for (final CloudProvisioningRecord expected : expectedRecords) { + final CloudProvisioningRecord actual = CloudProvisioningAlgorithm.findTemplateWithMostFreeCapacity(records); + i++; + assertThat("findTemplateWithMostFreeCapacity(" + records + ")#" + i, actual, sameInstance(expected)); + final String nodeName = "PlannedInStep" + i; + if (actual != null) { + actual.addCurrentlyPlanned(nodeName); + } + } + } + + @Test + public void toBigIntegerGivenTwoPow128MinusOneThenReturnsTwoPow128MinusOne() { + testToBigInteger(-1, -1, "340282366920938463463374607431768211455"); + } + + @Test + public void toBigIntegerGivenTwoPow64PlusOneThenReturnsTwoPow64PlusOne() { + testToBigInteger(1, 1, "18446744073709551617"); + } + + @Test + public void toBigIntegerGivenZeroThenReturnsZero() { + testToBigInteger(0, 0, "0"); + } + + @Test + public void toBigIntegerGivenPowersOfTwoThenReturnsPowersOfTwo() { + long lsb = 1; + long msb = 0; + BigInteger big = new BigInteger("1"); + for (int bit = 0; bit < 64; bit++) { + testToBigInteger(msb, lsb, big.toString()); + lsb <<= 1; + big = big.add(big); + } + msb = 1; + for (int bit = 0; bit < 64; bit++) { + testToBigInteger(msb, lsb, big.toString()); + msb <<= 1; + big = big.add(big); + } + } + + private void testToBigInteger(long msb, long lsb, String expected) { + // Given + final BigInteger expectedValue = new BigInteger(expected); + final byte[] expectedBytes = expectedValue.toByteArray(); + // When + final BigInteger actual = CloudProvisioningAlgorithm.toBigInteger(msb, lsb); + final byte[] actualBytes = actual.toByteArray(); + final String scenario = "toBigInteger(" + msb + "," + lsb + ") == " + expected + "\ni.e. " + + toHexString(actualBytes) + " ~= " + toHexString(expectedBytes); + // Then + assertThat(scenario, actual, equalTo(expectedValue)); + } + + @Test + public void findUnusedNameGivenZeroOfTwoExistsThenReturnsOneThenTwo() { + // Given + final CloudProvisioningRecord record = createInstance(2, 0, 0); + final String prefix = record.getTemplate().getCloneNamePrefix(); + final String expected1 = prefix + "_1"; + final String expected2 = prefix + "_2"; + + // When + final String actual1 = CloudProvisioningAlgorithm.findUnusedName(record); + record.addCurrentlyActive(actual1); + final String actual2 = CloudProvisioningAlgorithm.findUnusedName(record); + record.addCurrentlyActive(actual2); + + // Then + assertThat(actual1, equalTo(expected1)); + assertThat(actual2, equalTo(expected2)); + } + + @Test + public void findUnusedNameGivenMiddleOfThreeStillExistsThenReturnsOneThenThree() { + // Given + final CloudProvisioningRecord record = createInstance(3, 0, 0); + final String prefix = record.getTemplate().getCloneNamePrefix(); + final String expected1 = prefix + "_1"; + final String unwanted = prefix + "_2"; + record.setCurrentlyUnwanted(unwanted, false); + final String expected2 = prefix + "_3"; + + // When + final String actual1 = CloudProvisioningAlgorithm.findUnusedName(record); + record.addCurrentlyActive(actual1); + final String actual2 = CloudProvisioningAlgorithm.findUnusedName(record); + record.addCurrentlyActive(actual2); + + // Then + assertThat(actual1, equalTo(expected1)); + assertThat(actual2, equalTo(expected2)); + } + + @Test + public void findUnusedNameGivenNoSpaceThenThrowsIllegalStateException() { + // Given + final CloudProvisioningRecord record = createInstance(3, 0, 0); + final String prefix = record.getTemplate().getCloneNamePrefix(); + final String unwanted = prefix + "_1"; + final String active = prefix + "_2"; + final String planned = prefix + "_3"; + record.setCurrentlyUnwanted(unwanted, false); + record.addCurrentlyActive(active); + record.addCurrentlyPlanned(planned); + final List records = Arrays.asList(record); + final CloudProvisioningRecord shouldBeNull = CloudProvisioningAlgorithm.findTemplateWithMostFreeCapacity(records); + assertThat(shouldBeNull, nullValue()); + + // When + try { + final String unexpected = CloudProvisioningAlgorithm.findUnusedName(record); + fail("Expected IllegalStateException, got '" + unexpected + "'."); + } catch (IllegalStateException expected) { + // Then passed. + } + } + + @Test + public void findUnusedNameGivenOneOfTwoHasEndedThenReturnsOne() { + // Given + final CloudProvisioningRecord record = createInstance(2, 0, 0); + final String prefix = record.getTemplate().getCloneNamePrefix(); + final String expected = prefix + "_1"; + + // When + final String actual1 = CloudProvisioningAlgorithm.findUnusedName(record); + record.addCurrentlyActive(actual1); + final String actual2 = CloudProvisioningAlgorithm.findUnusedName(record); + record.addCurrentlyActive(actual2); + record.removeCurrentlyActive(actual1); + final String actual = CloudProvisioningAlgorithm.findUnusedName(record); + record.addCurrentlyActive(actual); + + // Then + assertThat(actual, equalTo(expected)); + } + + @Test + public void findUnusedNameGivenUncappedInstancesThenReturnsUniqueNames() { + // Given + final CloudProvisioningRecord record = createInstance(0, 5, 6); + final String prefix = record.getTemplate().getCloneNamePrefix(); + final List actuals = new ArrayList(); + + // When + for (int i = 0; i < 100; i++) { + final String actual = CloudProvisioningAlgorithm.findUnusedName(record); + record.addCurrentlyActive(actual); + actuals.add(actual); + } + + // Then + final List uniques = new ArrayList(new LinkedHashSet(actuals)); + assertThat(actuals, equalTo(uniques)); + assertThat(actuals, everyItem(startsWith(prefix + "_"))); + } + + private CloudProvisioningRecord createInstance(int capacity, int provisioned, int planned) { + final int iNum = ++instanceNumber; + final vSphereCloudSlaveTemplate template = stubTemplate(iNum + "cap" + capacity, capacity); + final CloudProvisioningRecord instance = new CloudProvisioningRecord(template); + for (int i = 0; i < provisioned; i++) { + final String nodeName = iNum + "provisioned#" + i; + instance.addCurrentlyActive(nodeName); + } + for (int i = 0; i < planned; i++) { + final String nodeName = iNum + "planned#" + i; + instance.addCurrentlyPlanned(nodeName); + } + return instance; + } + + private static vSphereCloudSlaveTemplate stubTemplate(String prefix, int templateInstanceCap) { + return new vSphereCloudSlaveTemplate(prefix, "", null, null, false, null, null, null, null, null, null, templateInstanceCap, 1, + null, null, null, false, false, 0, 0, false, null, null, null, new JNLPLauncher(), + RetentionStrategy.NOOP, null, null); + } + + private static String toHexString(byte[] bytes) { + final StringBuilder s = new StringBuilder("0x"); + for (final byte b : bytes) { + final int highDigit = (((int) b) >> 8) & 15; + final int lowDigit = ((int) b) & 15; + s.append(Integer.toString(highDigit, 16)); + s.append(Integer.toString(lowDigit, 16)); + } + return s.toString(); + } +} diff --git a/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningStateTest.java b/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningStateTest.java index 5ba51592..8fa88ffa 100644 --- a/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningStateTest.java +++ b/src/test/java/org/jenkinsci/plugins/vsphere/tools/CloudProvisioningStateTest.java @@ -1,411 +1,567 @@ -package org.jenkinsci.plugins.vsphere.tools; - -import static org.hamcrest.CoreMatchers.allOf; -import static org.hamcrest.CoreMatchers.any; -import static org.hamcrest.CoreMatchers.containsString; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.everyItem; -import static org.hamcrest.collection.IsArrayContainingInOrder.arrayContaining; -import static org.hamcrest.collection.IsIterableContainingInOrder.contains; -import static org.junit.Assert.assertThat; -import hudson.model.Node.Mode; -import hudson.slaves.NodeProperty; -import hudson.slaves.JNLPLauncher; -import hudson.slaves.RetentionStrategy; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.logging.Handler; -import java.util.logging.Level; -import java.util.logging.LogRecord; -import java.util.logging.Logger; - -import org.hamcrest.Description; -import org.hamcrest.Matcher; -import org.hamcrest.TypeSafeMatcher; -import org.hamcrest.collection.IsIterableWithSize; -import org.jenkinsci.plugins.vSphereCloud; -import org.jenkinsci.plugins.vSphereCloudSlaveTemplate; -import org.jenkinsci.plugins.vsphere.VSphereConnectionConfig; -import org.jenkinsci.plugins.vsphere.VSphereGuestInfoProperty; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -public class CloudProvisioningStateTest { - private static List stubVSphereCloudTemplates; - private static vSphereCloud stubVSphereCloud; - private int recordNumber; - private int nodeNumber; - private Logger testLogger; - private List loggedMessages; - - @BeforeClass - public static void setupClass() { - stubVSphereCloudTemplates = new ArrayList(); - final VSphereConnectionConfig vsConnectionConfig = new VSphereConnectionConfig("vsHost", "credentialsId"); - stubVSphereCloud = new vSphereCloud(vsConnectionConfig, "vsDescription", 0, 0, stubVSphereCloudTemplates); - } - - @Before - public void setup() { - stubVSphereCloudTemplates.clear(); - recordNumber = 0; - nodeNumber = 0; - loggedMessages = new ArrayList(); - Logger logger = Logger.getLogger("CloudProvisioningStateTest"); - logger.setLevel(Level.ALL); - final Handler[] handlers = logger.getHandlers(); - for (final Handler handler : handlers) { - logger.removeHandler(handler); - } - final Handler testHandler = new Handler() { - @Override - public void publish(LogRecord record) { - loggedMessages.add(record); - } - - @Override - public void flush() { - } - - @Override - public void close() { - } - }; - logger.addHandler(testHandler); - testLogger = logger; - } - - @Test - public void constructorGivenCalledThenLogsConstructions() { - // Given - final Object[] expectedArgs = { stubVSphereCloud.toString() }; - - // When - createInstance(); - - // Then - assertThat(loggedMessages, contains(logMessage(Level.FINE, expectedArgs))); - } - - @Test - public void provisioningStartedGivenNoPreviousStateThenLogs() { - // Given - final String nodeName = createNodeName(); - final Object[] expectedArgs = { nodeName }; - final CloudProvisioningState instance = createInstance(); - final CloudProvisioningRecord provisionable = createRecord(instance); - wipeLog(); - - // When - instance.provisioningStarted(provisionable, nodeName); - - // Then - assertThat(loggedMessages, contains(logMessage(Level.FINE, expectedArgs))); - } - - @Test - public void provisioningStartedGivenPreviouslyStartedThenWarns() { - // Given - final String nodeName = createNodeName(); - final Object[] expectedArgs = { nodeName }; - final CloudProvisioningState instance = createInstance(); - final CloudProvisioningRecord provisionable = createRecord(instance); - instance.provisioningStarted(provisionable, nodeName); - wipeLog(); - - // When - instance.provisioningStarted(provisionable, nodeName); - - // Then - assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); - } - - @Test - public void normalLifecycleGivenNoErrorsThenLogs() { - // Given - final String nodeName = createNodeName(); - final Object[] expectedArgs = { nodeName }; - final CloudProvisioningState instance = createInstance(); - final CloudProvisioningRecord provisionable = createRecord(instance); - wipeLog(); - - // When - instance.provisioningStarted(provisionable, nodeName); - instance.provisionedSlaveNowActive(provisionable, nodeName); - instance.provisionedSlaveNowTerminated(nodeName); - - // Then - assertThat(loggedMessages, everyItem(logMessage(Level.FINE, expectedArgs))); - assertThat(loggedMessages, IsIterableWithSize. iterableWithSize(3)); - } - - @SuppressWarnings("unchecked") - @Test - public void failedToProvisionGivenNothingOutOfSequenceThenLogs() { - // Given - final String nodeName = createNodeName(); - final Object[] expectedArgs = { nodeName }; - final CloudProvisioningState instance = createInstance(); - final CloudProvisioningRecord provisionable = createRecord(instance); - wipeLog(); - - // When - instance.provisioningStarted(provisionable, nodeName); - instance.provisioningEndedInError(provisionable, nodeName); - - // Then - assertThat( - loggedMessages, - contains(logMessage(Level.FINE, expectedArgs), - logMessage(containsString("failed"), Level.INFO, expectedArgs))); - assertThat(loggedMessages, IsIterableWithSize. iterableWithSize(2)); - } - - @Test - public void provisionGivenOutOfOrderSequenceThenComplains() { - // Given - final String nodeName = createNodeName(); - final Object[] expectedArgs = { nodeName }; - final CloudProvisioningState instance = createInstance(); - final CloudProvisioningRecord provisionable = createRecord(instance); - wipeLog(); - - // When/Then - wipeLog(); - instance.provisionedSlaveNowTerminated(nodeName); - assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); - - wipeLog(); - instance.provisioningStarted(provisionable, nodeName); - assertThat(loggedMessages, contains(logMessage(Level.FINE, expectedArgs))); - - wipeLog(); - instance.provisioningStarted(provisionable, nodeName); - assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); - - wipeLog(); - instance.provisioningEndedInError(provisionable, nodeName); - assertThat(loggedMessages, contains(logMessage(Level.INFO, expectedArgs))); - - wipeLog(); - instance.provisioningEndedInError(provisionable, nodeName); - assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); - - wipeLog(); - instance.provisionedSlaveNowActive(provisionable, nodeName); - assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); - - wipeLog(); - instance.provisionedSlaveNowActive(provisionable, nodeName); - assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); - - wipeLog(); - instance.provisioningStarted(provisionable, nodeName); - assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); - - wipeLog(); - instance.provisioningStarted(provisionable, nodeName); - assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); - - wipeLog(); - instance.provisionedSlaveNowTerminated(nodeName); - assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); - - wipeLog(); - instance.provisionedSlaveNowTerminated(nodeName); - assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); - } - - @Test - public void pruneUnwantedRecordsGivenUnknownTemplatesThenRemovesRecordsForEmptyDeletedTemplates() { - // Given - final String deletedAndInactiveNodeName = createNodeName(); - final String deletedButActiveNodeName = createNodeName(); - final String livedAndDiedNodeName = createNodeName(); - final CloudProvisioningState instance = createInstance(); - // A template which the user deleted but still has an active slave - final CloudProvisioningRecord deletedButActiveRecord = createRecord(instance); - instance.provisioningStarted(deletedButActiveRecord, deletedButActiveNodeName); - instance.provisionedSlaveNowActive(deletedButActiveRecord, deletedButActiveNodeName); - // A template which the user deleted and is no longer needed - final CloudProvisioningRecord deletedAndInactiveRecord = createRecord(instance); - instance.provisioningStarted(deletedAndInactiveRecord, deletedAndInactiveNodeName); - instance.provisionedSlaveNowActive(deletedAndInactiveRecord, deletedAndInactiveNodeName); - final vSphereCloudSlaveTemplate deletedAndInactiveTemplate = deletedAndInactiveRecord.getTemplate(); - instance.provisionedSlaveNowTerminated(deletedAndInactiveNodeName); - // A template which is current but has no active slaves right now - final CloudProvisioningRecord activeRecord = createRecord(instance); - instance.provisioningStarted(activeRecord, livedAndDiedNodeName); - instance.provisionedSlaveNowActive(activeRecord, livedAndDiedNodeName); - instance.provisionedSlaveNowTerminated(livedAndDiedNodeName); - - // When - userHasDeletedSlaveTemplate(deletedButActiveRecord); - userHasDeletedSlaveTemplate(deletedAndInactiveRecord); - wipeLog(); - instance.pruneUnwantedRecords(); - - // Then - assertThat( - loggedMessages, - contains(logMessage(containsString("Disposing"), Level.FINE, deletedAndInactiveTemplate.getCloneNamePrefix(), deletedAndInactiveTemplate.toString()))); - } - - @Test - public void countNodesGivenNoTemplatesOrSlavesThenReturnsZero() { - // Given - final CloudProvisioningState instance = createInstance(); - - // When - final int actual = instance.countNodes(); - - // Then - assertThat(actual, equalTo(0)); - } - - @Test - public void countNodesGivenNoSlavesInAnyTemplatesThenReturnsZero() { - // Given - final CloudProvisioningState instance = createInstance(); - createRecord(instance); - final String node = createNodeName(); - final CloudProvisioningRecord previouslyActiveRecord = createRecord(instance); - previouslyActiveRecord.addCurrentlyActive(node); - previouslyActiveRecord.removeCurrentlyActive(node); - createRecord(instance); - - // When - final int actual = instance.countNodes(); - - // Then - assertThat(actual, equalTo(0)); - } - - @Test - public void countNodesGiven2ActiveSlavesThenReturns2() { - // Given - final CloudProvisioningState instance = createInstance(); - final CloudProvisioningRecord activeRecord = createRecord(instance); - activeRecord.addCurrentlyActive(createNodeName()); - activeRecord.addCurrentlyActive(createNodeName()); - - // When - final int actual = instance.countNodes(); - - // Then - assertThat(actual, equalTo(2)); - } - - @Test - public void countNodesGiven3ActiveAnd4PendingSlavesThenReturns7() { - // Given - final CloudProvisioningState instance = createInstance(); - final CloudProvisioningRecord recordWith2Active = createRecord(instance); - recordWith2Active.addCurrentlyActive(createNodeName()); - recordWith2Active.addCurrentlyActive(createNodeName()); - final CloudProvisioningRecord recordWith1Active4Planned = createRecord(instance); - recordWith1Active4Planned.addCurrentlyActive(createNodeName()); - recordWith1Active4Planned.addCurrentlyPlanned(createNodeName()); - recordWith1Active4Planned.addCurrentlyPlanned(createNodeName()); - recordWith1Active4Planned.addCurrentlyPlanned(createNodeName()); - recordWith1Active4Planned.addCurrentlyPlanned(createNodeName()); - - // When - final int actual = instance.countNodes(); - - // Then - assertThat(actual, equalTo(7)); - } - - private void wipeLog() { - loggedMessages.clear(); - } - - private CloudProvisioningState createInstance() { - return new CloudProvisioningState(stubVSphereCloud, testLogger); - } - - private CloudProvisioningRecord createRecord(CloudProvisioningState instance) { - recordNumber++; - final String cloneNamePrefix = "prefix" + recordNumber; - final vSphereCloudSlaveTemplate template = new vSphereCloudSlaveTemplate(cloneNamePrefix, "masterImageName", - null, "snapshotName", false, "cluster", "resourcePool", "datastore", "folder", "customizationSpec", "templateDescription", 0, 1, "remoteFS", - "", Mode.NORMAL, false, false, 0, 0, false, "targetResourcePool", "targetHost", null, - new JNLPLauncher(), RetentionStrategy.NOOP, Collections.> emptyList(), - Collections. emptyList()); - stubVSphereCloudTemplates.add(template); - final List templates = new ArrayList(); - templates.add(template); - final List records = instance.calculateProvisionableTemplates(templates); - assertThat(records, IsIterableWithSize. iterableWithSize(1)); - final CloudProvisioningRecord record = records.get(0); - return record; - } - - private String createNodeName() { - nodeNumber++; - final String nodeName = "N#" + nodeNumber; - return nodeName; - } - - private void userHasDeletedSlaveTemplate(CloudProvisioningRecord record) { - stubVSphereCloudTemplates.remove(record.getTemplate()); - } - - private static Matcher logMessage(final Level expectedLevel, final Object... expectedArgs) { - final List> messageMatchers = new ArrayList>( - expectedArgs.length); - for (int i = 0; i < expectedArgs.length; i++) { - final String expectedString = "{" + i + "}"; - messageMatchers.add(containsString(expectedString)); - } - final Matcher messageMatcher; - if (messageMatchers.isEmpty()) { - messageMatcher = any(String.class); - } else { - messageMatcher = allOf(messageMatchers); - } - return logMessage(messageMatcher, expectedLevel, expectedArgs); - } - - private static Matcher logMessage(final Matcher messageMatcher, final Level expectedLevel, - final Object... expectedArgs) { - final Matcher levelMatcher = equalTo(expectedLevel); - final Matcher parametersMatcher = arrayContaining(expectedArgs); - final Matcher itemMatcher = new TypeSafeMatcher(LogRecord.class) { - @Override - public boolean matchesSafely(LogRecord actual) { - final String actualMessage = actual.getMessage(); - final Level actualLevel = actual.getLevel(); - final Object[] actualParameters = actual.getParameters(); - return messageMatcher.matches(actualMessage) && levelMatcher.matches(actualLevel) - && parametersMatcher.matches(actualParameters); - } - - @Override - public void describeTo(Description description) { - description.appendText("LogRecord("); - description.appendText("message ").appendDescriptionOf(messageMatcher); - description.appendText(" && level ").appendDescriptionOf(levelMatcher); - description.appendText(" && parameters ").appendDescriptionOf(parametersMatcher); - description.appendText(")"); - } - - @Override - protected void describeMismatchSafely(LogRecord actual, Description description) { - final String actualMessage = actual.getMessage(); - final Level actualLevel = actual.getLevel(); - final Object[] actualParameters = actual.getParameters(); - description.appendText("was LogRecord("); - description.appendText("message=\"").appendValue(actualMessage); - description.appendText("\", level ").appendValue(actualLevel); - description.appendText(", parameters ").appendValueList("[", ",", "]", actualParameters); - description.appendText(")"); - } - }; - return itemMatcher; - } -} +package org.jenkinsci.plugins.vsphere.tools; + +import static org.hamcrest.CoreMatchers.allOf; +import static org.hamcrest.CoreMatchers.either; +import static org.hamcrest.CoreMatchers.any; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.everyItem; +import static org.hamcrest.collection.IsArrayContainingInOrder.arrayContaining; +import static org.hamcrest.collection.IsIterableContainingInOrder.contains; +import static org.junit.Assert.assertThat; +import hudson.model.Node.Mode; +import hudson.slaves.NodeProperty; +import hudson.slaves.JNLPLauncher; +import hudson.slaves.RetentionStrategy; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; + +import org.hamcrest.Description; +import org.hamcrest.Matcher; +import org.hamcrest.TypeSafeMatcher; +import org.hamcrest.collection.IsIterableWithSize; +import org.jenkinsci.plugins.vSphereCloud; +import org.jenkinsci.plugins.vSphereCloudSlaveTemplate; +import org.jenkinsci.plugins.vsphere.VSphereConnectionConfig; +import org.jenkinsci.plugins.vsphere.VSphereGuestInfoProperty; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class CloudProvisioningStateTest { + private static List stubVSphereCloudTemplates; + private static vSphereCloud stubVSphereCloud; + private int recordNumber; + private int nodeNumber; + private Logger testLogger; + private List loggedMessages; + + @BeforeClass + public static void setupClass() { + stubVSphereCloudTemplates = new ArrayList(); + final VSphereConnectionConfig vsConnectionConfig = new VSphereConnectionConfig("vsHost", "credentialsId"); + stubVSphereCloud = new vSphereCloud(vsConnectionConfig, "vsDescription", 0, 0, stubVSphereCloudTemplates); + } + + @Before + public void setup() { + stubVSphereCloudTemplates.clear(); + recordNumber = 0; + nodeNumber = 0; + loggedMessages = new ArrayList(); + Logger logger = Logger.getLogger("CloudProvisioningStateTest"); + logger.setLevel(Level.ALL); + final Handler[] handlers = logger.getHandlers(); + for (final Handler handler : handlers) { + logger.removeHandler(handler); + } + final Handler testHandler = new Handler() { + @Override + public void publish(LogRecord record) { + loggedMessages.add(record); + } + + @Override + public void flush() { + } + + @Override + public void close() { + } + }; + logger.addHandler(testHandler); + testLogger = logger; + } + + @Test + public void constructorGivenCalledThenLogsConstructions() { + // Given + final Object[] expectedArgs = { stubVSphereCloud.toString() }; + + // When + createInstance(); + + // Then + assertThat(loggedMessages, contains(logMessage(Level.FINE, expectedArgs))); + } + + @Test + public void provisioningStartedGivenNoPreviousStateThenLogs() { + // Given + final String nodeName = createNodeName(); + final Object[] expectedArgs = { nodeName }; + final CloudProvisioningState instance = createInstance(); + final CloudProvisioningRecord provisionable = createRecord(instance); + wipeLog(); + + // When + instance.provisioningStarted(provisionable, nodeName); + + // Then + assertThat(loggedMessages, contains(logMessage(Level.FINE, expectedArgs))); + } + + @Test + public void provisioningStartedGivenPreviouslyStartedThenWarns() { + // Given + final String nodeName = createNodeName(); + final Object[] expectedArgs = { nodeName }; + final CloudProvisioningState instance = createInstance(); + final CloudProvisioningRecord provisionable = createRecord(instance); + instance.provisioningStarted(provisionable, nodeName); + wipeLog(); + + // When + instance.provisioningStarted(provisionable, nodeName); + + // Then + assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); + } + + @Test + public void normalLifecycleGivenNoErrorsThenLogs() { + // Given + final String nodeName = createNodeName(); + final Object[] expectedArgs = { nodeName }; + final CloudProvisioningState instance = createInstance(); + final CloudProvisioningRecord provisionable = createRecord(instance); + wipeLog(); + + // When + instance.provisioningStarted(provisionable, nodeName); + instance.provisionedSlaveNowActive(provisionable, nodeName); + instance.provisionedSlaveNowUnwanted(nodeName, true); + instance.unwantedSlaveNowDeleted(nodeName); + + // Then + assertThat(loggedMessages, everyItem(logMessage(Level.FINE, expectedArgs))); + assertThat(loggedMessages, IsIterableWithSize. iterableWithSize(4)); + } + + @SuppressWarnings("unchecked") + @Test + public void failedToProvisionGivenNothingOutOfSequenceThenLogs() { + // Given + final String nodeName = createNodeName(); + final Object[] expectedArgs = { nodeName }; + final CloudProvisioningState instance = createInstance(); + final CloudProvisioningRecord provisionable = createRecord(instance); + wipeLog(); + + // When + instance.provisioningStarted(provisionable, nodeName); + instance.provisioningEndedInError(provisionable, nodeName); + + // Then + assertThat( + loggedMessages, + contains(logMessage(Level.FINE, expectedArgs), + logMessage(containsString("failed"), Level.INFO, expectedArgs))); + assertThat(loggedMessages, IsIterableWithSize. iterableWithSize(2)); + } + + @Test + public void provisionGivenOutOfOrderSequenceThenComplains() { + // Given + final String nodeName = createNodeName(); + final Object[] expectedArgs = { nodeName }; + final CloudProvisioningState instance = createInstance(); + final CloudProvisioningRecord provisionable = createRecord(instance); + wipeLog(); + + // When/Then + wipeLog(); + instance.unwantedSlaveNowDeleted(nodeName); + assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); + + wipeLog(); + instance.provisionedSlaveNowUnwanted(nodeName, false); + assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); + + wipeLog(); + instance.provisioningStarted(provisionable, nodeName); + assertThat(loggedMessages, contains(logMessage(Level.FINE, expectedArgs))); + + wipeLog(); + instance.provisioningStarted(provisionable, nodeName); + assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); + + wipeLog(); + instance.provisioningEndedInError(provisionable, nodeName); + assertThat(loggedMessages, contains(logMessage(Level.INFO, expectedArgs))); + + wipeLog(); + instance.provisioningEndedInError(provisionable, nodeName); + assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); + + wipeLog(); + instance.provisionedSlaveNowActive(provisionable, nodeName); + assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); + + wipeLog(); + instance.provisionedSlaveNowActive(provisionable, nodeName); + assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); + + wipeLog(); + instance.provisioningStarted(provisionable, nodeName); + assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); + + wipeLog(); + instance.provisioningStarted(provisionable, nodeName); + assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); + + wipeLog(); + instance.provisionedSlaveNowUnwanted(nodeName, false); + assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); + + wipeLog(); + instance.provisionedSlaveNowUnwanted(nodeName, true); + assertThat(loggedMessages, contains(logMessage(Level.WARNING, expectedArgs))); + } + + @Test + public void pruneUnwantedRecordsGivenUnknownTemplatesThenRemovesRecordsForEmptyDeletedTemplates() { + // Given + final String deletedAndInactiveNodeName = createNodeName(); + final String deletedButActiveNodeName = createNodeName(); + final String livedAndDiedNodeName = createNodeName(); + final CloudProvisioningState instance = createInstance(); + // A template which the user deleted but still has an active slave + final CloudProvisioningRecord deletedButActiveRecord = createRecord(instance); + instance.provisioningStarted(deletedButActiveRecord, deletedButActiveNodeName); + instance.provisionedSlaveNowActive(deletedButActiveRecord, deletedButActiveNodeName); + // A template which the user deleted and is no longer needed + final CloudProvisioningRecord deletedAndInactiveRecord = createRecord(instance); + instance.provisioningStarted(deletedAndInactiveRecord, deletedAndInactiveNodeName); + instance.provisionedSlaveNowActive(deletedAndInactiveRecord, deletedAndInactiveNodeName); + final vSphereCloudSlaveTemplate deletedAndInactiveTemplate = deletedAndInactiveRecord.getTemplate(); + instance.provisionedSlaveNowUnwanted(deletedAndInactiveNodeName, true); + instance.unwantedSlaveNowDeleted(deletedAndInactiveNodeName); + // A template which is current but has no active slaves right now + final CloudProvisioningRecord activeRecord = createRecord(instance); + instance.provisioningStarted(activeRecord, livedAndDiedNodeName); + instance.provisionedSlaveNowActive(activeRecord, livedAndDiedNodeName); + instance.provisionedSlaveNowUnwanted(livedAndDiedNodeName, true); + instance.unwantedSlaveNowDeleted(livedAndDiedNodeName); + + // When + userHasDeletedSlaveTemplate(deletedButActiveRecord); + userHasDeletedSlaveTemplate(deletedAndInactiveRecord); + wipeLog(); + instance.pruneUnwantedRecords(); + + // Then + assertThat( + loggedMessages, + contains(logMessage(containsString("Disposing"), Level.FINE, deletedAndInactiveTemplate.getCloneNamePrefix(), deletedAndInactiveTemplate.toString()))); + } + + @Test + public void countNodesGivenNoTemplatesOrSlavesThenReturnsZero() { + // Given + final CloudProvisioningState instance = createInstance(); + + // When + final int actual = instance.countNodes(); + + // Then + assertThat(actual, equalTo(0)); + } + + @Test + public void countNodesGivenNoSlavesInAnyTemplatesThenReturnsZero() { + // Given + final CloudProvisioningState instance = createInstance(); + createRecord(instance); + final String node = createNodeName(); + final CloudProvisioningRecord previouslyActiveRecord = createRecord(instance); + previouslyActiveRecord.addCurrentlyActive(node); + previouslyActiveRecord.removeCurrentlyActive(node); + createRecord(instance); + + // When + final int actual = instance.countNodes(); + + // Then + assertThat(actual, equalTo(0)); + } + + @Test + public void countNodesGiven2ActiveSlavesThenReturns2() { + // Given + final CloudProvisioningState instance = createInstance(); + final CloudProvisioningRecord activeRecord = createRecord(instance); + activeRecord.addCurrentlyActive(createNodeName()); + activeRecord.addCurrentlyActive(createNodeName()); + + // When + final int actual = instance.countNodes(); + + // Then + assertThat(actual, equalTo(2)); + } + + @Test + public void countNodesGiven2UnwantedSlavesThenReturns2() { + // Given + final CloudProvisioningState instance = createInstance(); + final CloudProvisioningRecord activeRecord = createRecord(instance); + activeRecord.setCurrentlyUnwanted(createNodeName(), false); + activeRecord.setCurrentlyUnwanted(createNodeName(), false); + + // When + final int actual = instance.countNodes(); + + // Then + assertThat(actual, equalTo(2)); + } + + @Test + public void countNodesGiven3ActiveAnd4PendingSlavesThenReturns7() { + // Given + final CloudProvisioningState instance = createInstance(); + final CloudProvisioningRecord recordWith2Active = createRecord(instance); + recordWith2Active.addCurrentlyActive(createNodeName()); + recordWith2Active.addCurrentlyActive(createNodeName()); + final CloudProvisioningRecord recordWith1Active4Planned = createRecord(instance); + recordWith1Active4Planned.addCurrentlyActive(createNodeName()); + recordWith1Active4Planned.addCurrentlyPlanned(createNodeName()); + recordWith1Active4Planned.addCurrentlyPlanned(createNodeName()); + recordWith1Active4Planned.addCurrentlyPlanned(createNodeName()); + recordWith1Active4Planned.addCurrentlyPlanned(createNodeName()); + + // When + final int actual = instance.countNodes(); + + // Then + assertThat(actual, equalTo(7)); + } + + @Test + public void getUnwantedVMsThatNeedDeletingGivenNothingNeedsDeletingThenReturnsNothing() { + // Given + final CloudProvisioningState instance = createInstance(); + final CloudProvisioningRecord recordWith2Active = createRecord(instance); + recordWith2Active.addCurrentlyActive(createNodeName()); + recordWith2Active.addCurrentlyActive(createNodeName()); + final CloudProvisioningRecord recordWith1Active1Planned = createRecord(instance); + recordWith1Active1Planned.addCurrentlyActive(createNodeName()); + recordWith1Active1Planned.addCurrentlyPlanned(createNodeName()); + final List expected = Arrays.asList(); + + // When + final List actual = instance.getUnwantedVMsThatNeedDeleting(); + // Then + assertThat(actual, equalTo(expected)); + } + + @Test + public void getUnwantedVMsThatNeedDeletingGivenSeveralUnwantedThenReturnsDeletableVMsInCorrectOrder() { + // Given + final CloudProvisioningState instance = createInstance(); + final CloudProvisioningRecord r1 = createRecord(instance); + final String r1unwanted1 = createNodeName(); + final String r1unwanted2alreadyBeingDeleted = createNodeName(); + final String r1unwanted3 = createNodeName(); + r1.setCurrentlyUnwanted(r1unwanted1, false); + r1.setCurrentlyUnwanted(r1unwanted2alreadyBeingDeleted, true); + r1.setCurrentlyUnwanted(r1unwanted3, false); + final CloudProvisioningRecord r2 = createRecord(instance); + final String r2unwanted1alreadyBeingDeleted = createNodeName(); + final String r2unwanted2 = createNodeName(); + r2.setCurrentlyUnwanted(r2unwanted1alreadyBeingDeleted, true); + r2.setCurrentlyUnwanted(r2unwanted2, false); + // Expect to get the first not-being-deleted record from both before the + // second not-being-deleted record. However we don't mind the order that + // the records are processed, so there are two correct answers. + final List expectedA = Arrays.asList(r1unwanted1, r2unwanted2, r1unwanted3); + final List expectedB = Arrays.asList(r2unwanted2, r1unwanted1, r1unwanted3); + + // When + final List actual = instance.getUnwantedVMsThatNeedDeleting(); + // Then + assertThat(actual, either(equalTo(expectedA)).or(equalTo(expectedB))); + } + + @Test + public void unwantedSlaveNowDeletedGivenNothingElseToDeleteThenNothingRemains() { + // Given + final CloudProvisioningState instance = createInstance(); + final CloudProvisioningRecord r1 = createRecord(instance); + final String r1unwanted1 = createNodeName(); + final vSphereCloudSlaveTemplate r1t = r1.getTemplate(); + instance.recordExistingUnwantedVM(r1t, r1unwanted1); + final int numberOfNodesBeforeDeletion = instance.countNodes(); + assertThat(numberOfNodesBeforeDeletion, equalTo(1)); + final List noNodes = Arrays.asList(); + + // When + final Boolean actualIsOkToDelete = instance.isOkToDeleteUnwantedVM(r1unwanted1); + instance.unwantedSlaveNowDeleted(r1unwanted1); + final int actualNodesAfterDeletion = instance.countNodes(); + final List actualNodesToBeDeletedNext = instance.getUnwantedVMsThatNeedDeleting(); + + // Then + assertThat(actualIsOkToDelete, equalTo(Boolean.TRUE)); + assertThat(actualNodesAfterDeletion, equalTo(0)); + assertThat(actualNodesToBeDeletedNext, equalTo(noNodes)); + } + + @Test + public void unwantedSlaveNotDeletedGivenNothingElseToDeleteThenFailedDeletionRemains() { + // Given + final CloudProvisioningState instance = createInstance(); + final CloudProvisioningRecord r1 = createRecord(instance); + final String r1unwanted1 = createNodeName(); + final vSphereCloudSlaveTemplate r1t = r1.getTemplate(); + instance.recordExistingUnwantedVM(r1t, r1unwanted1); + final int numberOfNodesBeforeDeletion = instance.countNodes(); + assertThat(numberOfNodesBeforeDeletion, equalTo(1)); + final List sameNodeRemains = Arrays.asList(r1unwanted1); + + // When + final Boolean actualIsOkToDelete = instance.isOkToDeleteUnwantedVM(r1unwanted1); + instance.unwantedSlaveNotDeleted(r1unwanted1); + final int actualNodesAfterDeletion = instance.countNodes(); + final List actualNodesToBeDeletedNext = instance.getUnwantedVMsThatNeedDeleting(); + + // Then + assertThat(actualIsOkToDelete, equalTo(Boolean.TRUE)); + assertThat(actualNodesAfterDeletion, equalTo(1)); + assertThat(actualNodesToBeDeletedNext, equalTo(sameNodeRemains)); + } + + @Test + public void isOkToDeleteUnwantedVMGivenNobodyElseDeletingVMThenReturnsTrueOnceAndFalseThereafter() { + // Given + final CloudProvisioningState instance = createInstance(); + final CloudProvisioningRecord r1 = createRecord(instance); + final String r1unwanted1 = createNodeName(); + final vSphereCloudSlaveTemplate r1t = r1.getTemplate(); + instance.recordExistingUnwantedVM(r1t, r1unwanted1); + + // When + final Boolean actual1 = instance.isOkToDeleteUnwantedVM(r1unwanted1); + final Boolean actual2 = instance.isOkToDeleteUnwantedVM(r1unwanted1); + final Boolean actual3 = instance.isOkToDeleteUnwantedVM(r1unwanted1); + + // Then + assertThat(actual1, equalTo(Boolean.TRUE)); + assertThat(actual2, equalTo(Boolean.FALSE)); + assertThat(actual3, equalTo(Boolean.FALSE)); + } + + @Test + public void isOkToDeleteUnwantedVMGivenDeletionInProgressVMThenReturnsFalseUntilDeletionFails() { + // Given + final CloudProvisioningState instance = createInstance(); + final CloudProvisioningRecord r1 = createRecord(instance); + final String r1unwanted1 = createNodeName(); + r1.setCurrentlyUnwanted(r1unwanted1, true); + + // When + final Boolean actualBefore = instance.isOkToDeleteUnwantedVM(r1unwanted1); + instance.unwantedSlaveNotDeleted(r1unwanted1); + final Boolean actualAfter = instance.isOkToDeleteUnwantedVM(r1unwanted1); + + // Then + assertThat(actualBefore, equalTo(Boolean.FALSE)); + assertThat(actualAfter, equalTo(Boolean.TRUE)); + } + + private void wipeLog() { + loggedMessages.clear(); + } + + private CloudProvisioningState createInstance() { + return new CloudProvisioningState(stubVSphereCloud, testLogger); + } + + private CloudProvisioningRecord createRecord(CloudProvisioningState instance) { + recordNumber++; + final String cloneNamePrefix = "prefix" + recordNumber; + final vSphereCloudSlaveTemplate template = new vSphereCloudSlaveTemplate(cloneNamePrefix, "masterImageName", + null, "snapshotName", false, "cluster", "resourcePool", "datastore", "folder", "customizationSpec", "templateDescription", 0, 1, "remoteFS", + "", Mode.NORMAL, false, false, 0, 0, false, "targetResourcePool", "targetHost", null, + new JNLPLauncher(), RetentionStrategy.NOOP, Collections.> emptyList(), + Collections. emptyList()); + stubVSphereCloudTemplates.add(template); + final List templates = new ArrayList(); + templates.add(template); + final List records = instance.calculateProvisionableTemplates(templates); + assertThat(records, IsIterableWithSize. iterableWithSize(1)); + final CloudProvisioningRecord record = records.get(0); + return record; + } + + private String createNodeName() { + nodeNumber++; + final String nodeName = "N#" + nodeNumber; + return nodeName; + } + + private void userHasDeletedSlaveTemplate(CloudProvisioningRecord record) { + stubVSphereCloudTemplates.remove(record.getTemplate()); + } + + private static Matcher logMessage(final Level expectedLevel, final Object... expectedArgs) { + final List> messageMatchers = new ArrayList>( + expectedArgs.length); + for (int i = 0; i < expectedArgs.length; i++) { + final String expectedString = "{" + i + "}"; + messageMatchers.add(containsString(expectedString)); + } + final Matcher messageMatcher; + if (messageMatchers.isEmpty()) { + messageMatcher = any(String.class); + } else { + messageMatcher = allOf(messageMatchers); + } + return logMessage(messageMatcher, expectedLevel, expectedArgs); + } + + private static Matcher logMessage(final Matcher messageMatcher, final Level expectedLevel, + final Object... expectedArgs) { + final Matcher levelMatcher = equalTo(expectedLevel); + final Matcher parametersMatcher = arrayContaining(expectedArgs); + final Matcher itemMatcher = new TypeSafeMatcher(LogRecord.class) { + @Override + public boolean matchesSafely(LogRecord actual) { + final String actualMessage = actual.getMessage(); + final Level actualLevel = actual.getLevel(); + final Object[] actualParameters = actual.getParameters(); + return messageMatcher.matches(actualMessage) && levelMatcher.matches(actualLevel) + && parametersMatcher.matches(actualParameters); + } + + @Override + public void describeTo(Description description) { + description.appendText("LogRecord("); + description.appendText("message ").appendDescriptionOf(messageMatcher); + description.appendText(" && level ").appendDescriptionOf(levelMatcher); + description.appendText(" && parameters ").appendDescriptionOf(parametersMatcher); + description.appendText(")"); + } + + @Override + protected void describeMismatchSafely(LogRecord actual, Description description) { + final String actualMessage = actual.getMessage(); + final Level actualLevel = actual.getLevel(); + final Object[] actualParameters = actual.getParameters(); + description.appendText("was LogRecord("); + description.appendText("message=\"").appendValue(actualMessage); + description.appendText("\", level ").appendValue(actualLevel); + description.appendText(", parameters ").appendValueList("[", ",", "]", actualParameters); + description.appendText(")"); + } + }; + return itemMatcher; + } +}