From e2cecb599d30ae8f110da92e92909ff50d0d79aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20L=C3=A4ubrich?= Date: Fri, 28 Jan 2022 05:30:08 +0100 Subject: [PATCH] Enhance the incomplete soloution computation (#576) Fix #575 - add a predicate to customize filtering - Use loop instead of recursive call - break on incomplete prerequisite - Support NoExecutionEnvironmentResolutionHints - Support HardRequirement's - Also consider available units as not being a valid missing requirement --- .../AbstractSlicerResolutionStrategy.java | 130 +++++++++++++++++- .../ProjectorResolutionStrategy.java | 122 ++++++++-------- .../p2/util/resolution/ResolutionData.java | 7 + .../util/resolution/ResolutionDataImpl.java | 12 ++ 4 files changed, 216 insertions(+), 55 deletions(-) diff --git a/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/util/resolution/AbstractSlicerResolutionStrategy.java b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/util/resolution/AbstractSlicerResolutionStrategy.java index 56cf987d17..59b7f7bdc9 100644 --- a/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/util/resolution/AbstractSlicerResolutionStrategy.java +++ b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/util/resolution/AbstractSlicerResolutionStrategy.java @@ -18,16 +18,24 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.UUID; import org.eclipse.core.runtime.IProgressMonitor; import org.eclipse.core.runtime.MultiStatus; +import org.eclipse.equinox.internal.p2.director.Explanation; +import org.eclipse.equinox.internal.p2.director.Explanation.HardRequirement; +import org.eclipse.equinox.internal.p2.director.Explanation.IUToInstall; +import org.eclipse.equinox.internal.p2.director.Explanation.MissingIU; import org.eclipse.equinox.internal.p2.director.Slicer; import org.eclipse.equinox.internal.p2.metadata.IRequiredCapability; import org.eclipse.equinox.internal.p2.metadata.RequiredCapability; +import org.eclipse.equinox.internal.p2.metadata.RequiredPropertiesMatch; import org.eclipse.equinox.p2.metadata.IInstallableUnit; import org.eclipse.equinox.p2.metadata.IProvidedCapability; import org.eclipse.equinox.p2.metadata.IRequirement; @@ -35,9 +43,13 @@ import org.eclipse.equinox.p2.metadata.MetadataFactory.InstallableUnitDescription; import org.eclipse.equinox.p2.metadata.Version; import org.eclipse.equinox.p2.metadata.VersionRange; +import org.eclipse.equinox.p2.metadata.expression.ExpressionUtil; +import org.eclipse.equinox.p2.metadata.expression.IExpression; import org.eclipse.equinox.p2.metadata.expression.IMatchExpression; +import org.eclipse.equinox.p2.publisher.actions.JREAction; import org.eclipse.equinox.p2.query.IQueryable; import org.eclipse.tycho.core.shared.MavenLogger; +import org.eclipse.tycho.p2.target.ee.NoExecutionEnvironmentResolutionHints; import org.eclipse.tycho.repository.p2base.metadata.QueryableCollection; import org.eclipse.tycho.repository.util.StatusTool; @@ -132,7 +144,7 @@ protected IInstallableUnit createUnitProviding(String name, Collection matches = propertiesMatch.getMatches(); + Map properties = new HashMap<>(); + Object p = matches.getParameters()[1]; + if (p instanceof IExpression) { + IExpression expression = (IExpression) p; + IExpression operand = ExpressionUtil.getOperand(expression); + IExpression[] operands = ExpressionUtil.getOperands(operand); + for (IExpression eq : operands) { + IExpression lhs = ExpressionUtil.getLHS(eq); + IExpression rhs = ExpressionUtil.getRHS(eq); + Object value = ExpressionUtil.getValue(rhs); + String key = ExpressionUtil.getName(lhs); + if (IProvidedCapability.PROPERTY_VERSION.equals(key)) { + properties.put(key, Version.create(value.toString())); + } else { + properties.put(key, value.toString()); + } + } + } + IProvidedCapability providedCapability = MetadataFactory.createProvidedCapability( + RequiredPropertiesMatch.extractNamespace(matches), properties); + result.addProvidedCapabilities(Collections.singleton(providedCapability)); + } + } catch (RuntimeException e) { + logger.debug("can't convert requirement " + requirement + " to capability: " + e.toString(), e); + } } } return MetadataFactory.createInstallableUnit(result); } + /** + * Computes a list of current missing requirements. The list only contains requirements up to + * the point where it is known that this is a 'root' that means a requirement that prevents + * computation of a complete solution. + * + * @param explanation + * @return + */ + protected List computeMissingRequirements(Set explanation) { + List missingRequirements = new ArrayList<>(); + //We collect here all units that are available but maybe incomplete due to an missing requirement. + //This is important as otherwise we could generate false missing requirements as they might just be chained + // Here is an example: + // a) Bundle require an EE or package what is missing + // b) Feature requires the Bundle + // c) Updatesite requires feature + // When resolving the Updatesite, it now seem to miss the Bundle *and* the Feature because the feature itself + // is incomplete but actually on only the EE or package is missing. + Collection availableIUs = new HashSet<>(data.getAvailableIUs()); + for (Explanation exp : explanation) { + if (exp instanceof IUToInstall) { + IUToInstall iuToInstall = (IUToInstall) exp; + availableIUs.add(iuToInstall.iu); + } else if (exp instanceof MissingIU) { + MissingIU missingIU = (MissingIU) exp; + availableIUs.add(missingIU.iu); + if (isEERequirement(missingIU.req)) { + if (data.getEEResolutionHints() instanceof NoExecutionEnvironmentResolutionHints) { + //if NoEE is specified this is acceptable and should be recorded + missingRequirements.add(missingIU.req); + } + continue; + } + for (IInstallableUnit available : availableIUs) { + if (missingIU.req.isMatch(available)) { + if (logger.isExtendedDebugEnabled()) { + logger.debug("IU " + missingIU.iu + " requires an available or incomplete IU " + available + + " ..."); + } + return missingRequirements; + } + } + if (data.failOnMissingRequirements()) { + //we should not record those... + continue; + } + missingRequirements.add(missingIU.req); + } else if (exp instanceof HardRequirement) { + HardRequirement hardRequirement = (HardRequirement) exp; + availableIUs.add(hardRequirement.iu); + for (IInstallableUnit available : availableIUs) { + if (hardRequirement.req.isMatch(available)) { + if (logger.isExtendedDebugEnabled()) { + logger.debug("IU " + hardRequirement.iu + " has requirement on available or incomplete IU " + + available + " ..."); + } + return missingRequirements; + } + } + missingRequirements.add(hardRequirement.req); + } else { + if (logger.isExtendedDebugEnabled()) { + logger.debug("Ignoring Explanation of type " + exp.getClass() + + " in computation of missing requirements: " + exp); + } + } + } + missingRequirements.forEach(data::addMissingRequirement); + return missingRequirements; + } + + /** + * Check if this is an EE environment requirement + * + * @param requirement + * @return + */ + protected static boolean isEERequirement(IRequirement requirement) { + if (requirement instanceof RequiredPropertiesMatch) { + RequiredPropertiesMatch propertiesMatch = (RequiredPropertiesMatch) requirement; + String namespace = RequiredPropertiesMatch.extractNamespace(propertiesMatch.getMatches()); + return JREAction.NAMESPACE_OSGI_EE.equals(namespace); + } + return false; + } + private static IRequirement createStrictRequirementTo(IInstallableUnit unit) { VersionRange strictRange = new VersionRange(unit.getVersion(), true, unit.getVersion(), true); int min = 1; diff --git a/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/util/resolution/ProjectorResolutionStrategy.java b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/util/resolution/ProjectorResolutionStrategy.java index d13997f640..a57d521dcc 100644 --- a/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/util/resolution/ProjectorResolutionStrategy.java +++ b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/util/resolution/ProjectorResolutionStrategy.java @@ -20,6 +20,8 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeoutException; +import java.util.function.Predicate; import java.util.stream.Collectors; import org.eclipse.core.runtime.IProgressMonitor; @@ -27,7 +29,6 @@ import org.eclipse.core.runtime.MultiStatus; import org.eclipse.core.runtime.NullProgressMonitor; import org.eclipse.equinox.internal.p2.director.Explanation; -import org.eclipse.equinox.internal.p2.director.Explanation.MissingIU; import org.eclipse.equinox.internal.p2.director.Projector; import org.eclipse.equinox.internal.p2.director.QueryableArray; import org.eclipse.equinox.internal.p2.director.SimplePlanner; @@ -40,6 +41,12 @@ @SuppressWarnings("restriction") public class ProjectorResolutionStrategy extends AbstractSlicerResolutionStrategy { + /** + * Internal property to control the maximum number of iterations performed to resolve an + * incomplete solution + */ + private static final int MAX_ITERATIONS = Integer + .getInteger("tycho.internal.ProjectorResolutionStrategy.maxIterations", 100); public ProjectorResolutionStrategy(MavenLogger logger) { super(logger); @@ -47,7 +54,16 @@ public ProjectorResolutionStrategy(MavenLogger logger) { @Override protected Slicer newSlicer(IQueryable availableUnits, Map properties) { - return new Slicer(availableUnits, properties, false); + Predicate acceptor = data.getIInstallableUnitAcceptor(); + return new Slicer(availableUnits, properties, false) { + @Override + protected boolean isApplicable(IInstallableUnit iu) { + if (acceptor != null) { + return acceptor.test(iu); + } + return super.isApplicable(iu); + } + }; } @Override @@ -58,17 +74,8 @@ protected boolean isSlicerError(MultiStatus slicerStatus) { @Override public Collection resolve(Map properties, IProgressMonitor monitor) throws ResolverException { - List additionalUnits = new ArrayList<>(); - Collection newState = resolveInternal(properties, additionalUnits, monitor); - newState.removeAll(additionalUnits); //remove the tycho generated IUs if any - return newState; - } - - protected Collection resolveInternal(Map properties, - List additionalUnits, IProgressMonitor monitor) throws ResolverException { - Map newSelectionContext = SimplePlanner.createSelectionContext(properties); - - IQueryable slice = slice(properties, additionalUnits, monitor); + List generatedUnits = new ArrayList<>(); + Map selectionContext = SimplePlanner.createSelectionContext(properties); Set seedUnits = new LinkedHashSet<>(data.getRootIUs()); List seedRequires = new ArrayList<>(); @@ -80,54 +87,61 @@ protected Collection resolveInternal(Map prope seedUnits.addAll(data.getEEResolutionHints().getMandatoryUnits()); seedRequires.addAll(data.getEEResolutionHints().getMandatoryRequires()); - Projector projector = new Projector(slice, newSelectionContext, new HashSet(), false); - projector.encode(createUnitRequiring("tycho", seedUnits, seedRequires), - EMPTY_IU_ARRAY /* alreadyExistingRoots */, - new QueryableArray(EMPTY_IU_ARRAY) /* installedIUs */, seedUnits /* newRoots */, monitor); - IStatus s = projector.invokeSolver(monitor); - if (s.getSeverity() == IStatus.ERROR) { - Set explanation = projector.getExplanation(new NullProgressMonitor()); // suppress "Cannot complete the request. Generating details." - if (!data.failOnMissingRequirements()) { - List missingRequirements = new ArrayList<>(); - for (Explanation exp : explanation) { - if (exp instanceof MissingIU) { - MissingIU missingIU = (MissingIU) exp; - logger.debug("Recording missing requirement for IU " + missingIU.iu + ": " + missingIU.req); - data.addMissingRequirement(missingIU.req); - missingRequirements.add(missingIU.req); - } else { + int iteration = 0; + do { + Projector projector = new Projector(slice(properties, generatedUnits, monitor), selectionContext, + new HashSet(), false); + projector.encode(createUnitRequiring("tycho", seedUnits, seedRequires), + EMPTY_IU_ARRAY /* alreadyExistingRoots */, + new QueryableArray(EMPTY_IU_ARRAY) /* installedIUs */, seedUnits /* newRoots */, monitor); + IStatus s = projector.invokeSolver(monitor); + if (s.getSeverity() == IStatus.ERROR) { + Set explanation = projector.getExplanation(new NullProgressMonitor()); // suppress "Cannot complete the request. Generating details." + if (!data.failOnMissingRequirements()) { + List missingRequirements = computeMissingRequirements(explanation); + if (missingRequirements.size() > 0) { if (logger.isExtendedDebugEnabled()) { - logger.debug("Ignoring Explanation of type " + exp.getClass() - + " in computation of missing requirements: " + exp); + logger.debug( + "At iteration " + iteration + " the following requirements are not yet satisfied:"); + for (IRequirement requirement : missingRequirements) { + logger.debug("> " + requirement); + } + } + //only start a new resolve if we have collected additional requirements... + IInstallableUnit providing = createUnitProviding("tycho.unresolved.requirements", + missingRequirements); + int newCapabilities = providing.getProvidedCapabilities().size(); + if (newCapabilities > 0) { + //... and we could provide additional capabilities + if (logger.isExtendedDebugEnabled()) { + logger.debug(newCapabilities + + " new capabilities where created, starting next iteration..."); + } + generatedUnits.add(providing); + iteration++; + continue; } } } - if (missingRequirements.size() > 0) { - //only start a new resolve if we have collected additional requirements... - IInstallableUnit providing = createUnitProviding("tycho.unresolved.requirements", - missingRequirements); - if (providing.getProvidedCapabilities().size() > 0) { - //... and we could provide additional capabilities - additionalUnits.add(providing); - return resolveInternal(properties, additionalUnits, monitor); - } - } + // log all transitive requirements which cannot be satisfied; this doesn't print the dependency chain from the seed to the units with missing requirements, so this is less useful than the "explanation" + logger.debug(StatusTool.collectProblems(s)); + explainProblems(explanation, MavenLogger::error); + throw new ResolverException( + explanation.stream().map(Object::toString).collect(Collectors.joining("\n")), + selectionContext.toString(), StatusTool.findException(s)); } - // log all transitive requirements which cannot be satisfied; this doesn't print the dependency chain from the seed to the units with missing requirements, so this is less useful than the "explanation" - logger.debug(StatusTool.collectProblems(s)); - explainProblems(explanation, MavenLogger::error); - throw new ResolverException(explanation.stream().map(Object::toString).collect(Collectors.joining("\n")), - newSelectionContext.toString(), StatusTool.findException(s)); - } - Collection newState = projector.extractSolution(); + Collection newState = projector.extractSolution(); - // remove fake IUs from resolved state - newState.removeAll(data.getEEResolutionHints().getTemporaryAdditions()); + // remove fake IUs from resolved state + newState.removeAll(data.getEEResolutionHints().getTemporaryAdditions()); + newState.removeAll(generatedUnits); //remove the tycho generated IUs if any - if (logger.isExtendedDebugEnabled()) { - logger.debug("Resolved IUs:\n" + ResolverDebugUtils.toDebugString(newState, false)); - } - return newState; + if (logger.isExtendedDebugEnabled()) { + logger.debug("Resolved IUs:\n" + ResolverDebugUtils.toDebugString(newState, false)); + } + return newState; + } while (iteration < MAX_ITERATIONS); + throw new ResolverException("Maximum iterations reached", new TimeoutException()); } } diff --git a/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/util/resolution/ResolutionData.java b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/util/resolution/ResolutionData.java index 6da03451eb..436b531332 100644 --- a/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/util/resolution/ResolutionData.java +++ b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/util/resolution/ResolutionData.java @@ -16,6 +16,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.function.Predicate; import org.eclipse.equinox.p2.metadata.IInstallableUnit; import org.eclipse.equinox.p2.metadata.IRequirement; @@ -41,4 +42,10 @@ public interface ResolutionData { void addMissingRequirement(IRequirement requirement); Collection getMissingRequirements(); + + /** + * + * @return a predicate that us used to check if a given unit should be accepted by the slicer + */ + Predicate getIInstallableUnitAcceptor(); } diff --git a/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/util/resolution/ResolutionDataImpl.java b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/util/resolution/ResolutionDataImpl.java index 43e8e05e81..2984de6149 100644 --- a/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/util/resolution/ResolutionDataImpl.java +++ b/tycho-bundles/org.eclipse.tycho.p2.resolver.impl/src/main/java/org/eclipse/tycho/p2/util/resolution/ResolutionDataImpl.java @@ -20,6 +20,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.function.Predicate; import org.eclipse.core.runtime.NullProgressMonitor; import org.eclipse.equinox.p2.metadata.IInstallableUnit; @@ -39,6 +40,8 @@ public class ResolutionDataImpl implements ResolutionData { private Collection missing = new ArrayList<>(); private boolean failOnMissing = true; + private Predicate slicerPredicate; + public ResolutionDataImpl(ExecutionEnvironmentResolutionHints eeResolutionHints) { this.eeResolutionHints = eeResolutionHints; } @@ -141,4 +144,13 @@ public void clearMissingRequirements() { missing.clear(); } + @Override + public Predicate getIInstallableUnitAcceptor() { + return slicerPredicate; + } + + public void setSlicerPredicate(Predicate slicerPredicate) { + this.slicerPredicate = slicerPredicate; + } + }