Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Preserve requirements.psd1 and use it for managed dependencies snapshot comparison #676

Merged
merged 1 commit into from
Aug 31, 2021
Merged

Preserve requirements.psd1 and use it for managed dependencies snapshot comparison #676

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/DependencyManagement/DependencyManager.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ public DependencyManager(
ILogger logger = null)
{
_storage = storage ?? new DependencyManagerStorage(GetFunctionAppRootPath(requestMetadataDirectory));
_installedDependenciesLocator = installedDependenciesLocator ?? new InstalledDependenciesLocator(_storage);
_installedDependenciesLocator = installedDependenciesLocator ?? new InstalledDependenciesLocator(_storage, logger);
var snapshotContentLogger = new PowerShellModuleSnapshotLogger();
_installer = installer ?? new DependencySnapshotInstaller(
moduleProvider ?? new PowerShellGalleryModuleProvider(logger),
Expand Down
32 changes: 30 additions & 2 deletions src/DependencyManagement/DependencyManagerStorage.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,7 @@ public DependencyManagerStorage(string functionAppRootPath)

public IEnumerable<DependencyManifestEntry> GetDependencies()
{
var dependencyManifest = new DependencyManifest(_functionAppRootPath);
return dependencyManifest.GetEntries();
return GetAppDependencyManifest().GetEntries();
}

public bool SnapshotExists(string path)
Expand Down Expand Up @@ -127,6 +126,30 @@ public DateTime GetSnapshotAccessTimeUtc(string path)
return heartbeatLastWrite >= snapshotCreation ? heartbeatLastWrite : snapshotCreation;
}

public void PreserveDependencyManifest(string path)
{
var source = GetAppDependencyManifest().GetPath();
var destination = Path.Join(path, Path.GetFileName(source));
File.Copy(source, destination, overwrite: true);
}

public bool IsEquivalentDependencyManifest(string path)
{
var source = GetAppDependencyManifest().GetPath();
if (!File.Exists(source))
{
return false;
}

var destination = Path.Join(path, Path.GetFileName(source));
if (!File.Exists(destination))
{
return false;
}

return File.ReadAllText(source) == File.ReadAllText(destination);
}

private IEnumerable<string> GetInstalledSnapshots()
{
if (!Directory.Exists(_managedDependenciesRootPath))
Expand All @@ -138,5 +161,10 @@ private IEnumerable<string> GetInstalledSnapshots()
_managedDependenciesRootPath,
DependencySnapshotFolderNameTools.InstalledPattern);
}

private DependencyManifest GetAppDependencyManifest()
{
return new DependencyManifest(_functionAppRootPath);
}
}
}
7 changes: 6 additions & 1 deletion src/DependencyManagement/DependencyManifest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,11 @@ public DependencyManifest(string functionAppRootPath, int maxDependencyEntries =
_maxDependencyEntries = maxDependencyEntries;
}

public string GetPath()
{
return Path.Combine(_functionAppRootPath, RequirementsPsd1FileName);
}

public IEnumerable<DependencyManifestEntry> GetEntries()
{
var hashtable = ParsePowerShellDataFile();
Expand Down Expand Up @@ -93,7 +98,7 @@ private static DependencyManifestEntry CreateDependencyManifestEntry(string name
private Hashtable ParsePowerShellDataFile()
{
// Path to requirements.psd1 file.
var requirementsFilePath = Path.Join(_functionAppRootPath, RequirementsPsd1FileName);
var requirementsFilePath = GetPath();

if (!File.Exists(requirementsFilePath))
{
Expand Down
4 changes: 3 additions & 1 deletion src/DependencyManagement/DependencySnapshotInstaller.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,9 @@ private string CreateInstallingSnapshot(string path)
{
try
{
return _storage.CreateInstallingSnapshot(path);
var installingPath = _storage.CreateInstallingSnapshot(path);
_storage.PreserveDependencyManifest(installingPath);
return installingPath;
}
catch (Exception e)
{
Expand Down
4 changes: 4 additions & 0 deletions src/DependencyManagement/IDependencyManagerStorage.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,5 +37,9 @@ internal interface IDependencyManagerStorage
void SetSnapshotAccessTimeToUtcNow(string path);

DateTime GetSnapshotAccessTimeUtc(string path);

void PreserveDependencyManifest(string path);

bool IsEquivalentDependencyManifest(string path);
}
}
34 changes: 26 additions & 8 deletions src/DependencyManagement/InstalledDependenciesLocator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,28 +7,46 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.DependencyManagement
{
using System;
using System.Linq;
using Microsoft.Azure.Functions.PowerShellWorker.Utility;
using static Microsoft.Azure.WebJobs.Script.Grpc.Messages.RpcLog.Types;

internal class InstalledDependenciesLocator : IInstalledDependenciesLocator
{
private readonly IDependencyManagerStorage _storage;

public InstalledDependenciesLocator(IDependencyManagerStorage storage)
private readonly ILogger _logger;

public InstalledDependenciesLocator(IDependencyManagerStorage storage, ILogger logger)
{
_storage = storage;
_storage = storage ?? throw new ArgumentNullException(nameof(storage));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}

public string GetPathWithAcceptableDependencyVersionsInstalled()
{
var lastSnapshotPath = _storage.GetLatestInstalledSnapshot();
if (lastSnapshotPath != null)
if (lastSnapshotPath == null)
{
_logger.Log(isUserOnlyLog: false, Level.Information, string.Format(PowerShellWorkerStrings.NoInstalledDependencySnapshot, lastSnapshotPath));
return null;
}

_logger.Log(isUserOnlyLog: false, Level.Information, string.Format(PowerShellWorkerStrings.LastInstalledDependencySnapshotFound, lastSnapshotPath));

if (_storage.IsEquivalentDependencyManifest(lastSnapshotPath))
{
_logger.Log(isUserOnlyLog: false, Level.Information, string.Format(PowerShellWorkerStrings.EquivalentDependencySnapshotManifest, lastSnapshotPath));
return lastSnapshotPath;
}

var dependencies = _storage.GetDependencies();
if (dependencies.All(entry => IsAcceptableVersionInstalled(lastSnapshotPath, entry)))
{
var dependencies = _storage.GetDependencies();
if (dependencies.All(entry => IsAcceptableVersionInstalled(lastSnapshotPath, entry)))
{
return lastSnapshotPath;
}
_logger.Log(isUserOnlyLog: false, Level.Information, string.Format(PowerShellWorkerStrings.DependencySnapshotContainsAcceptableModuleVersions, lastSnapshotPath));
return lastSnapshotPath;
}

_logger.Log(isUserOnlyLog: false, Level.Information, string.Format(PowerShellWorkerStrings.DependencySnapshotDoesNotContainAcceptableModuleVersions, lastSnapshotPath));
return null;
}

Expand Down
15 changes: 15 additions & 0 deletions src/resources/PowerShellWorkerStrings.resx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -337,4 +337,19 @@
<data name="LogDependencySnapshotsInstalledAndSnapshotsToKeep" xml:space="preserve">
<value>Number of dependency snapshots installed: '{0}'. Dependency snapshots to keep: '{1}'.</value>
</data>
<data name="NoInstalledDependencySnapshot" xml:space="preserve">
<value>No installed dependency snapshot found.</value>
</data>
<data name="LastInstalledDependencySnapshotFound" xml:space="preserve">
<value>Last installed dependency snapshot found: '{0}'.</value>
</data>
<data name="EquivalentDependencySnapshotManifest" xml:space="preserve">
<value>Dependency snapshot '{0}' manifest is equivalent to the current manifest.</value>
</data>
<data name="DependencySnapshotContainsAcceptableModuleVersions" xml:space="preserve">
<value>Dependency snapshot '{0}' contains acceptable module versions.</value>
</data>
<data name="DependencySnapshotDoesNotContainAcceptableModuleVersions" xml:space="preserve">
<value>Dependency snapshot '{0}' does not contain acceptable module versions.</value>
</data>
</root>
12 changes: 7 additions & 5 deletions test/Unit/DependencyManagement/DependencyManagementTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ public void TestManagedDependencySuccessfulModuleDownload()
var mockModuleProvider = new MockModuleProvider { SuccessfulDownload = true };

// Create DependencyManager and process the requirements.psd1 file at the function app root.
using (var dependencyManager = new DependencyManager(functionLoadRequest.Metadata.Directory, mockModuleProvider))
using (var dependencyManager = new DependencyManager(functionLoadRequest.Metadata.Directory, mockModuleProvider, logger: _testLogger))
{
dependencyManager.Initialize(_testLogger);

Expand Down Expand Up @@ -268,7 +268,7 @@ public void TestManagedDependencySuccessfulModuleDownloadAfterTwoTries()
var mockModuleProvider = new MockModuleProvider { ShouldNotThrowAfterCount = 2 };

// Create DependencyManager and process the requirements.psd1 file at the function app root.
using (var dependencyManager = new DependencyManager(functionLoadRequest.Metadata.Directory, mockModuleProvider))
using (var dependencyManager = new DependencyManager(functionLoadRequest.Metadata.Directory, mockModuleProvider, logger: _testLogger))
{
dependencyManager.Initialize(_testLogger);

Expand Down Expand Up @@ -325,7 +325,7 @@ public void TestManagedDependencyRetryLogicMaxNumberOfTries()
var functionLoadRequest = GetFuncLoadRequest(functionFolderPath, true);

// Create DependencyManager and process the requirements.psd1 file at the function app root.
using (var dependencyManager = new DependencyManager(functionLoadRequest.Metadata.Directory, new MockModuleProvider()))
using (var dependencyManager = new DependencyManager(functionLoadRequest.Metadata.Directory, new MockModuleProvider(), logger: _testLogger))
{
dependencyManager.Initialize(_testLogger);

Expand Down Expand Up @@ -382,7 +382,8 @@ public void FunctionAppExecutionShouldStopIfNoPreviousDependenciesAreInstalled()
// the PSGallery to retrieve the latest module version
using (var dependencyManager = new DependencyManager(
functionLoadRequest.Metadata.Directory,
new MockModuleProvider { GetLatestModuleVersionThrows = true }))
new MockModuleProvider { GetLatestModuleVersionThrows = true },
logger: _testLogger))
{
dependencyManager.Initialize(_testLogger);
dependencyManager.StartDependencyInstallationIfNeeded(PowerShell.Create(), PowerShell.Create, _testLogger);
Expand Down Expand Up @@ -417,7 +418,8 @@ public void FunctionAppExecutionShouldContinueIfPreviousDependenciesExist()
// the PSGallery to retrive the latest module version
using (var dependencyManager = new DependencyManager(
functionLoadRequest.Metadata.Directory,
new MockModuleProvider { GetLatestModuleVersionThrows = true }))
new MockModuleProvider { GetLatestModuleVersionThrows = true },
logger: _testLogger))
{
// Create a path to mimic an existing installation of the Az module
AzModulePath = Path.Join(managedDependenciesFolderPath, "FakeDependenciesSnapshot", "Az");
Expand Down
3 changes: 2 additions & 1 deletion test/Unit/DependencyManagement/DependencyManagerTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,8 @@ private DependencyManager CreateDependencyManagerWithMocks()
installer: _mockInstaller.Object,
newerSnapshotDetector: _mockNewerDependencySnapshotDetector.Object,
maintainer: _mockBackgroundDependencySnapshotMaintainer.Object,
currentSnapshotContentLogger: _mockBackgroundDependencySnapshotContentLogger.Object);
currentSnapshotContentLogger: _mockBackgroundDependencySnapshotContentLogger.Object,
logger: _mockLogger.Object);
}
}
}
1 change: 1 addition & 0 deletions test/Unit/DependencyManagement/DependencySnapshotInstallerTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ public DependencySnapshotInstallerTests()
_targetPathInstalled = DependencySnapshotFolderNameTools.CreateUniqueName();
_targetPathInstalling = DependencySnapshotFolderNameTools.ConvertInstalledToInstalling(_targetPathInstalled);
_mockStorage.Setup(_ => _.CreateInstallingSnapshot(_targetPathInstalled)).Returns(_targetPathInstalling);
_mockStorage.Setup(_ => _.PreserveDependencyManifest(_targetPathInstalling));
_mockStorage.Setup(_ => _.PromoteInstallingSnapshotToInstalledAtomically(_targetPathInstalled));
_mockStorage.Setup(_ => _.GetLatestInstalledSnapshot()).Returns(default(string));
}
Expand Down
37 changes: 26 additions & 11 deletions test/Unit/DependencyManagement/InstalledDependenciesLocatorTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,12 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.Test.DependencyManagement
using Xunit;

using Microsoft.Azure.Functions.PowerShellWorker.DependencyManagement;
using Microsoft.Azure.Functions.PowerShellWorker.Utility;

public class InstalledDependenciesLocatorTests
{
private readonly Mock<IDependencyManagerStorage> _mockStorage = new Mock<IDependencyManagerStorage>(MockBehavior.Strict);
private readonly Mock<ILogger> _mockLogger = new Mock<ILogger>();

private readonly DependencyManifestEntry[] _dependencyManifestEntries =
{
Expand All @@ -21,18 +23,31 @@ public class InstalledDependenciesLocatorTests
};

[Fact]
public void ReturnsLatestSnapshotPath_WhenAllDependenciesHaveAcceptableVersionInstalled()
public void ReturnsLatestSnapshotPath_WhenSnapshotWithEquivalentDependencyManifestInstalled()
{
// Even though multiple snapshots can be currently installed, only the latest one will be considered
// (determined by name).
_mockStorage.Setup(_ => _.GetLatestInstalledSnapshot()).Returns("snapshot");
_mockStorage.Setup(_ => _.IsEquivalentDependencyManifest("snapshot")).Returns(true);

_mockStorage.Setup(_ => _.GetDependencies()).Returns(_dependencyManifestEntries);
var installedDependenciesLocator = new InstalledDependenciesLocator(_mockStorage.Object, _mockLogger.Object);
var result = installedDependenciesLocator.GetPathWithAcceptableDependencyVersionsInstalled();

Assert.Equal("snapshot", result);
}

[Fact]
public void ReturnsLatestSnapshotPath_WhenAllDependenciesHaveAcceptableVersionInstalled()
{
// Even though multiple snapshots can be currently installed, only the latest one will be considered
// (determined by name).
_mockStorage.Setup(_ => _.GetLatestInstalledSnapshot()).Returns("snapshot");
_mockStorage.Setup(_ => _.IsEquivalentDependencyManifest("snapshot")).Returns(false);
_mockStorage.Setup(_ => _.GetDependencies()).Returns(_dependencyManifestEntries);
_mockStorage.Setup(_ => _.IsModuleVersionInstalled("snapshot", "A", "exact version of A")).Returns(true);
_mockStorage.Setup(_ => _.GetInstalledModuleVersions("snapshot", "B", "major version of B")).Returns(new [] { "exact version of B" });

var installedDependenciesLocator = new InstalledDependenciesLocator(_mockStorage.Object);
var installedDependenciesLocator = new InstalledDependenciesLocator(_mockStorage.Object, _mockLogger.Object);
var result = installedDependenciesLocator.GetPathWithAcceptableDependencyVersionsInstalled();

Assert.Equal("snapshot", result);
Expand All @@ -43,7 +58,7 @@ public void ReturnsNull_WhenNoInstalledDependencySnapshotsFound()
{
_mockStorage.Setup(_ => _.GetLatestInstalledSnapshot()).Returns(default(string));

var installedDependenciesLocator = new InstalledDependenciesLocator(_mockStorage.Object);
var installedDependenciesLocator = new InstalledDependenciesLocator(_mockStorage.Object, _mockLogger.Object);
var result = installedDependenciesLocator.GetPathWithAcceptableDependencyVersionsInstalled();

Assert.Null(result);
Expand All @@ -55,14 +70,14 @@ public void ReturnsNull_WhenNoMajorVersionInstalled()
// Even though multiple snapshots can be currently installed, only the latest one will be considered
// (determined by name).
_mockStorage.Setup(_ => _.GetLatestInstalledSnapshot()).Returns("snapshot");

_mockStorage.Setup(_ => _.IsEquivalentDependencyManifest("snapshot")).Returns(false);
_mockStorage.Setup(_ => _.GetDependencies()).Returns(_dependencyManifestEntries);

// No version for module B detected!
_mockStorage.Setup(_ => _.IsModuleVersionInstalled("snapshot", "A", "exact version of A")).Returns(true);
_mockStorage.Setup(_ => _.GetInstalledModuleVersions("snapshot", "B", "major version of B")).Returns(new string[0]);

var installedDependenciesLocator = new InstalledDependenciesLocator(_mockStorage.Object);
var installedDependenciesLocator = new InstalledDependenciesLocator(_mockStorage.Object, _mockLogger.Object);
var result = installedDependenciesLocator.GetPathWithAcceptableDependencyVersionsInstalled();

Assert.Null(result);
Expand All @@ -74,14 +89,14 @@ public void ReturnsNull_WhenExactModuleVersionIsNotInstalled()
// Even though multiple snapshots can be currently installed, only the latest one will be considered
// (determined by name).
_mockStorage.Setup(_ => _.GetLatestInstalledSnapshot()).Returns("snapshot");

_mockStorage.Setup(_ => _.IsEquivalentDependencyManifest("snapshot")).Returns(false);
_mockStorage.Setup(_ => _.GetDependencies()).Returns(_dependencyManifestEntries);

// The specified module A version is not installed
_mockStorage.Setup(_ => _.IsModuleVersionInstalled("snapshot", "A", "exact version of A")).Returns(false);
_mockStorage.Setup(_ => _.GetInstalledModuleVersions("snapshot", "B", "major version of B")).Returns(new [] { "exact version of B" });

var installedDependenciesLocator = new InstalledDependenciesLocator(_mockStorage.Object);
var installedDependenciesLocator = new InstalledDependenciesLocator(_mockStorage.Object, _mockLogger.Object);
var result = installedDependenciesLocator.GetPathWithAcceptableDependencyVersionsInstalled();

Assert.Null(result);
Expand All @@ -106,16 +121,16 @@ public void ReturnsLatestSnapshotPath_WhenPreviewVersionInstalled(string postfix
new DependencyManifestEntry("A", VersionSpecificationType.ExactVersion, fullVersion)
};

_mockStorage.Setup(_ => _.GetDependencies()).Returns(dependencyManifestEntries);

_mockStorage.Setup(_ => _.GetLatestInstalledSnapshot()).Returns("snapshot");
_mockStorage.Setup(_ => _.IsEquivalentDependencyManifest("snapshot")).Returns(false);
_mockStorage.Setup(_ => _.GetDependencies()).Returns(dependencyManifestEntries);

// No exact match...
_mockStorage.Setup(_ => _.IsModuleVersionInstalled("snapshot", "A", fullVersion)).Returns(false);
// ...but the base version is here
_mockStorage.Setup(_ => _.IsModuleVersionInstalled("snapshot", "A", baseVersion)).Returns(true);

var installedDependenciesLocator = new InstalledDependenciesLocator(_mockStorage.Object);
var installedDependenciesLocator = new InstalledDependenciesLocator(_mockStorage.Object, _mockLogger.Object);
var result = installedDependenciesLocator.GetPathWithAcceptableDependencyVersionsInstalled();

Assert.Equal("snapshot", result);
Expand Down