diff --git a/Microsoft.DotNet.Cli.sln b/Microsoft.DotNet.Cli.sln index 5929564f2a..2b397c406c 100644 --- a/Microsoft.DotNet.Cli.sln +++ b/Microsoft.DotNet.Cli.sln @@ -227,6 +227,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "tool_fsc", "src\tool_fsharp EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.DotNet.MSBuildSdkResolver", "src\Microsoft.DotNet.MSBuildSdkResolver\Microsoft.DotNet.MSBuildSdkResolver.csproj", "{FCDFAF40-CC16-4D49-96C0-E49F195E7142}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "dotnet-cli-build.Tests", "build_projects\dotnet-cli-build.Tests\dotnet-cli-build.Tests.csproj", "{84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1563,6 +1565,30 @@ Global {FCDFAF40-CC16-4D49-96C0-E49F195E7142}.RelWithDebInfo|x64.Build.0 = Release|Any CPU {FCDFAF40-CC16-4D49-96C0-E49F195E7142}.RelWithDebInfo|x86.ActiveCfg = Release|Any CPU {FCDFAF40-CC16-4D49-96C0-E49F195E7142}.RelWithDebInfo|x86.Build.0 = Release|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Debug|Any CPU.Build.0 = Debug|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Debug|x64.ActiveCfg = Debug|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Debug|x64.Build.0 = Debug|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Debug|x86.ActiveCfg = Debug|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Debug|x86.Build.0 = Debug|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.MinSizeRel|x64.Build.0 = Debug|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.MinSizeRel|x86.Build.0 = Debug|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Release|Any CPU.ActiveCfg = Release|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Release|Any CPU.Build.0 = Release|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Release|x64.ActiveCfg = Release|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Release|x64.Build.0 = Release|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Release|x86.ActiveCfg = Release|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Release|x86.Build.0 = Release|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.RelWithDebInfo|Any CPU.ActiveCfg = Release|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.RelWithDebInfo|Any CPU.Build.0 = Release|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.RelWithDebInfo|x64.ActiveCfg = Release|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.RelWithDebInfo|x64.Build.0 = Release|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.RelWithDebInfo|x86.ActiveCfg = Release|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.RelWithDebInfo|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1633,6 +1659,7 @@ Global {08A40B6A-F695-4EA9-AC8D-CF88FADEA796} = {17735A9D-BFD9-4585-A7CB-3208CA6EA8A7} {602976C5-2477-4B4C-AD9A-1EAFB250529A} = {ED2FE3E2-F7E7-4389-8231-B65123F2076F} {FCDFAF40-CC16-4D49-96C0-E49F195E7142} = {ED2FE3E2-F7E7-4389-8231-B65123F2076F} + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94} = {88278B81-7649-45DC-8A6A-D3A645C5AFC3} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {B526D2CE-EE2D-4AD4-93EF-1867D90FF1F5} diff --git a/build/Microsoft.DotNet.Cli.tasks b/build/Microsoft.DotNet.Cli.tasks index 901a48f846..6ada95a1cc 100644 --- a/build/Microsoft.DotNet.Cli.tasks +++ b/build/Microsoft.DotNet.Cli.tasks @@ -32,6 +32,7 @@ + diff --git a/build/Test.targets b/build/Test.targets index 86cd8ea7e7..9b86bcca42 100644 --- a/build/Test.targets +++ b/build/Test.targets @@ -88,6 +88,8 @@ + diff --git a/build/publish/PublishDebian.targets b/build/publish/PublishDebian.targets index 41ea7f4bd6..6dd7771805 100644 --- a/build/publish/PublishDebian.targets +++ b/build/publish/PublishDebian.targets @@ -6,28 +6,23 @@ $(DotnetBlobRootUrl)/$(Product)/$(FullNugetVersion)/$(DistroSpecificArtifactNameWithVersionCombinedHostHostFxrFrameworkSdk)-$(Architecture)$(InstallerExtension) - $(SdkDebianIntermediateDirectory)/package_upload.json - 1 - - - { - "name":"$(SdkDebianPackageName)", - "version":"$(NugetVersion)-$(DebianRevisionNumber)", - "repositoryId":"$(REPO_ID)", - "sourceUrl": "$(SdkDebianUploadUrl)" - } - - - - - - - - - + + + + + - + diff --git a/build_projects/dotnet-cli-build.Tests/GivenActionAndRetryTimes.cs b/build_projects/dotnet-cli-build.Tests/GivenActionAndRetryTimes.cs new file mode 100644 index 0000000000..47c83e0a72 --- /dev/null +++ b/build_projects/dotnet-cli-build.Tests/GivenActionAndRetryTimes.cs @@ -0,0 +1,92 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository; +using Xunit; + +namespace dotnet_cli_build.Tests +{ + public class GivenActionAndRetryTimes + { + public static IEnumerable NoWaitTimer() + { + while (true) + { + yield return Task.CompletedTask; + } + } + + [Fact] + public void ExponentialRetryShouldProvideIntervalSequence() + { + ExponentialRetry.Intervals.First().Should().Be(TimeSpan.FromSeconds(5)); + ExponentialRetry.Intervals.Skip(1).First().Should().Be(TimeSpan.FromSeconds(10)); + ExponentialRetry.Intervals.Skip(2).First().Should().Be(TimeSpan.FromSeconds(20)); + ExponentialRetry.Intervals.Skip(3).First().Should().Be(TimeSpan.FromSeconds(40)); + ExponentialRetry.Intervals.Skip(4).First().Should().Be(TimeSpan.FromSeconds(80)); + } + + [Fact] + public void ExponentialShouldNotRetryAfterFirstSucceess() + { + var fakeAction = new FakeAction(0); + ExponentialRetry.ExecuteWithRetry( + fakeAction.Run, + s => s == "success", + 10, + NoWaitTimer).Wait(); + fakeAction.Count.Should().Be(0); + } + + [Fact] + public void ExponentialShouldRetryUntilSuccess() + { + var fakeAction = new FakeAction(5); + ExponentialRetry.ExecuteWithRetry( + fakeAction.Run, + s => s == "success", + 10, + NoWaitTimer).Wait(); + fakeAction.Count.Should().Be(5); + } + + [Fact] + public void ExponentialShouldThrowAfterMaximumAmountReached() + { + var fakeAction = new FakeAction(10); + Action a = () => ExponentialRetry.ExecuteWithRetry( + fakeAction.Run, + s => s == "success", + 5, + NoWaitTimer, + "testing retry").Wait(); + a.ShouldThrow() + .WithMessage("Retry failed for testing retry after 5 times with result: fail"); + } + } + + public class FakeAction + { + private readonly int _successAfter; + + public FakeAction(int successAfter) + { + _successAfter = successAfter; + } + + public int Count { get; private set; } + + public Task Run() + { + if (_successAfter == Count) + { + return Task.FromResult("success"); + } + + Count++; + return Task.FromResult("fail"); + } + } +} \ No newline at end of file diff --git a/build_projects/dotnet-cli-build.Tests/dotnet-cli-build.Tests.csproj b/build_projects/dotnet-cli-build.Tests/dotnet-cli-build.Tests.csproj new file mode 100644 index 0000000000..e28340dda3 --- /dev/null +++ b/build_projects/dotnet-cli-build.Tests/dotnet-cli-build.Tests.csproj @@ -0,0 +1,22 @@ + + + + + $(CliTargetFramework) + 1.0.0 + $(AssetTargetFallback);portable-net45+win8+wp8+wpa81 + false + + + + + + + + + + + + + + diff --git a/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/AddPackageStrategy.cs b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/AddPackageStrategy.cs new file mode 100644 index 0000000000..cb2ec81013 --- /dev/null +++ b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/AddPackageStrategy.cs @@ -0,0 +1,62 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; +using NuGet.Protocol; + +namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository +{ + internal class AddPackageStrategy : IAzurelinuxRepositoryServiceHttpStrategy + { + private readonly IdInRepositoryService _idInRepositoryService; + private readonly string _packageName; + private readonly string _packageVersion; + private readonly string _repositoryId; + + public AddPackageStrategy( + IdInRepositoryService idInRepositoryService, + string packageName, + string packageVersion, + string repositoryId) + { + _idInRepositoryService = idInRepositoryService + ?? throw new ArgumentNullException(nameof(idInRepositoryService)); + _packageName = packageName; + _packageVersion = packageVersion; + _repositoryId = repositoryId; + } + + public async Task Execute(HttpClient client, Uri baseAddress) + { + var debianUploadJsonContent = new Dictionary + { + ["name"] = _packageName, + ["version"] = AppendDebianRevisionNumber(_packageVersion), + ["fileId"] = _idInRepositoryService.Id, + ["repositoryId"] = _repositoryId + }.ToJson(); + var content = new StringContent(debianUploadJsonContent, + Encoding.UTF8, + "application/json"); + + using (var response = await client.PostAsync(new Uri(baseAddress, "/v1/packages"), content)) + { + if (!response.IsSuccessStatusCode) + throw new FailedToAddPackageToPackageRepositoryException( + $"request:{debianUploadJsonContent} response:{response.ToJson()}"); + return response.Headers.GetValues("Location").Single(); + } + } + + private static string AppendDebianRevisionNumber(string packageVersion) + { + return packageVersion + "-1"; + } + } +} diff --git a/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/ExponentialRetry.cs b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/ExponentialRetry.cs new file mode 100644 index 0000000000..df3590940d --- /dev/null +++ b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/ExponentialRetry.cs @@ -0,0 +1,53 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository +{ + public static class ExponentialRetry + { + public static IEnumerable Intervals + { + get + { + var seconds = 5; + while (true) + { + yield return TimeSpan.FromSeconds(seconds); + seconds *= 2; + } + } + } + + public static async Task ExecuteWithRetry(Func> action, + Func isSuccess, + int maxRetryCount, + Func> timer, + string taskDescription = "") + { + var count = 0; + foreach (var t in timer()) + { + await t; + var result = await action(); + if (isSuccess(result)) + return; + count++; + if (count == maxRetryCount) + throw new RetryFailedException( + $"Retry failed for {taskDescription} after {count} times with result: {result}"); + } + throw new Exception("Timer should not be exhausted"); + } + + public static IEnumerable Timer(IEnumerable interval) + { + return interval.Select(Task.Delay); + } + } +} diff --git a/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/FailedToAddPackageToPackageRepositoryException.cs b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/FailedToAddPackageToPackageRepositoryException.cs new file mode 100644 index 0000000000..68c314c08c --- /dev/null +++ b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/FailedToAddPackageToPackageRepositoryException.cs @@ -0,0 +1,24 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; + +namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository +{ + public class FailedToAddPackageToPackageRepositoryException : Exception + { + public FailedToAddPackageToPackageRepositoryException(string message) : base(message) + { + } + + public FailedToAddPackageToPackageRepositoryException() + { + } + + public FailedToAddPackageToPackageRepositoryException(string message, Exception innerException) : base(message, + innerException) + { + } + } +} diff --git a/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/FileUploadStrategy.cs b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/FileUploadStrategy.cs new file mode 100644 index 0000000000..59be2c16c7 --- /dev/null +++ b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/FileUploadStrategy.cs @@ -0,0 +1,49 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Net.Http; +using System.Threading.Tasks; +using NuGet.Protocol; + +namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository +{ + internal class FileUploadStrategy : IAzurelinuxRepositoryServiceHttpStrategy + { + private readonly string _pathToPackageToUpload; + + public FileUploadStrategy(string pathToPackageToUpload) + { + _pathToPackageToUpload = pathToPackageToUpload + ?? throw new ArgumentNullException(nameof(pathToPackageToUpload)); + } + + public async Task Execute(HttpClient client, Uri baseAddress) + { + var fileName = Path.GetFileName(_pathToPackageToUpload); + + using (var content = + new MultipartFormDataContent()) + { + var url = new Uri(baseAddress, "/v1/files"); + content.Add( + new StreamContent( + new MemoryStream( + File.ReadAllBytes(_pathToPackageToUpload))), + "file", + fileName); + using (var message = await client.PostAsync(url, content)) + { + if (!message.IsSuccessStatusCode) + { + throw new FailedToAddPackageToPackageRepositoryException( + $"{message.ToJson()} failed to post file to {url} file name:{fileName} pathToPackageToUpload:{_pathToPackageToUpload}"); + } + return await message.Content.ReadAsStringAsync(); + } + } + } + } +} diff --git a/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/IAzurelinuxRepositoryServiceHttpStrategy.cs b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/IAzurelinuxRepositoryServiceHttpStrategy.cs new file mode 100644 index 0000000000..a829f398a3 --- /dev/null +++ b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/IAzurelinuxRepositoryServiceHttpStrategy.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Net.Http; +using System.Threading.Tasks; + +namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository +{ + internal interface IAzurelinuxRepositoryServiceHttpStrategy + { + Task Execute(HttpClient client, Uri baseAddress); + } +} diff --git a/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/IdInRepositoryService.cs b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/IdInRepositoryService.cs new file mode 100644 index 0000000000..b51ebb3bef --- /dev/null +++ b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/IdInRepositoryService.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; + +namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository +{ + internal class IdInRepositoryService + { + public IdInRepositoryService(string id) + { + Id = id ?? throw new ArgumentNullException(nameof(id)); + } + + public string Id { get; } + } +} diff --git a/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/LinuxPackageRepositoryDestiny.cs b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/LinuxPackageRepositoryDestiny.cs new file mode 100644 index 0000000000..aa2e6d9bc6 --- /dev/null +++ b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/LinuxPackageRepositoryDestiny.cs @@ -0,0 +1,38 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; + +namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository +{ + internal class LinuxPackageRepositoryDestiny + { + private readonly string _password; + private readonly string _server; + private readonly string _username; + + public LinuxPackageRepositoryDestiny(string username, + string password, + string server, + string repositoryId) + { + _username = username ?? throw new ArgumentNullException(nameof(username)); + _password = password ?? throw new ArgumentNullException(nameof(password)); + _server = server ?? throw new ArgumentNullException(nameof(server)); + RepositoryId = repositoryId ?? throw new ArgumentNullException(nameof(repositoryId)); + } + + public string RepositoryId { get; } + + public Uri GetBaseAddress() + { + return new Uri($"https://{_server}"); + } + + public string GetSimpleAuth() + { + return $"{_username}:{_password}"; + } + } +} diff --git a/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/LinuxPackageRepositoryHttpPrepare.cs b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/LinuxPackageRepositoryHttpPrepare.cs new file mode 100644 index 0000000000..7e1f9ade71 --- /dev/null +++ b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/LinuxPackageRepositoryHttpPrepare.cs @@ -0,0 +1,46 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository +{ + internal class LinuxPackageRepositoryHttpPrepare + { + private readonly IAzurelinuxRepositoryServiceHttpStrategy _httpStrategy; + private readonly LinuxPackageRepositoryDestiny _linuxPackageRepositoryDestiny; + + public LinuxPackageRepositoryHttpPrepare( + LinuxPackageRepositoryDestiny linuxPackageRepositoryDestiny, + IAzurelinuxRepositoryServiceHttpStrategy httpStrategy + ) + { + _linuxPackageRepositoryDestiny = linuxPackageRepositoryDestiny + ?? throw new ArgumentNullException(nameof(linuxPackageRepositoryDestiny)); + _httpStrategy = httpStrategy ?? throw new ArgumentNullException(nameof(httpStrategy)); + } + + public async Task RemoteCall() + { + using (var handler = new HttpClientHandler()) + { + using (var client = new HttpClient(handler)) + { + var authHeader = + Convert.ToBase64String(Encoding.UTF8.GetBytes((string) _linuxPackageRepositoryDestiny.GetSimpleAuth())); + client.DefaultRequestHeaders.Authorization = + new AuthenticationHeaderValue("Basic", authHeader); + client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + client.Timeout = TimeSpan.FromMinutes(10); + + return await _httpStrategy.Execute(client, _linuxPackageRepositoryDestiny.GetBaseAddress()); + } + } + } + } +} diff --git a/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/PullQueuedPackageStatus.cs b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/PullQueuedPackageStatus.cs new file mode 100644 index 0000000000..393a160aee --- /dev/null +++ b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/PullQueuedPackageStatus.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Net.Http; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; + +namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository +{ + internal class PullQueuedPackageStatus : IAzurelinuxRepositoryServiceHttpStrategy + { + private readonly QueueResourceLocation _queueResourceLocation; + + public PullQueuedPackageStatus(QueueResourceLocation queueResourceLocation) + { + _queueResourceLocation = queueResourceLocation + ?? throw new ArgumentNullException(nameof(queueResourceLocation)); + } + + public async Task Execute(HttpClient client, Uri baseAddress) + { + using (var response = await client.GetAsync(new Uri(baseAddress, _queueResourceLocation.Location))) + { + if (!response.IsSuccessStatusCode) + throw new FailedToAddPackageToPackageRepositoryException( + "Failed to make request to " + _queueResourceLocation.Location); + var body = await response.Content.ReadAsStringAsync(); + return !body.Contains("status") ? "" : JObject.Parse(body)["status"].ToString(); + } + } + } +} diff --git a/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/QueueResourceLocation.cs b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/QueueResourceLocation.cs new file mode 100644 index 0000000000..cbf11e48dd --- /dev/null +++ b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/QueueResourceLocation.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; + +namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository +{ + internal class QueueResourceLocation + { + public QueueResourceLocation(string location) + { + Location = location ?? throw new ArgumentNullException(nameof(location)); + } + + public string Location { get; } + } +} diff --git a/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/RetryFailedException.cs b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/RetryFailedException.cs new file mode 100644 index 0000000000..b251ec4b83 --- /dev/null +++ b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/RetryFailedException.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; + +namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository +{ + public class RetryFailedException : Exception + { + public RetryFailedException(string message) : base(message) + { + } + + public RetryFailedException() + { + } + + public RetryFailedException(string message, Exception innerException) : base(message, innerException) + { + } + } +} diff --git a/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/UploadToLinuxPackageRepository.cs b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/UploadToLinuxPackageRepository.cs new file mode 100644 index 0000000000..3e14557d69 --- /dev/null +++ b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/UploadToLinuxPackageRepository.cs @@ -0,0 +1,108 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Net.Http; +using System.Threading.Tasks; +using Microsoft.Build.Framework; +using Newtonsoft.Json.Linq; +using Task = Microsoft.Build.Utilities.Task; + +namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository +{ + public class UploadToLinuxPackageRepository : Task + { + /// + /// The Azure repository service user name. + /// + [Required] + public string Username { get; set; } + + /// + /// The Azure repository service Password. + /// + [Required] + public string Password { get; set; } + + /// + /// The Azure repository service URL ex: "tux-devrepo.corp.microsoft.com". + /// + [Required] + public string Server { get; set; } + + [Required] + public string RepositoryId { get; set; } + + [Required] + public string PathOfPackageToUpload { get; set; } + + [Required] + public string PackageNameInLinuxPackageRepository { get; set; } + + + [Required] + public string PackageVersionInLinuxPackageRepository { get; set; } + + + public override bool Execute() + { + ExecuteAsyncWithRetry().GetAwaiter().GetResult(); + return true; + } + + private async System.Threading.Tasks.Task ExecuteAsyncWithRetry() + { + await ExponentialRetry.ExecuteWithRetry( + UploadAndAddpackageAndEnsureItIsReady, + s => s == "", + maxRetryCount: 3, + timer: () => ExponentialRetry.Timer(ExponentialRetry.Intervals), + taskDescription: $"running {nameof(UploadAndAddpackageAndEnsureItIsReady)}"); + } + + private async Task UploadAndAddpackageAndEnsureItIsReady() + { + try + { + var linuxPackageRepositoryDestiny = + new LinuxPackageRepositoryDestiny(Username, Password, Server, RepositoryId); + var uploadResponse = await new LinuxPackageRepositoryHttpPrepare( + linuxPackageRepositoryDestiny, + new FileUploadStrategy(PathOfPackageToUpload)).RemoteCall(); + + var idInRepositoryService = new IdInRepositoryService(JObject.Parse(uploadResponse)["id"].ToString()); + + var addPackageResponse = await new LinuxPackageRepositoryHttpPrepare( + linuxPackageRepositoryDestiny, + new AddPackageStrategy( + idInRepositoryService, + PackageNameInLinuxPackageRepository, + PackageVersionInLinuxPackageRepository, + linuxPackageRepositoryDestiny.RepositoryId)).RemoteCall(); + + var queueResourceLocation = new QueueResourceLocation(addPackageResponse); + + Func> pullQueuedPackageStatus = new LinuxPackageRepositoryHttpPrepare( + linuxPackageRepositoryDestiny, + new PullQueuedPackageStatus(queueResourceLocation)).RemoteCall; + + await ExponentialRetry.ExecuteWithRetry( + pullQueuedPackageStatus, + s => s == "fileReady", + 5, + () => ExponentialRetry.Timer(ExponentialRetry.Intervals), + $"PullQueuedPackageStatus location: {queueResourceLocation.Location}"); + return ""; + } + catch (FailedToAddPackageToPackageRepositoryException e) + { + return e.ToString(); + } + catch (HttpRequestException e) + { + return e.ToString(); + } + } + } +} diff --git a/scripts/publish/repoapi_client.sh b/scripts/publish/repoapi_client.sh deleted file mode 100755 index 875210bff2..0000000000 --- a/scripts/publish/repoapi_client.sh +++ /dev/null @@ -1,194 +0,0 @@ -#!/bin/bash -# -# Copyright (c) .NET Foundation and contributors. All rights reserved. -# Licensed under the MIT license. See LICENSE file in the project root for full license information. -# - -# This is a VERY basic script for Create/Delete operations on repos and packages -# -# Environment Dependencies: -# $REPO_SERVER -# $REPO_USER -# $REPO_PASS - -cmd=$1 -urls=urls.txt -defaultPackageFile=new_package.json -defaultRepoFile=new_repo.json -repositoryId=$REPO_ID -server=$REPO_SERVER -user=$REPO_USER -pass=$REPO_PASS -protocol=https -port=443 -baseurl="$protocol://$user:$pass@$server:$port" - -echo $baseurl - -function BailIf -{ - if [ $1 -ne 0 ]; then - echo "Failure occurred communicating with $server" - exit 1 - fi -} - -# List packages, using $1 as a regex to filter results -function ListPackages -{ - curl -k "$baseurl/v1/packages" | sed 's/{/\n{/g' | egrep "$1" | sed 's/,/,\n/g' | sed 's/^"/\t"/g' - echo "" -} - -# Create a new Repo using the specified JSON file -function AddRepo -{ - repoFile=$1 - if [ -z $repoFile ]; then - echo "Error: Must specify a JSON-formatted file. Reference $defaultRepoFile.template" - exit 1 - fi - if [ ! -f $repoFile ]; then - echo "Error: Cannot create repo - $repoFile does not exist" - exit 1 - fi - packageUrl=$(grep "url" $repoFile | head -n 1 | awk '{print $2}' | tr -d ',') - echo "Creating new repo on $server [$packageUrl]" - curl -i -k "$baseurl/v1/repositories" --data @./$repoFile -H "Content-Type: application/json" - BailIf $? - echo "" -} - -# Upload a single package using the specified JSON file -function AddPackage -{ - packageFile=$1 - if [ -z $packageFile ]; then - echo "Error: Must specify a JSON-formatted file. Reference $defaultPackageFile.template" - exit 1 - fi - if [ ! -f $packageFile ]; then - echo "Error: Cannot add package - $packageFile does not exist" - exit 1 - fi - packageUrl=$(grep "sourceUrl" $packageFile | head -n 1 | awk '{print $2}') - echo "Adding package to $server [$packageUrl]" - curl -i -k "$baseurl/v1/packages" --data @$packageFile -H "Content-Type: application/json" - BailIf $? - echo "" -} - -# Upload a single package by dynamically creating a JSON file using a provided URL -function AddPackageByUrl -{ - # Parse URL - url=$(echo "$1") - if [ -z $url ]; then - return - fi - escapedUrl=$(echo "$url" | sed 's/\//\\\//g') - set -- "$1" - oldIFS=$IFS - IFS="/"; declare -a splitUrl=($*) - index=${#splitUrl[@]} - let "index -= 1" - filename=${splitUrl[$index]} - set -- "$filename" - IFS="_"; declare -a splitFile=($*) - IFS=$oldIFS - pkgName=${splitFile[0]} - pkgVer=${splitFile[1]} - if [ -z $pkgName ] || [ -z $pkgVer ]; then - echo "ERROR parsing $url" - return - fi - # Create Package .json file - cp $defaultPackageFile.template $defaultPackageFile - sed -i "s/PACKAGENAME/$pkgName/g" $defaultPackageFile - sed -i "s/PACKAGEVERSION/$pkgVer/g" $defaultPackageFile - sed -i "s/PACKAGEURL/$escapedUrl/g" $defaultPackageFile - sed -i "s/REPOSITORYID/$repositoryId/g" $defaultPackageFile - # Test that URL is ok - wget -q --spider "$url" - if [[ $? -eq 0 ]]; then - echo "Ready to upload $pkgName [$pkgVer]" - else - echo "ERROR testing URL $url" - return - fi - # Perform Upload - AddPackage $defaultPackageFile - # Cleanup - # rm $defaultPackageFile -} - -# Upload multiple packages by reading urls line-by-line from the specified file -function AddPackages -{ - urlFile=$1 - if [ -z $urlFile ]; then - echo "Error: Must specify a flat text file containing one or more URLs" - exit 1 - fi - if [ ! -f $urlFile ]; then - echo "Error: Cannot add packages. File $urlFile does not exist" - exit 1 - fi - for url in $(cat $urlFile); do - AddPackageByUrl "$url" - sleep 5 - done -} - -# Delete the specified repo -function DeleteRepo -{ - repoId=$1 - if [ -z $repoId ]; then - echo "Error: Please specify repository ID. Run -listrepos for a list of IDs" - exit 1 - fi - curl -I -k -X DELETE "$baseurl/v1/repositories/$repoId" - BailIf $? -} - -# Delete the specified package -function DeletePackage -{ - packageId=$1 - if [ -z $packageId ]; then - echo "Error: Please specify package ID. Run -listpkgs for a list of IDs" - exit 1 - fi - echo Removing pkgId $packageId from repo $repositoryId - curl -I -k -X DELETE "$baseurl/v1/packages/$packageId" - BailIf $? -} - -if [[ "$1" == "-listrepos" ]]; then - echo "Fetching repo list from $server..." - curl -k "$baseurl/v1/repositories" | sed 's/,/,\n/g' | sed 's/^"/\t"/g' - echo "" -elif [[ "$1" == "-listpkgs" ]]; then - echo "Fetching package list from $server" - ListPackages $2 -elif [[ "$1" == "-addrepo" ]]; then - AddRepo $2 -elif [[ "$1" == "-addpkg" ]]; then - AddPackage $2 -elif [[ "$1" == "-addpkgs" ]]; then - AddPackages $2 -elif [[ "$1" == "-delrepo" ]]; then - DeleteRepo $2 -elif [[ "$1" == "-delpkg" ]]; then - DeletePackage $2 -else - echo "USAGE: ./repotool.sh -OPTION" - echo "-listrepos: Gather a list of repos" - echo "-listpkgs: Gather a list of packages" - echo "-addrepo [FILENAME] : Create a new repo using the specified JSON file" - echo "-addpkg [FILENAME] : Add package to repo using the specified JSON file" - echo "-addpkgs [FILENAME] : Add packages to repo using urls contained in FILENAME" - echo "-delrepo REPOID : Delete the specified repo by ID" - echo "-delpkg PKGID : Delete the specified package by ID" -fi