From ac82646af8b64399d9407d216684e83b929f6a64 Mon Sep 17 00:00:00 2001 From: William Li Date: Wed, 9 Aug 2017 16:57:47 -0700 Subject: [PATCH] Use Rest Api to upload to the feed Add pulling logic to make sure it is uploaded to the feed. Add retry logic for the whole upload process Remove the old upload script --- Microsoft.DotNet.Cli.sln | 27 +++ build/Microsoft.DotNet.Cli.tasks | 1 + build/Test.targets | 2 + build/publish/PublishDebian.targets | 35 ++-- .../GivenActionAndRetryTimes.cs | 92 +++++++++ .../dotnet-cli-build.Tests.csproj | 21 ++ .../AddPackageStrategy.cs | 62 ++++++ .../ExponentialRetry.cs | 53 +++++ ...oAddPackageToPackageRepositoryException.cs | 24 +++ .../FileUploadStrategy.cs | 49 +++++ ...AzurelinuxRepositoryServiceHttpStrategy.cs | 15 ++ .../IdInRepositoryService.cs | 18 ++ .../LinuxPackageRepositoryDestiny.cs | 38 ++++ .../LinuxPackageRepositoryHttpPrepare.cs | 46 +++++ .../PullQueuedPackageStatus.cs | 34 +++ .../QueueResourceLocation.cs | 18 ++ .../RetryFailedException.cs | 23 +++ .../UploadToLinuxPackageRepository.cs | 108 ++++++++++ scripts/publish/repoapi_client.sh | 194 ------------------ 19 files changed, 646 insertions(+), 214 deletions(-) create mode 100644 build_projects/dotnet-cli-build.Tests/GivenActionAndRetryTimes.cs create mode 100644 build_projects/dotnet-cli-build.Tests/dotnet-cli-build.Tests.csproj create mode 100644 build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/AddPackageStrategy.cs create mode 100644 build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/ExponentialRetry.cs create mode 100644 build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/FailedToAddPackageToPackageRepositoryException.cs create mode 100644 build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/FileUploadStrategy.cs create mode 100644 build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/IAzurelinuxRepositoryServiceHttpStrategy.cs create mode 100644 build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/IdInRepositoryService.cs create mode 100644 build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/LinuxPackageRepositoryDestiny.cs create mode 100644 build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/LinuxPackageRepositoryHttpPrepare.cs create mode 100644 build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/PullQueuedPackageStatus.cs create mode 100644 build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/QueueResourceLocation.cs create mode 100644 build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/RetryFailedException.cs create mode 100644 build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/UploadToLinuxPackageRepository.cs delete mode 100755 scripts/publish/repoapi_client.sh diff --git a/Microsoft.DotNet.Cli.sln b/Microsoft.DotNet.Cli.sln index 5929564f2ad..2b397c406cc 100644 --- a/Microsoft.DotNet.Cli.sln +++ b/Microsoft.DotNet.Cli.sln @@ -227,6 +227,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "tool_fsc", "src\tool_fsharp EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.DotNet.MSBuildSdkResolver", "src\Microsoft.DotNet.MSBuildSdkResolver\Microsoft.DotNet.MSBuildSdkResolver.csproj", "{FCDFAF40-CC16-4D49-96C0-E49F195E7142}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "dotnet-cli-build.Tests", "build_projects\dotnet-cli-build.Tests\dotnet-cli-build.Tests.csproj", "{84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1563,6 +1565,30 @@ Global {FCDFAF40-CC16-4D49-96C0-E49F195E7142}.RelWithDebInfo|x64.Build.0 = Release|Any CPU {FCDFAF40-CC16-4D49-96C0-E49F195E7142}.RelWithDebInfo|x86.ActiveCfg = Release|Any CPU {FCDFAF40-CC16-4D49-96C0-E49F195E7142}.RelWithDebInfo|x86.Build.0 = Release|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Debug|Any CPU.Build.0 = Debug|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Debug|x64.ActiveCfg = Debug|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Debug|x64.Build.0 = Debug|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Debug|x86.ActiveCfg = Debug|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Debug|x86.Build.0 = Debug|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.MinSizeRel|Any CPU.ActiveCfg = Debug|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.MinSizeRel|Any CPU.Build.0 = Debug|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.MinSizeRel|x64.ActiveCfg = Debug|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.MinSizeRel|x64.Build.0 = Debug|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.MinSizeRel|x86.ActiveCfg = Debug|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.MinSizeRel|x86.Build.0 = Debug|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Release|Any CPU.ActiveCfg = Release|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Release|Any CPU.Build.0 = Release|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Release|x64.ActiveCfg = Release|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Release|x64.Build.0 = Release|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Release|x86.ActiveCfg = Release|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.Release|x86.Build.0 = Release|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.RelWithDebInfo|Any CPU.ActiveCfg = Release|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.RelWithDebInfo|Any CPU.Build.0 = Release|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.RelWithDebInfo|x64.ActiveCfg = Release|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.RelWithDebInfo|x64.Build.0 = Release|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.RelWithDebInfo|x86.ActiveCfg = Release|Any CPU + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94}.RelWithDebInfo|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1633,6 +1659,7 @@ Global {08A40B6A-F695-4EA9-AC8D-CF88FADEA796} = {17735A9D-BFD9-4585-A7CB-3208CA6EA8A7} {602976C5-2477-4B4C-AD9A-1EAFB250529A} = {ED2FE3E2-F7E7-4389-8231-B65123F2076F} {FCDFAF40-CC16-4D49-96C0-E49F195E7142} = {ED2FE3E2-F7E7-4389-8231-B65123F2076F} + {84BB2DD5-B2D8-4C85-AAF9-29D2A74FBF94} = {88278B81-7649-45DC-8A6A-D3A645C5AFC3} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {B526D2CE-EE2D-4AD4-93EF-1867D90FF1F5} diff --git a/build/Microsoft.DotNet.Cli.tasks b/build/Microsoft.DotNet.Cli.tasks index 901a48f8465..6ada95a1cc2 100644 --- a/build/Microsoft.DotNet.Cli.tasks +++ b/build/Microsoft.DotNet.Cli.tasks @@ -32,6 +32,7 @@ + diff --git a/build/Test.targets b/build/Test.targets index 86cd8ea7e7a..9b86bcca423 100644 --- a/build/Test.targets +++ b/build/Test.targets @@ -88,6 +88,8 @@ + diff --git a/build/publish/PublishDebian.targets b/build/publish/PublishDebian.targets index 41ea7f4bd6a..6dd77718057 100644 --- a/build/publish/PublishDebian.targets +++ b/build/publish/PublishDebian.targets @@ -6,28 +6,23 @@ $(DotnetBlobRootUrl)/$(Product)/$(FullNugetVersion)/$(DistroSpecificArtifactNameWithVersionCombinedHostHostFxrFrameworkSdk)-$(Architecture)$(InstallerExtension) - $(SdkDebianIntermediateDirectory)/package_upload.json - 1 - - - { - "name":"$(SdkDebianPackageName)", - "version":"$(NugetVersion)-$(DebianRevisionNumber)", - "repositoryId":"$(REPO_ID)", - "sourceUrl": "$(SdkDebianUploadUrl)" - } - - - - - - - - - + + + + + - + diff --git a/build_projects/dotnet-cli-build.Tests/GivenActionAndRetryTimes.cs b/build_projects/dotnet-cli-build.Tests/GivenActionAndRetryTimes.cs new file mode 100644 index 00000000000..47c83e0a72b --- /dev/null +++ b/build_projects/dotnet-cli-build.Tests/GivenActionAndRetryTimes.cs @@ -0,0 +1,92 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository; +using Xunit; + +namespace dotnet_cli_build.Tests +{ + public class GivenActionAndRetryTimes + { + public static IEnumerable NoWaitTimer() + { + while (true) + { + yield return Task.CompletedTask; + } + } + + [Fact] + public void ExponentialRetryShouldProvideIntervalSequence() + { + ExponentialRetry.Intervals.First().Should().Be(TimeSpan.FromSeconds(5)); + ExponentialRetry.Intervals.Skip(1).First().Should().Be(TimeSpan.FromSeconds(10)); + ExponentialRetry.Intervals.Skip(2).First().Should().Be(TimeSpan.FromSeconds(20)); + ExponentialRetry.Intervals.Skip(3).First().Should().Be(TimeSpan.FromSeconds(40)); + ExponentialRetry.Intervals.Skip(4).First().Should().Be(TimeSpan.FromSeconds(80)); + } + + [Fact] + public void ExponentialShouldNotRetryAfterFirstSucceess() + { + var fakeAction = new FakeAction(0); + ExponentialRetry.ExecuteWithRetry( + fakeAction.Run, + s => s == "success", + 10, + NoWaitTimer).Wait(); + fakeAction.Count.Should().Be(0); + } + + [Fact] + public void ExponentialShouldRetryUntilSuccess() + { + var fakeAction = new FakeAction(5); + ExponentialRetry.ExecuteWithRetry( + fakeAction.Run, + s => s == "success", + 10, + NoWaitTimer).Wait(); + fakeAction.Count.Should().Be(5); + } + + [Fact] + public void ExponentialShouldThrowAfterMaximumAmountReached() + { + var fakeAction = new FakeAction(10); + Action a = () => ExponentialRetry.ExecuteWithRetry( + fakeAction.Run, + s => s == "success", + 5, + NoWaitTimer, + "testing retry").Wait(); + a.ShouldThrow() + .WithMessage("Retry failed for testing retry after 5 times with result: fail"); + } + } + + public class FakeAction + { + private readonly int _successAfter; + + public FakeAction(int successAfter) + { + _successAfter = successAfter; + } + + public int Count { get; private set; } + + public Task Run() + { + if (_successAfter == Count) + { + return Task.FromResult("success"); + } + + Count++; + return Task.FromResult("fail"); + } + } +} \ No newline at end of file diff --git a/build_projects/dotnet-cli-build.Tests/dotnet-cli-build.Tests.csproj b/build_projects/dotnet-cli-build.Tests/dotnet-cli-build.Tests.csproj new file mode 100644 index 00000000000..aee1809873f --- /dev/null +++ b/build_projects/dotnet-cli-build.Tests/dotnet-cli-build.Tests.csproj @@ -0,0 +1,21 @@ + + + + + $(CliTargetFramework) + + false + + + + + + + + + + + + + + diff --git a/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/AddPackageStrategy.cs b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/AddPackageStrategy.cs new file mode 100644 index 00000000000..cb2ec81013e --- /dev/null +++ b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/AddPackageStrategy.cs @@ -0,0 +1,62 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; +using NuGet.Protocol; + +namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository +{ + internal class AddPackageStrategy : IAzurelinuxRepositoryServiceHttpStrategy + { + private readonly IdInRepositoryService _idInRepositoryService; + private readonly string _packageName; + private readonly string _packageVersion; + private readonly string _repositoryId; + + public AddPackageStrategy( + IdInRepositoryService idInRepositoryService, + string packageName, + string packageVersion, + string repositoryId) + { + _idInRepositoryService = idInRepositoryService + ?? throw new ArgumentNullException(nameof(idInRepositoryService)); + _packageName = packageName; + _packageVersion = packageVersion; + _repositoryId = repositoryId; + } + + public async Task Execute(HttpClient client, Uri baseAddress) + { + var debianUploadJsonContent = new Dictionary + { + ["name"] = _packageName, + ["version"] = AppendDebianRevisionNumber(_packageVersion), + ["fileId"] = _idInRepositoryService.Id, + ["repositoryId"] = _repositoryId + }.ToJson(); + var content = new StringContent(debianUploadJsonContent, + Encoding.UTF8, + "application/json"); + + using (var response = await client.PostAsync(new Uri(baseAddress, "/v1/packages"), content)) + { + if (!response.IsSuccessStatusCode) + throw new FailedToAddPackageToPackageRepositoryException( + $"request:{debianUploadJsonContent} response:{response.ToJson()}"); + return response.Headers.GetValues("Location").Single(); + } + } + + private static string AppendDebianRevisionNumber(string packageVersion) + { + return packageVersion + "-1"; + } + } +} diff --git a/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/ExponentialRetry.cs b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/ExponentialRetry.cs new file mode 100644 index 00000000000..df3590940d5 --- /dev/null +++ b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/ExponentialRetry.cs @@ -0,0 +1,53 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository +{ + public static class ExponentialRetry + { + public static IEnumerable Intervals + { + get + { + var seconds = 5; + while (true) + { + yield return TimeSpan.FromSeconds(seconds); + seconds *= 2; + } + } + } + + public static async Task ExecuteWithRetry(Func> action, + Func isSuccess, + int maxRetryCount, + Func> timer, + string taskDescription = "") + { + var count = 0; + foreach (var t in timer()) + { + await t; + var result = await action(); + if (isSuccess(result)) + return; + count++; + if (count == maxRetryCount) + throw new RetryFailedException( + $"Retry failed for {taskDescription} after {count} times with result: {result}"); + } + throw new Exception("Timer should not be exhausted"); + } + + public static IEnumerable Timer(IEnumerable interval) + { + return interval.Select(Task.Delay); + } + } +} diff --git a/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/FailedToAddPackageToPackageRepositoryException.cs b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/FailedToAddPackageToPackageRepositoryException.cs new file mode 100644 index 00000000000..68c314c08c1 --- /dev/null +++ b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/FailedToAddPackageToPackageRepositoryException.cs @@ -0,0 +1,24 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; + +namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository +{ + public class FailedToAddPackageToPackageRepositoryException : Exception + { + public FailedToAddPackageToPackageRepositoryException(string message) : base(message) + { + } + + public FailedToAddPackageToPackageRepositoryException() + { + } + + public FailedToAddPackageToPackageRepositoryException(string message, Exception innerException) : base(message, + innerException) + { + } + } +} diff --git a/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/FileUploadStrategy.cs b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/FileUploadStrategy.cs new file mode 100644 index 00000000000..59be2c16c7d --- /dev/null +++ b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/FileUploadStrategy.cs @@ -0,0 +1,49 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Net.Http; +using System.Threading.Tasks; +using NuGet.Protocol; + +namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository +{ + internal class FileUploadStrategy : IAzurelinuxRepositoryServiceHttpStrategy + { + private readonly string _pathToPackageToUpload; + + public FileUploadStrategy(string pathToPackageToUpload) + { + _pathToPackageToUpload = pathToPackageToUpload + ?? throw new ArgumentNullException(nameof(pathToPackageToUpload)); + } + + public async Task Execute(HttpClient client, Uri baseAddress) + { + var fileName = Path.GetFileName(_pathToPackageToUpload); + + using (var content = + new MultipartFormDataContent()) + { + var url = new Uri(baseAddress, "/v1/files"); + content.Add( + new StreamContent( + new MemoryStream( + File.ReadAllBytes(_pathToPackageToUpload))), + "file", + fileName); + using (var message = await client.PostAsync(url, content)) + { + if (!message.IsSuccessStatusCode) + { + throw new FailedToAddPackageToPackageRepositoryException( + $"{message.ToJson()} failed to post file to {url} file name:{fileName} pathToPackageToUpload:{_pathToPackageToUpload}"); + } + return await message.Content.ReadAsStringAsync(); + } + } + } + } +} diff --git a/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/IAzurelinuxRepositoryServiceHttpStrategy.cs b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/IAzurelinuxRepositoryServiceHttpStrategy.cs new file mode 100644 index 00000000000..a829f398a3e --- /dev/null +++ b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/IAzurelinuxRepositoryServiceHttpStrategy.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Net.Http; +using System.Threading.Tasks; + +namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository +{ + internal interface IAzurelinuxRepositoryServiceHttpStrategy + { + Task Execute(HttpClient client, Uri baseAddress); + } +} diff --git a/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/IdInRepositoryService.cs b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/IdInRepositoryService.cs new file mode 100644 index 00000000000..b51ebb3beff --- /dev/null +++ b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/IdInRepositoryService.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; + +namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository +{ + internal class IdInRepositoryService + { + public IdInRepositoryService(string id) + { + Id = id ?? throw new ArgumentNullException(nameof(id)); + } + + public string Id { get; } + } +} diff --git a/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/LinuxPackageRepositoryDestiny.cs b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/LinuxPackageRepositoryDestiny.cs new file mode 100644 index 00000000000..aa2e6d9bc64 --- /dev/null +++ b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/LinuxPackageRepositoryDestiny.cs @@ -0,0 +1,38 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; + +namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository +{ + internal class LinuxPackageRepositoryDestiny + { + private readonly string _password; + private readonly string _server; + private readonly string _username; + + public LinuxPackageRepositoryDestiny(string username, + string password, + string server, + string repositoryId) + { + _username = username ?? throw new ArgumentNullException(nameof(username)); + _password = password ?? throw new ArgumentNullException(nameof(password)); + _server = server ?? throw new ArgumentNullException(nameof(server)); + RepositoryId = repositoryId ?? throw new ArgumentNullException(nameof(repositoryId)); + } + + public string RepositoryId { get; } + + public Uri GetBaseAddress() + { + return new Uri($"https://{_server}"); + } + + public string GetSimpleAuth() + { + return $"{_username}:{_password}"; + } + } +} diff --git a/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/LinuxPackageRepositoryHttpPrepare.cs b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/LinuxPackageRepositoryHttpPrepare.cs new file mode 100644 index 00000000000..7e1f9ade71f --- /dev/null +++ b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/LinuxPackageRepositoryHttpPrepare.cs @@ -0,0 +1,46 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository +{ + internal class LinuxPackageRepositoryHttpPrepare + { + private readonly IAzurelinuxRepositoryServiceHttpStrategy _httpStrategy; + private readonly LinuxPackageRepositoryDestiny _linuxPackageRepositoryDestiny; + + public LinuxPackageRepositoryHttpPrepare( + LinuxPackageRepositoryDestiny linuxPackageRepositoryDestiny, + IAzurelinuxRepositoryServiceHttpStrategy httpStrategy + ) + { + _linuxPackageRepositoryDestiny = linuxPackageRepositoryDestiny + ?? throw new ArgumentNullException(nameof(linuxPackageRepositoryDestiny)); + _httpStrategy = httpStrategy ?? throw new ArgumentNullException(nameof(httpStrategy)); + } + + public async Task RemoteCall() + { + using (var handler = new HttpClientHandler()) + { + using (var client = new HttpClient(handler)) + { + var authHeader = + Convert.ToBase64String(Encoding.UTF8.GetBytes((string) _linuxPackageRepositoryDestiny.GetSimpleAuth())); + client.DefaultRequestHeaders.Authorization = + new AuthenticationHeaderValue("Basic", authHeader); + client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + client.Timeout = TimeSpan.FromMinutes(10); + + return await _httpStrategy.Execute(client, _linuxPackageRepositoryDestiny.GetBaseAddress()); + } + } + } + } +} diff --git a/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/PullQueuedPackageStatus.cs b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/PullQueuedPackageStatus.cs new file mode 100644 index 00000000000..393a160aee1 --- /dev/null +++ b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/PullQueuedPackageStatus.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Net.Http; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; + +namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository +{ + internal class PullQueuedPackageStatus : IAzurelinuxRepositoryServiceHttpStrategy + { + private readonly QueueResourceLocation _queueResourceLocation; + + public PullQueuedPackageStatus(QueueResourceLocation queueResourceLocation) + { + _queueResourceLocation = queueResourceLocation + ?? throw new ArgumentNullException(nameof(queueResourceLocation)); + } + + public async Task Execute(HttpClient client, Uri baseAddress) + { + using (var response = await client.GetAsync(new Uri(baseAddress, _queueResourceLocation.Location))) + { + if (!response.IsSuccessStatusCode) + throw new FailedToAddPackageToPackageRepositoryException( + "Failed to make request to " + _queueResourceLocation.Location); + var body = await response.Content.ReadAsStringAsync(); + return !body.Contains("status") ? "" : JObject.Parse(body)["status"].ToString(); + } + } + } +} diff --git a/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/QueueResourceLocation.cs b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/QueueResourceLocation.cs new file mode 100644 index 00000000000..cbf11e48dda --- /dev/null +++ b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/QueueResourceLocation.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; + +namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository +{ + internal class QueueResourceLocation + { + public QueueResourceLocation(string location) + { + Location = location ?? throw new ArgumentNullException(nameof(location)); + } + + public string Location { get; } + } +} diff --git a/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/RetryFailedException.cs b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/RetryFailedException.cs new file mode 100644 index 00000000000..b251ec4b83a --- /dev/null +++ b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/RetryFailedException.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; + +namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository +{ + public class RetryFailedException : Exception + { + public RetryFailedException(string message) : base(message) + { + } + + public RetryFailedException() + { + } + + public RetryFailedException(string message, Exception innerException) : base(message, innerException) + { + } + } +} diff --git a/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/UploadToLinuxPackageRepository.cs b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/UploadToLinuxPackageRepository.cs new file mode 100644 index 00000000000..3e14557d69a --- /dev/null +++ b/build_projects/dotnet-cli-build/UploadToLinuxPackageRepository/UploadToLinuxPackageRepository.cs @@ -0,0 +1,108 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Net.Http; +using System.Threading.Tasks; +using Microsoft.Build.Framework; +using Newtonsoft.Json.Linq; +using Task = Microsoft.Build.Utilities.Task; + +namespace Microsoft.DotNet.Cli.Build.UploadToLinuxPackageRepository +{ + public class UploadToLinuxPackageRepository : Task + { + /// + /// The Azure repository service user name. + /// + [Required] + public string Username { get; set; } + + /// + /// The Azure repository service Password. + /// + [Required] + public string Password { get; set; } + + /// + /// The Azure repository service URL ex: "tux-devrepo.corp.microsoft.com". + /// + [Required] + public string Server { get; set; } + + [Required] + public string RepositoryId { get; set; } + + [Required] + public string PathOfPackageToUpload { get; set; } + + [Required] + public string PackageNameInLinuxPackageRepository { get; set; } + + + [Required] + public string PackageVersionInLinuxPackageRepository { get; set; } + + + public override bool Execute() + { + ExecuteAsyncWithRetry().GetAwaiter().GetResult(); + return true; + } + + private async System.Threading.Tasks.Task ExecuteAsyncWithRetry() + { + await ExponentialRetry.ExecuteWithRetry( + UploadAndAddpackageAndEnsureItIsReady, + s => s == "", + maxRetryCount: 3, + timer: () => ExponentialRetry.Timer(ExponentialRetry.Intervals), + taskDescription: $"running {nameof(UploadAndAddpackageAndEnsureItIsReady)}"); + } + + private async Task UploadAndAddpackageAndEnsureItIsReady() + { + try + { + var linuxPackageRepositoryDestiny = + new LinuxPackageRepositoryDestiny(Username, Password, Server, RepositoryId); + var uploadResponse = await new LinuxPackageRepositoryHttpPrepare( + linuxPackageRepositoryDestiny, + new FileUploadStrategy(PathOfPackageToUpload)).RemoteCall(); + + var idInRepositoryService = new IdInRepositoryService(JObject.Parse(uploadResponse)["id"].ToString()); + + var addPackageResponse = await new LinuxPackageRepositoryHttpPrepare( + linuxPackageRepositoryDestiny, + new AddPackageStrategy( + idInRepositoryService, + PackageNameInLinuxPackageRepository, + PackageVersionInLinuxPackageRepository, + linuxPackageRepositoryDestiny.RepositoryId)).RemoteCall(); + + var queueResourceLocation = new QueueResourceLocation(addPackageResponse); + + Func> pullQueuedPackageStatus = new LinuxPackageRepositoryHttpPrepare( + linuxPackageRepositoryDestiny, + new PullQueuedPackageStatus(queueResourceLocation)).RemoteCall; + + await ExponentialRetry.ExecuteWithRetry( + pullQueuedPackageStatus, + s => s == "fileReady", + 5, + () => ExponentialRetry.Timer(ExponentialRetry.Intervals), + $"PullQueuedPackageStatus location: {queueResourceLocation.Location}"); + return ""; + } + catch (FailedToAddPackageToPackageRepositoryException e) + { + return e.ToString(); + } + catch (HttpRequestException e) + { + return e.ToString(); + } + } + } +} diff --git a/scripts/publish/repoapi_client.sh b/scripts/publish/repoapi_client.sh deleted file mode 100755 index 875210bff2c..00000000000 --- a/scripts/publish/repoapi_client.sh +++ /dev/null @@ -1,194 +0,0 @@ -#!/bin/bash -# -# Copyright (c) .NET Foundation and contributors. All rights reserved. -# Licensed under the MIT license. See LICENSE file in the project root for full license information. -# - -# This is a VERY basic script for Create/Delete operations on repos and packages -# -# Environment Dependencies: -# $REPO_SERVER -# $REPO_USER -# $REPO_PASS - -cmd=$1 -urls=urls.txt -defaultPackageFile=new_package.json -defaultRepoFile=new_repo.json -repositoryId=$REPO_ID -server=$REPO_SERVER -user=$REPO_USER -pass=$REPO_PASS -protocol=https -port=443 -baseurl="$protocol://$user:$pass@$server:$port" - -echo $baseurl - -function BailIf -{ - if [ $1 -ne 0 ]; then - echo "Failure occurred communicating with $server" - exit 1 - fi -} - -# List packages, using $1 as a regex to filter results -function ListPackages -{ - curl -k "$baseurl/v1/packages" | sed 's/{/\n{/g' | egrep "$1" | sed 's/,/,\n/g' | sed 's/^"/\t"/g' - echo "" -} - -# Create a new Repo using the specified JSON file -function AddRepo -{ - repoFile=$1 - if [ -z $repoFile ]; then - echo "Error: Must specify a JSON-formatted file. Reference $defaultRepoFile.template" - exit 1 - fi - if [ ! -f $repoFile ]; then - echo "Error: Cannot create repo - $repoFile does not exist" - exit 1 - fi - packageUrl=$(grep "url" $repoFile | head -n 1 | awk '{print $2}' | tr -d ',') - echo "Creating new repo on $server [$packageUrl]" - curl -i -k "$baseurl/v1/repositories" --data @./$repoFile -H "Content-Type: application/json" - BailIf $? - echo "" -} - -# Upload a single package using the specified JSON file -function AddPackage -{ - packageFile=$1 - if [ -z $packageFile ]; then - echo "Error: Must specify a JSON-formatted file. Reference $defaultPackageFile.template" - exit 1 - fi - if [ ! -f $packageFile ]; then - echo "Error: Cannot add package - $packageFile does not exist" - exit 1 - fi - packageUrl=$(grep "sourceUrl" $packageFile | head -n 1 | awk '{print $2}') - echo "Adding package to $server [$packageUrl]" - curl -i -k "$baseurl/v1/packages" --data @$packageFile -H "Content-Type: application/json" - BailIf $? - echo "" -} - -# Upload a single package by dynamically creating a JSON file using a provided URL -function AddPackageByUrl -{ - # Parse URL - url=$(echo "$1") - if [ -z $url ]; then - return - fi - escapedUrl=$(echo "$url" | sed 's/\//\\\//g') - set -- "$1" - oldIFS=$IFS - IFS="/"; declare -a splitUrl=($*) - index=${#splitUrl[@]} - let "index -= 1" - filename=${splitUrl[$index]} - set -- "$filename" - IFS="_"; declare -a splitFile=($*) - IFS=$oldIFS - pkgName=${splitFile[0]} - pkgVer=${splitFile[1]} - if [ -z $pkgName ] || [ -z $pkgVer ]; then - echo "ERROR parsing $url" - return - fi - # Create Package .json file - cp $defaultPackageFile.template $defaultPackageFile - sed -i "s/PACKAGENAME/$pkgName/g" $defaultPackageFile - sed -i "s/PACKAGEVERSION/$pkgVer/g" $defaultPackageFile - sed -i "s/PACKAGEURL/$escapedUrl/g" $defaultPackageFile - sed -i "s/REPOSITORYID/$repositoryId/g" $defaultPackageFile - # Test that URL is ok - wget -q --spider "$url" - if [[ $? -eq 0 ]]; then - echo "Ready to upload $pkgName [$pkgVer]" - else - echo "ERROR testing URL $url" - return - fi - # Perform Upload - AddPackage $defaultPackageFile - # Cleanup - # rm $defaultPackageFile -} - -# Upload multiple packages by reading urls line-by-line from the specified file -function AddPackages -{ - urlFile=$1 - if [ -z $urlFile ]; then - echo "Error: Must specify a flat text file containing one or more URLs" - exit 1 - fi - if [ ! -f $urlFile ]; then - echo "Error: Cannot add packages. File $urlFile does not exist" - exit 1 - fi - for url in $(cat $urlFile); do - AddPackageByUrl "$url" - sleep 5 - done -} - -# Delete the specified repo -function DeleteRepo -{ - repoId=$1 - if [ -z $repoId ]; then - echo "Error: Please specify repository ID. Run -listrepos for a list of IDs" - exit 1 - fi - curl -I -k -X DELETE "$baseurl/v1/repositories/$repoId" - BailIf $? -} - -# Delete the specified package -function DeletePackage -{ - packageId=$1 - if [ -z $packageId ]; then - echo "Error: Please specify package ID. Run -listpkgs for a list of IDs" - exit 1 - fi - echo Removing pkgId $packageId from repo $repositoryId - curl -I -k -X DELETE "$baseurl/v1/packages/$packageId" - BailIf $? -} - -if [[ "$1" == "-listrepos" ]]; then - echo "Fetching repo list from $server..." - curl -k "$baseurl/v1/repositories" | sed 's/,/,\n/g' | sed 's/^"/\t"/g' - echo "" -elif [[ "$1" == "-listpkgs" ]]; then - echo "Fetching package list from $server" - ListPackages $2 -elif [[ "$1" == "-addrepo" ]]; then - AddRepo $2 -elif [[ "$1" == "-addpkg" ]]; then - AddPackage $2 -elif [[ "$1" == "-addpkgs" ]]; then - AddPackages $2 -elif [[ "$1" == "-delrepo" ]]; then - DeleteRepo $2 -elif [[ "$1" == "-delpkg" ]]; then - DeletePackage $2 -else - echo "USAGE: ./repotool.sh -OPTION" - echo "-listrepos: Gather a list of repos" - echo "-listpkgs: Gather a list of packages" - echo "-addrepo [FILENAME] : Create a new repo using the specified JSON file" - echo "-addpkg [FILENAME] : Add package to repo using the specified JSON file" - echo "-addpkgs [FILENAME] : Add packages to repo using urls contained in FILENAME" - echo "-delrepo REPOID : Delete the specified repo by ID" - echo "-delpkg PKGID : Delete the specified package by ID" -fi