diff --git a/Tasks/DownloadBuildArtifactsV0/DownloadHandlers/DownloadHandler.ts b/Tasks/DownloadBuildArtifactsV0/DownloadHandlers/DownloadHandler.ts new file mode 100644 index 000000000000..b9fefe5b50c2 --- /dev/null +++ b/Tasks/DownloadBuildArtifactsV0/DownloadHandlers/DownloadHandler.ts @@ -0,0 +1,73 @@ +import { IBaseHandlerConfig } from './HandlerConfigs'; +import { handlerCheckDownloadedFiles } from '../download_helper'; +import { ArtifactEngine } from 'artifact-engine/Engine'; +import { IArtifactProvider, ArtifactDownloadTicket } from 'artifact-engine/Models'; +import * as tl from 'azure-pipelines-task-lib/task'; + +/** + * Base class for artifact download handlers + */ +export abstract class DownloadHandler { + /** + * @member {IBaseHandlerConfig} - contains info for generate source and destination providers + * @access protected + */ + protected config: IBaseHandlerConfig; + + constructor(handlerConfig: IBaseHandlerConfig) { + this.config = handlerConfig; + } + + /** + * Pure abstract method for getting Source Provider. + * Source Provider is an object that contains info about from where we will download the artifact. + * @access protected + * @returns {IArtifactProvider} Objects that implement the IArtifactProvider interface + */ + protected abstract getSourceProvider(): IArtifactProvider; + + /** + * Pure abstract method for getting Destination Provider. + * Destination Provider is an object that contains info about where we will download artifacts. + * @access protected + * @returns {IArtifactProvider} Objects that implement the IArtifactProvider interface + */ + protected abstract getDestinationProvider(): IArtifactProvider; + + /** + * Method to download Build Artifact. + * Since the logic for downloading builds artifacts is the same for all + * types of source and destination providers, we will implement this logic in the base class. + * @access public + * @returns {Promise>} an array of Download Tickets + */ + public async downloadResources(): Promise> { + const downloader: ArtifactEngine = new ArtifactEngine(); + const sourceProvider: IArtifactProvider = this.getSourceProvider(); + const destinationProvider: IArtifactProvider = this.getDestinationProvider(); + + const downloadPromise: Promise> = new Promise>(async (downloadComplete, downloadFailed) => { + try { + // First attempt to download artifact + const downloadTickets: Array = await downloader.processItems(sourceProvider, destinationProvider, this.config.downloaderOptions); + + // We will proceed with the files check only if the "Check download files" option enabled + if (this.config.checkDownloadedFiles && Array.isArray(downloadTickets)) { + try { + // Launch the files check, if all files are fully downloaded no exceptions will be thrown. + handlerCheckDownloadedFiles(downloadTickets); + downloadComplete(downloadTickets); + } catch (error) { + downloadFailed(error); + } + } else { + downloadComplete(downloadTickets); + } + } catch (error) { + downloadFailed(error); + } + }); + + return downloadPromise; + } +} diff --git a/Tasks/DownloadBuildArtifactsV0/DownloadHandlers/DownloadHandlerContainer.ts b/Tasks/DownloadBuildArtifactsV0/DownloadHandlers/DownloadHandlerContainer.ts new file mode 100644 index 000000000000..23f76215507e --- /dev/null +++ b/Tasks/DownloadBuildArtifactsV0/DownloadHandlers/DownloadHandlerContainer.ts @@ -0,0 +1,62 @@ +import { DownloadHandler } from './DownloadHandler'; +import { IContainerHandlerConfig } from './HandlerConfigs'; +import { WebProvider, FilesystemProvider } from 'artifact-engine/Providers'; +import * as tl from 'azure-pipelines-task-lib/task'; + +/** + * Handler for download artifact from related container resource. + * Build Artifact will be downloaded via `_apis/resources/Containers/` resource. + * @extends DownloadHandler + * @example + * const config: IContainerHandlerConfig = {...}; + * const downloadHandler: IContainerHandlerConfig = new IContainerHandlerConfig(config); + * downloadHandler.downloadResources(); + */ +export class DownloadHandlerContainer extends DownloadHandler { + protected config: IContainerHandlerConfig; + + constructor(handlerConfig: IContainerHandlerConfig) { + super(handlerConfig); + } + + /** + * To download artifact from container resource we will use `WebProvider` as source provider + * @access protected + * @returns {WebProvider} Configured Web Provider + */ + protected getSourceProvider(): WebProvider { + console.log(tl.loc('DownloadingContainerResource', this.config.artifactInfo.resource.data)); + const containerParts: Array = this.config.artifactInfo.resource.data.split('/'); + + if (containerParts.length < 3) { + throw new Error(tl.loc('FileContainerInvalidArtifactData')); + } + + const containerId: number = parseInt(containerParts[1]); + let containerPath: string = containerParts.slice(2, containerParts.length).join('/'); + + if (containerPath === '/') { + //container REST api oddity. Passing '/' as itemPath downloads the first file instead of returning the meta data about the all the files in the root level. + //This happens only if the first item is a file. + containerPath = ''; + } + + const variables = {}; + const itemsUrl: string = `${this.config.endpointUrl}/_apis/resources/Containers/${containerId}?itemPath=${encodeURIComponent(containerPath)}&isShallow=true&api-version=4.1-preview.4`; + + console.log(tl.loc('DownloadArtifacts', this.config.artifactInfo.name, itemsUrl)); + + const provider: WebProvider = new WebProvider(itemsUrl, this.config.templatePath, variables, this.config.handler); + return provider; + } + + /** + * Since we download artifact to local storage we will use a `FilesystemProvider` as destination provider + * @access protected + * @returns {FilesystemProvider} Configured Filesystem Provider + */ + protected getDestinationProvider(): FilesystemProvider { + const provider: FilesystemProvider = new FilesystemProvider(this.config.downloadPath); + return provider; + } +} diff --git a/Tasks/DownloadBuildArtifactsV0/DownloadHandlers/DownloadHandlerContainerZip.ts b/Tasks/DownloadBuildArtifactsV0/DownloadHandlers/DownloadHandlerContainerZip.ts new file mode 100644 index 000000000000..233af73261eb --- /dev/null +++ b/Tasks/DownloadBuildArtifactsV0/DownloadHandlers/DownloadHandlerContainerZip.ts @@ -0,0 +1,121 @@ +import { DownloadHandler } from './DownloadHandler'; +import { IContainerHandlerZipConfig } from './HandlerConfigs'; +import { FilesystemProvider, ZipProvider } from 'artifact-engine/Providers'; +import * as tl from 'azure-pipelines-task-lib/task'; +import * as path from 'path'; +import * as DecompressZip from 'decompress-zip'; + +/** + * Handler for download artifact via build API + * Build Artifact will be downloaded as zip archive via `/_apis/build/builds/` resource. + * This handler was designed to work only on windows system. + * @extends DownloadHandler + * @example + * const config: IContainerHandlerZipConfig = {...}; + * const downloadHandler: DownloadHandlerContainerZip = new DownloadHandlerContainerZip(config); + * downloadHandler.downloadResources(); + */ +export class DownloadHandlerContainerZip extends DownloadHandler { + protected config: IContainerHandlerZipConfig; + private readonly archiveUrl: string; + private readonly zipLocation: string; + + constructor(handlerConfig: IContainerHandlerZipConfig) { + super(handlerConfig); + this.archiveUrl = `${this.config.endpointUrl}/${this.config.projectId}/_apis/build/builds/${this.config.buildId}/artifacts?artifactName=${this.config.artifactInfo.name}&$format=zip`; + this.zipLocation = path.join(this.config.downloadPath, `${this.config.artifactInfo.name}.zip`); + } + + /** + * Unpack an archive with an artifact + * @param unzipLocation path to the target artifact + * @access private + * @returns {Promise} promise that will be resolved once the archive will be unpacked + */ + private unzipContainer(unzipLocation: string): Promise { + const unZipPromise: Promise = new Promise((resolve, reject) => { + if (!tl.exist(this.zipLocation)) { + return resolve(); + } + + tl.debug(`Extracting ${this.zipLocation} to ${unzipLocation}`); + + const unzipper = new DecompressZip(this.zipLocation); + + unzipper.on('error', err => { + return reject(tl.loc('ExtractionFailed', err)); + }); + + unzipper.on('extract', log => { + tl.debug(`Extracted ${this.zipLocation} to ${unzipLocation} successfully`); + return resolve(); + }); + + unzipper.extract({ + path: unzipLocation + }); + + }); + + return unZipPromise; + } + + /** + * Get zip provider. + * Since we will download archived artifact we will use `ZipProvider` as source provider. + * @access protected + * @returns {ZipProvider} Configured Zip Provider + */ + protected getSourceProvider(): ZipProvider { + console.log(tl.loc('DownloadArtifacts', this.config.artifactInfo.name, this.archiveUrl)); + const provider: ZipProvider = new ZipProvider(this.archiveUrl, this.config.handler); + return provider; + } + + /** + * Get filesystem provider. + * Since we download artifact to local storage we will use a `FilesystemProvider` as destination provider. + * @access protected + * @returns {FilesystemProvider} Configured Filesystem Provider + */ + protected getDestinationProvider(): FilesystemProvider { + const provider: FilesystemProvider = new FilesystemProvider(this.zipLocation); + return provider; + } + + /** + * Download and unpack an archive with an artifact. + * @access public + */ + public downloadResources(): Promise { + const downloadProcess: Promise = new Promise((resolve, reject) => { + tl.debug('Starting downloadZip action'); + + if (tl.exist(this.zipLocation)) { + tl.rmRF(this.zipLocation); + } + + super.downloadResources().then(() => { + tl.debug(`Successfully downloaded from ${this.archiveUrl}`); + + this.unzipContainer(this.config.downloadPath).then(() => { + tl.debug(`Successfully extracted ${this.zipLocation}`); + + if (tl.exist(this.zipLocation)) { + tl.rmRF(this.zipLocation); + } + + resolve(); + + }).catch((error) => { + reject(error); + }); + + }).catch((error) => { + reject(error); + }); + }); + + return downloadProcess; + } +} diff --git a/Tasks/DownloadBuildArtifactsV0/DownloadHandlers/DownloadHandlerFilePath.ts b/Tasks/DownloadBuildArtifactsV0/DownloadHandlers/DownloadHandlerFilePath.ts new file mode 100644 index 000000000000..884c43683893 --- /dev/null +++ b/Tasks/DownloadBuildArtifactsV0/DownloadHandlers/DownloadHandlerFilePath.ts @@ -0,0 +1,46 @@ +import { DownloadHandler } from './DownloadHandler'; +import { FilesystemProvider } from 'artifact-engine/Providers'; +import * as tl from 'azure-pipelines-task-lib/task'; +import * as fs from 'fs'; +import * as path from 'path'; + +/** + * Handler for download artifact via local file share + * @extends DownloadHandler + * @example + * const config: IBaseHandlerConfig = {...}; + * const downloadHandler: DownloadHandlerFilePath = new DownloadHandlerFilePath(config); + * downloadHandler.downloadResources(); + */ +export class DownloadHandlerFilePath extends DownloadHandler { + /** + * Get source provider with source folder. + * Since we will work with local files we use `Filesystem Provider` as source provider. + * @access protected + * @returns {FilesystemProvider} Configured Filesystem Provider + */ + protected getSourceProvider(): FilesystemProvider { + const downloadUrl = this.config.artifactInfo.resource.data; + const artifactName = this.config.artifactInfo.name.replace('/', '\\'); + let artifactLocation = path.join(downloadUrl, artifactName); + + if (!fs.existsSync(artifactLocation)) { + console.log(tl.loc('ArtifactNameDirectoryNotFound', artifactLocation, downloadUrl)); + artifactLocation = downloadUrl; + } + + const provider: FilesystemProvider = new FilesystemProvider(artifactLocation, artifactName); + return provider; + } + + /** + * Get destination provider with destination folder. + * Since we will work with local files we use `Filesystem Provider` as source provider. + * @access protected + * @returns {FilesystemProvider} Configured Filesystem Provider + */ + protected getDestinationProvider(): FilesystemProvider { + const provider: FilesystemProvider = new FilesystemProvider(this.config.downloadPath); + return provider; + } +} diff --git a/Tasks/DownloadBuildArtifactsV0/DownloadHandlers/HandlerConfigs.ts b/Tasks/DownloadBuildArtifactsV0/DownloadHandlers/HandlerConfigs.ts new file mode 100644 index 000000000000..5162c2b9ec18 --- /dev/null +++ b/Tasks/DownloadBuildArtifactsV0/DownloadHandlers/HandlerConfigs.ts @@ -0,0 +1,23 @@ +import { ArtifactEngineOptions } from 'artifact-engine/Engine'; +import { BuildArtifact } from 'azure-devops-node-api/interfaces/BuildInterfaces'; +import { PersonalAccessTokenCredentialHandler } from 'artifact-engine/Providers/typed-rest-client/Handlers'; + +export interface IBaseHandlerConfig { + artifactInfo: BuildArtifact; + downloadPath: string; + downloaderOptions: ArtifactEngineOptions; + checkDownloadedFiles: boolean; +} + +export interface IContainerHandlerConfig extends IBaseHandlerConfig { + endpointUrl: string; + templatePath: string; + handler: PersonalAccessTokenCredentialHandler; +} + +export interface IContainerHandlerZipConfig extends IBaseHandlerConfig { + endpointUrl: string; + projectId: string; + buildId: number; + handler: PersonalAccessTokenCredentialHandler; +} diff --git a/Tasks/DownloadBuildArtifactsV0/Strings/resources.resjson/en-US/resources.resjson b/Tasks/DownloadBuildArtifactsV0/Strings/resources.resjson/en-US/resources.resjson index 643c2a5b2061..e94bd73fca95 100644 --- a/Tasks/DownloadBuildArtifactsV0/Strings/resources.resjson/en-US/resources.resjson +++ b/Tasks/DownloadBuildArtifactsV0/Strings/resources.resjson/en-US/resources.resjson @@ -31,6 +31,10 @@ "loc.input.help.downloadPath": "Path on the agent machine where the artifacts will be downloaded", "loc.input.label.parallelizationLimit": "Parallelization limit", "loc.input.help.parallelizationLimit": "Number of files to download simultaneously", + "loc.input.label.checkDownloadedFiles": "Check downloaded files", + "loc.input.help.checkDownloadedFiles": "If checked, this build task will check that all files are fully downloaded.", + "loc.input.label.retryDownloadCount": "Retry count", + "loc.input.help.retryDownloadCount": "Optional number of times to retry downloading a build artifact if the download fails.", "loc.messages.DownloadArtifacts": "Downloading artifact %s from: %s", "loc.messages.DownloadingArtifactsForBuild": "Downloading artifacts for build: %s", "loc.messages.LinkedArtifactCount": "Linked artifacts count: %s", @@ -51,5 +55,9 @@ "loc.messages.DownloadingContainerResource": "Downloading items from container resource %s", "loc.messages.DefinitionNameMatchFound": "Definition Name %s resolved to id %d", "loc.messages.InvalidBuildDefinitionName": "Definition name %s didn't correspond to a valid definition", - "loc.messages.UnresolvedDefinitionId": "Could not resolve build definition id" + "loc.messages.UnresolvedDefinitionId": "Could not resolve build definition id", + "loc.messages.BeginArtifactItemsIntegrityCheck": "Starting artifact items integrity check", + "loc.messages.CorruptedArtifactItemsList": "The following items are not passed integrity check:", + "loc.messages.IntegrityCheckNotPassed": "Artifact items integrity check failed", + "loc.messages.IntegrityCheckPassed": "Artifact items integrity check successfully finished" } \ No newline at end of file diff --git a/Tasks/DownloadBuildArtifactsV0/download_helper.ts b/Tasks/DownloadBuildArtifactsV0/download_helper.ts new file mode 100644 index 000000000000..bf88a9add8e8 --- /dev/null +++ b/Tasks/DownloadBuildArtifactsV0/download_helper.ts @@ -0,0 +1,98 @@ +import { debug, loc } from 'azure-pipelines-task-lib/task'; +import { ArtifactDownloadTicket, ItemType, TicketState } from 'artifact-engine/Models'; +import { getFileSizeInBytes } from './file_helper'; + +/** + * Just a Promise wrapper for setTimeout function + * @param {number} interval - timeout interval in milliseconds + */ +export function timeoutPromise(interval: number): Promise<{}> { + debug(`Wait for ${interval} milliseconds`); + return new Promise(resolve => setTimeout(resolve, interval)); +} + +/** + * This function checks a result of artifact download + * @param {Array} downloadTickets + * @throws Exception if downloaded build artifact is not healthy + * @returns void + */ +export function handlerCheckDownloadedFiles(downloadTickets: Array): void { + console.log(loc('BeginArtifactItemsIntegrityCheck')); + debug(`Items count: ${downloadTickets.length}`); + + const corruptedItems: Array = downloadTickets.filter(ticket => isItemCorrupted(ticket)); + + if (corruptedItems.length > 0) { + console.log(loc('CorruptedArtifactItemsList')); + corruptedItems.map(item => console.log(item.artifactItem.metadata.destinationUrl)); + + throw new Error(loc('IntegrityCheckNotPassed')); + } + + console.log(loc('IntegrityCheckPassed')); +} + +/** + * This function investigates the download ticket of the artifact item. + * + * Since artifact's items stored as compressed files the only appropriate way (at the moment) + * to make sure that the item fully downloaded is to compare bytes length before compress + * that provided by Azure DevOps and actual bytes length from local storage. + * + * @param {ArtifactDownloadTicket} ticket - download ticket of artifact item + * @returns {boolean} `true` if item corrupted, `false` if item healthy + */ +function isItemCorrupted(ticket: ArtifactDownloadTicket): boolean { + let isCorrupted: boolean = false; + + // We check the tickets only with processed status and File item type + if (ticket.state === TicketState.Processed && + ticket.artifactItem.itemType === ItemType.File) { + debug(`Start check for item: ${ticket.artifactItem.path}`); + debug(`Getting info from download ticket`); + + const localPathToFile: string = ticket.artifactItem.metadata.destinationUrl; + debug(`Local path to the item: ${localPathToFile}`); + + if (ticket.artifactItem.fileLength) { + const expectedBytesLength: number = Number(ticket.artifactItem.fileLength); + + if (isNaN(expectedBytesLength)) { + debug('Incorrect data in related download ticket, skip item validation.'); + isCorrupted = true; + } else { + debug(`Expected length in bytes ${expectedBytesLength}`); + + try { + const actualBytesLength = getFileSizeInBytes(localPathToFile); + debug(`Actual length in bytes ${actualBytesLength}`); + isCorrupted = (expectedBytesLength !== actualBytesLength); + } catch (error) { + debug('Unable to get file stats from local storage due to the following error:'); + debug(error); + debug('Skip item validation'); + isCorrupted = true; + } + } + } else if (ticket.artifactItem.metadata.downloadUrl.endsWith('format=zip')) { + // When we use a Zip Provider the Artifact Engine returns only "fileSizeInBytes" + try { + const expectedBytesLength: number = Number(ticket.fileSizeInBytes); + const actualBytesLength: number = getFileSizeInBytes(localPathToFile); + + debug(`Expected length in bytes ${expectedBytesLength}`); + debug(`Actual length in bytes ${actualBytesLength}`); + + return (expectedBytesLength !== actualBytesLength); + } catch (error) { + debug('Unable to get file stats from local storage due to the following error:'); + debug(error); + debug('Skip item validation'); + isCorrupted = true; + } + } + } + + return isCorrupted; +} diff --git a/Tasks/DownloadBuildArtifactsV0/file_helper.ts b/Tasks/DownloadBuildArtifactsV0/file_helper.ts new file mode 100644 index 000000000000..6931fbad1b09 --- /dev/null +++ b/Tasks/DownloadBuildArtifactsV0/file_helper.ts @@ -0,0 +1,21 @@ +import { Stats, statSync as getFile } from 'fs'; + +/** + * Get size of file on local storage + * @param {string} path - path to the file in local storage + * @throws Exception if path to the file is empty + * @returns Size of the file in bytes + */ +export function getFileSizeInBytes(path: string): number { + let fileSize: number = 0; + + if (path) { + // TODO: Add support of BigInt after migration on Node10 + const file: Stats = getFile(path); + fileSize = file.size; + } else { + throw 'Path to the file is empty'; + } + + return fileSize; +} diff --git a/Tasks/DownloadBuildArtifactsV0/main.ts b/Tasks/DownloadBuildArtifactsV0/main.ts index 3897e1543c5e..65dc0ad54b57 100644 --- a/Tasks/DownloadBuildArtifactsV0/main.ts +++ b/Tasks/DownloadBuildArtifactsV0/main.ts @@ -10,10 +10,12 @@ import { BuildStatus, BuildResult, BuildQueryOrder, Build, BuildDefinitionRefere import * as models from 'artifact-engine/Models'; import * as engine from 'artifact-engine/Engine'; -import * as providers from 'artifact-engine/Providers'; import * as webHandlers from 'artifact-engine/Providers/typed-rest-client/Handlers'; +import { IBaseHandlerConfig, IContainerHandlerConfig, IContainerHandlerZipConfig } from './DownloadHandlers/HandlerConfigs'; -var DecompressZip = require('decompress-zip'); +import { DownloadHandlerContainer } from './DownloadHandlers/DownloadHandlerContainer'; +import { DownloadHandlerContainerZip } from './DownloadHandlers/DownloadHandlerContainerZip'; +import { DownloadHandlerFilePath } from './DownloadHandlers/DownloadHandlerFilePath'; var taskJson = require('./task.json'); @@ -79,15 +81,17 @@ async function main(): Promise { if (!!tagFiltersInput) { tagFilters = tagFiltersInput.split(","); } + const checkDownloadedFiles: boolean = tl.getBoolInput('checkDownloadedFiles', false); var endpointUrl: string = tl.getVariable("System.TeamFoundationCollectionUri"); var accessToken: string = tl.getEndpointAuthorizationParameter('SYSTEMVSSCONNECTION', 'AccessToken', false); var credentialHandler: IRequestHandler = getHandlerFromToken(accessToken); var webApi: WebApi = new WebApi(endpointUrl, credentialHandler); - var retryLimit = parseInt(tl.getVariable("VSTS_HTTP_RETRY")) ? parseInt(tl.getVariable("VSTS_HTTP_RETRY")) : 4; + const retryLimitRequest: number = parseInt(tl.getVariable('VSTS_HTTP_RETRY')) ? parseInt(tl.getVariable("VSTS_HTTP_RETRY")) : 4; + const retryLimitDownload: number = parseInt(tl.getInput('retryDownloadCount', false)) ? parseInt(tl.getInput('retryDownloadCount', false)) : 4; var templatePath: string = path.join(__dirname, 'vsts.handlebars.txt'); - var buildApi: IBuildApi = await executeWithRetries("getBuildApi", () => webApi.getBuildApi(), retryLimit).catch((reason) => { + var buildApi: IBuildApi = await executeWithRetries("getBuildApi", () => webApi.getBuildApi(), retryLimitRequest).catch((reason) => { reject(reason); return; }); @@ -145,7 +149,7 @@ async function main(): Promise { // if the definition name includes a variable then definitionIdSpecified is a name vs a number if (!!definitionIdSpecified && Number.isNaN(parseInt(definitionIdSpecified))) { - var definitions: BuildDefinitionReference[] = await executeWithRetries("getBuildDefinitions", () => buildApi.getDefinitions(projectId, definitionIdSpecified), retryLimit).catch((reason) => { + var definitions: BuildDefinitionReference[] = await executeWithRetries("getBuildDefinitions", () => buildApi.getDefinitions(projectId, definitionIdSpecified), retryLimitRequest).catch((reason) => { reject(reason); return; }); @@ -176,7 +180,7 @@ async function main(): Promise { var branchNameFilter = (buildVersionToDownload == "latest") ? null : branchName; // get latest successful build filtered by branch - var buildsForThisDefinition = await executeWithRetries("getBuildId", () => buildApi.getBuilds(projectId, [parseInt(definitionId)], null, null, null, null, null, null, BuildStatus.Completed, resultFilter, tagFilters, null, null, null, null, null, BuildQueryOrder.FinishTimeDescending, branchNameFilter), retryLimit).catch((reason) => { + var buildsForThisDefinition = await executeWithRetries("getBuildId", () => buildApi.getBuilds(projectId, [parseInt(definitionId)], null, null, null, null, null, null, BuildStatus.Completed, resultFilter, tagFilters, null, null, null, null, null, BuildQueryOrder.FinishTimeDescending, branchNameFilter), retryLimitRequest).catch((reason) => { reject(reason); return; }); @@ -193,7 +197,7 @@ async function main(): Promise { } if (!build) { - build = await executeWithRetries("getBuild", () => buildApi.getBuild(buildId, projectId), retryLimit).catch((reason) => { + build = await executeWithRetries("getBuild", () => buildApi.getBuild(buildId, projectId), retryLimitRequest).catch((reason) => { reject(reason); return; }); @@ -219,7 +223,7 @@ async function main(): Promise { // populate itempattern and artifacts based on downloadType if (downloadType === 'single') { var artifactName = tl.getInput("artifactName", true); - var artifact = await executeWithRetries("getArtifact", () => buildApi.getArtifact(buildId, artifactName, projectId), retryLimit).catch((reason) => { + var artifact = await executeWithRetries("getArtifact", () => buildApi.getArtifact(buildId, artifactName, projectId), retryLimitRequest).catch((reason) => { reject(reason); return; }); @@ -232,7 +236,7 @@ async function main(): Promise { artifacts.push(artifact); } else { - var buildArtifacts = await executeWithRetries("getArtifacts", () => buildApi.getArtifacts(buildId, projectId), retryLimit).catch((reason) => { + var buildArtifacts = await executeWithRetries("getArtifacts", () => buildApi.getArtifacts(buildId, projectId), retryLimitRequest).catch((reason) => { reject(reason); }); @@ -251,6 +255,13 @@ async function main(): Promise { artifacts.forEach(async function (artifact, index, artifacts) { let downloaderOptions = configureDownloaderOptions(); + const config: IBaseHandlerConfig = { + artifactInfo: artifact, + downloadPath: downloadPath, + downloaderOptions: downloaderOptions, + checkDownloadedFiles: checkDownloadedFiles + }; + if (artifact.resource.type.toLowerCase() === "container") { var handler = new webHandlers.PersonalAccessTokenCredentialHandler(accessToken); var isPullRequestFork = tl.getVariable("SYSTEM.PULLREQUEST.ISFORK"); @@ -265,70 +276,55 @@ async function main(): Promise { } if (!isZipDownloadDisabledBool && isWin && isPullRequestForkBool) { - const archiveUrl: string = endpointUrl + "/" + projectId + "/_apis/build/builds/" + buildId + "/artifacts?artifactName=" + artifact.name + "&$format=zip"; - console.log(tl.loc("DownloadArtifacts", artifact.name, archiveUrl)); + const operationName: string = `Download zip - ${artifact.name}`; + + const handlerConfig: IContainerHandlerZipConfig = { ...config, projectId, buildId, handler, endpointUrl }; + const downloadHandler: DownloadHandlerContainerZip = new DownloadHandlerContainerZip(handlerConfig); - var zipLocation = path.join(downloadPath, artifact.name + ".zip"); - var operationName = "downloadZip-" + artifact.name; - var downloadZipPromise = executeWithRetries(operationName, () => downloadZip(archiveUrl, downloadPath, zipLocation, handler, downloaderOptions), retryLimit).catch((reason) => { + const downloadPromise: Promise = executeWithRetries( + operationName, + () => downloadHandler.downloadResources(), + retryLimitDownload + ).catch((reason) => { reject(reason); return; }); - - downloadPromises.push(downloadZipPromise); - await downloadZipPromise; - - } - else { - let downloader = new engine.ArtifactEngine(); - - console.log(tl.loc("DownloadingContainerResource", artifact.resource.data)); - var containerParts = artifact.resource.data.split('/'); - - if (containerParts.length < 3) { - throw new Error(tl.loc("FileContainerInvalidArtifactData")); - } - var containerId = parseInt(containerParts[1]); - var containerPath = containerParts.slice(2, containerParts.length).join('/'); - - if (containerPath == "/") { - //container REST api oddity. Passing '/' as itemPath downloads the first file instead of returning the meta data about the all the files in the root level. - //This happens only if the first item is a file. - containerPath = "" - } - - var itemsUrl = endpointUrl + "/_apis/resources/Containers/" + containerId + "?itemPath=" + encodeURIComponent(containerPath) + "&isShallow=true&api-version=4.1-preview.4"; - console.log(tl.loc("DownloadArtifacts", artifact.name, itemsUrl)); - - var variables = {}; - var webProvider = new providers.WebProvider(itemsUrl, templatePath, variables, handler); - var fileSystemProvider = new providers.FilesystemProvider(downloadPath); - - downloadPromises.push(downloader.processItems(webProvider, fileSystemProvider, downloaderOptions).catch((reason) => { + downloadPromises.push(downloadPromise); + await downloadPromise; + } else { + const operationName: string = `Download container - ${artifact.name}`; + + const handlerConfig: IContainerHandlerConfig = { ...config, endpointUrl, templatePath, handler }; + const downloadHandler: DownloadHandlerContainer = new DownloadHandlerContainer(handlerConfig); + const downloadPromise: Promise = executeWithRetries( + operationName, + () => downloadHandler.downloadResources(), + retryLimitDownload + ).catch((reason) => { reject(reason); - })); - } - } - else if (artifact.resource.type.toLowerCase() === "filepath") { - let downloader = new engine.ArtifactEngine(); - let downloadUrl = artifact.resource.data; - let artifactName = artifact.name.replace('/', '\\'); - let artifactLocation = path.join(downloadUrl, artifactName); - if (!fs.existsSync(artifactLocation)) { - console.log(tl.loc("ArtifactNameDirectoryNotFound", artifactLocation, downloadUrl)); - artifactLocation = downloadUrl; - } - - console.log(tl.loc("DownloadArtifacts", artifact.name, artifactLocation)); - var fileShareProvider = new providers.FilesystemProvider(artifactLocation, artifactName); - var fileSystemProvider = new providers.FilesystemProvider(downloadPath); + return; + }); - downloadPromises.push(downloader.processItems(fileShareProvider, fileSystemProvider, downloaderOptions).catch((reason) => { + downloadPromises.push(downloadPromise); + await downloadPromise; + } + } else if (artifact.resource.type.toLowerCase() === "filepath") { + const operationName: string = `Download by FilePath - ${artifact.name}`; + + const downloadHandler: DownloadHandlerFilePath = new DownloadHandlerFilePath(config); + const downloadPromise: Promise = executeWithRetries( + operationName, + () => downloadHandler.downloadResources(), + retryLimitDownload + ).catch((reason) => { reject(reason); - })); - } - else { + return; + }); + + downloadPromises.push(downloadPromise); + await downloadPromise; + } else { console.log(tl.loc("UnsupportedArtifactType", artifact.resource.type)); } }); @@ -370,51 +366,11 @@ function executeWithRetriesImplementation(operationName: string, operation: () = function getRetryIntervalInSeconds(retryCount: number): number { let MaxRetryLimitInSeconds = 360; - let baseRetryIntervalInSeconds = 5; + let baseRetryIntervalInSeconds = 5; var exponentialBackOff = baseRetryIntervalInSeconds * Math.pow(3, (retryCount + 1)); return exponentialBackOff < MaxRetryLimitInSeconds ? exponentialBackOff : MaxRetryLimitInSeconds ; } -async function downloadZip(artifactArchiveUrl: string, downloadPath: string, zipLocation: string, handler: webHandlers.PersonalAccessTokenCredentialHandler, downloaderOptions: engine.ArtifactEngineOptions) { - var executePromise = new Promise((resolve, reject) => { - tl.debug("Starting downloadZip action"); - - if (tl.exist(zipLocation)) { - tl.rmRF(zipLocation); - } - - getZipFromUrl(artifactArchiveUrl, zipLocation, handler, downloaderOptions).then(() => { - tl.debug("Successfully downloaded from " + artifactArchiveUrl); - unzip(zipLocation, downloadPath).then(() => { - - tl.debug("Successfully extracted " + zipLocation); - if (tl.exist(zipLocation)) { - tl.rmRF(zipLocation); - } - - resolve(); - - }).catch((error) => { - reject(error); - }); - - }).catch((error) => { - reject(error); - }); - }); - - return executePromise; -} - -function getZipFromUrl(artifactArchiveUrl: string, localPathRoot: string, handler: webHandlers.PersonalAccessTokenCredentialHandler, downloaderOptions: engine.ArtifactEngineOptions): Promise { - var downloader = new engine.ArtifactEngine(); - var zipProvider = new providers.ZipProvider(artifactArchiveUrl, handler); - var filesystemProvider = new providers.FilesystemProvider(localPathRoot); - - tl.debug("Starting download from " + artifactArchiveUrl); - return downloader.processItems(zipProvider, filesystemProvider, downloaderOptions) -} - function configureDownloaderOptions(): engine.ArtifactEngineOptions { var downloaderOptions = new engine.ArtifactEngineOptions(); downloaderOptions.itemPattern = tl.getInput('itemPattern', false) || "**"; @@ -425,28 +381,6 @@ function configureDownloaderOptions(): engine.ArtifactEngineOptions { return downloaderOptions; } -export function unzip(zipLocation: string, unzipLocation: string): Promise { - return new Promise(function (resolve, reject) { - if (!tl.exist(zipLocation)) { - return resolve(); - } - - tl.debug('Extracting ' + zipLocation + ' to ' + unzipLocation); - - var unzipper = new DecompressZip(zipLocation); - unzipper.on('error', err => { - return reject(tl.loc("ExtractionFailed", err)) - }); - unzipper.on('extract', log => { - tl.debug('Extracted ' + zipLocation + ' to ' + unzipLocation + ' successfully'); - return resolve(); - }); - unzipper.extract({ - path: unzipLocation - }); - }); -} - main() .then((result) => tl.setResult(tl.TaskResult.Succeeded, "")) .catch((err) => { diff --git a/Tasks/DownloadBuildArtifactsV0/task.json b/Tasks/DownloadBuildArtifactsV0/task.json index eee47c9ff586..1f0473e79557 100644 --- a/Tasks/DownloadBuildArtifactsV0/task.json +++ b/Tasks/DownloadBuildArtifactsV0/task.json @@ -9,7 +9,7 @@ "author": "Microsoft Corporation", "version": { "Major": 0, - "Minor": 178, + "Minor": 183, "Patch": 0 }, "groups": [ @@ -178,6 +178,24 @@ "groupName": "advanced", "required": false, "helpMarkDown": "Number of files to download simultaneously" + }, + { + "name": "checkDownloadedFiles", + "type": "boolean", + "label": "Check downloaded files", + "defaultValue": "false", + "groupName": "advanced", + "required": false, + "helpMarkDown": "If checked, this build task will check that all files are fully downloaded." + }, + { + "name": "retryDownloadCount", + "type": "string", + "label": "Retry count", + "defaultValue": "4", + "groupName": "advanced", + "required": false, + "helpMarkDown": "Optional number of times to retry downloading a build artifact if the download fails." } ], "dataSourceBindings": [ @@ -256,7 +274,11 @@ "DownloadingContainerResource": "Downloading items from container resource %s", "DefinitionNameMatchFound": "Definition Name %s resolved to id %d", "InvalidBuildDefinitionName": "Definition name %s didn't correspond to a valid definition", - "UnresolvedDefinitionId": "Could not resolve build definition id" + "UnresolvedDefinitionId": "Could not resolve build definition id", + "BeginArtifactItemsIntegrityCheck": "Starting artifact items integrity check", + "CorruptedArtifactItemsList": "The following items are not passed integrity check:", + "IntegrityCheckNotPassed": "Artifact items integrity check failed", + "IntegrityCheckPassed": "Artifact items integrity check successfully finished" }, "outputVariables": [ { @@ -264,4 +286,4 @@ "description": "Stores the build number of the build artifact source.
Please note that in fact it returns BuildId due to backward compatibility

[More Information](https://docs.microsoft.com/azure/devops/pipelines/build/variables?view=azure-devops&tabs=yaml#build-variables-devops-services)

" } ] -} \ No newline at end of file +} diff --git a/Tasks/DownloadBuildArtifactsV0/task.loc.json b/Tasks/DownloadBuildArtifactsV0/task.loc.json index ca0e05cdf179..04261432414a 100644 --- a/Tasks/DownloadBuildArtifactsV0/task.loc.json +++ b/Tasks/DownloadBuildArtifactsV0/task.loc.json @@ -9,7 +9,7 @@ "author": "Microsoft Corporation", "version": { "Major": 0, - "Minor": 178, + "Minor": 183, "Patch": 0 }, "groups": [ @@ -178,6 +178,24 @@ "groupName": "advanced", "required": false, "helpMarkDown": "ms-resource:loc.input.help.parallelizationLimit" + }, + { + "name": "checkDownloadedFiles", + "type": "boolean", + "label": "ms-resource:loc.input.label.checkDownloadedFiles", + "defaultValue": "false", + "groupName": "advanced", + "required": false, + "helpMarkDown": "ms-resource:loc.input.help.checkDownloadedFiles" + }, + { + "name": "retryDownloadCount", + "type": "string", + "label": "ms-resource:loc.input.label.retryDownloadCount", + "defaultValue": "4", + "groupName": "advanced", + "required": false, + "helpMarkDown": "ms-resource:loc.input.help.retryDownloadCount" } ], "dataSourceBindings": [ @@ -256,7 +274,11 @@ "DownloadingContainerResource": "ms-resource:loc.messages.DownloadingContainerResource", "DefinitionNameMatchFound": "ms-resource:loc.messages.DefinitionNameMatchFound", "InvalidBuildDefinitionName": "ms-resource:loc.messages.InvalidBuildDefinitionName", - "UnresolvedDefinitionId": "ms-resource:loc.messages.UnresolvedDefinitionId" + "UnresolvedDefinitionId": "ms-resource:loc.messages.UnresolvedDefinitionId", + "BeginArtifactItemsIntegrityCheck": "ms-resource:loc.messages.BeginArtifactItemsIntegrityCheck", + "CorruptedArtifactItemsList": "ms-resource:loc.messages.CorruptedArtifactItemsList", + "IntegrityCheckNotPassed": "ms-resource:loc.messages.IntegrityCheckNotPassed", + "IntegrityCheckPassed": "ms-resource:loc.messages.IntegrityCheckPassed" }, "outputVariables": [ {