[DownloadBuildArtifactsV0] Add download post-check (#14065)

Changes:
* Added the possibility to check the integrity of the downloaded artifact' items

Additions: 
* Introduced new task optional parameter - "Check downloaded files" (Can be found in advanced settings)
* Introduced new task optional parameter - "Retry count" (Can be found in advanced settings)
* Added "handlerCheckDownloadedFiles" function for validation results of artifact download
* Added "isItemCorrupted" function for checking information from download ticket of artifact item
* Added check of the file size in local storage

Refactoring:
* The logic for download build artifacts was extracted to the download handlers classes
* Introduced new interfaces to contain all needed information to generate Source and Destination providers
* All download operations are now executed via the "executeWithRetry" handler

Others:
* Bumped up task minor version to "183"
* Added loc strings for messages

Tasks/DownloadBuildArtifactsV0/DownloadHandlers/DownloadHandler.ts

@@ -0,0 +1,73 @@
+import { IBaseHandlerConfig } from './HandlerConfigs';
+import { handlerCheckDownloadedFiles } from '../download_helper';
+import { ArtifactEngine } from 'artifact-engine/Engine';
+import { IArtifactProvider, ArtifactDownloadTicket } from 'artifact-engine/Models';
+import * as tl from 'azure-pipelines-task-lib/task';
+
+/**
+ * Base class for artifact download handlers
+ */
+export abstract class DownloadHandler {
+    /**
+     * @member {IBaseHandlerConfig} - contains info for generate source and destination providers
+     * @access protected
+     */
+    protected config: IBaseHandlerConfig;
+
+    constructor(handlerConfig: IBaseHandlerConfig) {
+        this.config = handlerConfig;
+    }
+
+    /**
+     * Pure abstract method for getting Source Provider.
+     * Source Provider is an object that contains info about from where we will download the artifact.
+     * @access protected
+     * @returns {IArtifactProvider} Objects that implement the IArtifactProvider interface
+    */
+    protected abstract getSourceProvider(): IArtifactProvider;
+
+    /**
+     * Pure abstract method for getting Destination Provider.
+     * Destination Provider is an object that contains info about where we will download artifacts.
+     * @access protected
+     * @returns {IArtifactProvider} Objects that implement the IArtifactProvider interface
+    */
+    protected abstract getDestinationProvider(): IArtifactProvider;
+
+    /**
+     * Method to download Build Artifact.
+     * Since the logic for downloading builds artifacts is the same for all
+     * types of source and destination providers, we will implement this logic in the base class.
+     * @access public
+     * @returns {Promise<Array<ArtifactDownloadTicket>>} an array of Download Tickets
+    */
+    public async downloadResources(): Promise<Array<ArtifactDownloadTicket>> {
+        const downloader: ArtifactEngine = new ArtifactEngine();
+        const sourceProvider: IArtifactProvider = this.getSourceProvider();
+        const destinationProvider: IArtifactProvider = this.getDestinationProvider();
+
+        const downloadPromise: Promise<Array<ArtifactDownloadTicket>> = new Promise<Array<ArtifactDownloadTicket>>(async (downloadComplete, downloadFailed) => {
+            try {
+                // First attempt to download artifact
+                const downloadTickets: Array<ArtifactDownloadTicket> = await downloader.processItems(sourceProvider, destinationProvider, this.config.downloaderOptions);
+
+                // We will proceed with the files check only if the "Check download files" option enabled
+                if (this.config.checkDownloadedFiles && Array.isArray(downloadTickets)) {
+                    try {
+                        // Launch the files check, if all files are fully downloaded no exceptions will be thrown.
+                        handlerCheckDownloadedFiles(downloadTickets);
+                        downloadComplete(downloadTickets);
+                    } catch (error) {
+                        downloadFailed(error);
+                    }
+                } else {
+                    downloadComplete(downloadTickets);
+                }
+            } catch (error) {
+                downloadFailed(error);
+            }
+        });
+
+        return downloadPromise;
+    }
+}

Tasks/DownloadBuildArtifactsV0/DownloadHandlers/DownloadHandlerContainer.ts

@@ -0,0 +1,62 @@
+import { DownloadHandler } from './DownloadHandler';
+import { IContainerHandlerConfig } from './HandlerConfigs';
+import { WebProvider, FilesystemProvider } from 'artifact-engine/Providers';
+import * as tl from 'azure-pipelines-task-lib/task';
+
+/**
+ * Handler for download artifact from related container resource.
+ * Build Artifact will be downloaded via `_apis/resources/Containers/` resource.
+ * @extends DownloadHandler
+ * @example
+ * const config: IContainerHandlerConfig = {...};
+ * const downloadHandler: IContainerHandlerConfig = new IContainerHandlerConfig(config);
+ * downloadHandler.downloadResources();
+ */
+export class DownloadHandlerContainer extends DownloadHandler {
+    protected config: IContainerHandlerConfig;
+
+    constructor(handlerConfig: IContainerHandlerConfig) {
+        super(handlerConfig);
+    }
+
+    /**
+     * To download artifact from container resource we will use `WebProvider` as source provider
+     * @access protected
+     * @returns {WebProvider} Configured Web Provider
+    */
+    protected getSourceProvider(): WebProvider {
+        console.log(tl.loc('DownloadingContainerResource', this.config.artifactInfo.resource.data));
+        const containerParts: Array<string> = this.config.artifactInfo.resource.data.split('/');
+
+        if (containerParts.length < 3) {
+            throw new Error(tl.loc('FileContainerInvalidArtifactData'));
+        }
+
+        const containerId: number = parseInt(containerParts[1]);
+        let containerPath: string = containerParts.slice(2, containerParts.length).join('/');
+
+        if (containerPath === '/') {
+            //container REST api oddity. Passing '/' as itemPath downloads the first file instead of returning the meta data about the all the files in the root level.
+            //This happens only if the first item is a file.
+            containerPath = '';
+        }
+
+        const variables = {};
+        const itemsUrl: string = `${this.config.endpointUrl}/_apis/resources/Containers/${containerId}?itemPath=${encodeURIComponent(containerPath)}&isShallow=true&api-version=4.1-preview.4`;
+
+        console.log(tl.loc('DownloadArtifacts', this.config.artifactInfo.name, itemsUrl));
+
+        const provider: WebProvider = new WebProvider(itemsUrl, this.config.templatePath, variables, this.config.handler);
+        return provider;
+    }
+
+    /**
+     * Since we download artifact to local storage we will use a `FilesystemProvider` as destination provider
+     * @access protected
+     * @returns {FilesystemProvider} Configured Filesystem Provider
+    */
+    protected getDestinationProvider(): FilesystemProvider {
+        const provider: FilesystemProvider = new FilesystemProvider(this.config.downloadPath);
+        return provider;
+    }
+}

Tasks/DownloadBuildArtifactsV0/DownloadHandlers/DownloadHandlerContainerZip.ts

@@ -0,0 +1,121 @@
+import { DownloadHandler } from './DownloadHandler';
+import { IContainerHandlerZipConfig } from './HandlerConfigs';
+import { FilesystemProvider, ZipProvider } from 'artifact-engine/Providers';
+import * as tl from 'azure-pipelines-task-lib/task';
+import * as path from 'path';
+import * as DecompressZip from 'decompress-zip';
+
+/**
+ * Handler for download artifact via build API
+ * Build Artifact will be downloaded as zip archive via `/_apis/build/builds/` resource.
+ * This handler was designed to work only on windows system.
+ * @extends DownloadHandler
+ * @example
+ * const config: IContainerHandlerZipConfig = {...};
+ * const downloadHandler: DownloadHandlerContainerZip = new DownloadHandlerContainerZip(config);
+ * downloadHandler.downloadResources();
+ */
+export class DownloadHandlerContainerZip extends DownloadHandler {
+    protected config: IContainerHandlerZipConfig;
+    private readonly archiveUrl: string;
+    private readonly zipLocation: string;
+
+    constructor(handlerConfig: IContainerHandlerZipConfig) {
+        super(handlerConfig);
+        this.archiveUrl = `${this.config.endpointUrl}/${this.config.projectId}/_apis/build/builds/${this.config.buildId}/artifacts?artifactName=${this.config.artifactInfo.name}&$format=zip`;
+        this.zipLocation = path.join(this.config.downloadPath, `${this.config.artifactInfo.name}.zip`);
+    }
+
+    /**
+     * Unpack an archive with an artifact
+     * @param unzipLocation path to the target artifact
+     * @access private
+     * @returns {Promise<void>} promise that will be resolved once the archive will be unpacked
+    */
+    private unzipContainer(unzipLocation: string): Promise<void> {
+        const unZipPromise: Promise<void> = new Promise<void>((resolve, reject) => {
+            if (!tl.exist(this.zipLocation)) {
+                return resolve();
+            }
+
+            tl.debug(`Extracting ${this.zipLocation} to ${unzipLocation}`);
+
+            const unzipper = new DecompressZip(this.zipLocation);
+
+            unzipper.on('error', err => {
+                return reject(tl.loc('ExtractionFailed', err));
+            });
+
+            unzipper.on('extract', log => {
+                tl.debug(`Extracted ${this.zipLocation} to ${unzipLocation} successfully`);
+                return resolve();
+            });
+
+            unzipper.extract({
+                path: unzipLocation
+            });
+
+        });
+
+        return unZipPromise;
+    }
+
+    /**
+     * Get zip provider.
+     * Since we will download archived artifact we will use `ZipProvider` as source provider.
+     * @access protected
+     * @returns {ZipProvider} Configured Zip Provider
+    */
+    protected getSourceProvider(): ZipProvider {
+        console.log(tl.loc('DownloadArtifacts', this.config.artifactInfo.name, this.archiveUrl));
+        const provider: ZipProvider = new ZipProvider(this.archiveUrl, this.config.handler);
+        return provider;
+    }
+
+    /**
+     * Get filesystem provider.
+     * Since we download artifact to local storage we will use a `FilesystemProvider` as destination provider.
+     * @access protected
+     * @returns {FilesystemProvider} Configured Filesystem Provider
+    */
+    protected getDestinationProvider(): FilesystemProvider {
+        const provider: FilesystemProvider = new FilesystemProvider(this.zipLocation);
+        return provider;
+    }
+
+    /**
+     * Download and unpack an archive with an artifact.
+     * @access public
+    */
+    public downloadResources(): Promise<any> {
+        const downloadProcess: Promise<any> = new Promise((resolve, reject) => {
+            tl.debug('Starting downloadZip action');
+
+            if (tl.exist(this.zipLocation)) {
+                tl.rmRF(this.zipLocation);
+            }
+
+            super.downloadResources().then(() => {
+                tl.debug(`Successfully downloaded from ${this.archiveUrl}`);
+
+                this.unzipContainer(this.config.downloadPath).then(() => {
+                    tl.debug(`Successfully extracted ${this.zipLocation}`);
+
+                    if (tl.exist(this.zipLocation)) {
+                        tl.rmRF(this.zipLocation);
+                    }
+
+                    resolve();
+
+                }).catch((error) => {
+                    reject(error);
+                });
+
+            }).catch((error) => {
+                reject(error);
+            });
+        });
+
+        return downloadProcess;
+    }
+}

Tasks/DownloadBuildArtifactsV0/DownloadHandlers/DownloadHandlerFilePath.ts

@@ -0,0 +1,46 @@
+import { DownloadHandler } from './DownloadHandler';
+import { FilesystemProvider } from 'artifact-engine/Providers';
+import * as tl from 'azure-pipelines-task-lib/task';
+import * as fs from 'fs';
+import * as path from 'path';
+
+/**
+ * Handler for download artifact via local file share
+ * @extends DownloadHandler
+ * @example
+ * const config: IBaseHandlerConfig = {...};
+ * const downloadHandler: DownloadHandlerFilePath = new DownloadHandlerFilePath(config);
+ * downloadHandler.downloadResources();
+ */
+export class DownloadHandlerFilePath extends DownloadHandler {
+    /**
+     * Get source provider with source folder.
+     * Since we will work with local files we use `Filesystem Provider` as source provider.
+     * @access protected
+     * @returns {FilesystemProvider} Configured Filesystem Provider
+     */
+    protected getSourceProvider(): FilesystemProvider {
+        const downloadUrl = this.config.artifactInfo.resource.data;
+        const artifactName = this.config.artifactInfo.name.replace('/', '\\');
+        let artifactLocation = path.join(downloadUrl, artifactName);
+
+        if (!fs.existsSync(artifactLocation)) {
+            console.log(tl.loc('ArtifactNameDirectoryNotFound', artifactLocation, downloadUrl));
+            artifactLocation = downloadUrl;
+        }
+
+        const provider: FilesystemProvider = new FilesystemProvider(artifactLocation, artifactName);
+        return provider;
+    }
+
+    /**
+     * Get destination provider with destination folder.
+     * Since we will work with local files we use `Filesystem Provider` as source provider.
+     * @access protected
+     * @returns {FilesystemProvider} Configured Filesystem Provider
+     */
+    protected getDestinationProvider(): FilesystemProvider {
+        const provider: FilesystemProvider = new FilesystemProvider(this.config.downloadPath);
+        return provider;
+    }
+}

Tasks/DownloadBuildArtifactsV0/DownloadHandlers/HandlerConfigs.ts

@@ -0,0 +1,23 @@
+import { ArtifactEngineOptions } from 'artifact-engine/Engine';
+import { BuildArtifact } from 'azure-devops-node-api/interfaces/BuildInterfaces';
+import { PersonalAccessTokenCredentialHandler } from 'artifact-engine/Providers/typed-rest-client/Handlers';
+
+export interface IBaseHandlerConfig {
+    artifactInfo: BuildArtifact;
+    downloadPath: string;
+    downloaderOptions: ArtifactEngineOptions;
+    checkDownloadedFiles: boolean;
+}
+
+export interface IContainerHandlerConfig extends IBaseHandlerConfig {
+    endpointUrl: string;
+    templatePath: string;
+    handler: PersonalAccessTokenCredentialHandler;
+}
+
+export interface IContainerHandlerZipConfig extends IBaseHandlerConfig {
+    endpointUrl: string;
+    projectId: string;
+    buildId: number;
+    handler: PersonalAccessTokenCredentialHandler;
+}

Tasks/DownloadBuildArtifactsV0/Strings/resources.resjson/en-US/resources.resjson

@@ -31,6 +31,10 @@
   "loc.input.help.downloadPath": "Path on the agent machine where the artifacts will be downloaded",
   "loc.input.label.parallelizationLimit": "Parallelization limit",
   "loc.input.help.parallelizationLimit": "Number of files to download simultaneously",
+  "loc.input.label.checkDownloadedFiles": "Check downloaded files",
+  "loc.input.help.checkDownloadedFiles": "If checked, this build task will check that all files are fully downloaded.",
+  "loc.input.label.retryDownloadCount": "Retry count",
+  "loc.input.help.retryDownloadCount": "Optional number of times to retry downloading a build artifact if the download fails.",
   "loc.messages.DownloadArtifacts": "Downloading artifact %s from: %s",
   "loc.messages.DownloadingArtifactsForBuild": "Downloading artifacts for build: %s",
   "loc.messages.LinkedArtifactCount": "Linked artifacts count:  %s",
@@ -51,5 +55,9 @@
   "loc.messages.DownloadingContainerResource": "Downloading items from container resource %s",
   "loc.messages.DefinitionNameMatchFound": "Definition Name %s resolved to id %d",
   "loc.messages.InvalidBuildDefinitionName": "Definition name %s didn't correspond to a valid definition",
-  "loc.messages.UnresolvedDefinitionId": "Could not resolve build definition id"
+  "loc.messages.UnresolvedDefinitionId": "Could not resolve build definition id",
+  "loc.messages.BeginArtifactItemsIntegrityCheck": "Starting artifact items integrity check",
+  "loc.messages.CorruptedArtifactItemsList": "The following items are not passed integrity check:",
+  "loc.messages.IntegrityCheckNotPassed": "Artifact items integrity check failed",
+  "loc.messages.IntegrityCheckPassed": "Artifact items integrity check successfully finished"
 }