From 445bbfd24a282290cee1148f87d8ecee6566dab4 Mon Sep 17 00:00:00 2001 From: Ningxin Zheng <49771382+zheng-ningxin@users.noreply.github.com> Date: Tue, 4 Aug 2020 17:06:23 +0800 Subject: [PATCH 01/28] Speedup enhancement (#2719) --- .../compression/torch/speedup/compressor.py | 8 ++++++ .../compression/torch/speedup/infer_shape.py | 27 ++++++++++++------- src/sdk/pynni/tests/test_model_speedup.py | 6 ++--- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/src/sdk/pynni/nni/compression/torch/speedup/compressor.py b/src/sdk/pynni/nni/compression/torch/speedup/compressor.py index b31acfe664..41753e1c9f 100644 --- a/src/sdk/pynni/nni/compression/torch/speedup/compressor.py +++ b/src/sdk/pynni/nni/compression/torch/speedup/compressor.py @@ -141,6 +141,14 @@ def infer_modules_masks(self): """ for module_name, mask in self.masks.items(): _logger.debug('Start mask inference from %s', module_name) + if module_name not in self.torch_graph.name_to_node: + # this module is not traced in the torch_graph, + # jit.trace only correctly records functions and + # modules which are not data dependent (e.g., do + # not have conditionals on data in tensors) + # so, if a node is not traced, we just skip it. + _logger.warning('%s has mask, but not found in the traced graph, just skip it.', module_name) + continue self.infer_module_mask(module_name, None, mask=mask) def replace_compressed_modules(self): diff --git a/src/sdk/pynni/nni/compression/torch/speedup/infer_shape.py b/src/sdk/pynni/nni/compression/torch/speedup/infer_shape.py index 47aa8087df..2635617031 100644 --- a/src/sdk/pynni/nni/compression/torch/speedup/infer_shape.py +++ b/src/sdk/pynni/nni/compression/torch/speedup/infer_shape.py @@ -222,6 +222,7 @@ def __repr__(self): 'ReLU': lambda module_masks, mask: relu_inshape(module_masks, mask), 'ReLU6': lambda module_masks, mask: relu_inshape(module_masks, mask), 'aten::relu': lambda module_masks, mask: relu_inshape(module_masks, mask), + 'aten::relu_': lambda module_masks, mask: relu_inshape(module_masks, mask), 'Conv2d': lambda module_masks, mask: conv2d_inshape(module_masks, mask), 'MaxPool2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), 'aten::max_pool2d': lambda module_masks, mask: maxpool2d_inshape(module_masks, mask), @@ -241,7 +242,8 @@ def __repr__(self): 'aten::cat': lambda module_mask, mask, cat_info, last_visited: cat_inshape(module_mask, mask, cat_info, last_visited), 'aten::mean': lambda module_masks, mask, shape: mean_inshape(module_masks, mask, shape), 'Dropout': lambda module_masks, mask: dropout_inshape(module_masks, mask), - 'Dropout2d': lambda module_masks, mask: dropout_inshape(module_masks, mask) + 'Dropout2d': lambda module_masks, mask: dropout_inshape(module_masks, mask), + 'aten::dropout': lambda module_masks, mask: dropout_inshape(module_masks, mask) } """ @@ -258,8 +260,14 @@ def dropout_inshape(module_masks, mask): return module_masks.output_mask # if alreay visited assert module_masks.input_mask <= mask - if module_masks.input_mask == mask: - return None + # It should be the same, we pass the masks by the reference(not the value), + # so they acutually are two references of the same object(mask, + # module_masks.input_mask). So we should continue pass the mask + # to the following nodes even module_masks.input_mask == mask. + # if pass the mask by copy.deepcopy(), then we can stop when + # module_masks.input_mask == mask. + # if module_masks.input_mask == mask: + # return None module_masks.set_input_mask(mask) module_masks.set_output_mask(mask) return module_masks.output_mask @@ -413,7 +421,8 @@ def linear_inshape(module_masks, mask): """ assert isinstance(mask, CoarseMask) assert mask.mask_index[0] is None - assert module_masks.input_mask is None + if module_masks.input_mask is not None: + assert module_masks.input_mask <= mask module_masks.set_input_mask(mask) return None @@ -451,7 +460,10 @@ def view_inshape(module_masks, mask, shape): assert mask.mask_index[0] is None assert mask.mask_index[2] is None assert mask.mask_index[3] is None - assert module_masks.input_mask is None + # due to the cat operation, the same node may be + # accessed more than once + if module_masks.input_mask is not None: + assert module_masks.input_mask <= mask module_masks.set_input_mask(mask) output_cmask = CoarseMask(num_dim=2) index = [] @@ -535,12 +547,9 @@ def relu_inshape(module_masks, mask): The mask of its output tensor """ assert isinstance(mask, CoarseMask) - # TODO: double check this assert, is it possible that a module is passed twice if module_masks.input_mask is not None: # check if has a mask conflict - assert module_masks.input_mask == mask - # No need to pass the mask again - return None + assert module_masks.input_mask <= mask # assert module_masks.input_mask is None, "A relu op can only be processed once" module_masks.set_input_mask(mask) module_masks.set_output_mask(mask) diff --git a/src/sdk/pynni/tests/test_model_speedup.py b/src/sdk/pynni/tests/test_model_speedup.py index a06f991c97..845ed793ff 100644 --- a/src/sdk/pynni/tests/test_model_speedup.py +++ b/src/sdk/pynni/tests/test_model_speedup.py @@ -145,18 +145,18 @@ def test_speedup_bigmodel(self): assert model.backbone2.fc1.in_features == int(orig_model.backbone2.fc1.in_features * SPARSITY) def test_speedup_integration(self): - for model_name in ['resnet18', 'squeezenet1_1', 'mobilenet_v2']: + for model_name in ['resnet18', 'squeezenet1_1', 'mobilenet_v2', 'densenet121', 'inception_v3']: Model = getattr(models, model_name) net = Model(pretrained=True, progress=False).to(device) + speedup_model = Model().to(device) net.eval() # this line is necessary + speedup_model.eval() # random generate the prune config for the pruner cfgs = generate_random_sparsity(net) pruner = L1FilterPruner(net, cfgs) pruner.compress() pruner.export_model(MODEL_FILE, MASK_FILE) pruner._unwrap_model() - speedup_model = Model().to(device) - speedup_model.eval() state_dict = torch.load(MODEL_FILE) speedup_model.load_state_dict(state_dict) zero_bn_bias(net) From 2d026a13f97612bc7961d5a55f5088a169de9ee9 Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Wed, 5 Aug 2020 10:54:02 +0800 Subject: [PATCH 02/28] Merge v1.7.1 back to master (#2761) --- .../rest_server/restValidationSchemas.ts | 1 - .../pai/paiJobInfoCollector.ts | 7 +- .../pai/paiK8S/paiK8STrainingService.ts | 11 +-- .../pai/paiTrainingService.ts | 4 +- .../environments/amlEnvironmentService.ts | 8 +- .../environments/openPaiEnvironmentService.ts | 99 +++---------------- 6 files changed, 30 insertions(+), 100 deletions(-) diff --git a/src/nni_manager/rest_server/restValidationSchemas.ts b/src/nni_manager/rest_server/restValidationSchemas.ts index a480501a79..b845dcc30e 100644 --- a/src/nni_manager/rest_server/restValidationSchemas.ts +++ b/src/nni_manager/rest_server/restValidationSchemas.ts @@ -103,7 +103,6 @@ export namespace ValidationSchemas { }), pai_config: joi.object({ // eslint-disable-line @typescript-eslint/camelcase userName: joi.string().min(1).required(), - passWord: joi.string().min(1), token: joi.string().min(1), host: joi.string().min(1).required(), reuse: joi.boolean(), diff --git a/src/nni_manager/training_service/pai/paiJobInfoCollector.ts b/src/nni_manager/training_service/pai/paiJobInfoCollector.ts index 2590547849..5f6ccf4d9c 100644 --- a/src/nni_manager/training_service/pai/paiJobInfoCollector.ts +++ b/src/nni_manager/training_service/pai/paiJobInfoCollector.ts @@ -52,7 +52,7 @@ export class PAIJobInfoCollector { // Rest call to get PAI job info and update status // Refer https://github.com/Microsoft/pai/blob/master/docs/rest-server/API.md for more detail about PAI Rest API const getJobInfoRequest: request.Options = { - uri: `${protocol}://${paiClusterConfig.host}/rest-server/api/v1/user/${paiClusterConfig.userName}/jobs/${paiTrialJob.paiJobName}`, + uri: `${protocol}://${paiClusterConfig.host}/rest-server/api/v2/jobs/${paiClusterConfig.userName}~${paiTrialJob.paiJobName}`, method: 'GET', json: true, headers: { @@ -63,8 +63,9 @@ export class PAIJobInfoCollector { //TODO : pass in request timeout param? request(getJobInfoRequest, (error: Error, response: request.Response, _body: any) => { - if ((error !== undefined && error !== null) || response.statusCode >= 500) { - this.log.error(`PAI Training service: get job info for trial ${paiTrialJob.id} from PAI Cluster failed!`); + // Status code 200 for success + if ((error !== undefined && error !== null) || response.statusCode >= 400) { + // The job refresh time could be ealier than job submission, so it might return 404 error code, need refactor // Queried PAI job info failed, set job status to UNKNOWN if (paiTrialJob.status === 'WAITING' || paiTrialJob.status === 'RUNNING') { paiTrialJob.status = 'UNKNOWN'; diff --git a/src/nni_manager/training_service/pai/paiK8S/paiK8STrainingService.ts b/src/nni_manager/training_service/pai/paiK8S/paiK8STrainingService.ts index 59bd994535..6d6169a6b2 100644 --- a/src/nni_manager/training_service/pai/paiK8S/paiK8STrainingService.ts +++ b/src/nni_manager/training_service/pai/paiK8S/paiK8STrainingService.ts @@ -55,12 +55,7 @@ class PAIK8STrainingService extends PAITrainingService { this.paiJobRestServer = new PAIJobRestServer(component.get(PAIK8STrainingService)); this.paiClusterConfig = JSON.parse(value); this.paiClusterConfig.host = this.formatPAIHost(this.paiClusterConfig.host); - if (this.paiClusterConfig.passWord) { - // Get PAI authentication token - await this.updatePaiToken(); - } else if (this.paiClusterConfig.token) { - this.paiToken = this.paiClusterConfig.token; - } + this.paiToken = this.paiClusterConfig.token; break; case TrialConfigMetadataKey.TRIAL_CONFIG: { @@ -290,18 +285,20 @@ class PAIK8STrainingService extends PAITrainingService { uri: `${this.protocol}://${this.paiClusterConfig.host}/rest-server/api/v2/jobs`, method: 'POST', body: paiJobConfig, + followAllRedirects: true, headers: { 'Content-Type': 'text/yaml', Authorization: `Bearer ${this.paiToken}` } }; request(submitJobRequest, (error: Error, response: request.Response, body: any) => { + // If submit success, will get status code 202. refer: https://github.com/microsoft/pai/blob/master/src/rest-server/docs/swagger.yaml if ((error !== undefined && error !== null) || response.statusCode >= 400) { const errorMessage: string = (error !== undefined && error !== null) ? error.message : `Submit trial ${trialJobId} failed, http code:${response.statusCode}, http body: ${body}`; - this.log.error(errorMessage); trialJobDetail.status = 'FAILED'; + deferred.reject(errorMessage); } else { trialJobDetail.submitTime = Date.now(); } diff --git a/src/nni_manager/training_service/pai/paiTrainingService.ts b/src/nni_manager/training_service/pai/paiTrainingService.ts index e26c16ecee..aff583de54 100644 --- a/src/nni_manager/training_service/pai/paiTrainingService.ts +++ b/src/nni_manager/training_service/pai/paiTrainingService.ts @@ -162,8 +162,7 @@ abstract class PAITrainingService implements TrainingService { } const stopJobRequest: request.Options = { - uri: `${this.protocol}://${this.paiClusterConfig.host}/rest-server/api/v1/user/${this.paiClusterConfig.userName}\ -/jobs/${trialJobDetail.paiJobName}/executionType`, + uri: `${this.protocol}://${this.paiClusterConfig.host}/rest-server/api/v2/jobs/${this.paiClusterConfig.userName}~${trialJobDetail.paiJobName}/executionType`, method: 'PUT', json: true, body: { value: 'STOP' }, @@ -178,6 +177,7 @@ abstract class PAITrainingService implements TrainingService { const deferred: Deferred = new Deferred(); request(stopJobRequest, (error: Error, response: request.Response, _body: any) => { + // Status code 202 for success. if ((error !== undefined && error !== null) || response.statusCode >= 400) { this.log.error(`PAI Training service: stop trial ${trialJobId} to PAI Cluster failed!`); deferred.reject((error !== undefined && error !== null) ? error.message : diff --git a/src/nni_manager/training_service/reusable/environments/amlEnvironmentService.ts b/src/nni_manager/training_service/reusable/environments/amlEnvironmentService.ts index fea393e75d..aefd94fbb5 100644 --- a/src/nni_manager/training_service/reusable/environments/amlEnvironmentService.ts +++ b/src/nni_manager/training_service/reusable/environments/amlEnvironmentService.ts @@ -16,7 +16,7 @@ import { AMLClient } from '../aml/amlClient'; import { AMLClusterConfig, AMLEnvironmentInformation, AMLTrialConfig } from '../aml/amlConfig'; import { AMLCommandChannel } from '../channels/amlCommandChannel'; import { CommandChannel } from "../commandChannel"; -import { EnvironmentInformation, EnvironmentService, EnvironmentStatus } from '../environment'; +import { EnvironmentInformation, EnvironmentService } from '../environment'; /** @@ -74,7 +74,7 @@ export class AMLEnvironmentService extends EnvironmentService { environments.forEach(async (environment) => { const amlClient = (environment as AMLEnvironmentInformation).amlClient; if (!amlClient) { - throw new Error('AML client not initialized!'); + return Promise.reject('AML client not initialized!'); } const newStatus = await amlClient.updateStatus(environment.status); switch (newStatus.toUpperCase()) { @@ -90,8 +90,8 @@ export class AMLEnvironmentService extends EnvironmentService { environment.setStatus('SUCCEEDED'); break; case 'FAILED': - environment.setStatus(newStatus.toUpperCase() as EnvironmentStatus); - break; + environment.setStatus('FAILED'); + return Promise.reject(`AML: job ${environment.envId} is failed!`); case 'STOPPED': case 'STOPPING': environment.setStatus('USER_CANCELED'); diff --git a/src/nni_manager/training_service/reusable/environments/openPaiEnvironmentService.ts b/src/nni_manager/training_service/reusable/environments/openPaiEnvironmentService.ts index 3d92df5c99..596c81dbe9 100644 --- a/src/nni_manager/training_service/reusable/environments/openPaiEnvironmentService.ts +++ b/src/nni_manager/training_service/reusable/environments/openPaiEnvironmentService.ts @@ -28,15 +28,12 @@ export class OpenPaiEnvironmentService extends EnvironmentService { private paiTrialConfig: NNIPAIK8STrialConfig | undefined; private paiJobConfig: any; private paiToken?: string; - private paiTokenUpdateTime?: number; - private readonly paiTokenUpdateInterval: number; private protocol: string = 'http'; private experimentId: string; constructor() { super(); - this.paiTokenUpdateInterval = 7200000; //2hours this.experimentId = getExperimentId(); } @@ -53,12 +50,7 @@ export class OpenPaiEnvironmentService extends EnvironmentService { case TrialConfigMetadataKey.PAI_CLUSTER_CONFIG: this.paiClusterConfig = JSON.parse(value); this.paiClusterConfig.host = this.formatPAIHost(this.paiClusterConfig.host); - if (this.paiClusterConfig.passWord) { - // Get PAI authentication token - await this.updatePaiToken(); - } else if (this.paiClusterConfig.token) { - this.paiToken = this.paiClusterConfig.token; - } + this.paiToken = this.paiClusterConfig.token; break; case TrialConfigMetadataKey.TRIAL_CONFIG: { @@ -95,7 +87,6 @@ export class OpenPaiEnvironmentService extends EnvironmentService { public async refreshEnvironmentsStatus(environments: EnvironmentInformation[]): Promise { const deferred: Deferred = new Deferred(); - await this.refreshPlatform(); if (this.paiClusterConfig === undefined) { throw new Error('PAI Cluster config is not initialized'); @@ -115,9 +106,12 @@ export class OpenPaiEnvironmentService extends EnvironmentService { }; request(getJobInfoRequest, async (error: any, response: request.Response, body: any) => { + // Status code 200 for success if ((error !== undefined && error !== null) || response.statusCode >= 400) { - this.log.error(`OpenPAI: get environment list from PAI Cluster failed!\nerror: ${error}`); - deferred.reject(error); + const errorMessage: string = (error !== undefined && error !== null) ? error.message : + `OpenPAI: get environment list from PAI Cluster failed!, http code:${response.statusCode}, http body: ${JSON.stringify(body)}`; + this.log.error(`${errorMessage}`); + deferred.reject(errorMessage); } else { const jobInfos = new Map(); body.forEach((jobInfo: any) => { @@ -133,8 +127,11 @@ export class OpenPaiEnvironmentService extends EnvironmentService { case 'RUNNING': case 'WAITING': case 'SUCCEEDED': + environment.setStatus(jobResponse.state); + break; case 'FAILED': environment.setStatus(jobResponse.state); + deferred.reject(`OpenPAI: job ${environment.envId} is failed!`); break; case 'STOPPED': case 'STOPPING': @@ -166,8 +163,6 @@ export class OpenPaiEnvironmentService extends EnvironmentService { public async startEnvironment(environment: EnvironmentInformation): Promise { const deferred: Deferred = new Deferred(); - await this.refreshPlatform(); - if (this.paiClusterConfig === undefined) { throw new Error('PAI Cluster config is not initialized'); } @@ -195,18 +190,21 @@ export class OpenPaiEnvironmentService extends EnvironmentService { uri: `${this.protocol}://${this.paiClusterConfig.host}/rest-server/api/v2/jobs`, method: 'POST', body: paiJobConfig, + followAllRedirects: true, headers: { 'Content-Type': 'text/yaml', Authorization: `Bearer ${this.paiToken}` } }; request(submitJobRequest, (error, response, body) => { + // Status code 202 for success, refer https://github.com/microsoft/pai/blob/master/src/rest-server/docs/swagger.yaml if ((error !== undefined && error !== null) || response.statusCode >= 400) { const errorMessage: string = (error !== undefined && error !== null) ? error.message : `start environment ${environment.envId} failed, http code:${response.statusCode}, http body: ${body}`; this.log.error(errorMessage); environment.status = 'FAILED'; + deferred.reject(errorMessage); } deferred.resolve(); }); @@ -241,8 +239,11 @@ export class OpenPaiEnvironmentService extends EnvironmentService { try { request(stopJobRequest, (error, response, _body) => { try { + // Status code 202 for success. if ((error !== undefined && error !== null) || (response && response.statusCode >= 400)) { - this.log.error(`OpenPAI: stop job ${environment.envId} failed with ${response.statusCode}\n${error}`); + const errorMessage: string = (error !== undefined && error !== null) ? error.message : + `OpenPAI: stop job ${environment.envId} failed, http code:${response.statusCode}, http body: ${_body}`; + this.log.error(`${errorMessage}`); deferred.reject((error !== undefined && error !== null) ? error : `Stop trial failed, http code: ${response.statusCode}`); } else { @@ -262,19 +263,6 @@ export class OpenPaiEnvironmentService extends EnvironmentService { return deferred.promise; } - private async refreshPlatform(): Promise { - if (this.paiClusterConfig && this.paiClusterConfig.passWord) { - try { - await this.updatePaiToken(); - } catch (error) { - this.log.error(`${error}`); - if (this.paiToken === undefined) { - throw new Error(error); - } - } - } - } - private generateJobConfigInYamlFormat(environment: EnvironmentInformation): any { if (this.paiTrialConfig === undefined) { throw new Error('trial config is not initialized'); @@ -386,59 +374,4 @@ export class OpenPaiEnvironmentService extends EnvironmentService { return host; } } - /** - * Update pai token by the interval time or initialize the pai token - */ - protected async updatePaiToken(): Promise { - const deferred: Deferred = new Deferred(); - - const currentTime: number = new Date().getTime(); - //If pai token initialized and not reach the interval time, do not update - if (this.paiTokenUpdateTime !== undefined && (currentTime - this.paiTokenUpdateTime) < this.paiTokenUpdateInterval) { - return Promise.resolve(); - } - - if (this.paiClusterConfig === undefined) { - const paiClusterConfigError: string = `pai cluster config not initialized!`; - this.log.error(`${paiClusterConfigError}`); - throw Error(`${paiClusterConfigError}`); - } - - const authenticationReq: request.Options = { - uri: `${this.protocol}://${this.paiClusterConfig.host}/rest-server/api/v1/token`, - method: 'POST', - json: true, - body: { - username: this.paiClusterConfig.userName, - password: this.paiClusterConfig.passWord - } - }; - - request(authenticationReq, (error: any, response: request.Response, body: any) => { - if (error !== undefined && error !== null) { - this.log.error(`Get PAI token failed: ${error.message}, authenticationReq: ${authenticationReq}`); - deferred.reject(new Error(`Get PAI token failed: ${error.message}`)); - } else { - if (response.statusCode !== 200) { - this.log.error(`Get PAI token failed: get PAI Rest return code ${response.statusCode}, authenticationReq: ${authenticationReq}`); - deferred.reject(new Error(`Get PAI token failed code: ${response.statusCode}, body: ${response.body}, authenticationReq: ${authenticationReq}, please check paiConfig username or password`)); - } else { - this.paiToken = body.token; - this.paiTokenUpdateTime = new Date().getTime(); - deferred.resolve(); - } - } - }); - - let timeoutId: NodeJS.Timer; - const timeoutDelay: Promise = new Promise((_resolve: Function, reject: Function): void => { - // Set timeout and reject the promise once reach timeout (5 seconds) - timeoutId = setTimeout( - () => reject(new Error('Get PAI token timeout. Please check your PAI cluster.')), - 5000); - }); - - return Promise.race([timeoutDelay, deferred.promise]) - .finally(() => { clearTimeout(timeoutId); }); - } } From 2488aa653ee318b1cb4332e1f0be26465e7199fb Mon Sep 17 00:00:00 2001 From: Lijiaoa <61399850+Lijiaoa@users.noreply.github.com> Date: Fri, 7 Aug 2020 10:40:15 +0800 Subject: [PATCH 03/28] refactor experiment summary file(to fix componentWillReceiveProps waring) and click other areas to close panel (#2734) --- ...perimentDrawer.tsx => ExperimentPanel.tsx} | 96 +++++++++---------- .../Modals/{LogDrawer.tsx => LogPanel.tsx} | 2 + src/webui/src/components/NavCon.tsx | 13 +-- .../src/components/overview/Progress.tsx | 2 +- src/webui/src/static/model/trialmanager.ts | 16 +++- 5 files changed, 71 insertions(+), 58 deletions(-) rename src/webui/src/components/Modals/{ExperimentDrawer.tsx => ExperimentPanel.tsx} (56%) rename src/webui/src/components/Modals/{LogDrawer.tsx => LogPanel.tsx} (98%) diff --git a/src/webui/src/components/Modals/ExperimentDrawer.tsx b/src/webui/src/components/Modals/ExperimentPanel.tsx similarity index 56% rename from src/webui/src/components/Modals/ExperimentDrawer.tsx rename to src/webui/src/components/Modals/ExperimentPanel.tsx index 142af89f59..cbc674ec9a 100644 --- a/src/webui/src/components/Modals/ExperimentDrawer.tsx +++ b/src/webui/src/components/Modals/ExperimentPanel.tsx @@ -1,17 +1,16 @@ import * as React from 'react'; -import axios from 'axios'; import { downFile } from '../../static/function'; import { Stack, PrimaryButton, DefaultButton, Panel, StackItem, Pivot, PivotItem } from 'office-ui-fabric-react'; -import { MANAGER_IP, DRAWEROPTION } from '../../static/const'; +import { DRAWEROPTION } from '../../static/const'; +import { EXPERIMENT, TRIALS } from '../../static/datamodel'; import MonacoEditor from 'react-monaco-editor'; import '../../static/style/logDrawer.scss'; -import { TrialManager } from '../../static/model/trialmanager'; interface ExpDrawerProps { - isVisble: boolean; closeExpDrawer: () => void; + experimentProfile: object; } interface ExpDrawerState { @@ -21,7 +20,9 @@ interface ExpDrawerState { class ExperimentDrawer extends React.Component { - public _isCompareMount!: boolean; + public _isExperimentMount!: boolean; + private refreshId!: number | undefined; + constructor(props: ExpDrawerProps) { super(props); @@ -32,42 +33,40 @@ class ExperimentDrawer extends React.Component { } getExperimentContent = (): void => { - axios - .all([ - axios.get(`${MANAGER_IP}/experiment`), - axios.get(`${MANAGER_IP}/trial-jobs`), - axios.get(`${MANAGER_IP}/metric-data`) - ]) - .then(axios.spread((resExperiment, resTrialJobs, resMetricData) => { - if (resExperiment.status === 200 && resTrialJobs.status === 200 && resMetricData.status === 200) { - if (resExperiment.data.params.searchSpace) { - resExperiment.data.params.searchSpace = JSON.parse(resExperiment.data.params.searchSpace); - } - const trialMessagesArr = TrialManager.expandJobsToTrials(resTrialJobs.data); - const interResultList = resMetricData.data; - Object.keys(trialMessagesArr).map(item => { - // not deal with trial's hyperParameters - const trialId = trialMessagesArr[item].id; - // add intermediate result message - trialMessagesArr[item].intermediate = []; - Object.keys(interResultList).map(key => { - const interId = `${interResultList[key].trialJobId}-${interResultList[key].parameterId}`; - if (trialId === interId) { - trialMessagesArr[item].intermediate.push(interResultList[key]); - } - }); - }); - const result = { - experimentParameters: resExperiment.data, - trialMessage: trialMessagesArr - }; - if (this._isCompareMount === true) { - this.setState({ experiment: JSON.stringify(result, null, 4) }); - } + const experimentData = JSON.parse(JSON.stringify(this.props.experimentProfile)); + if (experimentData.params.searchSpace) { + experimentData.params.searchSpace = JSON.parse(experimentData.params.searchSpace); + } + const trialMessagesArr = TRIALS.getTrialJobList(); + const interResultList = TRIALS.getMetricsList(); + Object.keys(trialMessagesArr).map(item => { + // not deal with trial's hyperParameters + const trialId = trialMessagesArr[item].jobId; + // add intermediate result message + trialMessagesArr[item].intermediate = []; + Object.keys(interResultList).map(key => { + const interId = interResultList[key].trialJobId; + if (trialId === interId) { + trialMessagesArr[item].intermediate.push(interResultList[key]); } - })); - } + }); + }); + const result = { + experimentParameters: experimentData, + trialMessage: trialMessagesArr + }; + if (this._isExperimentMount === true) { + this.setState({ experiment: JSON.stringify(result, null, 4) }); + } + if (['DONE', 'ERROR', 'STOPPED'].includes(EXPERIMENT.status)) { + if(this.refreshId !== null || this.refreshId !== undefined){ + window.clearInterval(this.refreshId); + } + } + + } + downExperimentParameters = (): void => { const { experiment } = this.state; downFile(experiment, 'experiment.json'); @@ -78,31 +77,28 @@ class ExperimentDrawer extends React.Component { } componentDidMount(): void { - this._isCompareMount = true; + this._isExperimentMount = true; this.getExperimentContent(); + this.refreshId = window.setInterval(this.getExperimentContent, 10000); window.addEventListener('resize', this.onWindowResize); } - componentWillReceiveProps(nextProps: ExpDrawerProps): void { - const { isVisble } = nextProps; - if (isVisble === true) { - this.getExperimentContent(); - } - } - componentWillUnmount(): void { - this._isCompareMount = false; + this._isExperimentMount = false; + window.clearTimeout(this.refreshId); window.removeEventListener('resize', this.onWindowResize); } render(): React.ReactNode { - const { isVisble, closeExpDrawer } = this.props; + const { closeExpDrawer } = this.props; const { experiment, expDrawerHeight } = this.state; return ( diff --git a/src/webui/src/components/Modals/LogDrawer.tsx b/src/webui/src/components/Modals/LogPanel.tsx similarity index 98% rename from src/webui/src/components/Modals/LogDrawer.tsx rename to src/webui/src/components/Modals/LogPanel.tsx index a54b0f4c25..97a408fe9d 100644 --- a/src/webui/src/components/Modals/LogDrawer.tsx +++ b/src/webui/src/components/Modals/LogPanel.tsx @@ -92,6 +92,8 @@ class LogDrawer extends React.Component { isOpen={true} hasCloseButton={false} isFooterAtBottom={true} + isLightDismiss={true} + onLightDismissClick={closeDrawer} >
{ openDocs = (): void => { window.open(WEBUIDOC); } - + openGithubNNI = (): void => { - const {version} = this.state; + const { version } = this.state; const nniLink = `https://github.com/Microsoft/nni/tree/${version}`; window.open(nniLink); } @@ -178,8 +179,8 @@ class NavCon extends React.Component { {/* the drawer for dispatcher & nnimanager log message */} - {isvisibleLogDrawer && } - + {isvisibleLogDrawer && } + {isvisibleExperimentDrawer && } ); } diff --git a/src/webui/src/components/overview/Progress.tsx b/src/webui/src/components/overview/Progress.tsx index c63e23827c..c21e9e8921 100644 --- a/src/webui/src/components/overview/Progress.tsx +++ b/src/webui/src/components/overview/Progress.tsx @@ -9,7 +9,7 @@ import { EXPERIMENT, TRIALS } from '../../static/datamodel'; import { convertTime } from '../../static/function'; import ConcurrencyInput from './NumInput'; import ProgressBar from './ProgressItem'; -import LogDrawer from '../Modals/LogDrawer'; +import LogDrawer from '../Modals/LogPanel'; import MessageInfo from '../Modals/MessageInfo'; import { infoIcon } from "../Buttons/Icon"; import '../../static/style/progress.scss'; diff --git a/src/webui/src/static/model/trialmanager.ts b/src/webui/src/static/model/trialmanager.ts index bc613e1ba1..b860cfa042 100644 --- a/src/webui/src/static/model/trialmanager.ts +++ b/src/webui/src/static/model/trialmanager.ts @@ -48,6 +48,16 @@ class TrialManager { private latestMetricdataErrorMessage: string = ''; // metric-data-latest error message private isMetricdataRangeError: boolean = false; // metric-data-range api error filed private metricdataRangeErrorMessage: string = ''; // metric-data-latest error message + private metricsList: Array = []; + private trialJobList: Array = []; + + public getMetricsList(): Array { + return this.metricsList; + } + + public getTrialJobList(): Array { + return this.trialJobList; + } public async init(): Promise { while (!this.infoInitialized || !this.metricInitialized) { @@ -230,6 +240,7 @@ class TrialManager { requestAxios(`${MANAGER_IP}/trial-jobs`) .then(data => { const newTrials = TrialManager.expandJobsToTrials(data as any); + this.trialJobList = newTrials; for (const trialInfo of newTrials as TrialJobInfo[]) { if (this.trials.has(trialInfo.id)) { // eslint-disable-next-line @typescript-eslint/no-non-null-assertion @@ -265,7 +276,10 @@ class TrialManager { private async updateAllMetrics(): Promise { return requestAxios(`${MANAGER_IP}/metric-data`) - .then(data => this.doUpdateMetrics(data as any, false)) + .then(data => { + this.metricsList = data; + return this.doUpdateMetrics(data as any, false); + }) .catch(error => { this.isMetricdataError = true; this.MetricdataErrorMessage = `${error.message}`; From 109d9a3210f618d1803cc751436cdf3dfa2ba589 Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Fri, 7 Aug 2020 11:12:27 +0800 Subject: [PATCH 04/28] Fix remote machine connection logic (#2725) --- .../remoteMachineTrainingService.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts b/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts index c997b03a01..b291690a0b 100644 --- a/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts +++ b/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts @@ -57,6 +57,7 @@ class RemoteMachineTrainingService implements TrainingService { private nniManagerIpConfig?: NNIManagerIpConfig; private versionCheck: boolean = true; private logCollection: string; + private sshConnectionPromises: any[]; constructor(@component.Inject timer: ObservableTimer) { this.metricsEmitter = new EventEmitter(); @@ -65,6 +66,7 @@ class RemoteMachineTrainingService implements TrainingService { this.machineCopyExpCodeDirPromiseMap = new Map>(); this.machineExecutorManagerMap = new Map(); this.jobQueue = []; + this.sshConnectionPromises = []; this.expRootDir = getExperimentRootDir(); this.timer = timer; this.log = getLogger(); @@ -80,6 +82,12 @@ class RemoteMachineTrainingService implements TrainingService { await restServer.start(); restServer.setEnableVersionCheck = this.versionCheck; this.log.info('Run remote machine training service.'); + if (this.sshConnectionPromises.length > 0) { + await Promise.all(this.sshConnectionPromises); + this.log.info('ssh connection initialized!'); + // set sshConnectionPromises to [] to avoid log information duplicated + this.sshConnectionPromises = []; + } while (!this.stopping) { while (this.jobQueue.length > 0) { this.updateGpuReservation(); @@ -408,7 +416,6 @@ class RemoteMachineTrainingService implements TrainingService { //TO DO: verify if value's format is wrong, and json parse failed, how to handle error const rmMetaList: RemoteMachineMeta[] = JSON.parse(machineList); - const connectionPromises = []; for (const rmMeta of rmMetaList) { rmMeta.occupiedGpuIndexMap = new Map(); const executorManager: ExecutorManager = new ExecutorManager(rmMeta); @@ -417,11 +424,9 @@ class RemoteMachineTrainingService implements TrainingService { this.log.debug(`reached ${executor.name}`); this.machineExecutorManagerMap.set(rmMeta, executorManager); this.log.debug(`initializing ${executor.name}`); - connectionPromises.push(this.initRemoteMachineOnConnected(rmMeta, executor)); - this.log.info(`connected to ${executor.name}`); + this.sshConnectionPromises.push(this.initRemoteMachineOnConnected(rmMeta, executor)); + this.log.info(`connecting to ${executor.name}`); } - - await Promise.all(connectionPromises); } private async initRemoteMachineOnConnected(rmMeta: RemoteMachineMeta, executor: ShellExecutor): Promise { From 214a8e18a9a19d7f2b13bcd7a8d6decf7e439b80 Mon Sep 17 00:00:00 2001 From: Lijiaoa <61399850+Lijiaoa@users.noreply.github.com> Date: Fri, 7 Aug 2020 14:45:49 +0800 Subject: [PATCH 05/28] delete multiphase in webui (#2760) --- src/webui/src/components/Modals/Killjob.tsx | 2 +- .../src/components/trial-detail/TableList.tsx | 10 +-- src/webui/src/static/interface.ts | 6 -- src/webui/src/static/model/trial.ts | 2 - src/webui/src/static/model/trialmanager.ts | 77 +------------------ 5 files changed, 8 insertions(+), 89 deletions(-) diff --git a/src/webui/src/components/Modals/Killjob.tsx b/src/webui/src/components/Modals/Killjob.tsx index 580ff5ff24..2f4c7a1833 100644 --- a/src/webui/src/components/Modals/Killjob.tsx +++ b/src/webui/src/components/Modals/Killjob.tsx @@ -77,7 +77,7 @@ class KillJob extends React.Component { onKill = (): void => { this.setState({ isCalloutVisible: false }, () => { const { trial } = this.props; - killJob(trial.key, trial.jobId, trial.status); + killJob(trial.key, trial.id, trial.status); }); } diff --git a/src/webui/src/components/trial-detail/TableList.tsx b/src/webui/src/components/trial-detail/TableList.tsx index 94af7e8fb9..4ce1ccd7bb 100644 --- a/src/webui/src/components/trial-detail/TableList.tsx +++ b/src/webui/src/components/trial-detail/TableList.tsx @@ -269,7 +269,7 @@ class TableList extends React.Component { showIntermediateModal = async (record: TrialJobInfo, event: React.SyntheticEvent): Promise => { event.preventDefault(); event.stopPropagation(); - const res = await axios.get(`${MANAGER_IP}/metric-data/${record.jobId}`); + const res = await axios.get(`${MANAGER_IP}/metric-data/${record.id}`); if (res.status === 200) { const intermediateArr: number[] = []; // support intermediate result is dict because the last intermediate result is @@ -277,14 +277,10 @@ class TableList extends React.Component { // get intermediate result dict keys array const { intermediateKey } = this.state; const otherkeys: string[] = []; - // One trial job may contains multiple parameter id - // only show current trial's metric data - const metricDatas = res.data.filter(item => { - return item.parameterId == record.parameterId; - }); + const metricDatas = res.data; if (metricDatas.length !== 0) { // just add type=number keys - const intermediateMetrics = parseMetrics(res.data[0].data); + const intermediateMetrics = parseMetrics(metricDatas[0].data); for (const key in intermediateMetrics) { if (typeof intermediateMetrics[key] === 'number') { otherkeys.push(key); diff --git a/src/webui/src/static/interface.ts b/src/webui/src/static/interface.ts index c033c225f4..493ffc4b41 100644 --- a/src/webui/src/static/interface.ts +++ b/src/webui/src/static/interface.ts @@ -43,8 +43,6 @@ interface TableRecord { startTime: number; endTime?: number; id: string; - jobId: string; - parameterId: string; duration: number; status: string; intermediateCount: number; @@ -126,8 +124,6 @@ interface Intermedia { interface MetricDataRecord { timestamp: number; trialJobId: string; - trialId: string; - parameterId: string; type: string; sequence: number; data: string; @@ -135,8 +131,6 @@ interface MetricDataRecord { interface TrialJobInfo { id: string; - jobId: string; - parameterId: string; sequenceId: number; status: string; startTime?: number; diff --git a/src/webui/src/static/model/trial.ts b/src/webui/src/static/model/trial.ts index ebdd35bc77..b1431a0a5f 100644 --- a/src/webui/src/static/model/trial.ts +++ b/src/webui/src/static/model/trial.ts @@ -115,8 +115,6 @@ class Trial implements TableObj { key: this.info.id, sequenceId: this.info.sequenceId, id: this.info.id, - jobId: this.info.jobId, - parameterId: this.info.parameterId, // eslint-disable-next-line @typescript-eslint/no-non-null-assertion startTime: this.info.startTime!, endTime: this.info.endTime, diff --git a/src/webui/src/static/model/trialmanager.ts b/src/webui/src/static/model/trialmanager.ts index b860cfa042..ffc0f85f55 100644 --- a/src/webui/src/static/model/trialmanager.ts +++ b/src/webui/src/static/model/trialmanager.ts @@ -7,29 +7,13 @@ import { requestAxios } from '../function'; function groupMetricsByTrial(metrics: MetricDataRecord[]): Map { const ret = new Map(); for (const metric of metrics) { - const trialId = `${metric.trialJobId}-${metric.parameterId}`; - metric.trialId = trialId; - if (ret.has(trialId)) { + if (ret.has(metric.trialJobId)) { // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - ret.get(trialId)!.push(metric); + ret.get(metric.trialJobId)!.push(metric); } else { - ret.set(trialId, [metric]); + ret.set(metric.trialJobId, [ metric ]); } } - // to compatiable with multi-trial in same job, fix offset of sequence - ret.forEach((trialMetrics) => { - let minSequenceNumber = Number.POSITIVE_INFINITY; - trialMetrics.map((item) => { - if (item.sequence < minSequenceNumber && item.type !== "FINAL") { - minSequenceNumber = item.sequence; - } - }); - trialMetrics.map((item) => { - if (item.type !== "FINAL") { - item.sequence -= minSequenceNumber; - } - }); - }); return ret; } @@ -145,57 +129,6 @@ class TrialManager { return new MetricSpace([...this.trials.values()]); } - public static expandJobsToTrials(jobs: TrialJobInfo[]): TrialJobInfo[] { - const trials: TrialJobInfo[] = []; - - for (const jobInfo of jobs as TrialJobInfo[]) { - if (jobInfo.hyperParameters) { - let trial: TrialJobInfo | undefined; - let lastTrial: TrialJobInfo | undefined; - for (let i = 0; i < jobInfo.hyperParameters.length; i++) { - const hyperParameters = jobInfo.hyperParameters[i] - const hpObject = JSON.parse(hyperParameters); - const parameterId = hpObject["parameter_id"]; - trial = { - id: `${jobInfo.id}-${parameterId}`, - jobId: jobInfo.id, - parameterId: parameterId, - sequenceId: parameterId, - status: "SUCCEEDED", - startTime: jobInfo.startTime, - endTime: jobInfo.startTime, - hyperParameters: [hyperParameters], - logPath: jobInfo.logPath, - stderrPath: jobInfo.stderrPath, - }; - if (jobInfo.finalMetricData) { - for (const metricData of jobInfo.finalMetricData) { - if (metricData.parameterId == parameterId) { - trial.finalMetricData = [metricData]; - trial.endTime = metricData.timestamp; - break; - } - } - } - if (lastTrial) { - trial.startTime = lastTrial.endTime; - } else { - trial.startTime = jobInfo.startTime; - } - lastTrial = trial; - trials.push(trial); - } - if (lastTrial !== undefined) { - lastTrial.status = jobInfo.status; - lastTrial.endTime = jobInfo.endTime; - } - } else { - trials.push(jobInfo); - } - } - return trials; - } - // if this.jobListError = true, show trial error message [/trial-jobs] public jobListError(): boolean { return this.isJobListError; @@ -239,9 +172,7 @@ class TrialManager { let updated = false; requestAxios(`${MANAGER_IP}/trial-jobs`) .then(data => { - const newTrials = TrialManager.expandJobsToTrials(data as any); - this.trialJobList = newTrials; - for (const trialInfo of newTrials as TrialJobInfo[]) { + for (const trialInfo of data as TrialJobInfo[]) { if (this.trials.has(trialInfo.id)) { // eslint-disable-next-line @typescript-eslint/no-non-null-assertion updated = this.trials.get(trialInfo.id)!.updateTrialJobInfo(trialInfo) || updated; From 654e8242b47b4bce4586feb3aa1aa01a0d3ce3a9 Mon Sep 17 00:00:00 2001 From: Yuge Zhang Date: Fri, 7 Aug 2020 17:08:29 +0800 Subject: [PATCH 06/28] Change line color range and rendering order (#2759) --- .../src/components/trial-detail/Para.tsx | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/webui/src/components/trial-detail/Para.tsx b/src/webui/src/components/trial-detail/Para.tsx index f2f130949a..6a018b71e3 100644 --- a/src/webui/src/components/trial-detail/Para.tsx +++ b/src/webui/src/components/trial-detail/Para.tsx @@ -162,21 +162,32 @@ class Para extends React.Component { const scale = this.convertToD3Scale(v); if (k === primaryMetricKey && scale !== undefined && scale.interpolate) { // set color for primary metrics - colorScale = this.convertToD3Scale(v, false) - .range(['green', 'red']) - .interpolate(d3.interpolateHsl); - colorDim = k; + // `colorScale` is used to produce a color range, while `scale` is to produce a pixel range + colorScale = this.convertToD3Scale(v, false); + convertedTrials.sort((a, b) => EXPERIMENT.optimizeMode === 'minimize' ? a[k] - b[k] : b[k] - a[k]); // filter top trials if (percent != 1) { const keptTrialNum = Math.max(Math.ceil(convertedTrials.length * percent), 1); - convertedTrials.sort((a, b) => EXPERIMENT.optimizeMode === 'minimize' ? a[k] - b[k] : b[k] - a[k]); convertedTrials = convertedTrials.slice(0, keptTrialNum); const domain = d3.extent(convertedTrials, item => item[k]); scale.domain([domain[0], domain[1]]); + colorScale.domain([domain[0], domain[1]]); if (colorScale !== undefined) { colorScale.domain(domain); } } + // reverse the converted trials to show the top ones upfront + convertedTrials.reverse(); + const assignColors = (scale: any): void => { + scale.range([0, 1]); // fake a range to perform invert + const [scaleMin, scaleMax] = scale.domain(); + const pivot = scale.invert(0.5); + scale.domain([scaleMin, pivot, scaleMax]) + .range(['#90EE90', '#FFC400', '#CA0000']) + .interpolate(d3.interpolateHsl); + }; + assignColors(colorScale); + colorDim = k; } dimensions.push([k, { type: 'number', From d0a9b106a11b9fc0975c3c63015c231e7359c81b Mon Sep 17 00:00:00 2001 From: Guoxin Date: Tue, 11 Aug 2020 10:39:59 +0800 Subject: [PATCH 07/28] fix IT pruning example issue (#2772) --- .../nni/compression/torch/pruning/one_shot.py | 36 +++++++++++++------ src/sdk/pynni/tests/test_compressor.py | 10 +++--- src/sdk/pynni/tests/test_pruners.py | 4 +-- 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/src/sdk/pynni/nni/compression/torch/pruning/one_shot.py b/src/sdk/pynni/nni/compression/torch/pruning/one_shot.py index f74eba2a52..b58477a653 100644 --- a/src/sdk/pynni/nni/compression/torch/pruning/one_shot.py +++ b/src/sdk/pynni/nni/compression/torch/pruning/one_shot.py @@ -94,9 +94,11 @@ class LevelPruner(OneshotPruner): Supported keys: - sparsity : This is to specify the sparsity operations to be compressed to. - op_types : Operation types to prune. + optimizer: torch.optim.Optimizer + Optimizer used to train model """ - def __init__(self, model, config_list): - super().__init__(model, config_list, pruning_algorithm='level') + def __init__(self, model, config_list, optimizer=None): + super().__init__(model, config_list, pruning_algorithm='level', optimizer=optimizer) class SlimPruner(OneshotPruner): """ @@ -108,9 +110,11 @@ class SlimPruner(OneshotPruner): Supported keys: - sparsity : This is to specify the sparsity operations to be compressed to. - op_types : Only BatchNorm2d is supported in Slim Pruner. + optimizer: torch.optim.Optimizer + Optimizer used to train model """ - def __init__(self, model, config_list): - super().__init__(model, config_list, pruning_algorithm='slim') + def __init__(self, model, config_list, optimizer=None): + super().__init__(model, config_list, pruning_algorithm='slim', optimizer=optimizer) def validate_config(self, model, config_list): schema = CompressorSchema([{ @@ -147,9 +151,11 @@ class L1FilterPruner(_StructuredFilterPruner): Supported keys: - sparsity : This is to specify the sparsity operations to be compressed to. - op_types : Only Conv2d is supported in L1FilterPruner. + optimizer: torch.optim.Optimizer + Optimizer used to train model """ - def __init__(self, model, config_list): - super().__init__(model, config_list, pruning_algorithm='l1') + def __init__(self, model, config_list, optimizer=None): + super().__init__(model, config_list, pruning_algorithm='l1', optimizer=optimizer) class L2FilterPruner(_StructuredFilterPruner): """ @@ -161,9 +167,11 @@ class L2FilterPruner(_StructuredFilterPruner): Supported keys: - sparsity : This is to specify the sparsity operations to be compressed to. - op_types : Only Conv2d is supported in L2FilterPruner. + optimizer: torch.optim.Optimizer + Optimizer used to train model """ - def __init__(self, model, config_list): - super().__init__(model, config_list, pruning_algorithm='l2') + def __init__(self, model, config_list, optimizer=None): + super().__init__(model, config_list, pruning_algorithm='l2', optimizer=optimizer) class FPGMPruner(_StructuredFilterPruner): """ @@ -175,9 +183,11 @@ class FPGMPruner(_StructuredFilterPruner): Supported keys: - sparsity : This is to specify the sparsity operations to be compressed to. - op_types : Only Conv2d is supported in FPGM Pruner. + optimizer: torch.optim.Optimizer + Optimizer used to train model """ - def __init__(self, model, config_list): - super().__init__(model, config_list, pruning_algorithm='fpgm') + def __init__(self, model, config_list, optimizer=None): + super().__init__(model, config_list, pruning_algorithm='fpgm', optimizer=optimizer) class TaylorFOWeightFilterPruner(_StructuredFilterPruner): """ @@ -189,6 +199,8 @@ class TaylorFOWeightFilterPruner(_StructuredFilterPruner): Supported keys: - sparsity : How much percentage of convolutional filters are to be pruned. - op_types : Currently only Conv2d is supported in TaylorFOWeightFilterPruner. + optimizer: torch.optim.Optimizer + Optimizer used to train model """ def __init__(self, model, config_list, optimizer=None, statistics_batch_num=1): super().__init__(model, config_list, pruning_algorithm='taylorfo', optimizer=optimizer, statistics_batch_num=statistics_batch_num) @@ -203,6 +215,8 @@ class ActivationAPoZRankFilterPruner(_StructuredFilterPruner): Supported keys: - sparsity : How much percentage of convolutional filters are to be pruned. - op_types : Only Conv2d is supported in ActivationAPoZRankFilterPruner. + optimizer: torch.optim.Optimizer + Optimizer used to train model """ def __init__(self, model, config_list, optimizer=None, activation='relu', statistics_batch_num=1): super().__init__(model, config_list, pruning_algorithm='apoz', optimizer=optimizer, \ @@ -218,6 +232,8 @@ class ActivationMeanRankFilterPruner(_StructuredFilterPruner): Supported keys: - sparsity : How much percentage of convolutional filters are to be pruned. - op_types : Only Conv2d is supported in ActivationMeanRankFilterPruner. + optimizer: torch.optim.Optimizer + Optimizer used to train model """ def __init__(self, model, config_list, optimizer=None, activation='relu', statistics_batch_num=1): super().__init__(model, config_list, pruning_algorithm='mean_activation', optimizer=optimizer, \ diff --git a/src/sdk/pynni/tests/test_compressor.py b/src/sdk/pynni/tests/test_compressor.py index 6a8727c9e4..87afb5f23c 100644 --- a/src/sdk/pynni/tests/test_compressor.py +++ b/src/sdk/pynni/tests/test_compressor.py @@ -88,8 +88,9 @@ def test_torch_quantizer_modules_detection(self): def test_torch_level_pruner(self): model = TorchModel() + optimizer = torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.5) configure_list = [{'sparsity': 0.8, 'op_types': ['default']}] - torch_compressor.LevelPruner(model, configure_list).compress() + torch_compressor.LevelPruner(model, configure_list, optimizer).compress() @tf2 def test_tf_level_pruner(self): @@ -128,7 +129,7 @@ def test_torch_fpgm_pruner(self): model = TorchModel() config_list = [{'sparsity': 0.6, 'op_types': ['Conv2d']}, {'sparsity': 0.2, 'op_types': ['Conv2d']}] - pruner = torch_compressor.FPGMPruner(model, config_list) + pruner = torch_compressor.FPGMPruner(model, config_list, torch.optim.SGD(model.parameters(), lr=0.01)) model.conv2.module.weight.data = torch.tensor(w).float() masks = pruner.calc_mask(model.conv2) @@ -314,7 +315,7 @@ def test_torch_QAT_quantizer(self): def test_torch_pruner_validation(self): # test bad configuraiton pruner_classes = [torch_compressor.__dict__[x] for x in \ - ['LevelPruner', 'SlimPruner', 'FPGMPruner', 'L1FilterPruner', 'L2FilterPruner', \ + ['LevelPruner', 'SlimPruner', 'FPGMPruner', 'L1FilterPruner', 'L2FilterPruner', 'AGPPruner',\ 'ActivationMeanRankFilterPruner', 'ActivationAPoZRankFilterPruner']] bad_configs = [ @@ -336,10 +337,11 @@ def test_torch_pruner_validation(self): ] ] model = TorchModel() + optimizer = torch.optim.SGD(model.parameters(), lr=0.01) for pruner_class in pruner_classes: for config_list in bad_configs: try: - pruner_class(model, config_list) + pruner_class(model, config_list, optimizer) print(config_list) assert False, 'Validation error should be raised for bad configuration' except schema.SchemaError: diff --git a/src/sdk/pynni/tests/test_pruners.py b/src/sdk/pynni/tests/test_pruners.py index 1fab9b2b2a..9bba85e3e4 100644 --- a/src/sdk/pynni/tests/test_pruners.py +++ b/src/sdk/pynni/tests/test_pruners.py @@ -192,9 +192,7 @@ def pruners_test(pruner_names=['level', 'agp', 'slim', 'fpgm', 'l1', 'l2', 'tayl pruner = prune_config[pruner_name]['pruner_class'](model, config_list, trainer=prune_config[pruner_name]['trainer']) elif pruner_name == 'autocompress': pruner = prune_config[pruner_name]['pruner_class'](model, config_list, trainer=prune_config[pruner_name]['trainer'], evaluator=prune_config[pruner_name]['evaluator'], dummy_input=x) - elif pruner_name in ['level', 'slim', 'fpgm', 'l1', 'l2']: - pruner = prune_config[pruner_name]['pruner_class'](model, config_list) - elif pruner_name in ['agp', 'taylorfo', 'mean_activation', 'apoz']: + else: pruner = prune_config[pruner_name]['pruner_class'](model, config_list, optimizer) pruner.compress() From accb40f9d901b628c1b1408d8fcea025be5894ac Mon Sep 17 00:00:00 2001 From: Guoxin Date: Tue, 11 Aug 2020 11:32:41 +0800 Subject: [PATCH 08/28] compression benchmark (#2742) --- .../ModelCompressionComparison.md | 89 ++++ docs/en_US/CommunitySharings/perf_compare.rst | 3 +- docs/en_US/Compressor/Overview.md | 1 + examples/model_compress/auto_pruners_torch.py | 332 ++++++++------- .../comparison_of_pruners/analyze.py | 107 +++++ .../cifar10/comparison_result_resnet18.json | 392 ++++++++++++++++++ .../cifar10/comparison_result_resnet50.json | 356 ++++++++++++++++ .../cifar10/comparison_result_vgg16.json | 392 ++++++++++++++++++ .../img/performance_comparison_resnet18.png | Bin 0 -> 87873 bytes .../img/performance_comparison_resnet50.png | Bin 0 -> 86829 bytes .../img/performance_comparison_vgg16.png | Bin 0 -> 84565 bytes .../model_compress/models/cifar10/resnet.py | 115 +++++ 12 files changed, 1644 insertions(+), 143 deletions(-) create mode 100644 docs/en_US/CommunitySharings/ModelCompressionComparison.md create mode 100644 examples/model_compress/comparison_of_pruners/analyze.py create mode 100644 examples/model_compress/comparison_of_pruners/cifar10/comparison_result_resnet18.json create mode 100644 examples/model_compress/comparison_of_pruners/cifar10/comparison_result_resnet50.json create mode 100644 examples/model_compress/comparison_of_pruners/cifar10/comparison_result_vgg16.json create mode 100644 examples/model_compress/comparison_of_pruners/img/performance_comparison_resnet18.png create mode 100644 examples/model_compress/comparison_of_pruners/img/performance_comparison_resnet50.png create mode 100644 examples/model_compress/comparison_of_pruners/img/performance_comparison_vgg16.png create mode 100644 examples/model_compress/models/cifar10/resnet.py diff --git a/docs/en_US/CommunitySharings/ModelCompressionComparison.md b/docs/en_US/CommunitySharings/ModelCompressionComparison.md new file mode 100644 index 0000000000..ba273f9581 --- /dev/null +++ b/docs/en_US/CommunitySharings/ModelCompressionComparison.md @@ -0,0 +1,89 @@ +# Comparison of Filter Pruning Algorithms + +To provide an initial insight into the performance of various filter pruning algorithms, +we conduct extensive experiments with various pruning algorithms on some benchmark models and datasets. +We present the experiment result in this document. +In addition, we provide friendly instructions on the re-implementation of these experiments to facilitate further contributions to this effort. + +## Experiment Setting + +The experiments are performed with the following pruners/datasets/models: + +* Models: [VGG16, ResNet18, ResNet50](https://github.com/microsoft/nni/tree/master/examples/model_compress/models/cifar10) + +* Datasets: CIFAR-10 + +* Pruners: + - These pruners are included: + - Pruners with scheduling : `SimulatedAnnealing Pruner`, `NetAdapt Pruner`, `AutoCompress Pruner`. + Given the overal sparsity requirement, these pruners can automatically generate a sparsity distribution among different layers. + - One-shot pruners: `L1Filter Pruner`, `L2Filter Pruner`, `FPGM Pruner`. + The sparsity of each layer is set the same as the overall sparsity in this experiment. + - Only **filter pruning** performances are compared here. + + For the pruners with scheduling, `L1Filter Pruner` is used as the base algorithm. That is to say, after the sparsities distribution is decided by the scheduling algorithm, `L1Filter Pruner` is used to performn real pruning. + + - All the pruners listed above are implemented in [nni](https://github.com/microsoft/nni/tree/master/docs/en_US/Compressor/Overview.md). + +## Experiment Result + +For each dataset/model/pruner combination, we prune the model to different levels by setting a series of target sparsities for the pruner. + +Here we plot both **Number of Weights - Performances** curve and **FLOPs - Performance** curve. +As a reference, we also plot the result declared in the paper [AutoCompress: An Automatic DNN Structured Pruning Framework for Ultra-High Compression Rates](http://arxiv.org/abs/1907.03141) for models VGG16 and ResNet18 on CIFAR-10. + +The experiment result are shown in the following figures: + +CIFAR-10, VGG16: + +![](../../../examples/model_compress/comparison_of_pruners/img/performance_comparison_vgg16.png) + +CIFAR-10, ResNet18: + +![](../../../examples/model_compress/comparison_of_pruners/img/performance_comparison_resnet18.png) + +CIFAR-10, ResNet50: + +![](../../../examples/model_compress/comparison_of_pruners/img/performance_comparison_resnet50.png) + +## Analysis + +From the experiment result, we get the following conclusions: + +* Given the constraint on the number of parameters, the pruners with scheduling ( `AutoCompress Pruner` , `SimualatedAnnealing Pruner` ) performs better than the others when the constraint is strict. However, they have no such advantage in FLOPs/Performances comparison since only number of parameters constraint is considered in the optimization process; +* The basic algorithms `L1Filter Pruner` , `L2Filter Pruner` , `FPGM Pruner` performs very similarly in these experiments; +* `NetAdapt Pruner` can not achieve very high compression rate. This is caused by its mechanism that it prunes only one layer each pruning iteration. This leads to un-acceptable complexity if the sparsity per iteration is much lower than the overall sparisity constraint. + +## Experiments Reproduction + +### Implementation Details + +* The experiment results are all collected with the default configuration of the pruners in nni, which means that when we call a pruner class in nni, we don't change any default class arguments. + +* Both FLOPs and the number of parameters are counted with [Model FLOPs/Parameters Counter](https://github.com/microsoft/nni/blob/master/docs/en_US/Compressor/CompressionUtils.md#model-flopsparameters-counter) after [model speed up](https://github.com/microsoft/nni/blob/master/docs/en_US/Compressor/ModelSpeedup.md). This avoids potential issues of counting them of masked models. + +* The experiment code can be found [here]( https://github.com/microsoft/nni/tree/master/examples/model_compress/auto_pruners_torch.py). + +### Experiment Result Rendering + +* If you follow the practice in the [example]( https://github.com/microsoft/nni/tree/master/examples/model_compress/auto_pruners_torch.py), for every single pruning experiment, the experiment result will be saved in JSON format as follows: + ``` json + { + "performance": {"original": 0.9298, "pruned": 0.1, "speedup": 0.1, "finetuned": 0.7746}, + "params": {"original": 14987722.0, "speedup": 167089.0}, + "flops": {"original": 314018314.0, "speedup": 38589922.0} + } + ``` + +* The experiment results are saved [here](https://github.com/microsoft/nni/tree/master/examples/model_compress/experiment_data). +You can refer to [analyze](https://github.com/microsoft/nni/tree/master/examples/model_compress/experiment_data/analyze.py) to plot new performance comparison figures. + +## Contribution + +### TODO Items + +* Pruners constrained by FLOPS/latency +* More pruning algorithms/datasets/models + +### Issues +For algorithm implementation & experiment issues, please [create an issue](https://github.com/microsoft/nni/issues/new/). diff --git a/docs/en_US/CommunitySharings/perf_compare.rst b/docs/en_US/CommunitySharings/perf_compare.rst index b87fd167c8..2b80ccdc6c 100644 --- a/docs/en_US/CommunitySharings/perf_compare.rst +++ b/docs/en_US/CommunitySharings/perf_compare.rst @@ -8,4 +8,5 @@ Performance comparison and analysis can help users decide a proper algorithm (e. :maxdepth: 1 Neural Architecture Search Comparison - Hyper-parameter Tuning Algorithm Comparsion \ No newline at end of file + Hyper-parameter Tuning Algorithm Comparsion + Model Compression Algorithm Comparsion \ No newline at end of file diff --git a/docs/en_US/Compressor/Overview.md b/docs/en_US/Compressor/Overview.md index 2d68496545..73298ee8af 100644 --- a/docs/en_US/Compressor/Overview.md +++ b/docs/en_US/Compressor/Overview.md @@ -42,6 +42,7 @@ Pruning algorithms compress the original network by removing redundant weights o | [SimulatedAnnealing Pruner](https://nni.readthedocs.io/en/latest/Compressor/Pruner.html#simulatedannealing-pruner) | Automatic pruning with a guided heuristic search method, Simulated Annealing algorithm [Reference Paper](https://arxiv.org/abs/1907.03141) | | [AutoCompress Pruner](https://nni.readthedocs.io/en/latest/Compressor/Pruner.html#autocompress-pruner) | Automatic pruning by iteratively call SimulatedAnnealing Pruner and ADMM Pruner [Reference Paper](https://arxiv.org/abs/1907.03141) | +You can refer to this [benchmark](https://github.com/microsoft/nni/tree/master/docs/en_US/Benchmark.md) for the performance of these pruners on some benchmark problems. ### Quantization Algorithms diff --git a/examples/model_compress/auto_pruners_torch.py b/examples/model_compress/auto_pruners_torch.py index 9f0678b6f0..33ecfb8f5f 100644 --- a/examples/model_compress/auto_pruners_torch.py +++ b/examples/model_compress/auto_pruners_torch.py @@ -9,78 +9,81 @@ import json import torch from torch.optim.lr_scheduler import StepLR, MultiStepLR -from torchvision import datasets, transforms, models +from torchvision import datasets, transforms from models.mnist.lenet import LeNet from models.cifar10.vgg import VGG -from nni.compression.torch import L1FilterPruner, SimulatedAnnealingPruner, ADMMPruner, NetAdaptPruner, AutoCompressPruner +from models.cifar10.resnet import ResNet18, ResNet50 +from nni.compression.torch import L1FilterPruner, L2FilterPruner, FPGMPruner +from nni.compression.torch import SimulatedAnnealingPruner, ADMMPruner, NetAdaptPruner, AutoCompressPruner from nni.compression.torch import ModelSpeedup +from nni.compression.torch.utils.counter import count_flops_params -def get_data(args): +def get_data(dataset, data_dir, batch_size, test_batch_size): ''' get data ''' kwargs = {'num_workers': 1, 'pin_memory': True} if torch.cuda.is_available() else { } - if args.dataset == 'mnist': + if dataset == 'mnist': train_loader = torch.utils.data.DataLoader( - datasets.MNIST(args.data_dir, train=True, download=True, + datasets.MNIST(data_dir, train=True, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ])), - batch_size=args.batch_size, shuffle=True, **kwargs) + batch_size=batch_size, shuffle=True, **kwargs) val_loader = torch.utils.data.DataLoader( - datasets.MNIST(args.data_dir, train=False, + datasets.MNIST(data_dir, train=False, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ])), - batch_size=args.test_batch_size, shuffle=True, **kwargs) + batch_size=test_batch_size, shuffle=True, **kwargs) criterion = torch.nn.NLLLoss() - elif args.dataset == 'cifar10': + elif dataset == 'cifar10': normalize = transforms.Normalize( (0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)) train_loader = torch.utils.data.DataLoader( - datasets.CIFAR10(args.data_dir, train=True, transform=transforms.Compose([ + datasets.CIFAR10(data_dir, train=True, transform=transforms.Compose([ transforms.RandomHorizontalFlip(), transforms.RandomCrop(32, 4), transforms.ToTensor(), normalize, ]), download=True), - batch_size=args.batch_size, shuffle=True, **kwargs) + batch_size=batch_size, shuffle=True, **kwargs) val_loader = torch.utils.data.DataLoader( - datasets.CIFAR10(args.data_dir, train=False, transform=transforms.Compose([ + datasets.CIFAR10(data_dir, train=False, transform=transforms.Compose([ transforms.ToTensor(), normalize, ])), - batch_size=args.batch_size, shuffle=False, **kwargs) + batch_size=batch_size, shuffle=False, **kwargs) criterion = torch.nn.CrossEntropyLoss() - elif args.dataset == 'imagenet': + elif dataset == 'imagenet': normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_loader = torch.utils.data.DataLoader( - datasets.ImageFolder(os.path.join(args.data_dir, 'train'), + datasets.ImageFolder(os.path.join(data_dir, 'train'), transform=transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])), - batch_size=args.batch_size, shuffle=True, **kwargs) + batch_size=batch_size, shuffle=True, **kwargs) val_loader = torch.utils.data.DataLoader( - datasets.ImageFolder(os.path.join(args.data_dir, 'val'), + datasets.ImageFolder(os.path.join(data_dir, 'val'), transform=transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])), - batch_size=args.test_batch_size, shuffle=True, **kwargs) + batch_size=test_batch_size, shuffle=True, **kwargs) criterion = torch.nn.CrossEntropyLoss() return train_loader, val_loader, criterion @@ -127,65 +130,91 @@ def test(model, device, criterion, val_loader): return accuracy -def get_trained_model(args, device, train_loader, val_loader, criterion): +def get_trained_model_optimizer(args, device, train_loader, val_loader, criterion): if args.model == 'LeNet': model = LeNet().to(device) - optimizer = torch.optim.Adadelta(model.parameters(), lr=1) - scheduler = StepLR(optimizer, step_size=1, gamma=0.7) - for epoch in range(args.pretrain_epochs): - train(args, model, device, train_loader, - criterion, optimizer, epoch) - scheduler.step() + if args.load_pretrained_model: + model.load_state_dict(torch.load(args.pretrained_model_dir)) + optimizer = torch.optim.Adadelta(model.parameters(), lr=1e-4) + else: + optimizer = torch.optim.Adadelta(model.parameters(), lr=1) + scheduler = StepLR(optimizer, step_size=1, gamma=0.7) elif args.model == 'vgg16': model = VGG(depth=16).to(device) - optimizer = torch.optim.SGD(model.parameters(), lr=0.01, - momentum=0.9, - weight_decay=5e-4) - scheduler = MultiStepLR( - optimizer, milestones=[int(args.pretrain_epochs*0.5), int(args.pretrain_epochs*0.75)], gamma=0.1) - for epoch in range(args.pretrain_epochs): - train(args, model, device, train_loader, - criterion, optimizer, epoch) - scheduler.step() + if args.load_pretrained_model: + model.load_state_dict(torch.load(args.pretrained_model_dir)) + optimizer = torch.optim.SGD(model.parameters(), lr=1e-4, momentum=0.9, weight_decay=5e-4) + else: + optimizer = torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.9, weight_decay=5e-4) + scheduler = MultiStepLR( + optimizer, milestones=[int(args.pretrain_epochs*0.5), int(args.pretrain_epochs*0.75)], gamma=0.1) elif args.model == 'resnet18': - model = models.resnet18(pretrained=False, num_classes=10).to(device) - optimizer = torch.optim.SGD(model.parameters(), lr=0.01, - momentum=0.9, - weight_decay=5e-4) - scheduler = MultiStepLR( - optimizer, milestones=[int(args.pretrain_epochs*0.5), int(args.pretrain_epochs*0.75)], gamma=0.1) + model = ResNet18().to(device) + if args.load_pretrained_model: + model.load_state_dict(torch.load(args.pretrained_model_dir)) + optimizer = torch.optim.SGD(model.parameters(), lr=1e-4, momentum=0.9, weight_decay=5e-4) + else: + optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9, weight_decay=5e-4) + scheduler = MultiStepLR( + optimizer, milestones=[int(args.pretrain_epochs*0.5), int(args.pretrain_epochs*0.75)], gamma=0.1) + elif args.model == 'resnet50': + model = ResNet50().to(device) + if args.load_pretrained_model: + model.load_state_dict(torch.load(args.pretrained_model_dir)) + optimizer = torch.optim.SGD(model.parameters(), lr=1e-4, momentum=0.9, weight_decay=5e-4) + else: + optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9, weight_decay=5e-4) + scheduler = MultiStepLR( + optimizer, milestones=[int(args.pretrain_epochs*0.5), int(args.pretrain_epochs*0.75)], gamma=0.1) + else: + raise ValueError("model not recognized") + + if not args.load_pretrained_model: + best_acc = 0 + best_epoch = 0 for epoch in range(args.pretrain_epochs): - train(args, model, device, train_loader, - criterion, optimizer, epoch) + train(args, model, device, train_loader, criterion, optimizer, epoch) scheduler.step() - elif args.model == 'mobilenet_v2': - model = models.mobilenet_v2(pretrained=True).to(device) - - if args.save_model: - torch.save(model.state_dict(), os.path.join( - args.experiment_data_dir, 'model_trained.pth')) - print('Model trained saved to %s', args.experiment_data_dir) + acc = test(model, device, criterion, val_loader) + if acc > best_acc: + best_acc = acc + best_epoch = epoch + state_dict = model.state_dict() + model.load_state_dict(state_dict) + print('Best acc:', best_acc) + print('Best epoch:', best_epoch) + + if args.save_model: + torch.save(state_dict, os.path.join(args.experiment_data_dir, 'model_trained.pth')) + print('Model trained saved to %s', args.experiment_data_dir) return model, optimizer def get_dummy_input(args, device): if args.dataset == 'mnist': - dummy_input = torch.randn( - [args.test_batch_size, 1, 28, 28]).to(device) + dummy_input = torch.randn([args.test_batch_size, 1, 28, 28]).to(device) elif args.dataset in ['cifar10', 'imagenet']: - dummy_input = torch.randn( - [args.test_batch_size, 3, 32, 32]).to(device) - + dummy_input = torch.randn([args.test_batch_size, 3, 32, 32]).to(device) return dummy_input +def get_input_size(dataset): + if dataset == 'mnist': + input_size = (1, 1, 28, 28) + elif dataset == 'cifar10': + input_size = (1, 3, 32, 32) + elif dataset == 'imagenet': + input_size = (1, 3, 256, 256) + return input_size + + def main(args): # prepare dataset torch.manual_seed(0) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - train_loader, val_loader, criterion = get_data(args) - model, optimizer = get_trained_model(args, device, train_loader, val_loader, criterion) + train_loader, val_loader, criterion = get_data(args.dataset, args.data_dir, args.batch_size, args.test_batch_size) + model, optimizer = get_trained_model_optimizer(args, device, train_loader, val_loader, criterion) def short_term_fine_tuner(model, epochs=1): for epoch in range(epochs): @@ -198,11 +227,15 @@ def evaluator(model): return test(model, device, criterion, val_loader) # used to save the performance of the original & pruned & finetuned models - result = {} + result = {'flops': {}, 'params': {}, 'performance':{}} + + flops, params = count_flops_params(model, get_input_size(args.dataset)) + result['flops']['original'] = flops + result['params']['original'] = params evaluation_result = evaluator(model) print('Evaluation result (original model): %s' % evaluation_result) - result['original'] = evaluation_result + result['performance']['original'] = evaluation_result # module types to prune, only "Conv2d" supported for channel pruning if args.base_algo in ['l1', 'l2']: @@ -218,6 +251,10 @@ def evaluator(model): if args.pruner == 'L1FilterPruner': pruner = L1FilterPruner(model, config_list) + elif args.pruner == 'L2FilterPruner': + pruner = L2FilterPruner(model, config_list) + elif args.pruner == 'FPGMPruner': + pruner = FPGMPruner(model, config_list) elif args.pruner == 'NetAdaptPruner': pruner = NetAdaptPruner(model, config_list, short_term_fine_tuner=short_term_fine_tuner, evaluator=evaluator, base_algo=args.base_algo, experiment_data_dir=args.experiment_data_dir) @@ -263,99 +300,123 @@ def evaluator(model): experiment_data_dir=args.experiment_data_dir) else: raise ValueError( - "Please use L1FilterPruner, NetAdaptPruner, SimulatedAnnealingPruner, ADMMPruner or AutoCompressPruner in this example.") + "Pruner not supported.") # Pruner.compress() returns the masked model # but for AutoCompressPruner, Pruner.compress() returns directly the pruned model - model_masked = pruner.compress() - evaluation_result = evaluator(model_masked) + model = pruner.compress() + evaluation_result = evaluator(model) print('Evaluation result (masked model): %s' % evaluation_result) - result['pruned'] = evaluation_result + result['performance']['pruned'] = evaluation_result if args.save_model: pruner.export_model( os.path.join(args.experiment_data_dir, 'model_masked.pth'), os.path.join(args.experiment_data_dir, 'mask.pth')) print('Masked model saved to %s', args.experiment_data_dir) + # model speed up + if args.speed_up: + if args.pruner != 'AutoCompressPruner': + if args.model == 'LeNet': + model = LeNet().to(device) + elif args.model == 'vgg16': + model = VGG(depth=16).to(device) + elif args.model == 'resnet18': + model = ResNet18().to(device) + elif args.model == 'resnet50': + model = ResNet50().to(device) + + model.load_state_dict(torch.load(os.path.join(args.experiment_data_dir, 'model_masked.pth'))) + masks_file = os.path.join(args.experiment_data_dir, 'mask.pth') + + m_speedup = ModelSpeedup(model, dummy_input, masks_file, device) + m_speedup.speedup_model() + evaluation_result = evaluator(model) + print('Evaluation result (speed up model): %s' % evaluation_result) + result['performance']['speedup'] = evaluation_result + + torch.save(model.state_dict(), os.path.join(args.experiment_data_dir, 'model_speed_up.pth')) + print('Speed up model saved to %s', args.experiment_data_dir) + flops, params = count_flops_params(model, get_input_size(args.dataset)) + result['flops']['speedup'] = flops + result['params']['speedup'] = params + if args.fine_tune: if args.dataset == 'mnist': - optimizer = torch.optim.Adadelta(model_masked.parameters(), lr=1) - scheduler = StepLR(optimizer, step_size=1, gamma=0.7) - for epoch in range(args.fine_tune_epochs): - train(args, model_masked, device, train_loader, criterion, optimizer, epoch) - scheduler.step() - test(model_masked, device, criterion, val_loader) - elif args.dataset == 'cifar10': - optimizer = torch.optim.SGD(model_masked.parameters(), lr=0.01, momentum=0.9, weight_decay=5e-4) + optimizer = torch.optim.Adadelta(model.parameters(), lr=1) scheduler = StepLR(optimizer, step_size=1, gamma=0.7) - for epoch in range(args.fine_tune_epochs): - train(args, model_masked, device, train_loader, criterion, optimizer, epoch) - scheduler.step() - test(model_masked, device, criterion, val_loader) - elif args.dataset == 'imagenet': - for epoch in range(args.fine_tune_epochs): - optimizer = torch.optim.SGD(model_masked.parameters(), lr=0.05, momentum=0.9, weight_decay=5e-4) - train(args, model_masked, device, train_loader, criterion, optimizer, epoch) - test(model_masked, device, criterion, val_loader) - - evaluation_result = evaluator(model_masked) - print('Evaluation result (fine tuned): %s' % evaluation_result) - result['finetuned'] = evaluation_result + elif args.dataset == 'cifar10' and args.model == 'vgg16': + optimizer = torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.9, weight_decay=5e-4) + scheduler = MultiStepLR( + optimizer, milestones=[int(args.fine_tune_epochs*0.5), int(args.fine_tune_epochs*0.75)], gamma=0.1) + elif args.dataset == 'cifar10' and args.model == 'resnet18': + optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9, weight_decay=5e-4) + scheduler = MultiStepLR( + optimizer, milestones=[int(args.fine_tune_epochs*0.5), int(args.fine_tune_epochs*0.75)], gamma=0.1) + elif args.dataset == 'cifar10' and args.model == 'resnet50': + optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9, weight_decay=5e-4) + scheduler = MultiStepLR( + optimizer, milestones=[int(args.fine_tune_epochs*0.5), int(args.fine_tune_epochs*0.75)], gamma=0.1) + best_acc = 0 + for epoch in range(args.fine_tune_epochs): + train(args, model, device, train_loader, criterion, optimizer, epoch) + scheduler.step() + acc = evaluator(model) + if acc > best_acc: + best_acc = acc + torch.save(model.state_dict(), os.path.join(args.experiment_data_dir, 'model_fine_tuned.pth')) - if args.save_model: - pruner.export_model(os.path.join( - args.experiment_data_dir, 'model_fine_tuned.pth'), os.path.join(args.experiment_data_dir, 'mask.pth')) - print('Fined tuned model saved to %s', args.experiment_data_dir) + print('Evaluation result (fine tuned): %s' % best_acc) + print('Fined tuned model saved to %s', args.experiment_data_dir) + result['performance']['finetuned'] = best_acc - # model speed up - if args.speed_up and args.pruner != 'AutoCompressPruner': - if args.model == 'LeNet': - model = LeNet().to(device) - elif args.model == 'vgg16': - model = VGG(depth=16).to(device) - elif args.model == 'resnet18': - model = models.resnet18(pretrained=False, num_classes=10).to(device) - elif args.model == 'mobilenet_v2': - model = models.mobilenet_v2(pretrained=False).to(device) - - model.load_state_dict(torch.load(os.path.join(args.experiment_data_dir, 'model_fine_tuned.pth'))) - masks_file = os.path.join(args.experiment_data_dir, 'mask.pth') - - m_speedup = ModelSpeedup(model, dummy_input, masks_file, device) - m_speedup.speedup_model() - evaluation_result = evaluator(model) - print('Evaluation result (speed up model): %s' % evaluation_result) - result['speedup'] = evaluation_result - - torch.save(model.state_dict(), os.path.join(args.experiment_data_dir, 'model_speed_up.pth')) - print('Speed up model saved to %s', args.experiment_data_dir) - - with open(os.path.join(args.experiment_data_dir, 'performance.json'), 'w+') as f: + with open(os.path.join(args.experiment_data_dir, 'result.json'), 'w+') as f: json.dump(result, f) if __name__ == '__main__': - def str2bool(v): - if isinstance(v, bool): - return v - if v.lower() in ('yes', 'true', 't', 'y', '1'): + def str2bool(s): + if isinstance(s, bool): + return s + if s.lower() in ('yes', 'true', 't', 'y', '1'): return True - elif v.lower() in ('no', 'false', 'f', 'n', '0'): + if s.lower() in ('no', 'false', 'f', 'n', '0'): return False - else: - raise argparse.ArgumentTypeError('Boolean value expected.') + raise argparse.ArgumentTypeError('Boolean value expected.') parser = argparse.ArgumentParser(description='PyTorch Example for SimulatedAnnealingPruner') + # dataset and model + parser.add_argument('--dataset', type=str, default='cifar10', + help='dataset to use, mnist, cifar10 or imagenet') + parser.add_argument('--data-dir', type=str, default='./data/', + help='dataset directory') + parser.add_argument('--model', type=str, default='vgg16', + help='model to use, LeNet, vgg16, resnet18 or resnet50') + parser.add_argument('--load-pretrained-model', type=str2bool, default=False, + help='whether to load pretrained model') + parser.add_argument('--pretrained-model-dir', type=str, default='./', + help='path to pretrained model') + parser.add_argument('--pretrain-epochs', type=int, default=100, + help='number of epochs to pretrain the model') + parser.add_argument('--batch-size', type=int, default=64, + help='input batch size for training (default: 64)') + parser.add_argument('--test-batch-size', type=int, default=64, + help='input batch size for testing (default: 64)') + parser.add_argument('--fine-tune', type=str2bool, default=True, + help='whether to fine-tune the pruned model') + parser.add_argument('--fine-tune-epochs', type=int, default=5, + help='epochs to fine tune') + parser.add_argument('--experiment-data-dir', type=str, default='./experiment_data', + help='For saving experiment data') + + # pruner parser.add_argument('--pruner', type=str, default='SimulatedAnnealingPruner', - help='pruner to use, L1FilterPruner, NetAdaptPruner, SimulatedAnnealingPruner, ADMMPruner or AutoCompressPruner') + help='pruner to use') parser.add_argument('--base-algo', type=str, default='l1', help='base pruning algorithm. level, l1 or l2') - parser.add_argument('--sparsity', type=float, default=0.3, - help='overall target sparsity') - parser.add_argument('--speed-up', type=str2bool, default=False, - help='Whether to speed-up the pruned model') - + parser.add_argument('--sparsity', type=float, default=0.1, + help='target overall target sparsity') # param for SimulatedAnnealingPruner parser.add_argument('--cool-down-rate', type=float, default=0.9, help='cool down rate') @@ -363,29 +424,16 @@ def str2bool(v): parser.add_argument('--sparsity-per-iteration', type=float, default=0.05, help='sparsity_per_iteration of NetAdaptPruner') - parser.add_argument('--dataset', type=str, default='mnist', - help='dataset to use, mnist, cifar10 or imagenet (default MNIST)') - parser.add_argument('--model', type=str, default='LeNet', - help='model to use, LeNet, vgg16, resnet18 or mobilenet_v2') - parser.add_argument('--fine-tune', type=str2bool, default=True, - help='whether to fine-tune the pruned model') - parser.add_argument('--fine-tune-epochs', type=int, default=10, - help='epochs to fine tune') - parser.add_argument('--data-dir', type=str, default='/datasets/', - help='dataset directory') - parser.add_argument('--experiment-data-dir', type=str, default='./', - help='For saving experiment data') + # speed-up + parser.add_argument('--speed-up', type=str2bool, default=False, + help='Whether to speed-up the pruned model') - parser.add_argument('--batch-size', type=int, default=64, - help='input batch size for training (default: 64)') - parser.add_argument('--test-batch-size', type=int, default=64, - help='input batch size for testing (default: 64)') - parser.add_argument('--pretrain-epochs', type=int, default=1, - help='number of epochs to pretrain the model') + # others parser.add_argument('--log-interval', type=int, default=200, help='how many batches to wait before logging training status') parser.add_argument('--save-model', type=str2bool, default=True, help='For Saving the current Model') + args = parser.parse_args() if not os.path.exists(args.experiment_data_dir): diff --git a/examples/model_compress/comparison_of_pruners/analyze.py b/examples/model_compress/comparison_of_pruners/analyze.py new file mode 100644 index 0000000000..c7cd13f72a --- /dev/null +++ b/examples/model_compress/comparison_of_pruners/analyze.py @@ -0,0 +1,107 @@ +import argparse +import json +import matplotlib.pyplot as plt + + +def plot_performance_comparison(args): + # reference data, performance of the original model and the performance declared in the AutoCompress Paper + references = { + 'original':{ + 'cifar10':{ + 'vgg16':{ + 'performance': 0.9298, + 'params':14987722.0, + 'flops':314018314.0 + }, + 'resnet18':{ + 'performance': 0.9433, + 'params':11173962.0, + 'flops':556651530.0 + }, + 'resnet50':{ + 'performance': 0.9488, + 'params':23520842.0, + 'flops':1304694794.0 + } + } + }, + 'AutoCompressPruner':{ + 'cifar10':{ + 'vgg16':{ + 'performance': 0.9321, + 'params':52.2, # times + 'flops':8.8 + }, + 'resnet18':{ + 'performance': 0.9381, + 'params':54.2, # times + 'flops':12.2 + } + } + } + } + + markers = ['v', '^', '<', '1', '2', '3', '4', '8', '*', '+', 'o'] + + with open('cifar10/comparison_result_{}.json'.format(args.model), 'r') as jsonfile: + result = json.load(jsonfile) + + pruners = result.keys() + + performances = {} + flops = {} + params = {} + sparsities = {} + for pruner in pruners: + performances[pruner] = [val['performance'] for val in result[pruner]] + flops[pruner] = [val['flops'] for val in result[pruner]] + params[pruner] = [val['params'] for val in result[pruner]] + sparsities[pruner] = [val['sparsity'] for val in result[pruner]] + + fig, axs = plt.subplots(2, 1, figsize=(8, 10)) + fig.suptitle('Channel Pruning Comparison on {}/CIFAR10'.format(args.model)) + fig.subplots_adjust(hspace=0.5) + + for idx, pruner in enumerate(pruners): + axs[0].scatter(params[pruner], performances[pruner], marker=markers[idx], label=pruner) + axs[1].scatter(flops[pruner], performances[pruner], marker=markers[idx], label=pruner) + + # references + params_original = references['original']['cifar10'][args.model]['params'] + performance_original = references['original']['cifar10'][args.model]['performance'] + axs[0].plot(params_original, performance_original, 'rx', label='original model') + if args.model in ['vgg16', 'resnet18']: + axs[0].plot(params_original/references['AutoCompressPruner']['cifar10'][args.model]['params'], + references['AutoCompressPruner']['cifar10'][args.model]['performance'], + 'bx', label='AutoCompress Paper') + + axs[0].set_title("Performance v.s. Number of Parameters") + axs[0].set_xlabel("Number of Parameters") + axs[0].set_ylabel('Accuracy') + axs[0].legend() + + # references + flops_original = references['original']['cifar10'][args.model]['flops'] + performance_original = references['original']['cifar10'][args.model]['performance'] + axs[1].plot(flops_original, performance_original, 'rx', label='original model') + if args.model in ['vgg16', 'resnet18']: + axs[1].plot(flops_original/references['AutoCompressPruner']['cifar10'][args.model]['flops'], + references['AutoCompressPruner']['cifar10'][args.model]['performance'], + 'bx', label='AutoCompress Paper') + + axs[1].set_title("Performance v.s. FLOPs") + axs[1].set_xlabel("FLOPs") + axs[1].set_ylabel('Accuracy') + axs[1].legend() + + plt.savefig('img/performance_comparison_{}.png'.format(args.model)) + plt.close() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='PyTorch MNIST Example') + parser.add_argument('--model', type=str, default='vgg16', + help='vgg16, resnet18 or resnet50') + args = parser.parse_args() + + plot_performance_comparison(args) diff --git a/examples/model_compress/comparison_of_pruners/cifar10/comparison_result_resnet18.json b/examples/model_compress/comparison_of_pruners/cifar10/comparison_result_resnet18.json new file mode 100644 index 0000000000..0ef5a6119d --- /dev/null +++ b/examples/model_compress/comparison_of_pruners/cifar10/comparison_result_resnet18.json @@ -0,0 +1,392 @@ +{ + "L1FilterPruner": [ + { + "sparsity": 0.1, + "params": 9642085.0, + "flops": 496882684.0, + "performance": 0.9436 + }, + { + "sparsity": 0.2, + "params": 8149126.0, + "flops": 436381222.0, + "performance": 0.9472 + }, + { + "sparsity": 0.3, + "params": 6705269.0, + "flops": 371666312.0, + "performance": 0.9391 + }, + { + "sparsity": 0.4, + "params": 5335138.0, + "flops": 307050934.0, + "performance": 0.9433 + }, + { + "sparsity": 0.5, + "params": 3998122.0, + "flops": 237900244.0, + "performance": 0.9379 + }, + { + "sparsity": 0.6, + "params": 2767325.0, + "flops": 175308326.0, + "performance": 0.9326 + }, + { + "sparsity": 0.7, + "params": 1617817.0, + "flops": 108532198.0, + "performance": 0.928 + }, + { + "sparsity": 0.8, + "params": 801338.0, + "flops": 53808728.0, + "performance": 0.9145 + }, + { + "sparsity": 0.9, + "params": 229372.0, + "flops": 15304972.0, + "performance": 0.8858 + }, + { + "sparsity": 0.95, + "params": 61337.0, + "flops": 4305146.0, + "performance": 0.8441 + }, + { + "sparsity": 0.975, + "params": 17763.0, + "flops": 1561644.0, + "performance": 0.7294 + } + ], + "L2FilterPruner": [ + { + "sparsity": 0.1, + "params": 9680242.0, + "flops": 497492746.0, + "performance": 0.9423 + }, + { + "sparsity": 0.2, + "params": 8137784.0, + "flops": 436199900.0, + "performance": 0.9471 + }, + { + "sparsity": 0.3, + "params": 6702679.0, + "flops": 369733768.0, + "performance": 0.9415 + }, + { + "sparsity": 0.4, + "params": 5330426.0, + "flops": 305512736.0, + "performance": 0.9411 + }, + { + "sparsity": 0.5, + "params": 3961076.0, + "flops": 236467814.0, + "performance": 0.9349 + }, + { + "sparsity": 0.6, + "params": 2776512.0, + "flops": 175872204.0, + "performance": 0.9393 + }, + { + "sparsity": 0.7, + "params": 1622571.0, + "flops": 107994906.0, + "performance": 0.9295 + }, + { + "sparsity": 0.8, + "params": 797075.0, + "flops": 53534414.0, + "performance": 0.9187 + }, + { + "sparsity": 0.9, + "params": 232153.0, + "flops": 15385078.0, + "performance": 0.8838 + }, + { + "sparsity": 0.95, + "params": 58180.0, + "flops": 4510072.0, + "performance": 0.8396 + }, + { + "sparsity": 0.975, + "params": 16836.0, + "flops": 1429752.0, + "performance": 0.7482 + } + ], + "FPGMPruner": [ + { + "sparsity": 0.1, + "params": 9705680.0, + "flops": 497899454.0, + "performance": 0.9443 + }, + { + "sparsity": 0.2, + "params": 8160468.0, + "flops": 436562544.0, + "performance": 0.946 + }, + { + "sparsity": 0.3, + "params": 6710052.0, + "flops": 367960482.0, + "performance": 0.9452 + }, + { + "sparsity": 0.4, + "params": 5334205.0, + "flops": 306166432.0, + "performance": 0.9412 + }, + { + "sparsity": 0.5, + "params": 4007259.0, + "flops": 237702210.0, + "performance": 0.9385 + }, + { + "sparsity": 0.6, + "params": 2782236.0, + "flops": 175813620.0, + "performance": 0.9304 + }, + { + "sparsity": 0.7, + "params": 1634603.0, + "flops": 108904676.0, + "performance": 0.9249 + }, + { + "sparsity": 0.8, + "params": 799610.0, + "flops": 53645918.0, + "performance": 0.9203 + }, + { + "sparsity": 0.9, + "params": 233644.0, + "flops": 15408784.0, + "performance": 0.8856 + }, + { + "sparsity": 0.95, + "params": 56518.0, + "flops": 4266910.0, + "performance": 0.83 + }, + { + "sparsity": 0.975, + "params": 17610.0, + "flops": 1441836.0, + "performance": 0.7356 + } + ], + "NetAdaptPruner": [ + { + "sparsity": 0.1, + "params": 11173962.0, + "flops": 556651530.0, + "performance": 0.9474 + }, + { + "sparsity": 0.2, + "params": 10454958.0, + "flops": 545147466.0, + "performance": 0.9482 + }, + { + "sparsity": 0.3, + "params": 9299986.0, + "flops": 526681564.0, + "performance": 0.9469 + }, + { + "sparsity": 0.4, + "params": 8137618.0, + "flops": 508087276.0, + "performance": 0.9451 + }, + { + "sparsity": 0.5, + "params": 6267654.0, + "flops": 478185102.0, + "performance": 0.947 + }, + { + "sparsity": 0.6, + "params": 5277444.0, + "flops": 462341742.0, + "performance": 0.9469 + }, + { + "sparsity": 0.7, + "params": 4854190.0, + "flops": 455580628.0, + "performance": 0.9466 + }, + { + "sparsity": 0.8, + "params": 3531098.0, + "flops": 434411156.0, + "performance": 0.9472 + } + ], + "SimulatedAnnealingPruner": [ + { + "sparsity": 0.1, + "params": 10307424.0, + "flops": 537697098.0, + "performance": 0.942 + }, + { + "sparsity": 0.2, + "params": 9264598.0, + "flops": 513101368.0, + "performance": 0.9456 + }, + { + "sparsity": 0.3, + "params": 7999316.0, + "flops": 489260738.0, + "performance": 0.946 + }, + { + "sparsity": 0.4, + "params": 6996176.0, + "flops": 450768626.0, + "performance": 0.9413 + }, + { + "sparsity": 0.5, + "params": 5412616.0, + "flops": 408698434.0, + "performance": 0.9477 + }, + { + "sparsity": 0.6, + "params": 5106924.0, + "flops": 391735326.0, + "performance": 0.9483 + }, + { + "sparsity": 0.7, + "params": 3032105.0, + "flops": 269777978.0, + "performance": 0.9414 + }, + { + "sparsity": 0.8, + "params": 2423230.0, + "flops": 294783862.0, + "performance": 0.9384 + }, + { + "sparsity": 0.9, + "params": 1151046.0, + "flops": 209639226.0, + "performance": 0.939 + }, + { + "sparsity": 0.95, + "params": 394406.0, + "flops": 108776618.0, + "performance": 0.923 + }, + { + "sparsity": 0.975, + "params": 250649.0, + "flops": 84645050.0, + "performance": 0.917 + } + ], + "AutoCompressPruner": [ + { + "sparsity": 0.1, + "params": 10238286.0, + "flops": 536590794.0, + "performance": 0.9406 + }, + { + "sparsity": 0.2, + "params": 9272049.0, + "flops": 512333916.0, + "performance": 0.9392 + }, + { + "sparsity": 0.3, + "params": 8099915.0, + "flops": 485418056.0, + "performance": 0.9398 + }, + { + "sparsity": 0.4, + "params": 6864547.0, + "flops": 449359492.0, + "performance": 0.9406 + }, + { + "sparsity": 0.5, + "params": 6106994.0, + "flops": 430766432.0, + "performance": 0.9397 + }, + { + "sparsity": 0.6, + "params": 5338096.0, + "flops": 415085278.0, + "performance": 0.9384 + }, + { + "sparsity": 0.7, + "params": 3701330.0, + "flops": 351057878.0, + "performance": 0.938 + }, + { + "sparsity": 0.8, + "params": 2229760.0, + "flops": 269058346.0, + "performance": 0.9388 + }, + { + "sparsity": 0.9, + "params": 1108564.0, + "flops": 189355930.0, + "performance": 0.9348 + }, + { + "sparsity": 0.95, + "params": 616893.0, + "flops": 159314256.0, + "performance": 0.93 + }, + { + "sparsity": 0.975, + "params": 297368.0, + "flops": 113398292.0, + "performance": 0.9072 + } + ] +} \ No newline at end of file diff --git a/examples/model_compress/comparison_of_pruners/cifar10/comparison_result_resnet50.json b/examples/model_compress/comparison_of_pruners/cifar10/comparison_result_resnet50.json new file mode 100644 index 0000000000..dcea274149 --- /dev/null +++ b/examples/model_compress/comparison_of_pruners/cifar10/comparison_result_resnet50.json @@ -0,0 +1,356 @@ +{ + "L1FilterPruner": [ + { + "sparsity": 0.1, + "params": 20378141.0, + "flops": 1134740738.0, + "performance": 0.9456 + }, + { + "sparsity": 0.2, + "params": 17286560.0, + "flops": 966734852.0, + "performance": 0.9433 + }, + { + "sparsity": 0.3, + "params": 14403947.0, + "flops": 807114812.0, + "performance": 0.9396 + }, + { + "sparsity": 0.4, + "params": 11558288.0, + "flops": 656314106.0, + "performance": 0.9402 + }, + { + "sparsity": 0.5, + "params": 8826728.0, + "flops": 507965924.0, + "performance": 0.9394 + }, + { + "sparsity": 0.6, + "params": 6319902.0, + "flops": 374211960.0, + "performance": 0.9372 + }, + { + "sparsity": 0.7, + "params": 4063713.0, + "flops": 246788556.0, + "performance": 0.9304 + }, + { + "sparsity": 0.8, + "params": 2120717.0, + "flops": 133614422.0, + "performance": 0.9269 + }, + { + "sparsity": 0.9, + "params": 652524.0, + "flops": 41973714.0, + "performance": 0.9081 + }, + { + "sparsity": 0.95, + "params": 195468.0, + "flops": 13732020.0, + "performance": 0.8723 + }, + { + "sparsity": 0.975, + "params": 58054.0, + "flops": 4268104.0, + "performance": 0.7941 + } + ], + "L2FilterPruner": [ + { + "sparsity": 0.1, + "params": 20378141.0, + "flops": 1134740738.0, + "performance": 0.9442 + }, + { + "sparsity": 0.2, + "params": 17275244.0, + "flops": 966400928.0, + "performance": 0.9463 + }, + { + "sparsity": 0.3, + "params": 14415409.0, + "flops": 807710914.0, + "performance": 0.9367 + }, + { + "sparsity": 0.4, + "params": 11564310.0, + "flops": 656653008.0, + "performance": 0.9391 + }, + { + "sparsity": 0.5, + "params": 8843266.0, + "flops": 508086256.0, + "performance": 0.9381 + }, + { + "sparsity": 0.6, + "params": 6316815.0, + "flops": 373882614.0, + "performance": 0.9368 + }, + { + "sparsity": 0.7, + "params": 4054272.0, + "flops": 246477678.0, + "performance": 0.935 + }, + { + "sparsity": 0.8, + "params": 2129321.0, + "flops": 134527520.0, + "performance": 0.9275 + }, + { + "sparsity": 0.9, + "params": 667500.0, + "flops": 42927060.0, + "performance": 0.9129 + }, + { + "sparsity": 0.95, + "params": 192464.0, + "flops": 13669430.0, + "performance": 0.8757 + }, + { + "sparsity": 0.975, + "params": 58250.0, + "flops": 4365620.0, + "performance": 0.7978 + } + ], + "FPGMPruner": [ + { + "sparsity": 0.1, + "params": 20401570.0, + "flops": 1135114552.0, + "performance": 0.9438 + }, + { + "sparsity": 0.2, + "params": 17321414.0, + "flops": 967137398.0, + "performance": 0.9427 + }, + { + "sparsity": 0.3, + "params": 14418221.0, + "flops": 807755756.0, + "performance": 0.9422 + }, + { + "sparsity": 0.4, + "params": 11565000.0, + "flops": 655412124.0, + "performance": 0.9403 + }, + { + "sparsity": 0.5, + "params": 8829840.0, + "flops": 506715294.0, + "performance": 0.9355 + }, + { + "sparsity": 0.6, + "params": 6308085.0, + "flops": 374231682.0, + "performance": 0.9359 + }, + { + "sparsity": 0.7, + "params": 4054237.0, + "flops": 246511714.0, + "performance": 0.9285 + }, + { + "sparsity": 0.8, + "params": 2134187.0, + "flops": 134456366.0, + "performance": 0.9275 + }, + { + "sparsity": 0.9, + "params": 665931.0, + "flops": 42859752.0, + "performance": 0.9083 + }, + { + "sparsity": 0.95, + "params": 191590.0, + "flops": 13641052.0, + "performance": 0.8762 + }, + { + "sparsity": 0.975, + "params": 57767.0, + "flops": 4350074.0, + "performance": 0.789 + } + ], + "NetAdaptPruner": [ + { + "sparsity": 0.1, + "params": 22348970.0, + "flops": 1275701258.0, + "performance": 0.9404 + }, + { + "sparsity": 0.2, + "params": 21177162.0, + "flops": 1256952330.0, + "performance": 0.9445 + }, + { + "sparsity": 0.3, + "params": 18407434.0, + "flops": 1212636682.0, + "performance": 0.9433 + }, + { + "sparsity": 0.4, + "params": 16061284.0, + "flops": 1175098282.0, + "performance": 0.9401 + } + ], + "SimulatedAnnealingPruner": [ + { + "sparsity": 0.1, + "params": 20551755.0, + "flops": 1230145122.0, + "performance": 0.9438 + }, + { + "sparsity": 0.2, + "params": 17766048.0, + "flops": 1159924128.0, + "performance": 0.9432 + }, + { + "sparsity": 0.3, + "params": 15105146.0, + "flops": 1094478662.0, + "performance": 0.943 + }, + { + "sparsity": 0.4, + "params": 12378092.0, + "flops": 1008801158.0, + "performance": 0.9398 + }, + { + "sparsity": 0.5, + "params": 9890487.0, + "flops": 911941770.0, + "performance": 0.9426 + }, + { + "sparsity": 0.6, + "params": 7638262.0, + "flops": 831218770.0, + "performance": 0.9412 + }, + { + "sparsity": 0.7, + "params": 5469936.0, + "flops": 691881792.0, + "performance": 0.9405 + }, + { + "sparsity": 0.8, + "params": 3668951.0, + "flops": 580850666.0, + "performance": 0.941 + }, + { + "sparsity": 0.9, + "params": 1765284.0, + "flops": 389162310.0, + "performance": 0.9294 + } + ], + "AutoCompressPruner": [ + { + "sparsity": 0.1, + "params": 20660299.0, + "flops": 1228508590.0, + "performance": 0.9337 + }, + { + "sparsity": 0.2, + "params": 17940465.0, + "flops": 1152868146.0, + "performance": 0.9326 + }, + { + "sparsity": 0.3, + "params": 15335831.0, + "flops": 1084996094.0, + "performance": 0.9348 + }, + { + "sparsity": 0.4, + "params": 12821408.0, + "flops": 991305524.0, + "performance": 0.936 + }, + { + "sparsity": 0.5, + "params": 10695425.0, + "flops": 919638860.0, + "performance": 0.9349 + }, + { + "sparsity": 0.6, + "params": 8536821.0, + "flops": 802011678.0, + "performance": 0.9339 + }, + { + "sparsity": 0.7, + "params": 7276898.0, + "flops": 744248114.0, + "performance": 0.9337 + }, + { + "sparsity": 0.8, + "params": 5557721.0, + "flops": 643881710.0, + "performance": 0.9323 + }, + { + "sparsity": 0.9, + "params": 3925140.0, + "flops": 512545272.0, + "performance": 0.9304 + }, + { + "sparsity": 0.95, + "params": 2867004.0, + "flops": 365184762.0, + "performance": 0.9263 + }, + { + "sparsity": 0.975, + "params": 1773257.0, + "flops": 229320266.0, + "performance": 0.9175 + } + ] +} \ No newline at end of file diff --git a/examples/model_compress/comparison_of_pruners/cifar10/comparison_result_vgg16.json b/examples/model_compress/comparison_of_pruners/cifar10/comparison_result_vgg16.json new file mode 100644 index 0000000000..9e476488c1 --- /dev/null +++ b/examples/model_compress/comparison_of_pruners/cifar10/comparison_result_vgg16.json @@ -0,0 +1,392 @@ +{ + "L1FilterPruner": [ + { + "sparsity": 0.1, + "params": 12187336.0, + "flops": 256252606.0, + "performance": 0.9344 + }, + { + "sparsity": 0.2, + "params": 9660216.0, + "flops": 203049930.0, + "performance": 0.9371 + }, + { + "sparsity": 0.3, + "params": 7435417.0, + "flops": 155477470.0, + "performance": 0.9341 + }, + { + "sparsity": 0.4, + "params": 5493954.0, + "flops": 114721578.0, + "performance": 0.9317 + }, + { + "sparsity": 0.5, + "params": 3820010.0, + "flops": 79155722.0, + "performance": 0.9309 + }, + { + "sparsity": 0.6, + "params": 2478632.0, + "flops": 51618494.0, + "performance": 0.9229 + }, + { + "sparsity": 0.7, + "params": 1420600.0, + "flops": 29455306.0, + "performance": 0.9031 + }, + { + "sparsity": 0.8, + "params": 658553.0, + "flops": 13290974.0, + "performance": 0.8756 + }, + { + "sparsity": 0.9, + "params": 186178.0, + "flops": 3574570.0, + "performance": 0.8145 + }, + { + "sparsity": 0.95, + "params": 58680.0, + "flops": 1050570.0, + "performance": 0.6983 + }, + { + "sparsity": 0.975, + "params": 23408.0, + "flops": 329918.0, + "performance": 0.5573 + } + ], + "L2FilterPruner": [ + { + "sparsity": 0.1, + "params": 12187336.0, + "flops": 256252606.0, + "performance": 0.9357 + }, + { + "sparsity": 0.2, + "params": 9660216.0, + "flops": 203049930.0, + "performance": 0.9355 + }, + { + "sparsity": 0.3, + "params": 7435417.0, + "flops": 155477470.0, + "performance": 0.9337 + }, + { + "sparsity": 0.4, + "params": 5493954.0, + "flops": 114721578.0, + "performance": 0.9308 + }, + { + "sparsity": 0.5, + "params": 3820010.0, + "flops": 79155722.0, + "performance": 0.9285 + }, + { + "sparsity": 0.6, + "params": 2478632.0, + "flops": 51618494.0, + "performance": 0.9208 + }, + { + "sparsity": 0.7, + "params": 1420600.0, + "flops": 29455306.0, + "performance": 0.909 + }, + { + "sparsity": 0.8, + "params": 658553.0, + "flops": 13290974.0, + "performance": 0.8698 + }, + { + "sparsity": 0.9, + "params": 186178.0, + "flops": 3574570.0, + "performance": 0.8203 + }, + { + "sparsity": 0.95, + "params": 58680.0, + "flops": 1050570.0, + "performance": 0.7063 + }, + { + "sparsity": 0.975, + "params": 23408.0, + "flops": 329918.0, + "performance": 0.5455 + } + ], + "FPGMPruner": [ + { + "sparsity": 0.1, + "params": 12187336.0, + "flops": 256252606.0, + "performance": 0.937 + }, + { + "sparsity": 0.2, + "params": 9660216.0, + "flops": 203049930.0, + "performance": 0.936 + }, + { + "sparsity": 0.3, + "params": 7435417.0, + "flops": 155477470.0, + "performance": 0.9359 + }, + { + "sparsity": 0.4, + "params": 5493954.0, + "flops": 114721578.0, + "performance": 0.9302 + }, + { + "sparsity": 0.5, + "params": 3820010.0, + "flops": 79155722.0, + "performance": 0.9233 + }, + { + "sparsity": 0.6, + "params": 2478632.0, + "flops": 51618494.0, + "performance": 0.922 + }, + { + "sparsity": 0.7, + "params": 1420600.0, + "flops": 29455306.0, + "performance": 0.9022 + }, + { + "sparsity": 0.8, + "params": 658553.0, + "flops": 13290974.0, + "performance": 0.8794 + }, + { + "sparsity": 0.9, + "params": 186178.0, + "flops": 3574570.0, + "performance": 0.8276 + }, + { + "sparsity": 0.95, + "params": 58680.0, + "flops": 1050570.0, + "performance": 0.6967 + }, + { + "sparsity": 0.975, + "params": 23408.0, + "flops": 329918.0, + "performance": 0.3683 + } + ], + "NetAdaptPruner": [ + { + "sparsity": 0.1, + "params": 13492098.0, + "flops": 308484330.0, + "performance": 0.9376 + }, + { + "sparsity": 0.2, + "params": 11998408.0, + "flops": 297641410.0, + "performance": 0.9374 + }, + { + "sparsity": 0.3, + "params": 10504344.0, + "flops": 281928834.0, + "performance": 0.9369 + }, + { + "sparsity": 0.4, + "params": 8263221.0, + "flops": 272964342.0, + "performance": 0.9382 + }, + { + "sparsity": 0.5, + "params": 6769885.0, + "flops": 249070966.0, + "performance": 0.9388 + }, + { + "sparsity": 0.6, + "params": 6022137.0, + "flops": 237106998.0, + "performance": 0.9383 + }, + { + "sparsity": 0.7, + "params": 4526754.0, + "flops": 222152490.0, + "performance": 0.936 + }, + { + "sparsity": 0.8, + "params": 3032759.0, + "flops": 162401210.0, + "performance": 0.9362 + } + ], + "SimulatedAnnealingPruner": [ + { + "sparsity": 0.1, + "params": 12691704.0, + "flops": 301467870.0, + "performance": 0.9366 + }, + { + "sparsity": 0.2, + "params": 10318461.0, + "flops": 275724450.0, + "performance": 0.9362 + }, + { + "sparsity": 0.3, + "params": 8217127.0, + "flops": 246321046.0, + "performance": 0.9371 + }, + { + "sparsity": 0.4, + "params": 6458368.0, + "flops": 232948294.0, + "performance": 0.9378 + }, + { + "sparsity": 0.5, + "params": 4973079.0, + "flops": 217675254.0, + "performance": 0.9362 + }, + { + "sparsity": 0.6, + "params": 3131526.0, + "flops": 151576878.0, + "performance": 0.9347 + }, + { + "sparsity": 0.7, + "params": 1891036.0, + "flops": 76575574.0, + "performance": 0.9289 + }, + { + "sparsity": 0.8, + "params": 1170751.0, + "flops": 107532322.0, + "performance": 0.9325 + }, + { + "sparsity": 0.9, + "params": 365978.0, + "flops": 46241354.0, + "performance": 0.9167 + }, + { + "sparsity": 0.95, + "params": 167089.0, + "flops": 38589922.0, + "performance": 0.7746 + }, + { + "sparsity": 0.975, + "params": 96779.0, + "flops": 26838230.0, + "performance": 0.1 + } + ], + "AutoCompressPruner": [ + { + "sparsity": 0.1, + "params": 12460277.0, + "flops": 290311730.0, + "performance": 0.9352 + }, + { + "sparsity": 0.2, + "params": 10138147.0, + "flops": 269180938.0, + "performance": 0.9324 + }, + { + "sparsity": 0.3, + "params": 8033350.0, + "flops": 241789714.0, + "performance": 0.9357 + }, + { + "sparsity": 0.4, + "params": 6105156.0, + "flops": 213573294.0, + "performance": 0.9367 + }, + { + "sparsity": 0.5, + "params": 4372604.0, + "flops": 185826362.0, + "performance": 0.9387 + }, + { + "sparsity": 0.6, + "params": 3029629.0, + "flops": 166285498.0, + "performance": 0.9334 + }, + { + "sparsity": 0.7, + "params": 1897060.0, + "flops": 134897806.0, + "performance": 0.9359 + }, + { + "sparsity": 0.8, + "params": 1145509.0, + "flops": 111766450.0, + "performance": 0.9334 + }, + { + "sparsity": 0.9, + "params": 362546.0, + "flops": 50777246.0, + "performance": 0.9261 + }, + { + "sparsity": 0.95, + "params": 149735.0, + "flops": 39201770.0, + "performance": 0.8924 + }, + { + "sparsity": 0.975, + "params": 45378.0, + "flops": 13213974.0, + "performance": 0.8193 + } + ] +} \ No newline at end of file diff --git a/examples/model_compress/comparison_of_pruners/img/performance_comparison_resnet18.png b/examples/model_compress/comparison_of_pruners/img/performance_comparison_resnet18.png new file mode 100644 index 0000000000000000000000000000000000000000..87a99e85bd9c87a6b7b5fb21bd0166bd2be72792 GIT binary patch literal 87873 zcmeFZbyQV*_&0b6X{5VB5fJH6T3Sj#1f)x8kVYgWl@OE^5h($YQo0)iln?}@JEf6^ zdG`IScg_4eYt77h{}|Tw-YXo=-rstjPwn$kQ$vXmj}{MwLJ{6mme)q1Fqq-b3Jw;0 zhrQ=rDf|c1P4=D+4t)9ISVh3!ai1zbazmkr%#lB6dD6MI@Xg!q3i|F3oS(RRnY%tl zIhwmab#Qifu(e?JeC+CG>+B@R!_Ong&1~cD{#1;Y_y7DkkF%>aZy<_a9))5?-IKqo zj-{2kPWLY`kr zBX8+vDYE&u7s*>jYU_~W*Y|D_`B%V%)+>%l5wxBg$Wqdl^78V+ z#KKB6Q;$q)Y?MAbKgUnF&;Isa09OKDLqkKdp!L}^if zN6yTc2{}zr>+0&pez(#uwFv#`IygAEeSSEdWjFkpPgpoKD~rvb+%8o5@=S2rL$BC` zK`#92u=04d+l@aJj`&JSN*v^PqobqD%*-hJ-(T)3Du%_w&CW*hHpG5%o~+5)IgURq ze)8j;*vNE|u{hidCN}o^`^JDv|7ceNLPB$A=hy%Kj?ed|2pSyo8dU{02V-ZM)cYWp zR%351wd~xTyb!kk-83>nrIJJX^XE@$Dc{>~-n`lUaerUL5+%EpeO)#2qhd5|L{wDU zh-pA{{qsYM_1}d@D_O5gYzA2Ox27Aj3-r0-c}*E_XuNNFNp_>eycO%VxVYtLiTTk| zsRr{Zd{r>U@BUCo%ccCGb{?9-(48AA5)b1<& zXz+_|XUBUxGtC&tl;>)s4R#0YEvd^ZC}g-T|3sl!Fi-ydjr9EcheceR?wfUA$Hn<+ zsq3Q3(a8xb2S=Xf$Z?XXKc}B!{-ygB_0rPP(Qnp7C}wVMg4Na4OJjSnJd@G^j%KN&c@>|KU;t!~4Xp@E9*9FoIIP@P z-w4Zg17$Px=oTBB-0j=9BjE4$*~LcH+9=}2%Zuk36+tVGEtJGQ%W7$26apy(kyN~4 zaOK~=u;j3GwmDmqwN%3edf_TLvhA-ZN*%@oqK}CfC1f-;iMjNO2m}QM;cY*{{oMB1 z&r_807zujx?TPunjWMJcKqXikE(m{ne0$lB!rn&T|7BGMUL{( z{A+2=Mt{HLV64?vz8$j&gZ>tHaa%e{QEa$nwW>I znp#xH*Jrh6^}QFEt_F=uliSmGoeW7$flNi$npTqt+@ z>mx+>KS*%S;GA2JeSNI0M@2_R=e{vYqE}=z(6uKplE|~>*YLQ9(s#QVTazmR2NxGD zcF#Sa`Nb9FK6SV#z7!S?_yudO1Nh9BwDrtbodw#|n=Kg37%S2~;K7sQ?D(^8XfCN?!S zp|G&9VD5&LD@#fOgM!fFRdRj{q>4H#!epA8nFYcOBhyZCQ@8DCXWpYIOD#pf2F^sr z+5_sI#9P5{-;(Km(MQYC&ObOdCiX}wIy&->-f!6~p6`4;4;5);ys}vFKS)_J2C+93-Y8wP`zJBN-Rql9hA-jdQTbHsckyP7b$TLyIwk zy_9^%js4%&bn&;-Ut`~_N$BaL%v;0EVeomTjZ$G(8D2r9xc3a{&np9LQS^66$15CL zYdqHbOU&im*M@KlbPET$+~Aw~RZm&i)sh4KPXA@Va)?!W9Bt1c-)uNpySV`csz9%} zLy5^RBqRirv+qy2{Xm1?bA5e%v&k9{{Tg>pMrnV7fPjFs^Cf6nQFs(wW_wFNs0E)8 z{4O$%gnO!g+IT-zgz)_IP$_}yVF_&Zv;AN9ZwFjRAk_}x{KQ}jDKJG9G1si|~O@IRG) zFs}79pQ?KXrMVr7dCo^JFB!Mv-N(qjr{XnfBa%J~+}?hQ92`@>g9LKwhk#Ccl6bLE zzNddHwUcs^{%YJv>LDQ~4@tPGX94r#eX`H#G+7ghhJn>II7o_tMbO_BaJXswUd%(~ zk96++^o}g0*ZnXTT~=R{47^On#>TpOdxwt5WNj9Xp@2U9`$u?xbCM2l=t_SE2EfOs z(8y#9=i&BHfiO?x`7v;3n5gJ*GMS&PJiNTTwo(J)IoYXZ5&N7xJQ^tCbl=^38A+Ua zMem_D%)y@3Hc(M{RZ?0?f`NUt*Q>g>S1m3sjs+bZ9jSgf?j9p(&d$zpDJh&e>2Q}Q zV%QFE2LYbkLPbQ=iQ2`(lG)ELE?!qJNWBzwp867Ze=^zvijDT49?$0Pt}bF48X8u1 zc10+wiYQ_xzx_BxDb!q35Qf2H6fxNijUg3YxS5s7+Df5%-(y!a8O$qJY!_1RxP3w~ z!ztM-3@|^m*40hqct9^Y7QuPL8r)(gPvOJDOw1tb0w61`u^4o0HdF-CA-S38>A6u6H|~E>wq5K_AQMPYdoTH1m(3aW zSZHXdLWA+#(6Y_t#d$k6lkZ17lfr#?7r;g@$=6KMlw1ia1qD56;$9)p zw4qeg#_NvS-;_q~BP=Xz&_zewWiAB8?0+nQD zs>Pn4KPk!B)rg@yqX(jxCnhH3b#&r&r=ZN?*xTD9iylI30mlJ7P`}JZE6AgZ@Yub1 z@PLYhB%{=!*rXm4W9Y8vfpC1<3{T-jc zTYEu4O3I3gfWl()!~RcMejXscj6;lp@fWw4sJOUT(co}%a%X?_K{Sc4|LI>*A)9^_ zAg94QHSVk0Y0EahKB)^E)a*W}^>hRdgl*Xo)jZo8UT!dvnUzHXkKW$? zYB%fm63N{op})P5PqeTpMn7cD9ZC76wXBPfy744>|T#`hI0zD0jrr z;0kl+pk2e>Um5$B+flCnNsR{JyUNJi^R0SpSmr23Nm_tOEzt1xjPJi?S2uHV;!zhD z6GPzaXKOn^}e8j3}O~-D+t4^5`3%?cgd)8~A;Wj*gCd_l`@wv$KhSwmL##n1D(g@tS_ zCSd?$(YB$oT4p|*^F#~#DSfJGpG-!8!}OQi>jCLw1HJ?_P{zvY8Z_$Mhm4v>^I~3` zM##cIhmb>-1aRb41~Fyh4@2vH&H+HL=Xd5iX;&5QP??l{s5`CW8_~V%UU)-FicuEl zwwRcmaU>M8_#y>WRaN>su9(B0AGVH`T516yf&yhe*Y=XOwGW;5;_RNXvdohw?EV+W z)`cFQmA-MUSqJh!6>Sc|B?JEW0+lX z*xnt{zH4M;r1|ZMdUWGN^B-vbvG3nU!K+zeK4HGRcNcI0D!zgvVk?P*3IxQ!A|ODj z6hq%EI_(>roQzP>jHbA6c$rfE&-$d9bJv}w?BA~#Yo5Jki%{W*bB^FEz&%yZ4|(8H zuL?IxO8AaWPc3I#LTOi@ow_w%T}!HfQ&!;Szsb9hl%d15$M8V9^6bM54ISOOZ-(9o z)z12-2HdVwR?b+rxBIN>?}UPkcsikS^1;cLE#wsu(YA?p z;pOGRAU^D@=B8V8#j=R)!IjBt>ZtB`ZPJGj590r|bFHR_#&z3uAtJuQV_i4--Mh?Z zdrR5|fJhOZ0Tt5{>SA&E6*HiZs6eFJFW2g+Ch;i9$OJ0FM$X-@US3|t2EK&Q2oM;$ z4n^hh=KlUAWM1}?bFECy$<1Yh23gP^vtrzYFiPLOAELyvW=(-;V(B)It*ppbE9tES zQhpP5!U)P0kDoE1V_?L^#s+@*!d;jf1fobtQqqWhaWD^XC;(q|ihrOy?wlRF)sM9I z^x!}4j&JUWrV}3Any7v-^Ex*_f%5@Yy5Bz@M~gNSjsI z4Re7$BV;?69buR`o}-%dbaUcPbXmFmNDI&+=ndGhZZTZsc+qqXRaLjsvdkzFKpg@A z5DcJ-1&TNn%kACW7vH`K0#Ga%SiH{3i3@cpQL)44At+SAz=`$slNC=UYrR~yXYLl# z{TN~4<0F>EnW*);zg(WzK=VVL{cSUF9_foCwl814xL>Ro-&Ipn>(9PN20BS|XQ$G# z&v5ftPZ`&e(&Ve8N<%#Z1KOYO?_}+mSp$9mHZ^qQ5Z|*%E$JgvWK#d?``a7qMx6ju zUCxenSVu^@s%KwEMahp88N00XvlL1L%0)zgf`ePd&JD*{2(ui@e-OFf8U#=xJUBS` z_tsm7i7FCKPR^B~{9md`_x3+o;^Wl2AwV}2p9+K-9AUemrkUnoKe0|)*3dI!OBAMSZ z%-yY+QA7l0!??!18Mu8U%zPGH-hfTMK1$4EjTzQiN=gdIR|kOc5}yOR=q3vQg`n7A z!?xysIwP~1R7OTd#_xBsZ!v}qiX@^+b;K}4f*P0s^-LM6r?N6myvlU|c>3JyoXWvi zgxf&M5GD1t--*!K*_oN0-K)`2eR^?Ef~lz~Y6)+?me$r%vu1QefI~giH0EMp=R=(72Sp!%#v?Ji_Am|0FByctN_7cGGwKYp^ zf3c%t6-UM3njp}iP%vaPl#s(HF#w>J?0avaBojzWOG`Z4#YALNr>VM^aCk7fY*!^* z7jS+Tn-)Ku2AoWblzt$=E}+gKM6{@=2r1teNApD7CUw+M_=@-a6Kf7E_g4nAi9d`3 z0e1n~|HZggWIBX16~*Vaq_)5M3;XWfyMVK90^H&E*|#aP8^(i*c5_#Rw7yhjS+H}~xN`c1&(e1d{xAVQD>?TJ?m z7&eZcQ+)!B-5U7}8!0dl2or{I>bo>Pb1>3t5ZYluu2?gxI2c7WWzkitz z3v?}@$W0W>5JR!GfCFW40Eo2ash1ZO=u4S^MJ)i`iuK)Cpz+|?s+&0VfJT!Ex_8T;KL*rv zbdgYF5$gOBWCc6!N?^cRxf%@6JVQZo1~^6x(!fYpR*~uyN$}9PALznq9^HRIRil7Y zBb%X#1K$3FzGh&Z>L?lo)hB6x$(emzYn+wABMm2_pbs}f_B(eLc0Jx zHFUn{YjrKU7FxdaRZQ%zy-!L;)((Y4K1U@X1i%KuxIkaA05G8a!8G9fA%H{jqty|pq`m1-s_Di*!ur;(Q93HLy z>Dvm&3GJdeD1a@{^82?gZQKUj^YP}*`}gk`3beGegj7~mYI2ZgLTO>WeOq4wG>7RE zlKLN3vMbc+FCtbL;*ygifQf&EEuvrN?Q%~qS9xM`QmfqVVVbmtl8&}Pr4u12S0M5w zxj&4^i#Mm@X7#ad0|^rV_J8ot-aZmWFb@KH8>sMnP&(ceF%B3$hGAYezQW3t1Mbj( zcBJpz8v)!Gy{+1d2akds)~gDm6Y}DQ#hTHveZavDVM>)%}0#G5vS7|E*lF z|NoBU<^LbC7@c3j5tcy=K~OWFy13k(Zt!#2nyR-jHPz7}rxkS~fLZ}S4lK3Y(UH3+ zmEg6l(!Yv|8hSSkny3DwM+q`N0@p{18?~1I00D1 z9BE_d@4UdL;%OQhDjWU%tJqZf+V$(^P*$Mzht<|TFHKO-Y0cBhdI~(`O;HEk&1=^% z#XUEgpgsa~Ml&=VN8bbda5|{&?dvN+$*un%l?ARf?RS_a_1VY*I3_j~71cfcRp`TQUGIKTgc?A%{>ID`o$qp zp-SiJ#KF0_IX-|oRaI3o8JbP4tr*bdL1K*Gov87^2WbTq(>5r-GcXgt91v+7zL}De z0^)7P&|~?(phF{=5Hu8M9a^wHU%;Bm(txHC1`ZBP3BN@f4wQ4GI*k;YCe^%Qknko3 zp);_s@D?{WH{#F0da&^DTm^MCG$DbaJMmUz9Geo*Zn>;8(6j;mVuLwSQc@xVki8US z6C4(y-Rjy}Hb8LC4>uLyyBV-IN-Y$)4J*{_rKf7Wh|&Wt?m)xN(BwdDQXb%}0%-1a@<1C5;sgaPP)nfKGBlr=OI)zk<;5kiN9f>3k-ShUO82A-ZG zvT|~>YioEp+AqQMUUkTS857ficpcycLle&eH40e?MMjULZ$OJIz&8PX>xFiH8??*a z$tBR!3223I04Bm9IZPWQI$krq22T`q=|Le5H^O67#ca$5!fsEcnlU-v6{({c3}|8W zxVX5e|I~U|6gbgADmg8uTaB+jeq^?7_yaHmA4UgmO2&g+jgqflUx4m|nt_En0#|Q# zbMtC~$`!~Az$h@la2w#rx*dbiB4jL!i&tPDA&w(V7^o~6@nF_n85kJ2{d6X07^VZp^rPEF0S zXnzGD{Y;gCh?PsqcSw0Zu@($)RMXTHqm0J37Km)r)YN7`SO9_m(7nO;xSJf==pfRM z==*O@ROe}CpaVv_f{%|6WoZ^(lc!hA2rKh~giTTTSyon-7I>C$`)2UslJar|eSK<# z#egRjMJue5wSkZ}*pRJY(3_c=pFVlgf(y;g&j-UE*ch)Q0<}2|*zudUZwIPeEGO!p z>vg*Uc2i5{L!;s`igj}LSVbJHlhqr*FSX{wBtgId*wGowA_peao@56|-7mnuYXF}V zwAxvaf^Pet-3Dvs{gwc)g_TuE{)601*c2d<-TYm|kC-uFMwCLf;cP45vK6tYKqx?T zG}u(_AMXa2I8RF<#ukX-M{5PeLW${+BmpMM2lAKIlP4A(Q8YGC2xAx}5uAw8g&RUA zWP>6iBI^AhNf#3nGg|*#wBcwDU&3b}XTBr)7l+$Wo)#!NXq3Dr1gK-rBoz&fFmUk@ zhXo3W_w(oT;I$$x;b*$^YfwyDhleSU2nLuBac|zRdK7%t&goX&DnN`m&|4A{6H%^3 z%UFc8hz0_xe;a_bomL`gB*%aR2JFghq4q&Z4}ycS8g>qyAWtWs6htsAINfa>9Twmg zfm$q5WK@lAZ@+GF$K=tYN1zF`!J;B`2SI3n*9P#QUb7o}=^tUT0L>xt$4)|N_z)VBy>+AVMMX5jmmBnF! zEyw|UvJFw=6Vrw56H&}iNw0+v&%r>^0}&VxNe_c{4CXKbn#{O=-*2Sjz~OdT{>c!% z4|@<7Fesp34)_I-4&WE|_IkX;8_srBqxV72#6cmng?1H$2{NFgeO`^wCtm{cCZVO} za&UB1fN+%%BpqC3^L6YtBB7&$(8Y$_R1U%sfCsohGQ)R75UdU`l9{1n2oH)n{sB4$ zdd_m~7h+k21r8OPnzA{90fin|^>kLYp$BnDP{_($Ni@|5wE*0r8jk{SNMPMwK|y^9 z;S#VuU|oq}2WVfw9ihpneUNxmz-?`9Ju@E&Ftj?R70zOA+ zciEdNdLKm$iok&K`I4qz_aTeYU#%dWEsnlHjy!QkQ7PujRSCTsiBc}MMOi&`k zlD>nP1HRfER1*?Z8+57cXK*6OQ1Ui5EBGUY7jrqpG*}?a#V}uV8g- zH%2IL87NQc;|)Sg;S97I+WU`_cuiyAmown!$2``~&P33<)Y7FG+?INq!S;^y7|KSH za3E3w;4~>sIOgi4L?O)@_PXsKux;UlA*KyWh>W&2DMSXMphC;#Xxq}i29(Bj{rVLo zTE+40qZ@xEI7q-5(HX=&hm^MKYK#d(Y

^unAuu&Hg-U22Kk)UjDypAL&Tn#A7aF64?-8=)o51QRt0^FfyWmH$ z`1E`+D93~P0BsAhHv~x8gled~H$Oj*C{nA6O6R$ojBje4MNsj#3@iHAHI$Y6Q>MRv zzxU2`^sA$TgU}}mJcWi;kQ4^@p*?KRhBKj~61g9}0PTnk?4UK*w{PBLmev&&i2xX; zJ%+pjii#@9cO7qGV+2Bp_J8%<+}vhR+0l={6NrHTQ zO<*te*FAGngA@R$LumB$^u!QXSnf-ws&X(FvKzt)3JOXV^WgdbB3Sn#nu2tN9mvrG zDT(jiy#l0+j2=q!5m1Km7^PazzbNu^iM}m*-UpuutOUkxPEP3NX`;bQAi5rpNgd)A zn?d0MREmoIm6H#)26S6&=vcv^A0bd^u=e`1zQ1P`>tNG&w1na#f+Es}VK2yn*W=*e z;GU>8G*7|A^cos;p(XU#9e_GiCS+>@h<;pN=fknmLf;92s2NaQ+Cg{f&wp?OK~B)m zipjZ22MT7zndO~RKnlE*uln? zD&_KZDD5z>!`ZN{2e^iW)`aq8T*{1)qXLppGKf#VB|PvE9Zs=^7#$39pGA*#QYxxk zLG^qcun#c-#q;s;$pAa#x9Mk2dkhIk6U&;Cu#K*_$`B*Dcfe#4PhE+nsXxZ1p>>yJB zr1}!8@1$va!&?>kt70)gS`a(}CO%?EAf^~34dkI|K&kxK5{@sga|R+xy(K8WrOywY zK)151cR2YBsfQ9UTiV*%Y%m3oGz+c8zK0gkV(Rm*LFh+cOzLa89)nGR2l*96clR=( zH<~7Ey6A?6hKPS-D+yEB4vq)%mLZ{`h|bM>o)rL<7LXh|Vh6v6ScCFGR{^XcBpyT& z^aXoI$?}65doaXK;J_8fF@oW~XY1oKTK7!w)BSWFk5(U0Wf379-Wo`9Gk78X{{C=A zUlbQRXVel?DndmChw=r)3NpbP*RDBH*b58{v@p~&Sxa4?nCwH%AjwU48u0$bdO(#b z{mtQ0%msW3Qj{;$*yW;Wh4Fe*D~I{LXdoW*qP|`dRLjF|56~~=;3cteXaETyHVq_{ zRZtkjnOL)^5t_ow=kaTHhGd-_dM15;23x0`I^bQ5y!c?+uhdu~hvE2bw3n$w6S(wm@$L=^)lW z;3LQTv^2!0P4E0 z+I7HUAeR{m4XI^#_~G)eec&a4J@4HwNPGVtSs3uKW;Z8mk!l4Hli&I$z3)yt)iKZE zS8$-WhxJU&!6`D>L&`n07)x+mk*M6-@4`sf#PtoDTp4N87DLJHX4S*3PsIp|r!=^1Ci519!>ob&?c;YDO5K1BJos$Cy*Zb3E>3AW~^#)HKKLdR9;x`=6o+y!sr8LP1# zB$DKy7T^tY&B1N}2x$q%xcAu+KiJ&J!34Giz%GhzQ8a znn`s8ehEyP_TaEL8$bNG_s0*kh4=lGl$6(v=#ZQ0UWa`F$>oI2@s&m76+@FnAVmGo z6BY>;8$71&8&!(`dx*nz-_W|?F#tq;0|f|B+zeENtP~_2JHRO+rl1G~CnE!BWR=@;%+IrF|I=Ik z$BR^mWru(&fOjEKQCPXSRMp!(3ax|tf!uM2*XkFPu9vEwxA94$))|C|A(w>Vh#9ZYo`}$_^D=AqVNdS{d zG2V53L7xZpZpI~yQoa#TWRP%9HiWBKA$qwi00Flq2z?pEFp zkz(F326)6qHb?%JmwlB%LAGXARx|$wALHw*Se*OM;YLvB7VHszsSPHe>nu zRj0#AN4e_EA2tZUm%i`0*(O~$KC`K3$|^n*tm8LpeVMwIhA%UOY_c+9*_-Nrp%4D2 z3qlVDOK=9Bu|d$Cxy_O3o(sSwB(nxhBMU*t<@Qu4Gbkd*zh_gO8=8S+X2UZ#=atkI zQ7e<0w2kDd-37ih(tfd-JI*fgkKV037Z>;@NJ2w(tTAb%lpfVS)lNn1CqStfhDH6m z4jmIEXJwu|z>aF-%Tni)7bE!5C$l8LfblONOzx7AYwQ(OFuq|{iS-4IeEQi|&_b$r z8u_)o3%{-fSF7vSn3vXzUCOl|xA{uqJ$lV}mpzG1P|BqE0ERd|3qypOW@clPH~ixT+p#(+>N3B9foh8WM)cI@@Rrr%CsZ}PXD^ZlWz$8uQpQq7kNY$*oe1R{ z{m!RerLMPhkCXoXIf7gCo2yLph|CF#C>D37uH9|aTO+vp85^nKCPot-Q}*NBPjf%R z>Z@E>NDePx7YJHHQU+4(CC~PrxWlfTu7gKxs$3WGpz`QAc-+Xm%xfek zOT5(@d`J4;>c8gZlD{KV0->3h_PNr}8lx;5E)3pT+gL{T9(fnih1?)|Q)++lOd>}0 zxALWZC*heN*HtFZvcAe*tAsnA47VdYOI0MqQ-8JVFx2^OXV4KutMs`QW^SY=nJ9U( z|C^WyKDJ(-pW1sKdmgZyC3?Pe@{pQ_b|w{1Ku4k)cU)b)wEqw3;esNcs>TcZOoB_X zcz41`rKR)Jrpv<+Wiek&y`(Fudw6dC6icCgav01H4y3soN{A0XOt*gN_}1T2udw4N z&+;9f7K55Mq?)wNZ6OT;Oz`B#AfICFroM~Hc5$jeR1>f=B$tJF=ZF{#;$jq}(9QZk z-bDq1x!pWA_K21QQsoydP=*nU3c^D~5ZjXl`5p-lB;0|X42e`qNRmQgK?)&Mfc!E6 z3tj&jEx+|*a6d<1`_=Aa2RqxS4Q{lAYI?%jhS!JoVsi#o5;5nVgkz7;brUAdYaioh zHl$+@uU#K>y~$PH;(tG&3_k#?*5SvY)KIf(DvMM}ndmb=x-(Lf*5=nTs&6KeCihb?-dt<7e4h}yCYNFZ=8w=}FvRzeOy&aK+=|r8H zAjkrCMF(^f^8|aL2*?^Csus8dEP&~>DxHiW_^QqIHo_!bdnZT#g^38Upf+{pFtc;A z0gaq37dA>K<<;p+j7JG8S7k3rg-N3b!MSEm!mW=N)=pLkhY&bhB$HIn+IYAUSp# zWmnCaq>hBA7JD96*0}H%>(*KL`BlL>ESN3MA%fd78lnw()6VuD^AARq*i5c93)4{7 zu+hnRomKWeAZuEpA$B&FP2gc)b8LJjl}oW5uH+|?@|bUwmJYgWITK9puV;RA z^8Ni=fWwfZWZhHj&jM|szp*98Mjo47;%w^VN!)bBG{^K+A}datOXh0L!oW4Ef8ov& z*fLFBtau&&NUE~E2~v(O*OmKf*A$j>1`E)84a+#jGQ2up_Z@q+U?z)P62C?zZ%z}* zpM48ZE=~zcUti-Lpj$OJemz8Nb z&3D}%Si;V!Ton)b%4cCzwmluTDvnJ)UTPWa2E4wq^LQn zxsz@=%@K_yla&}f;ts~`hkZ2tjqxQ8(+vxa#Hv9fBoQe(`z>jIqMwR?Bj}(BHP21K z);!tY5?*?;bu>@?I8Rv9q+J+~?%Qy_R+xg$_UPvw^Fnm(#{xQh z8bJ>}X)ibU{92j#NBgftT>>lkk3a7)-jV4|Wr4v>KTOeZ!iXfg6+fo@zJY}soLwR| zS7r!Tev0`^@IA(}T3rp4ZJwDYgol!+XqzwFz{KrWl1&t;eux@?!ga_)wLv({%C)f zzjOzYtzlg)EDE==x*6xZ1Sysb4QD@0;K+6{zr~&KNye;8|E9JObIaR(%QRWvpt+ZQ zd8Bc=Y+aTrZSsg~DCp~Rj@XnKKu`1y6=*ajoHlbhShK>0=6&<3$xMgXn>o@v4@Z9X zbl3jP)Nrh-uzQrMF&vn;I zhQT*GcUesk=b0rRDg#~pUHo1{Qjz_MI-P^;&hZws zBEXwVvs}}1q=1n%*y2|91lKC37%3G{sRS{Sus1Q&9RBM zCpoFtd*Z(I$|jnx;>oyAu26}>q?CU+yDA#|-mTVATfpG@b<5ZJxUz*tv-?e`-+esu zUD7?V)Krn{aW_z?i+ZVGS=-Nhhk7dG#2pGxtO#_z)67Y^_FzeT?|o6Ka%COM(M;#B zW!aT}E^#ug!>k$mEyJ%Kx7dsSv^=)537kb+)Uxbk?+Q_y_kDv-()H>vQbP zJDxu;nWoWsoPJhPEGRSzYLUv7Q~qOTX^gXV#b#Wi$pM{y{o_GPxf)im1M^J4KfbQS zS+fId<+3i0(vw9mvc>6j!MxsH(;hLmY3?VuGz6EI*NNFsh3UB;yT5ggz8k~CUCxXR z_&_7!^Lpm(MK1~TFoIx9xd6`exF8a{g#3r-!oHHP*tQ+5k8TiLN`JiK z`twG#OxvR5gzA0n3f-&#jmyx+OR=H^C+1>~^TweMNza0bWair`290n3uPlIjv@KUT zb)aUDyYa7@qI1#a@&3iT>RX~F+Uwy2cf6*tu2j6JTk&F7vB>+~GIh-+MlFlRTUqk; z)WqgGZ_X=63bjJ%D|@f~BcF!_s^RF=o8uNN<7IPvjs4QwnEFm?!Heu+>D~E}nyEU` zNV_nBpxv*hHd^0Fa1#p9`s<{&OfK-p9(~sKFecA^BgS~5yOG}bD1}c#fMM=({Ell6 zuDtfZ+69@s$0PJrPCtDD_~j?0W!rFg%6b)OE* z%f(-jQq94}Aoj_axZ4%adB^zHboNY^kmnVjhrFi>m)*^Jo4<)rGM+(&%%P`gFGtoN zI+6Z4iEod}GFKKI3sP$rlFRcF;L#q@4vu3 zdV4%^)MH01p=*Z&t)08^6I zb!BYh4X*s|lzhJPK2h-r#pbYG-G|>zR1@6Y%&sr?#BZQpE207o{))|Zm$r8O><%$X z`RqZsvRC7srn2B2A80{TW@L}SREQG5Um3K`?)rZCPH6KO*D3vyZ0u2r_XP(Um*8II zQeF1oOEesc?o_eSko+9}uL6Q`DRm57f^zOBPb5g^ALdlrR>$J)DVREJncI~6=TS`H zX%Ct#vS++Xp!&p)wnvdd%SKo4w)s9Vch^*ZG#MR5Di~jN(R&rew`51D+j!&YeJ9NJ zdBf}a-!B&$2K1)oDQb;GBjYL=&3~ABWj~T!ugQ3FhpsSHL5}T?Sj;<=Ae|vK=hQ@8 zx=Er*j1n%#Kbe~&k*C)ejU|=x7oa9m`MkwW<Y>tON=HNUVxD>aK&Y$ug}&^9{P&KON1L`$g1 zOBpqi7`rme@g_6OXk54>%->?;%3ZQ{bXJa*>(p0BFrO^>1`xAHVJI_UeV8`*whs@@ zN^M1DE~icQd+ypLI6vWie@Y&nR^53Uc`MFc3_b4w1?Oeu3IeYqQTUhHh# zbNyV&lwpr=j+aTP_I-HH2P)>4Ys+X(Mp7}JlHm{gQuA3B{gpXH7SyO%!=!kP#%vts z@9{Y^hQ=>*65b%;oJ}3$`8y?RbmqXZAg~}3z+k*K`pKvKnE0V@^NYZsTN#Eby92Sp zA>n_*-q?^lyur?_!|rJ0keTtNZTIV&i>%@`iKyQn&P$JmB&@8Ll zFMbtfxZZspyFC~yh$~;#e@`jz?H#fDXT;%TMDy#T#c^-aGKcB%A5t<^=tR3aU$wr; zDOUHSXjM}=m{vAZK_?AUi=3`$#QP7+&71gO>Ovq4J`*J6Q|o|1&Oso-`BRf0g>9^_ znz9N43zh0qOY08TS{oCyZxwxT>bjAHmciKc%ZfbxDX%PcNm=;zNVnow3%Y#awre2~ zc`WmL2E(P;A!{MK8=A%@w2xjc`e2qg)AG5hroE3frf3!aW8t@u)+n-6uuQj`ATVp< zZQJ7KH?ABV5s@dtkH!*xbL;AG!b37L%snet`o@$P-)j07vSC3tUYMgZpW5)n^wiuP z8VGHWe}U$h+9IP@d!uGu#hveYpdcmVd6BV# zzvN%nd;7s+>1P2dFitF&Usn1GiuuR0h&;*EQd6zlH!MctH@97jE)I1J<+a+HpJSKa z+Oek=zMnr$w|idRD7Q*(cPb8|n^rqNT84pM0>kH<1vNKtJYC8{(sD?&)2-NKyhIl= zQ7@TgdGK)hC!>R}n3R+YDNrf(*ws5b*6ti(QKmi0mMBVksT3zxLPHI2CD()*VrT59 zpYz$~_{Xo>#uxQJnYvwHGPm4S?o4fYw$E_;!Ftl7!$=i_?mvTkqXXPb)X)gw@mkXp zGtlY?gFK}f;&fgAZD8F`ubf%kqcp1V=In7!lgw$(<8tyo{d6>HMzoM=R~cwTzFpeP zNMJjTlk|(n{s7bQF7eF#MsP42k;UWUgpz(|1AUWX_5fsfjuu2X!(!22v>#%)J8y;- zCKrj27?R5y2|o)fKJ;O`era2;pQ)>-L?!i8ihAQVwPDd_d*Ao*xHpr5A`2jLW!kke z5)W4Kv`))rEqV5f3dsITe=W?QaeP*8@kXEfG#2gXkzJIi>9=;)0(S5CLvk5OgG|c6 z5AsBUwMU%Kuz1XO$Myi?Y zO`k|RZ;y|ky|OBMWS`Gb%J^EFYmJ0vPDs9*g$V=@t!I2_8Foo=yY#qvmw~pHY-K#l z!o&3{52L@Wn94z>r5PKQh_Jsod3Nup*gzjFwgpHqXg4}NHvz$K3wVfAAZg)mwD0}Qnw!!{?*eN zAT9rq^8@YaW5hVi+?^rj{w(5}g3jOh<|%K*Q`4yDr2#YD8K+N~tcYGZExH+})S<#H zMP}Et<+>A+P(SW(@)%@#M*JeC*O2%!aWnX)9E^x0AqE%cVPvng>vqq`$j-G}N;zk9 zLC>q)I0;*ycm`G{xP<ex5D!kh_wwQ(`STsgf(u*LiJkVLu}Y3>3TMFg0Lpxo-Co z-r%AGGlc6)_G3?vgX`1+^^$7b1ZFmw*eILfVJrxo%pd=fSXsN*J$Vt`DO$49%eq)v zam%{re|}e`^WKV%xkAszlWYk+PUi`u&uUVysNbz*xVYJd{?XIe6gjqct<)bml~U0y zlfU)+zYj-3F1YsIw;ElnPjcsey=bX`Bu0h69n^um{HOk(ZT}65#S&eXLia(>24(fT zee!Q!CW_xy2&vztMFtZwB_u~o^m&`;3L@&Z4mO-!*S9_7RjwRbjw!~~YFdSdt8+Fl z1CmXG;OxcXu)3oiN%U!{%Ky2}q~z91Q9ggVYMT;02Q^Ll#tJqzq&#KGh-`Q0Mj50>N67RFP#e2!=gAwute3$GRJ1VR=dRPhRP#Ru<$GiABkk!>hr|SEhP`Ns$hcSS-C2=Bf zW+|37#t5IN7oz3;3!D&pj#pG~GGbzA8uLwb?W(Y5VGv5I=lxm@& zre+{mt4Kl`z74scHpo7I9(v-(IxyVYb(}tX9^glK^SP3G_<#0`hkme{47?#9Vo9E? z=0?1Hc^8uQ$OjA|A6uj^4N4O7;58_3$Wva31G>-^2d~O^Pd$hZRTo@YTelY@`0tkH zq01|%V{45gc}4Ji9sZQvd$hE+_6owK%-62T7dAjHL+X700c12EJQ4vLT{%bl3@pb4 zyKmPZEQ6$t;OSvV2y&hNL&l#Mw*9Pdjn;@r+23CZd7ukY9tpc(56^i z&YK{>#>R%^9tIsL%Tp5)a(0osVg1}kYlQwLE)K2Q=qfyyp5O}q0%In@JG!z`6g=o~ zPzAwbV1`GgF3-0D;IVCkRS`6}PfP$P-AQR_#uU>G$R5&~{5?Y&%ld*jk3KRpPgi#V zaV6k`WClK|)2*zpTf)#B^+zMFsWXjh_ z%Rju_-*+!(jADfc01-#^gLFU)e8K^e8rIGy2RkXji3~+RKmgCSefqzQB(u2qH3Sfl zj8l#T+%triIQ9%+_{hhKKpN%wzYX|!0Oy49Ysld3lxY_k zRftgxQAQ-I3*H<&8-+r_9+6X8-ujsV9}0qqj_?Fgunt!{{CEeF60I#QnG=pk^&!#1 zn8eO5nyVQ^-qM3EedKdd^A9xSgt4;zfK!5eTfRID_kTXRZkU$+e?GBJ$3FRgKJiYg z&^ESbTl;wN`@(I1L55@3XZar2#AN2#i3Zv`FiuF%4WeR@EmKBFM$GkVI6F7Fde4s2 zda+-)77L?5d1Xbq`CWtk%-{|oHDS#CdZwdeVlz?d&2C>i2*J)^tGl;{YsGw-XsU~Z~heCptI#-sbmx=B~J z>KZQ@$$!7fakV*fH?F7u?*(NL*d?Sy)>5(-C8Nl`6d@Fe%9iXZWlEALYl-X`Qe??m zX(CICq_jNmGxNLm*L~m5^T#uPx$DcCKIb*-R`DFtu^UWsfH?%e!ygcD^Y;?og&p3vV@6e;% zoOG-*k6XV6Cue`kc;iE@F z$TzL2T!!b;*he{bR+A^Uf{y_EzY*iZ+PBl!yl2R<(9mIzW~*8^`u_cU$DO0BhNKrH zo_LhCT%oZdKTQMTSPs%Z^o@<6U2hS$L;Fx{CF>HD)*!7)^I={3SR%Z9hL=~?wnK(^bJd%*Tc8J-S^l+HeLvIsev4jjheJMZ zfDIXRmxi7;EEZp=mf8#b3R_#N@7%q66G2e%Fx$HCio;wPVTr-3uPzQ+fA!X_<2ME& ztVG;2y@w^diSJky%-HXRrQnoDIjYv((E2JA@WjRFZ~yi9sHNPfi|BR?KN5}e2s?4U z^Eht?$LBrRf(Op#PCDWC-6{t5(AS?;$EI$=T#b_uwDN}5lm|^KUtj%ATfCIo@(q4b zxdT4=M}5glcfFr^KzR;q%z2p+Tc>$)FU6&>Y3(^MH`j>l+h`QT+(U;BJxZ9XlT?Mu4<6b;s9!iehZ#W++fqw4( z(4k|39WSqpwb@-RR4HMiv~4S@2Ut$a_m}ox@XVXzasAGnhIld|qrHk?Yfk|zGL_+~ zwhocw_I4%OH7T)Xhv9YW5;+D8o>8ocJ?u&b!zS&H5{$~)cEaBM&S z>E}+KPTG9sXWG4c*P%#c@LF=6c>tM=ohk9kI312NR_dciEuk=S=@YUac14l|U7I9U z2d?ZbXft1NAacg9dnav2Y{>7o1c+u=-DEX~sxj{@99gDk`ix=M=(kW{e+<&p>yi)Z;3`;b6_UtK~G~vliHll`t5sCb9)3m|- z&cKzQo^6n4jthSW3q8a|j-EBdOS#+-vVOkZ)Vvqwo_vxqp09C4K_)32I}x0SWr#s1 z^|bj=;zZLUh8-pzUlp9CRH&J!A&pm-+%rPLfHGm;#i1ea|KG-*f1OGRqJ}6PT}*k10>@%#U zY~p_L;ub@O46){JDR+E;6<>ZOk?sZ@)NRNQHMvzcudMy$|K!ONCkS-W?mefTF;@=# zk@olBC$kTBrhn+#zrTHw=eOg?$)=ay+PKL=P1Ry;RjT8rO);heNqt(0_@UIJ=AJToQE?3do6B$d7>W_{!HXZKG>7DWVwZr$)QoFsS zpI_KlVh6kDcC%*9a-n7IxsCM@^>_3I2#+?-unH2NaR6-cJf>P#BE+}f>+9p=r?>%A z+~qlL80L!>6)j!g6Sk=O%$YMSe9gj6p2bu`&HLxOyDh2e#atv8BC%cWK%R{#WNVd| zbKk!02($0X&iofnCa-^mO}yNRlq8B$k#` zJP6m{d+DELsCZ%BZ{V?#xx8Oc&~5SRgAXyg*zrxjrfqtgPfda9{IAuDoH4Nre)a|0 zyWIm97nf@f9-J~Oq6=&rdUG589vdpXhB>C(T5YQ6sH3A)uUC6IZSjgygkxA@4 ze7GqSi$DH2yENtYt}o8+?y>DN?UGK;>tpU|8iIgIoE7rRQ}5jylYaybyL=G1ZhKt9 z^`)~gHde#&LSvn_d2NytVHnhp9Q32mvaa91-;@V@i@ra$sQRl0i-(Z8?r_NNR-Ob1 z=(l5%=0Y}=0$$=^Xs>Ic+&Vt~us;@F>VpOqc6u(pF&w_$s}}d~)e9@k;TN6b8hQ2} z82Dza<&_x@Zdz-%D%Z{K=ewb2*Q}OmS&i=Ptvc-Wu0XTH(}^u~{dK)d)-L(e$*Cmm zhm~7_9ks&aY<;U~C^)^T2l9^ARCa}2Ivc0}M@uz457)X}c6nu~>eIVu`jPZVlcI59 z6K9cw$u8%zFYan}OXoy$Qt?z=82@>2Sd5qP6mo)mOrq{4HYYfcgk08IyLRpJFE865 zeGqqzGt$f8Lqs>Xd-t)ubLf!#!lx7t?dj?1>9=j$2zt?bS_iw#7`NJg|KPYQXI`J` z_9`cTMsa@NR==nN$tDHI-nWk0*%^0(dKDEF1D5^M1P*Oog}%Q2Ra{sM5*)W&ym;|U zgRu8CBQc#pzijY1V+jTz$R6H4nlRLEub!Ts-KFFbgY*LNUP>I;6Yb)}fj_=}y~^o) z3-2?}xMh|GwI4rXHtceRiv7W~`}ciq4J^?2daYe+jK$j0^BVZKin|pN4L1I>GcJwM zMG9M6aTRvDC`o*Ld=v^VFR#-JUk$o&`Lf}bSv~MTfst$9GjZ^u*CyD>ima5vX37<> zfq*oUZS22wwNAb?U|Q;f%!bX6z6+1AwNh0K9^<@xNy5p}XMTR~_4LLKFsex0ybI&& zmO|F%UKzu#QifoLnxO5#fdk2;Hon;v^x4=5*FjkRultR}m*^3p_pWp8cu+;EBCf3x zPM;xWihN-hSK`%E2)j_x)cAaa67fJHIvaczcS~a6Z8TlRXkNLna0JtA?Q`#!*zU z)qJDSxL>@uZ1rnyImXo3enV3>b8zTDzS&GZcXsXZX-mn8^qbr*y6)jm!1E6x)QO3j z2oNA*+VxXBp8Q|nkhzgv((U$+JvA()ueDopM~h_bn*QgFTB_=X8>}wQt%nwnfT9cz zFmB7Ft}yX1rgegi_p<6x;hr<=9Qu|fO+GqrTyv?|cM#4(7a#ue>C*!0wW~L8M%v6@ z`{<4Bn&IZlRlnq1dR8Y~zZP3NF^=+ro%c|6Q% z`Ni7<0^(^O_Vv5spRa%dBXszTA=@baJU4?SJ{yKQ?z zrE~^!G>CZ3MH`5QglEgv!Pa7Fo6dht9)82|VAtW^@1ELu&p4iQwYH2m&G6TmqS$k= zDOK7Gm*lVET9)~J6TZ*dRQzk27`FKU{z+y%43Y_H|Sh!M+|VyrY7>U%c=(pI?2Vw*A#Q-giPUw?1omLxh*f+J)+$ z>eWJyZZsHop#M$#ryQ}?W1G&UW22L`EuJ;5MM<(O=O59>i)KaQncQW=2Q-M-wJSX& z^OyCzIENkYzIjh7%J%n{k5Skyc51zkyeES=dM!~XiE$OOh(20Jew_KJlGLu*_Lnvl zA76b<#mePzCZ93e(HRFisKWS9JhP#0gH$Y=m6VhyJD&`*E%iR7 z^l)Ls@naoz!-A9i$ zr*xYE-I^ME1Sw@U#!nP0VxNpbQQUH;Ns~&xeG7|=bNpD=6lOXO6P)${%yUIe1|j_G zw?{8$JlMFg&ifA^Tt8LYmhnuCaz)j(_}342j?bdI>b+--zr5Yc^UCoNI=XIOJGXdy zHeF?RKw~cV?fw35%so6Vq?Pj%uoaHl^Z8ARk%OibF> zR2B>s26q1W?-uUhW7jcz1K7i(vvprL_3E;enr<(iQ(vsBDzz6odt^f9^o$R~EgFk4 zI6#^?A5uv{0+hauCQ*f_8#9o`=t?J_YgyY(iU}Y|*@ruvc8kk#(T?Q9^ z=N{VueJV%@;!aIZDK~{$78(P<0{AZf zkXcVE(2K7#GSv8_*KxS$7;GilQWQ|ZgP05NLsu!E8%#s|w8^@Yju;hwP-)zF1olJN zAcU4oRdXFzXDQC{!B1%<7QefrCdp4^u**G41OLbI*xTNB%*_v+^DtIzfAWBjx}Ns6 zYkFndZ9{0E#Ccx0q7=;_8b4Hq*jAL6&&G^FfEiL0xUdRVXMW-M(vN(LPUZ&bLA3Rh z&cTa5t)jINv*lN>X2#B>kS7r;6cW>UPTsy)lyVbc3+jcd*)#l1CFSk0bE3Zdc>g0FCYEK8)}+r9JsAlPsjQOvmlV*9Uf@^) z2lHB+w`^%3{d{oxK-_GGR()@Y9Dft5GgNMlZR#DRM`B-rm4M(D{N+She*Ah&EAF?h z!Wb>Lv$6V}#y9E~H!v@H^*Vg|g@eyblak-J`qHI?-_X!_rt5{9UcQ`B4AkL7`UrlZ z(yW!mF!j>X#_O9lJGEKi7-kvNA(tF&ZmoWk2R$BO%W2M>h>}Xel?A~lcs{@A+G_pd zDQD(v`_G?aO}^kG|Gu!$06$5W=b#7Xw2=AnI4%xg@)5ib`B@ik>a_ez zNl7LdxQO985%u$Dv9p?YYYh#%+%>(iJiC7V`aQ2VHwUt(L4@-b@n6&Dk@IvFPfhA9S{1 z&>3wd#f=sr|DeYjH!Z34!>0{OzBOJSd-q6lg(w+Nar)3b)KzeNMde4>FgUNJso8+c z&EAD>_hhFEe^E^<=~q5U{+Sk{C$6!>hbytJBf9&eM~_DEya#_){{VbYQbu1b@){m{ z+cfrV*q!?DI{C6y6V4Czt*6oq4jnmi#Na7FM<$4oMyxct%C4 zEy&mpb5Y!l8aK|)L7Q5Zyy%PvhiGDmPHB}*6ti%2;9C*0rBc)uQw@v%sX}E-hU_CK zFUJE=0E*ZKH)!`hUwHq=cAPF@eYE^H#Yf2eVKOdQ_;IlLsPz9=(2kU!rI|DR3Q~i>us$b z_Sauq(1eSN`yfOGR*{8b80E2WV!c>$KRmDV(XcT)e?zzl4p@L7b1j|bv3{313U^2uiA06rqJ_<#b>Hd*#*GP z2e%de=}N21&vrDBqcu1ENC28Yo7hTro1`+S>!WI2wK%W1sT`I%7H(I-p=Q!WT=jLvB}p{P;2E^NTE=0upU) zNqUB1r7q{5##`Y{Yo=e&V~_e9g#QwRX+8PrBRx-rhAOb8v_3aUL%O}e3o(W1?DBVI z8C@S2MZiIj{-`AXxGx`o^yxEl890gtmZ%+E07U2O?X3_wzcU6P-MXLP|HYSfr9n|OcOYuuO#?vGc~$N+67w;H@U zVIkfxyoLE4!z;pO=B=Fae7@d+F^yBQ3Rg>b?wTRd3=4Vg!Fw0#TE93jqzHQZAEEGb6L^@sNRUaSv;_bHi9b2p9y5d|2i=~ zl{E_(XD!C_UDO?1syp}Z-=v5Uu_2>UI)GNxLZ@tc=ObnXtTbqOhg-ep*`sB2_R{rY z5_L`9xzu85WrJp2Rd{%M&IY+zneD3t2tqs+z4|fU)Ny@gfZc$=WV3sKUYj~|q%&oZwy4a7) zD;HBD4ohI_;z;8|#&IT`O{vH{-dVn+e)A67*Hu+qql2X<-q^9BOn{~00^yZ=f`V_D zxw%itAPjHu7#D{X5mAeTUG(wyj8bj@qS7|kS4il3z3!L$cI}~@sSa|p3B0piU8m-! z&&B|VYi7ZZHpUj!i-8q2mD~QOoQym!47NN&Ua$VxgJv-mKW-or%t|)_S&O_L5luS~ z>{M=yxS5IrMvJY=H0`$p$v9zWC|Yru5->tZw@TT9knlQDO58Mpf|^UwkdqTOc@ZU) z)C89=55}-`v!K1m?1$yl>c|Dq#&P=?s!{PV=8iPS&B*HZ$fnymUNM<B-XTTwkDK6vv8mh^?3OXA!(utvJb4dWW1pg*T zOnj6xv0hSj#YaqzTT-@!ewgTXh?n=b2{E~VDF zfeTNs*{RA+E$wU6Czv~BIzPK^++Mvy)jC{QTd^*hIyt0y?!+xPFIF20o}&o63*473Q+5~1Ak=^k$2 zF*bjwXqUMNy%;m0XSRCTAc6^*Tf1691B?b{czzuldazVtnkPJi>{BOxcgy{$sx$G% z1+0qd+zv+^-22{V#e~%F1ksK@w(y_Hy8c+fK2VCmD%&E}HMQfwfb3 zzqAp=eNz24supKT+)*fM#cGsmA&)d0F&^c}MbNu(`}PP`TDa6!o@~IiMr7=jc1yzqRPcs1G)JI{wK-P3_4$pNbE0gmqX^|T&Cz+2{@jZH) z)^OFw4i?w{LWD046HFxcJga7!hJny-w>3s6q|#b3P1tgF0`{bjhh%cBE2 zQ!J{BYO0&ho;_PY8O&e4{&=y-8NY@D&0yw4jrCu{ z;>j$JIay=xv_&Lde&ck>6h}vG1+!SjsFU)Rq~$Lqz$lPQb|NL$0hz?SP0SzgwfRY# zFed37DT*qsZwKem{<%*wGENk2-c+PEM z%5;1Lh{Or&f2pSp{Vq%{oDT=}!hwj_b**_YN1_0e9(|^fKE#D_TPzq?$NYFE^t^A*M zUNu!^Y2UKi^xt{z#J3Tr1w^A@0U{RXxyV{hS!TnTTbtFXf6nafgr`;}M*hRDUfs*# zxnI=k4fy7$&g+{u{RP3m>ae|yIo?VG&qw_16)OT!-l50$_iBs=l+?R4k!y%L%W{pTwoE@9f`ToAdA2+W`xLI*$u*c8NnojXchbumH56F3RZL)KG zavFB*cB@<9b3FOOIapfak@@MeQ&b?XMqpl9@y7%2|!I2(?P zd~ol2Y}&p1_s6+hk;O@i`HXH`4@O;y`qsz~6G;53zuwezd`m1A3pz>)UY+h6yRgAb z7u*JoB`CGEpJp^_W>J+wA%#|X3p>6hl=7Kf5cxMsbZw$-%YWq#X?rt~ zVFilS<3if?sP-SZq=F96Z>Vp7SSo24xqR zzP2=ZSTqvd-(p;twe(F0$-6nv#gwyin=7RzB|iq|X*&m5xlj#zT3=$EzO#wYLxLOX6%g7bsM}e z{L&~@#|{riwcQ;YT#tU9l#>dHRW6iu-~SE^zix2;d|BZLn$Q@?UH&@JcWf<9*e~iyuiyN&@M(``wcNq9$RK3$N1CqjM@?tWtzyxPY&;NO zl*5O$@O^?Xw9O!~mQ(lmVV^t7MCn@Ug;@V^QYDj8R1IIcbm`VoL6sy*kJ;OuZ7A)5 z3O3WWK?6FS`I}DH*SzV{PN`ID*KpY7akkle5!|0d$R-i%n4y$S;`dD@kamV1K9?>p z%^}eL+|(8e7mP5sU2j)5(#PC(*#))P_uUo`=}7nxCr}_V9^=@^BsdeC z79|DVHCq_e%m%6wdLpx(9sd$7-9`m!4{bJFRfY=+d=Cu|^l=T(iSmo;()WUL=T4pK zQbf?i8|3pZ8Xv>r15~NbFjOcjG^LioDT#_Whao+bQUzOL>TqxKStfPLzldTYs$v|D zJe@{vpJ-Ua880R>hRb!u7=a>IT1(D=%i&*TMbr+7PYKux1q&92gYz>UwMtrreK`ck zn9s?$U5;Bx}|!pmUgL&yI8&)4Sr`KJemHQpGyw7Y-!w5dDUHFR=Hd>v%u=8Uab zc@!i|QR#oLa{~>tx+sK=1+y|O4+Uu#9nz0!ctev*D$iCcTq+$W?Uu|05;dFeknMLg zGQAiBilr!o)5+0mfQu65KOOJ6fUKf;`1I+PfPirUw+{eu6E_rB@v_(|p0C`KpNs`0 z8^3$^t|K7N=Rvh}ny{Fd{g%gbm$b~%Yy^d^5gkftd=`VljkUBq)4_#c{&8Lrgk`p;j46& zmgjhFrNF1!z<}4P`8Iw$k#E+bML|GVLPDYgO0hBli5r1UDYl#5H0#bUFDwyc!8Z3` z-}$X*u|iG0a9ntb#L87%j&T6N$7z3BUpvVb^ApHAmY4(t=C)7q5Km#(bWp z&|}9ctVhdOxuLZ>_m8SG@~+F(yTQIGLDQys%&=4+_jhbyhfQJ_g-@Q;WWQlweZ#WNyDGsklCVGbfUyn3N7qthHJFx@WaEd=PbY^{72U1CnBfWNf>7 zp@SZ274Odr%RjV`ayH%cR};V;WpB5jgQKSRy`wYb|AwlY4;ahQzXsMkP=527`EM90!e!qRKE8}X7+qQoG;sx0{)xw~_y9?Rs9^6W^F_oR#{IAEq z`l}9dT(iuc|I$&YD!o_#vGGE3(@}lO-mHcIDPvh8q->{fE1*rmyy=9=skB44sj0~k zCx(O;{PT~lj?Qq;Pw(E@=8mXjtRbAV-FNv^%q5_6|1^sn@6z?ks`n3v=U?Dx(XPbp zB?E4&zaPI2w)A;7H~krUV;Rtv!EMT15aiKYYeM&)A4mD&W}U>0BlMw%(H&`VSTM_! zL!23@-`!S-n-wgA9%YmC)jd8ZKDGT(VBqBH8eO^8^;=iGXZ=Wa%%oUId+vsZAoPTx} zTI~mfCj=?it-HypG=ls4Z_VN5l_WJscV;&CUX8TNqpEKEvh#8>T5RN}Dpkip)+aS1 zHgmmnSzeVNyl0P(@a&m0y- z*NtEyOB;>gc^dj(@dLYs`)qPu-@fSYNR_#X5v%|C3FcNVRY@ht6JS=e_MqvfOLdAi z*F962`SE?B=95mkC$85r*Inz*tQ-1j)HqA33711Y<2lCv2cG-?3v#o|0mzuj!zZq} zdHZ(EA%V{MpuOOd;J7{`GJtR{00utO?1D6wwID2m!g^BwBZGLdCFswIn_pqU3|Cdg zNa{qE(Q1I*CQ$3lzzhP`wlcR}QCsX3D(yGhe9tvxr%`&`&L2RJA_E8xA)u#$NPL+M zm-E?$C7met|4Ld3jE*DW2u7NV-(FQBkk`e2cmqB_80AJR)rO|R@_{^T)(!IMvY7DN zZ@Y(I3q`0jH!=sKYiXiw+J5OpcGbw7IV&hU)7%;qmO@()jo7`=Pw=y_`@Uf^>&W3eZW!&*@f16@XGG^p);OXoKt6VoI?O1bT$N2 zvB!kMD+4kSzQDcQk+znpg}~uZq`TegK&MSHASPfvdNSe0VG@Q~82oP@|3>R-s#XDB z7^c_!dQ-DGF3yN*LtF;Av@k}^A`dh99HHbAKk+|Y09EDuV(PFAJXfVjT)cd_OkQz5 z41jXj?IQz5MupPWQ!<*74#nd$znnMFaHIEsM@?PsQq$0{u<7s15ap2@kLt}NZJn~B z?~}SVYLF1qj%(X^v%(zD^7Lf!LYCzV`A>tcl8^bip>my4IGQE{93rRROUo54g_OX)Oz@U ze0qcQo_PRK#WoxIq^;{QG%4&+xJB`)h!S+Xu{b@7Wg^GcF1rz=0RpM` zb}}Xp`BiOR8vG?Dh8}F^ametS%9`c=);ZTv@*UEk?_m#hF}DX%iZJPCY;0Y9MStMV zyK|tOd2+3So19X9!6E zSYhw#6e)(lo~N)X8B$be<|dm%cwmZ68E*wg&X~?4Z#o-|hegoiLpRr@tBcEkBy1>_ z6nj#tnI+A8opjM^2sx;C(S*kjk-cyoTh03|E8hA~X6lw#;T?8d3e_LDgt^X|dFw&4 zgs(vpH96`&aLsItQ#n|+`>)3>U%s3Kx{>Iy=hyHz0Ow3fCQUUM(68S)vtnuKA#Aho zV)M3b+tQ?KFmqSy+vihlvaEw{=JxXjBMxnTyEu1taM+Z@SM6-nm>upLysD>g6d5K6 zANcgyvm@viC?e`JMLRaBq$~^!n~>(5n0`=L>Oym3*Sg^7Qo?Z*4Igjjf^c-UFbQJt zCHxFHTLW0H7-7>5zt<|afc;UXh+*WDXJlpg2Ayr*EYQR+IC*mFWVgnl3rqcy7SJO= z%k9eT`TEbnMY2)UgA@mRvADX@6LNU4$KjnjtpmnpFJz!8MYb@uVN&0QY<3zVh06J} z3!eA(`tkfv`o0Y@F@_xNV!9;5GE1HCU5%VZ|B*7P6UqCx-t%vp8sjdyx}U!zU6Gs$ z+Z$WP-5eHuce}rb$}KF|ljIH%Fa3YHu26r?q7k7^`c1c!!-j<436O?yxCv zvQ}TM7HZb}Ym-}6-AjWmy?1(K0Ut{(xkbT$q+0(kQrQ2T3?qW6FJ#Gsbl@5{l%@Yp z>%)2i;vf%v^qwL@)5eZ_E+4kn6VPNcg%9wff!2Jxe|bwbhN}p!9UH1DE8zhyV>|9s zUsau*KalN`e&goMo98Evr);^8-uWZ_(7;W=fp1@}dSihDU%<;_CjiypeTX9lS z3timmB-+Y}_cEf>($k;5uiBcHTKnD{TxFL7xF1g| z=AD}Awf^T4ho|-G`EE+9x_l~mPQl;%Js-50{NwrlWk24HX5e=XP8GJiGGJsAk3tj& zsX#fByIi~ExX?{Vdo7DZ$acdUhMgON@(*^55A7Flw{Z3Xswvt0SzLG(S%SP44#PAx z2JX$9HQS8CrYyAQ`9@4|jL9^7{No90WzFK^oQ>*v_wMe1;t8TtyFm$R9l7f^!!D)D zMgNF;BWhZrfryWM?ksyE`&d|_mbuK=Yy)GUZ~z5xU~}2M!}j>Zwc+96Hf;-lz8ToWzg5||G35fh_N+L z1DmDU&7~`A`@(4EC1rN1LG{*p9*h2P#B9DGP|3xQZS*2LF!kE$79r{ z(z0_!RXcPz_2dd{C_&6H(3RCsg8D(~%_^Md>YBrs-v)+zpu>lUJ9f(mN|D?J^MIns3g|Z@|1oWg5PO{BlMoh9oo0SL;qpUk| z3`Ssg-sx6Iw5@?Ce>kP7j4XQu|2cQAY!8P~qW`SX+O6Ci7$c1oHsPyCVKU$4M9H!i zP=bgGQvIfYfTohMIK+x!EUr9_TeVr)xX%4+D<|!Sc1wFhqrY=yy>`v`ectQ4j@@yt z+r41l!Y?!5PmZ=ZX)HL;zs#`5Y4nvi*T$UdW7TlOp+oIEckVo|b7P~%ET@2>9$7%2 zycv6SX}ln<0gCV$+ELEf90nE_lrr`(!W=3lS)NL%%k~V0Z*4efvjhezKi3kZjl7rq z<5^cWk0P@crO(5xteebS1*>7@Fqucmf>(JcT z)F~-hqL!R~L76et`?_J%BUMf;d{KW}#<9m~Jw^tN%yNI>KcrltfU3y=#tF+^w|tpd z^`6rZN>`*%su#mugdP9xg#+*|SY`liycwS+J3Ct@_y4+iDt4S|;Q31HybMqnbYI0t zTjq0=nCmKwEfoheeG0P15C8H{UnYK>?py$WMsnTA;>tZ@Ym$T0#w~R?9vKb|{s@iQKx*%z4^+~9nHO8v;L-ul||kR-O^}g<7G&Uuq*G5kjF}e2ehx#ojAFT|Fm=Ni6i^d4vr4! zAs?SwHG14Q3o|pbuxvSXP`OkXO3a)8EwR{H~N4A07T9?hu!`E|Z*-Y%()A zh^`GY-rB|2M;{c?5x?>J7GQh(!GmU?hBry>Y!2GU!*S9vr2`!t4ytFfooVt(gIBpd z6OWB-Du*3i7c#Xzou*s)E5C}I(M>&^O_ zfVe9aDzeC=MGM|92U!&GZ2yH+Q$F}`46##|{#E~b7gZCm5n^T3>Rc|pX^@OFtFWGc=^HwSw_u* zG9BF7Nn*E57PVPGG6w{$)zTs~VXSkS_~g0<@Rk*J_3$BppgZc`)z?pC?HnklWPZ3W z!@wvUFL&-?#I8YE-6a4v`2ynv8CYXLzdgG}ukhx+W=WTN?X((8G*8>pZ4=Z;*2pJB zU=9(k`<-ba7f@syh);@MzRs(uY%rnOmTM7-4bqku+A6E9tE7~W#Y~Bj)6G3SUwsTN zdiq*Rvjq%*wcdQm4QODdqYj;~ov!OX-;-S{JIzd(hAeLhaEaGPSq6cYE3f$ZpGQZp zU;}WUbNyX2%>yAR>xS2%)6Od~3@;lNJ$9I^f54c(tw@gv9z<~(iYz-E8VYpC=x(4)B4qQly_TX3U0!VF{7E;BBk z65;cL+p^ZU(Bxx?p?x#l3o`TQm`LT1M-3D4P$AFU_`vJk+G?(^`y`=$Pq){99Olr{ zL+`ny+?fW4SOux}@H9`TFN?va>m zxin$Yv%~1z#`#Y9(RjO1bbGs`je?tX@ciJYRV&Jyzs)Gs96EHYa?eFQ_79LB^8T28 zVT&CzxcZ+js+t|^ZbY7=AKiP!xNitd7t>##{@}D6!wOuKkfa)a+V1=pa?ZZO$g`bKBXDU<5rWP6cJh9zNo-DGjQS!W7_t zojJf)e2zWtY?-+O;u=VHID3kiQ0hOq;-86wA{0 zdw)H0W~R~}Wp;N6Ac@F%xM;Qs8ytAFu5}a(UtUn^(WCU`Cv1X6i~>j$mvMwmhv@dd zQl{iGs3K$ut63f`pwv4{$NC=c_iN|VRqHEjy8A8bKWwVO`f(f-JxbmYK+$A_iI@`$ zLyr0_705%O;6c%AKl|=;Mq~VR9iOW_id1&fGj%j%vP^5aE%Gx`aq)ka6e$6NtIC~_ zW$?!q^0KtR{(6BM#Wn5JDZmrQw!vv_xFuBV)Tgq5hMMmxL-g!c9XbSZ8xt!S1gn~b zQs=gHZ^MQ{h<)_PBj?WTA_@EL+qb(e$Jh;jIUqRwljW(JqxCK>vAPw?;S6=Ewn}~N z*i+}9xCp1Bk6Zz{GOc>fkR2XeHQ6^JswFx5io!9c`SCK4XKW0~ON1Ury-FI%lK>LV zDhv(!FGQT;V;!+hXp_8POzA6*2t(=ob}xuAvLspr;K&AudDodCDJA_}Q_V_@9L89g zT}x;k6b?xrAS>t^*qu#I+>yt0R^*W!Fg*@{?t$GI&apcu^4cnE_fEKG+4znL*?;>v39JlS@^XwDJ~0OK`3 z7DUHj8{WC7>c_ z@pAl#mv72;EFB#C>{nVTev}AG0=5y!zQ9f@a@v+lIL()Ci9Xr(j zUgbY-%$UX$3X&_f!)tGE>eU*Sm9j#Bbs-r6e9)1QPJFmSbrn}C3Jrxy0?d02Q9=g$ z+9^;*_YXoU+=s=aL0n9s8v|>x@9fCP8tPQA9d%&}e*PRl^QFuF)MKxO*vS9OLRv=6 zAdd=TGEp*dP<2y9jxN8J(Bkh*3oflQU{zJyE_G9M2+Z5shwZ3le-e)EnU->eli=G~1 zHZ6K1L3CuiM?8fkG(9$eD>85X{5(pA8676KP)_d8!piOnKMI48<@!v6Z%`paz}O>E zYS)jznkC~Zd%fE+p!#;Au4~aa;rq!VaHL8`9JwwiZ**m6s(XWu)s^2S==i9zrl^iL zM>aNLE%~F4w*b?&{nl4|@#r;WY2D?^=!o`AOF^L3f|PZkBcsI6?AdDTKJM@!J7LuIc+H88 z&Qj_qOQD9Whe5~pih!5ZjqKa_y*x1b0oaqW;zL;^KU0_eezC6r9?u z*zO}`>{BFMs03T-1+PFPjkvX;jt?Y*R*D-mxsjh?rnXc(zV$BOI4v!$1x!3XbD~@D z<2V8rk!td`{&DfK_HjpKW7GG_aRp1MAHzwnZtmo>NeWeL1K91A@+Wtw{78Qthnm&- zzY^OMv^w=-=3fK?d5VmyN+to`Czq~x}pC{BWS7D()&eiHO zPgS0yBS)&A6y(~`?QDjsw5?Nri9;N*V+`Oje-I0Uv(N(x3ak(k6}d5eRTFgb z*$e06N7#5D<-d`>+2U^ZH;&0iN~geaU%8K%M$Y&~FcMip(&L?b3o%5IvbhhC0^c`yn1J8%ZHC`K0RmP=zMfzV9iITzwh21E^`xhat2g6RJ-{k zRdg+{<>*ulGr1BHICH4Ojq8IyxaI$~>PBAS{=q*Nc+A`r-)m2gBV;Cx53APGCv@!g zVZCM+TRHQH>tGsG&JuZSqGkEphcCep+1*x9c=eszsqkrB5PB(u2gi64JkQOHncSli z#)g|qoGBDybN1EMj@?UyYSOFM=~0WuybK*)m-$;LKhJZa(VdGP_RQOSScNR*I;EV# z=u|#Ku$?1k(r0=(-%pRpI|>X5UCS&ngYOkpB=y?EC;z;7S*8&(%le4zK!Zcc%UZGU zLr+UZO|5F^vEHli+-=pXSH|!cN{PJ|BsR$Y5m&EnORaZv8n@p6L1Rn`o&%PEBLc9> zzE`4+5FG{GW_fV*oeI}im-@;oIG(i}2n^Rw&%R#bwd~+MB-#eSDv%t^y}V4G#5o2L zPDaND!fgUH^e15Ju$PiDR_0p7wu&F0xA8QA?ID zceDA>N^gn-tIx7yTWr`eAPi8%#SEcm+(N33dX@8W=6!bhrc_rLGCUEc81PCSBy}N( zH}>lJ+71+5yx62o{PV$~Yg|lYhVZn+^Zs6(6!1Scw61?oOuoxARQ&c@U0S%7eA93~ zlq^jqzs&C(Zo71!_l~x9fhsitzl=<4zAgLy=Bf7V{@1MrLA0n?pMA+HL3>(*}Xu%eA{WG-JYk;vnH3QK+EBtIMM=CFD5J+p;S*kVj1n~Ki|A) z6Y*WjMj+C^|7i1TV)B>LX}gM^nzowJ-aaiq*hArcr>wCG;WjE^5Yi?&rl8#&yLD?s zHRE&m@ZoIDK`oOTwcKF?rgyk_@P09Av)+Kc$ZVbjAe1dY+)@_O=eWJx@BeQ$C5~IuHCIm$H>QBSGvre^h9yK!`!5$q<^tczKb2;`XyLRdYrqV6#i9AXwm3@ii650MDbYi;376ILc zMjBg3R2FN^?N(OUz2Toj{rvQe*i9s~B@qq@MnvhDdG2>l9Gyi2HJCU2Uq&F`qt||^ z23i>g&q1{B|1NC;SFssPjVw|edQ%V}RuwDZ!@X0!3hf^dV&1dUI;hG1S{Y~vPGe|jC^{!8%^Bl=<(lQUoAhO%WUt2~5zlh1 z)a}1aK?B&`W1aNlfOK{Hx;-97vxjL&#E&kOEBIW!-~&BA zHM1Q8)=^eg8-ZzjRNq6&#f{UulVzZgrLdLR6ciMcP(*;0ydnF%kIiJRO@LHVbQyoh zPdWj52Zp%)8CBE=6~uCCn7cOg8HwKmJWD!bhS~`Jm1yXp-== zVJO8jp*=bC^5obD@JEGTK#e9^b#f&IhAbc!MJ1!DF^m3Tb`WM0Ev`g}`}6pE>5~wX zbk}odWE3^yVsC-4cmd&5LtZLB;*3yCQTRvGxLdXpc}3|;Pp_+5-G0bwbxx<4okV_? zouqiae3jW0&<$P;qTQ2fHCtBv2}KnAechTt(JjBfM$R0s=qNqS{FJFD7GyYG)0sWk z^{;sszn;Eh<8m*1D1Z=Hj1E&faQ%@x6Qk48)IZ*8qA}mHnfl$QE&WfvN}^F5%axhY zP{(}o8}n_R>5rEZjRU#fc|#F1%-0g@X>+H?bx(FvZ(Dr$u) zSy|1jA%xJ5yutG9XEK`v2)V&si#)?b#X8=}Ldq6eCU`{jNgiKAZT|$suH(%e-krt@ zd0m1;WC015QTJ$Aw4+?eZ7n#Dnadb42~w4~56H@bv;X+V8bqZO*Q$i0mJ~^v3i<_) zBe&)Fq5Xi}6|r(@R_t{9isy`5Apj7ZPE4D+bm^IF zQ^o?;+rsDq1-U1CNA`vsnz)uE!0@Xv)GZm(q6TKfaGesJCTI&|)N6S9KqnFAES%x6 zbAP*nTub+WetMHszc>TsU7&Pqf`DK*uD!FzWs8e7k0-n@n&IUg=2CMGT) zPXyh9hS!OqdghWg2w5?VX={ANz2lapa zQoFh>(mwvr2}+7-rCJT^I`4M+h$)sTH7ls%t~4K2gfG_^)rK*2if$9+DkuQ{}^i^8aSS@fUI zKWM@dRXS&B(= z%Umb!5})xTIB~zV!f5Vbp0*8KYbzIctvvU}(BTvcng0swizms%nZj=bqtnosm~9fF zUce<02fpaTt9GuLU3M=WDd4H%&A(h^411- z(=LH>*SUd%28nm0u%<7q{OpesEYTE3LeAT_JN_)$8S?T*LH_=|_P0NbLltWsZ6gA~ z`RzlJlh=l2hi7hUL_w9&AoW_J@r=PMedpgk74>8Er0B|{EiL_H(}hpQn5uQJSqhNW ztPc2T=m{cCA&%Fh3_JY-0&dD<_+?kUqzi_V*zwEMQ|4_vyvKl=qNbnki8Q&Gv3DMW zs`oF>8!>~5%pVK^ZzO1ptnJH1i*$oQoG1gxSOP{;U2GzEUa78_FUHCY=q;F?%H~rN zw6wy9)5TKLc;%-lw12aBp(z)a>M`yu&0fooli~=JJO?p09FnV`3%%SP>ti<1SXNep zwhDwUTok&73#&^FxJ`nFi>o$%Gq@1m2E|(pH)GZK&idY1S1B@VeK#5{lPH+}d3-?b+A+B8G$2Sue3nkuF;ACR+ z2&5x#%G*olgrS%iJ?yd46Xji^^Sy1r0+LK+9LW3xgR@3J#y;YgM*#yMcXen@A%b#u$@Z z;0>c){m$A={kbA{)Vnz`UvFM;Jsss_-}!T9jnqQajxwDYy~V8e>VJoae~SFbbfT5J z*7E%WTHeY}8!_Gfyv!pFRDTm2di`PeaaXeqeo+&DduQi&GBHVt7}2rf{nwtq{gyko ztNluI^M8NQJNJIqt*_U69d2b*`v<*=l}ZX|x*>}@n}6?eG49dJr#&=!I0${YbKOWA zF&m~Bt?N;2u>w_z&oSNbkVZNHS%HYtNIy)B_bwl$t@r!I;?JC^qv+O1C%kMor0wsk zBg1bCacWc-=z`n?se1`4R`}YdZ3+x*_NS?Tn_8{a{%*lmSFY)V+icSeli8u#&ISLP zf&VwZ@MwBxP3N*Pi;!%L8PI}6rJFY9%dbA`rslkGu2a&%lOv!IOJSc*%u87!F26wu zGAVPm*ADfKA6~2x4;_JqX=>+GEj>X~ieb%LdX-*%`v%84rnS4BSCv}qQL}jLYoqo- zWsPqi``Jk6E3TNfC#R@Qo0fDS$9-}AcSFJ>BYp8e3Yk*FNvljR9K!dcEH-&PwEb?r zM_xkT;ZV6p|FJKmiSrxP?_U+!sD4&tjGA4|Hd--yGRLDXH1@uoXHnh2+rNDCimP+> zB{>f0P^1~yAi2mxZIE0BwY!_gock;LuNu?g?5`aR2z6M4PulM6ejJ;`J6WU___YoqArYxZ{E>1i2z62$KvK;fpo? zh{^r}Q(E85sG<)a_Agp%aN&HLAp6L_{_2D~1J$8npz4_5^IAd7HLv2Ij22A)i~LTT zATYg93QUnMk>RBuf2ng_;?HzmlZ{5D_j2c*c$^C~o*o3@!^G#ek z3=Ly^!gcHX)Yz|manBxgz7y7I(w_`9)-Ohk8jP!I< z%_+L1U!E}KmRIxd{xou%sUb$dqX7b6>QiOC=AiM1jAIR? zF6CMcYp6uF5b6*`&E8IPR*zn{KHbhh11Vdy5+ww-_Di4T|M$ zOxFHT>SpHKW;d2|GV2mLBHlyWQA-HAMkJ-H+uS}Ub4*Fw%SmZl-`!At*XhrN3!6pS zL>G^?qrSiOm+nOhx4}xZ-1D;-N|-(c?j{F#$X?=uMgG$|D-xo!F?5 zu^T7~lrZ&054GdZd#;85EoqS_&tMtEJ<6^f7|_q0`N}Dv+_|iub8ozyX?`03C@L$M z1~#* zDuX+Sn=)dtTq2MvctmC%K$+f(hT6fYfwyBkG*5=xJuAL|>lo-rD+I1G({h586mI6L z;zZXYUwSw-@Ge&Zul}tF{LcbPYEr3ag0hN3X+*P}W5uK*YIRTfM`;-Da<+$tu6MYk zN9@>b!8?qoxOhHC>yNqH<^4loNos18tO?B4VrccxV+a8>c!%LknuM>F*$=L#;L?c| z9J$LN%bO+kGNe!^YXlF>4Vpgt)^!}J6%ReIcNkNBb*K%gA!&P~Q5lAxJPxr`nR0jv za<;96aepYclOfXWKURG~WA0dTyiZC>3Pw(7jjxD7oiPW?aYdCpdO&6W7QI zUu{&;y9Z{6^Ld*Af)U|^nInJqM{M^v#-0Jm*Xz?i4_#O30gf3xgmIuRp-6K{M5Lye zz?$(+Z!=xNHsJcMW|4Og@Legr!h{{NYdy@Uhn!UfB$}B{u?$g+aE10}^tMY<@j0&1$u9Tu8_? zTi#U=7GsM;B>P*97E`62btt@B2$Y0wTpo%ma!0VjIaESi&q5 zJ$?IA?G5ZtZ6^x3CI~hZiUyMpW=X=ys$N`Y7~RyM4Z7P>e1A@oX=K$AA3UjF)Rdhj9VMAfY02C0AB(H^_$+9Z*WjkF?~EMXJmMftGh; zTwGj2cU?3&%tuM`!lq`&=72d>T=|;%Dr*U(j$iTa}`{$6s zlj0Y>F$GT4Mv^j*IT~#7_&?J-hyxL>_RKHL2{#ExRqKcz5kiQlou#QDKq$UCbvp#p zND&HLK=xqBxh8L2+^xLALKW$h@EVhzp0A)#o#p1{kLpz{f69+c&|SSo3?;`O8nA&C zKW{+|-@i^C4Ac#iH__x{+|0|Zc-LY-^7qblNm4>F9(e(5W-b1k-YVw<@%eOY5CI#o%q~;cB-P;uvNfg&6=1*@tt;ecNdZf4cM|H#j&)Z0v$^Rg6|=F035xFeZN}{;*^3Th#M#T z65FUYO~E(kxVR{J?0j!~YDOF8uf#Di=o7~tpUs<3d|n33E)zkJUD48bYaX2gOs&b) zx(e+9p=w9Hx<=plKr(A2Y3EYgBUrt~fH&UtQI}8HlYvfNrDln&0F$>woGL0o;Hp{R z6rvIoiyn@>OkWi!S~tN<>o$A!t93>PY5IgnNb&hNcDm;O5~8hVdeq#u7Y`)y`wtNf zN{LPU7HXUr5deS$REV`Ku+P9>){m9d)HFt=tg8}FSRYk0$}n?0P#qiU++Z5p&7rzz zIq@VMLw~nlJ*S@=3(4X{qRagGhJx0e!T8$`B9;{uNiZ>|1vjh}fF;*J;Rg^mY6U^q zobpWOvA(Kie7+IlMG#xW7pf?9yL1@|O9O_<;w4Kw^NIoUtfkU$bAi>h6}H*F?@9b% zC*A<$%=~X%d{p5Sw?YP?`TF(i@-Khj2ATmrkl2qWTu6wHFP<`&QC3Kf`ai(5yc^jz zgW6;p_vl385U8iJktA^?hDZADMt`rPQ?QS5{*A^uu32Di>!GN9a3<0RhgrJtFfR0 z{~(SrQwsXJo^fg;f)&Y+vm1tt(e00B;)f~(N`M?VY;|dc+unO7%y1SiB%a}FT}Q>p zrx5Erk!dPqxolfeB+8>j#;xY2 z*r(SJ4vhgOgg;I^9!?D{;MyWb^Y1Ig4gr~rnwk1(x4fsWN6SNMWjTN3Z@T_L|(fQYaZ-I~ zXJg}s)4G!g4c5TsE=I%_i^S?F%6*1dG)BGw{`cJCEG zB__6c^W&WpMVBB`2TB~>I6^A`D}@^WV>#{`3{Iq!c0fH912@DkR)D7=+nj_x7NMQ# zx=|rfaRXHSR#dV#Cjo|FMU_gh+7(;zu7#$W7j2?KA2TgvZrsSsSs66rM3S==1wNsYsKJkfme7DGCf7&t z7d8-w1&wf&5TqK43qW#CbDW?r3eIRz%pq(CVl^vy7GTw)JqCfo$-|==yHYri{*n0- z;fn~?jU%sZqaY>`MC=6!c`K?V3Zv-TApPXU4yDo;i4W2(cT!0EX~D4+}77VxeMv5Q-Mh%=~cz_!R^u#z9Ghb0Vn}PxNgjWu<=p zSGbV|=+@UKC@(`=2+=$>m;}k-wG5RMok$s=UTMZii3LGThPp>B9d#6u8qs?h56N6NI&-3N&}bRjpR?>Pfi5M^ub z%T|Xg z%@!7IryPKeS&I2EG6{IV$E7jt@(qSD>c+tEL>9tBej*I`d@G% zDR)WGu_jEIFdaDJ;I% zSYXZtvCpcJ@zMAb)B!6Zh)?4ssTGZZB+Kg6tBq23Zf%i6s!kkr%9jYaiW3-R1EGY` zg`ut~m!M)G#?48%!Q{=tUpX$vawQp#88@J3~W zS-@(@r}^S&*47O>is!QfvVwTTWUti~1^wii9yO9*wPg zB_j;H81EwPdPxXHyi~PcxmqBqVMX_-rv4GFN19hhetN^LYvbrgmy%K5y78`Hsux98 zug-jHt&_Yv!pA&!~4<(C2_1J4Wx77 z5Viv61{qe=#%)mZ5{Jx^$h`EboRM12Z z0y+R?F@|3}40WV)7mF?d*1Z3=7vOKqzJMY&uRQkZRtRb$v)oY)ee+Qis-&A1QL_R9k;zu^Wbgg&Xpe7 zBkV^^Oh}tt+x`x#6eT)P4f8xrQ8F`xu$fJ)CT-{Q@CY*&d0`(qjP}BH0Ua&Z++j3v zR2kd|+N%b=(BxbQhp(1V$ngKJreN%!fI`h=~ebTSI@7pK;Sjofy*o7+p+YKmdlJW5^i#XpwhX;BBWP%Yl0RdMw z*n`m!z;hLYVkLSs*kFczZJe&uQ(?X>Ua)q7@KiTS94y&%_anJ#1|!eEdvOQOAUjNg!JfskdyP` zT|cH{ZqUmL3Yjlam<_CLbf#_o5vG>j0 z7ppqa*UGa+pJogS=2E8JzDzR+2>Kc2H;${=veTqhjx3@K04+2D=5KmV?{FJCKm_*gZ()w4U8A8+T)8EvK6HMDJO7!U~6mJ9(U>^ponvD z8tqN{iS{os|B)o4hi{b916@5GLMSl6X%WIinaX24LQMPCed+4@jK(eEQ}xtj+f?MH z)>MUM-eZX;FV$kReu8Tzx-M2Ys|Y&<#8}3d!!N+j*w7Lt4*@aAW*~zAo|prHXMPrl z`7Aix2m$~%doSv`0v}5#J<^QD0=33`mm0=|#m2aS#Kd-HG?Q`y9_;{dv{AqRTts8L z0~cJi|5y<@Pn@gBHiz>Xb%JTc0!#(DWg>CwNQpLA1-KN7l1`9|At4mx$J!j=PH%wM4ZMfH`im>fqDv91_8)V>8t{n3`IFBt-^;7PZcy=LwqVNRiWW7a>)SI z#8P|^4zM!-i0!V;gB{*X0@z4gKSlin%8-eCi^PW&hZNa&`7bn2HOkf^S#3zTQKgUL z4qn4tXqX{7pT}k~{-6=@sh~T)0;$XZ&Hw`F)5IBrAM$3D|LB&i>I%CAZffZVEb|UT zr$T7M2IGlV_Gj0TKP`p0CqMidOR=2Ljhc-u0`4bi6@kl@Wa8@<&x`+H&oYhqR;N&57X z`vMP7q;Cd^TOl6^`Hx)HhVuk3#INQ>)5s-|Ij#Ri9R2TK^qeV8qsd_dnO5LNLm%53 zQ$qaO!L9m5!;a3Xb%i_I%+gk@%UXJl?5<%o-a45e( zQoZdGB%T8U<0CLi#}7C@4yg-e*IJ#l_@u3!-85h-Fm*wP)A+@X(hUT25=meCFVrWs zYMSH*bkmam%t0ME|D#s@|LBVhkMV_3rxrC3@(_|621UdCdoc`np+~lrG=mWr@wjL> zvuR=#)P>+M*xd}F>7HWF>8rT1OPVhXc?9i7YKM}rEV@?&|HW36&!3{%W@og86=qpT zX~@`*cpfg-ebiG^>9Ne?y!5hivt#6r{ z?voL>hDJNI7+pP~>a z=lIj`Fy8%416!?G_UZ~T(e~qcO>L4>!dLfZ2s6pFerNH{WvkNa_t#=|c0yRk^1tLYn{=z zZ#6}OcKZJMkw*Qz+;c9dk1pjA_A`&Iq7xJ+!VS!1!k%`n^;H zM!p5S@Gyb~ijpK)piplI6}IX3Vn7&(lmOP# zUfWzgp71r>71!J}lFC&+sr=evU#Q<57OlVWqdQ{-!-50ey?ywXjI zyB24RNZS=H6<0XB$7Y`izWq^Dh(XAa`^`UOGMhA~U`4$1CTIS~x6$sNT za&rbw9IcSG*r-so%*b>BPXx1zvqZT91Oa4Dz(PKQn=D_qftRP5^$}lakco-7;KFD( zHY4vdx=TH0ry4WwejxkHt$_XN38(63HLMDQuk`Vka$E~iUK&4dCDEvt9{Vt}G zajmrAR%zeS`8%9+4u54%by}XE6!KNvGyIqG6!U7)hfdGr_!eXZSc>g9$L*`WD<;^% zV|CNs(DuJ&tT{^+*j`4d&m-i`<`%74$z)XF;Kl3j{2N3Yfd~^?%1xBClNwLi2;P z4iM0(`9r?(3iHNCJ|TlA?^Wc{pu=AzBP|pjEt}vV-sAe~xZmyM`F$y!i;g(!uJFB` zykXOCh1YIJnzh-^6g?c>t@}CAa!s^VX+%`c@f_cj5KhL6`-U^z@|>zVeksO0)#*1# zzqsPooh&hnvOi-M16QxGWD13S+4I*tR6KIznb+Quylkxx8uQ--HD&d`(%3R3dE)pO z&$l%OIhSMAeunucz1knto3~T z%bxb}r(tAjG&>uE6ieNDGaqsu-3-Ubh6!997&VWc{00DKw=rCiV*OeMjxq=)HhzS z*0xz>?UJXL4Y{*_PdqK-_{gi4(0jQ!R{GV$5mAXBV=q0zZ;g&roqg3-QNDrMq|-hu zu-wz(kePs5(SmnTYXbX}mc_b1sy*QDkyht2SLb(;<-0p8kIa~%YwjVhkU!9!q>z1qqmd%?b^piA0_#_%wE--t9P6A=A+ADb?HZV4aFoyS7=5r|9YHDP0%*KAeh!rBt;1$RFCH1%1u1SxId#UnJ*DR}S|9TS>^E3E%jekJxdI#$ySh4i1#1+o8rp8Aq6bki#IwAJ=?mU5qBjs_hTay zyX;R{aS?||-kEznojSu7Up-~UX(FA#Tk@ah;^tB??}LT2e}>5n3I5qv3X+PnwhPKF zYIKs@-n!SHRra2&#qC=U%U|}|bGqd@7a7?Wu}a!99lburF}~NmW_tolX?D}2p!xiq ztIRmGy|)NOuN#-4+_C1q**mW9@tmA(Z1)B2@5?w?DeSVB+LGqSG3t_K=@sxXCod;G zYSG0yub3Hmk^(yJ@_B+^Rs~E49U#B_aUY}O} z9o3vytZ%z)ZoPW@_iw$xk%mIf8-3fKuritr9dkE#xThLRaxQLMnsxM&3@bOw0pFe0 zJ}t&~cKjA#AGZ(F)fK$B(6gP}?>7&}(7c@r^FL9~4B`Mt~#?@(GPNgf{SISuY^r}8r`nx~~#JjzG;c$cwJOAF@W8KM5eSLX+n-of3 z(_I{)ZztMJTg~qZ?!63bnKL%fxf|?b;5F8R=gZ{M?O`tZ#cq#E2+CC%lN^a z^lkPR;<%YTk4**-M{n77He>dRme_~JOSXl)SpUanXlO${0+ChW4~^ib+shBXdtvYQ zK$EjUtL54g^*i`zw+oPV9NVS*3Bq>!OobV z?Fw_b{&p*su6Ux3O5}=xP5bP)HfhtbB*BR`E+M6?f#>$8e5Zb_caYX{ePuDIEkF3; zeo@i&bf+Ved>6L|8h6GA>{Mo6@p`N+_e5)ndzM4AvBtLxADWvoVtgbQjd@0BxmW#u zBNk~t7ONJr_pZRsPTd1*cv`l7b)T)9|0Q1a#-cNOpUa;qxIePRGr#bS#eK_b@ezUh z!*4S&o?kl6w_D?KXj1*N`o`R?nngn~KkWv)-(B(OJb1UO`JSoM%DU5zK2w5CUPI5A zKB#}OS6=bucJJ_?MLl^E8{VzFeYtP-&a%#!1idYcxC`Gl^{2f?14gXkMgKyzl7Q6s z7bD(UI~D4y>Qjq^n#UvTLVR|6Jvtym7x=6`{Wg!_3 zdEX`l|7co@Tn=KYU7SDm(&W&G8*v}o=WBndOFkaZkmKSKurH;mxh-2*x_1`iQ^A}1 z14r8}axe5rMz&A=+R(4-Cb6nsr6- zul^Pom#i|~{C9Rk-(SJ%;>jf)swJggE6wVrJd~9`wtdCaWm6Y3#Wwcp3Y~c=J^Rs#<4Y#RRxL6W){B} z9*y73m8Wyzj%J#_RIqWUbG?l$R7z?QkZ!h3BpMXsyKm3GQ+r!+hL7?@;`?29l#999 zUk)j^-ME{c+-Rp2%eQ$($g!fqejjm>luR$9SLt`!N7ZMnA8J^BL}_XEhd=ukN}o8? zdgJY^)LbLiiid|@h7C*P?Ml!{E8wi?UURm*c5#<;htE-+LOGY&H!24>-k2Fq-P+*r zzDTqK9TYMD*%mv^y#K0 z&lwo{#@=+c@EP@8Kc7C^ZJV5(Dl2|c)N3NcY+OsSQlt9zi#}S|F=a$|1 zklRwg)YKK~@W=jbbF+-3i)5PayMdtEcdo)0q=$(0ha;$_ktep(>=>=^E|D-#?EA^vK22Y zdB?ee)y;A;Wmv05C$+JC8$K<)@k!Q#)VlL+0}tjjzBN9xd&hedgA>i6bJ|*x*N8hC zd20D_yH2dl6(cE4OocIK_XV&zw<57chwanfnWf2Q&z`GhiJmme4|u0+?ww>b8oMaN zBU|;xjvEbvmBYTAUY^?{cjVu3mK4o-AP^=Pn#9*Xv!0&i1DTMy?sHq78L zFe@9`*O|-eSJPzr@6o^Kw%{X^_~z8}_eV;50+pw{g%XZ_PPD()o$;%CX45afr;^6? zPdv55FN}6&YA*45u>_^FBh=ArSiY<@R^X2dK6_SCacV!?5r&NbXPKp~9oxNfpDO3H zd6V(S&-uGQ%QUetu6w*B64}$b-y5n5&+Y#uSQX%Wf}{Bf-rJz=o?!4K_ck=RhCWRh zsf&G5HQjXUv=39ovrE=7icE|fHN^qRn@gSu4lBC|c*ne*X|^=lXM?!4)N=$@L*I8ky(ybvV5VPmaa_FXaGS}=CBfQy`FFllRG}r~z=wvR zSgEPQ`5k|bmG^#DbBr|-`LtSlTi84M?*^kXX`=E{_m&vX?q0CxwRP;tN^v&!1AcY~ zno@PP9DyE0s@&R;*Bt0J>`GGmKDBl72@9*fbYN)m#&~C48HK?|at^JD>o&0z1#)eb zF3+|$aU37MQmv!3TO?d4^*HwkS6h&+QdHU2zTJDw<7*)vpl(A32(mM$-ZN=?TsP<2 zuf>w}_Tbjfe~;-Y{)sM2u{)fcz9*!vwVr=#$hm>D&u71Nvs~yMGFMG?w`z!Ta44>* zERr?k>xj!j(dJlp3e2|v86`Z~GESwQMa zolItjmSbpOK=};D7(d#3N&7C+o0x^q4~t-oNZlE{m#+L>Fz;;0*s@P21YPLcV3dXfLIcFA&KWdXB3#gMAdme0~bnpKyw z>WrEljJghXc`aIog|@m%c#n~^M6-^%n8wvEZc&%nrj9S>{p|4Bs_OkHH<@Uk>GQ;d zjBc-N*7>vKr_`)|!C-b)@lQ1$mRE75sjk<3E@!l!h4Hq#fWi3w_b(ar*+?p|1of;) zNoPz*d!HFT-rf=!&Uh7fGjp%UgAzL-@85$ZHEu=50+29o)v|Hl(xVBx1;CkEUl^iZY z^1qUs@pIyA!a1WO=09HLiZc7MxC;Fk3kWHi*(Av%wSTckVJnONru>fI1?v1CyywVT zo~a(y(STPhEzZua53p zg*~M&x6NV<^nYh+W0=?|)>XO4O8@6g#1jqUZp7VmG*r;uFB1HIH2m_wqO}VJMbN&B zeBIl+Gf{ub`c;lLPNVlVt7i$kBaQ2SKN>E&zdh$Z(OjUGvufi^!MbwxcRR#HqWfz_Dk~|&&}PYxvbhDXlsg5hnL%`qZS@)P_y|a+z`-qMkg)n zfeSVEV|ll$-hQ+-#qZZ!4mK^Jm3K!X#dy9YM4nSy&{yNRg{|vlPwnFkPruv}we5{a znCz7Ms~GLK_0_(zkIj$vHR$7u+DOLu?)<>*vqURY?Rb#IUM-fhLsxD%>#)4IxldK* zzFyyd*BohfHph1sF%;n`uX#E%d^fPzm`wr3%>;Cjb%6E02=~$uCzaUHn$4j(!$%4~ zu1S70p7?WiHd9w~G}}!vn{TVG%w`mX&K^)un%gjEe)CCszx2gNvW*2jRZ@@2JY?+S zWM$l+v(tKHI5|H$Y6g>v2$P~fX0C(Eu^U&@Y|T^*=BJSd4-@YCrI)|XGQLk@ zFej3(ufXcpbEn;LX;N+mb7uSl`I8bG{wdr$zmDe$)4I6- z`olhhm9uj^ZP^|a$_vcx=wGy%pYDzl3dx?Z8jF+d2IXd7y1%Hje>w8qiN4V_I41TH-JLx~blJQE?8#?d11sIlJC2Su0CFOg>t%IscWmKYKhXr@Ee2R0=ppHswoB zedXLNe~R%)N$AZE!E-*Lb|H%v#cuRUphKI*Q@1zw+!i8 zrMoJy$E9+8Q~08xu;G_qYaPel&|M({T=cW?9@l)ydjJDb5@98UDiF1-MpsKnsK9Xn zv`bEa1@0gHumseRRHHcc=B*BEDh3LI27Zh8w)b>e|9f>!Pg!j6!!a9O#AsGZ(xw16 zlRyQ`R_FAUfG$9uyoAyLJ(;{72}}+9hdWMq&&tFyLz(EOAp*1LJ8k6^_LDiuy&)4g zV}i+(1dU^CNr^IArROnI@BPsa!&$IA4ge_Cau+?qtq{0<++UorGwwY4f8~H)0Rj!L zn~blWcte<`T6$&<^_b1o^7h^WIiDB6=fw@t#1rVJS)sL0pPqwO%Jx`5ufQ!bXy3G9 z7*LX)LV9k2J$F5#A<%pig7f4@cpEYq0^m5#3T1~@1pJvvA1QBN2sl?X#n#f(!xb^e z9}Cdy!xo@2M;4N&3L<_cI-W~tEL6m;nyuG?`L`MdsU$*G)9Fk~Rx~0tlFY=9-;Yp0Agf~kx6vTWL?C?mpJ1HrmOXQBQ0d@mqxD^`Hky%&h z%2T(bMek5eQ1X<+VEY^>eFW5c3;Q!XQhI_6;-it8yrnw>j0zeP~cZpb85UGU@?i>c@Ujec2l;hN3Nk@|RKyS?ffD`1{ z&(NIlhuKwZF9l3MX@2vEwOs7gA(e#YqX-@5Mc1Z-Gs5ow)JV`!I=VgO;?w^Ovm(oW zNG6s~UO&93Em8i+>N^e!)6XjX2O?v^zq6)AZiPlG z+H-mQeB6@Jm$qCQIB~zc=sy3vEh^jCEhZ|oP8M*kb6>K4N93UM<;81t=*y>PBGA-T zB6UIy@9)deb#mdSma*kG-T3`uSIvQC`n*27gJNr~BBt-xe&nTf8f;AK-b=id-#aJI zU2@2eA;c7OjMjp|Vi~4H=K?f0%Wf}lP~NCoUjJ{{Q_D#;-KX%d^seC37cZ)2FoYWQ zep8eak(_};cMb0a79>JBZ~9$V>25z0aMx+0xtPfG=Pf`~{9~jOv&`&FS*T0S=*LNM zUY_OE+vp$jmf$K&rhYJ9b_kvE-ROIx`W*&dq70Qakw8vPUt1(XOr2&5%cWIn?RQP4 zPMkW#8_pE!a=fvbFH^~5iInQ}t14k3*vU72;52?omT*aJ9JJ*B!fvo(=|Wr;Pb|xb z@6K_zlgxkf7Vqp@SQ1)L=%h?Pm?Vzt9a>8#g#5{Q^p$Lqkhsty4g7y;0{lOCFXrh;h0%gf6RKm!mOWJd!lIQmQY_Z|AqJxvu(o%c`hoF5KPa!jlb zfL>km{CNP-KA=4h0{GC8b~|47q}%+<7{7ytLZ;ooJiVRvmNrN;0PYwQ`x8nI!3U z?7%syZp6&W%1Y>2-V0COCN^n{UGW91QUnY$0uEtTPUBTFUO0HrKe>hki7)|A!Yp&! zrHH`58X5f|fHMfMD|a-|VX-jjY=&RI&dCtloPO)Ux>%MkurwuU9vSaHZupqePg@`$ zG4{JR`>+Cz)TEbi)^Qwy^a35isf$4UV)z0G2Q+h}0)LQXl?`tgriW3FLQU6HLsZIq zMk97tWaz<$0Fd^>mkTGoJoGni!Mu6EH_Q^d$FUohdfvaki5Cq)5|*9ZOn7K@N-A?b z7D=BR>%Z?2tH*p7&{0I`uaS=n3M`>6vTb?=ASHQ@o0Qpds#$Fix~)93s*AQ-EL*)QD)m0HgpIqk8mc*2xJ_W`)DGPMBBfM~NdM%>@?W z9DLO?O{tw;e8C40M=%*FP9HF{QfK@ zEj@SL!FpdvI{`lo*#BIJFgQFse15(F-~kw5lFJKFAzAXjw9|~NaAZONz!z?yu*f@I z2>3G5N>3(;2OJn58#{o7K@=^}x1c8K4`gM}lTk=0CIdnXm6>h$V%+@vrdi_87KW4n zLJ0PCxyy5<04!cYP2$p!-Bf^m2U`%ct_fxaf#l!`1JR2Yw|6W9x=OSE4=|fV*}k8= z3eWKj$+mCw7BRGzr8Iyn%#I^lAb=M!A;8(kqhcvAHSh9(%OcPG`Tod{dLsb3T!#0N zb<+dbm3#jDl+)$E0b=2S`vL(@zn}WqrBBM40LIXQ!O3lAwF(n#h_E!dBc#ulQdq>s z-l(s-zE)E!usuFpl6y{Ek%m1; zjO~zkz&kzxUPec0ihSgejS&Bvrh`P~f=Z6r;}5T2FTi%*cH+c(a%~`d=ceWJ>Kp*m z-t+LN0O8nycleA5Q-$q8F3$uA$M!4*dgiH@1tboG7Fu}m0J=Esi~*xU@)dotu^YlQ zX8=$u9Qz!EoyP%Q#U&`~5`L@mQbLVQpobVtml@np>1kRy1r)0~;AcTfeZY|{gr_)o zg0mSU%C_l!MMXs=0KQLvW5prn2hR0FI8-kLL$dPqI1Jx87#`UW*QzsHf&69@u#Awg z+y^~JqvE?^+`=Ke!%h(|$h%surd%rke-hGiD* zn0ZTof@-OOd@(4bL;LC&G$HS--e2rhzz~9s<9%D8(HB5w!?su&GHmfagphmu0Uk1^%=Zg!3z_b{J>h^vOn^jx6qSZ9Nl{FF2U( zh$Ey>jVp3};D7#nR7RHgn^Ox^=RNNFi4`)~sJRU3{v6UdMO5)8-|`(w!}HEg{e3{@ zge{YRV_yJsHVKHoF+4xQ7n~bzxF)~DZa)fE#Db@;0?0<^&SagT8_hgE?rUo9sI+XF z><-NN2?+S_7rRnRh-A~pC-v>xu+HY~-vMkC$*N)n**}HpCBp37ynSX?oEn~n>w^lB ziXVPT{!YO3eF6pyO;N}WKzG?JHoNhKI|_z8wMQ4D?6;{!bQa@n=RhN|(cS`F2;=VS zP?pGd?OgyH(3JE46PcZrr#Sj~Rst9A2T2Lq?m-5_&RFI1L85#}Ys=nRTz0)gFYsFW5y2NZ_SC?`$(?S#N2%*!PQ}P#D?{B1mbu&ChgOPqfQ83P~Y|0 z{i(A>owFnA0<0WT3IFwA@xDa?scU3nS!|G|kRd4PC*Z)L_<&Q+4Lf+Se71B%f#Clb zL4(?@q2aZzES`xUcjaGaRB{y%RRCk@xoq{pRSQ4AT>E2Uc1qu9#gjE(Tk`+t3i9z7 z%=d6^2?~INS-;N1#S}Ln`DZb}nIu~Xd6aaegGnvrczk$D*w@~S zo%tfP*rtwIDjT=OwnCPe3$)pjtAFl zsJPR8&1xhxdd|^>%rV1Wd@;KE2OHOmBt6njku7YA>^jY^W-nz8-xwgK7p*(UNi-y4 zT7!S!mux*=9OHp_2cC={&`PS3kreIV%X6p6Qy2Z6%Bye(iNN%3i4cYGSbMl;=7qAEf)W zwwgjFs{9{7YZcdRf`pS83NUE(w-XFAcfBkzv3~HThXW}WvMLRjGx>rgaPRKj9Jo5; z(HkGiJoYl>{cMpJI?hW?)R(gfTlTY}p1FV=m&|*?Y}tkde2qEImN+CWlksV_`nyrhol-c*q}cgm(%;;MES#^m}p0i@(tV4Byz70{L%G z$$CQcJXMGI=Ex(exXuLz*W2tv2z>AasZ5B#wm5lp4-WH#bD0<)57x5DZg%`KowKK| zS&IrIIdn@o)(dzE=3r5XtXpT;tzoS4Puum+Btuk7X!7*T8@rZll%ZN6T}sc2Z2{v) z#jF6%#Yy+f^Sdr2r#tNP$T1%$gaFso$5Lk)FZ&o%@7eEnIr~y6Yvhiq2>j6}Z-tv2 zB}JWV7Bkpy+~dw(>~vx=w+wUG)%t3G`aTXTuh<*Fpyxfk&S%v7tKq$c(^T_xsy3l% z<9Xq|=Wvg)T6oje$`S8FIsg9a`P%zp38zQ)7Try9*{Z;LllSwEg|8+O2Tw_B{jo27 zXs54%vJH1RtoF{?$f&4WKXtZmuOeYGP;~GhYu>iU%01s+S|T^2LVf`p$-BxYaXUZ4 zVPM|PU1=vRV}|wtk%!KYVWRcVpFc^%fck|1sDo%rq8x<4N08!Z-unjbI4UCH_+8r4 zLh{^W@FxlpSP(9VRhIkk;TtJW=m&ygbOswkSW>c%&7hg5Yk^8od)1}0fq^=?Zfk`i znaj$4zmHnQWRzZhI{hiZ`N8+-AcX<+rAV9qw78Urvhp(ICi!5yLKXHbr~(yGu^`V3 zPd` zq?~yWPq_uSnbqX(iz0!9{)ov*+)|u&#NK820zi$7YX(kE>g7_xzIbt3X7@jZBviwN zaC{JPL2#V0+J$)7t){uLpt4f188hq%Fi#iY@`+kMA{C( z#0Q^u-#@Tf;y+jbST};XEj`#@iv%7-7hfF9QE+SqtoZ=aSO(Fe>AjKM0PjfbGcKf} zE`^Z1r-zH98v}hlp!I<=U@ufB37ZW029tXJXe-9uom?z52qGuH0v7J@)^nx4YM@Xa; zB1l-1`-?nmWfzl53ImPPN;H55pUJmCEH$XZLl6C?OFgoY+@l{hUEk!&f#z@} z;&m{s4YFxQNsRtQ;+wC3k@}GnI#$MK6d~0RNFD@(RnsnkNU_*=kd{()a*{zir4i*K zTE>4+e3Fp~@)L8n_L*P0yVX#{f-*p|P+6u0XR#oxak#bI`~kzdO1P_pfT#8^+F#<- zjpA6iGQ(_hue;`oKr0Gu%b)a)&TMlO>gLMSaAZ9H80NUKZLaVg@d_!DxG_$}5 zSPs8nBo}HZ$z1z>qOKVL*MU?5;X}Wx!0MKVN(?xOBCu`nQx|D;cSZJ$T>9DDcH;}d z!JJGgXk;zgaEaOY%vk4ZX%Q#Q8*Fd+h4vjix{_IrHPGVdq*YTsI1_t8@&mhK0ZOaL zEU6XhY$|L8g#&h4TJu@<_wwI^=&Ql~v_a%^ z7jdlhekVaL;Yb|~FR#J}1l8X1wl<52kFaGU zDM9gDVBhQ`?}6@o?0HLDTTyVQC7=vElX3jlSA%nJY@m~c6Vvpw>%z_-KUNc!-cRDs zW{%mj`4~^I4lgQo#oER5(1uCt0DJ1iJ3AP25ElgwXyiFzX=f*nE~V=T=!&@%rTw}6 zeV*e|Q0zo+9BH|Q+AS}o=wO}RE!GvNSRNj0+Xk0!GMIx@K9N+hJ`vmGPrA9PHVTFU zO3>l``ZU8=CVxHJD~olVWQ3w;jF0;(XWZ+aK@OE?u;|+(P0X1XsF%qm1kFanl-^=M z;Ze51c_wYOC%!aAW_)cXi9fH4zSc#bC;m8MG4}_N!R$p84q%gEeyT@UNtoy{J3y@jEa-+zEpo`zIBGlIxXopJLNfupHD64Sa3GyWEXy^d2!m^jTkB7M)tL40o_UV;%cATAq;Rq+yI> zEH|@kEGIMqoM2Fea^Pdwox69o8=E>C+%Lq*;i2u9ILY+2**5iWQbo)P-U}ad#4e~a zqn>s6zT+k5%vVSRiQq&P>BH<>#|3eEFJ5<${6yZu8U_Mb2wrTI!ehu+g|lTk@kdh% z6h>j3tmoS+f-Dreibg+Hpy5Zxk+UiIcklns=HQC117Du;C?V< z28Be@pIvMiuX*|^fp1qcG}4JPOiJi@+AoXA1{mmh$Bq`11U{3)3sDZ=Q5vbuopLTP z?JqEUuK1G^0g4p-9QUGg1X|1|oQVuDE$^TFcs6kiujfRM3L`oqgg?&2V#8_m=C$Es zq!W#vZ8x>~v~**}+p^3yx59vy6vcl2eC0mh?87}Y?ucnajVC0nBu&)fNU8|X?Q%ls zpmq{W3#8U=hu=Li>avOoHR-`kApL@47yuq1*{wm$=uCs(*ySwabuBkhUy|1g#$+Ei z6y)beELs|V9=RX5BTe6SH-qH96m1AgzrdbL3mR7E+^LpbZ#2*QUdlIQ{7Eg9sQA?Q zUS+%ljGm+bKzW>jqhM2Zw7Ca6q@*`6ZeRE^fKzh(Kf?fK>unPnS4>>Syb?6Peyh*7|!ru ztAxY`XqZ8eDh+D-Uu!0b4p6+-%)fP#E)6QaFPhX_usVOYDu<&)TYQSqPszWVjJlKxcaXV znR7*31%Y?Ymf25`VWN?@EiknYeS0IO~15yJcVkt_R!=d-!HU)=zaAbl=-3$#UG7KEVM#8bQn2n8%!5|?K z(iBCf=mo7^m@rHiwaL~g{S`=2!mnI8hm~uI#NG|kh-;ye%^RAF)d9^9E?8>P>ZO7a z)0Z&#*(0d~6}tmOGx8d^D(6Vv1XTP|^^g7x?h&~{Kz0CjIDceGs>5e$qHTWh`Q{9h z3TW?=u`rSaoPOX>Z7!PpczVjzIeiZGosf2( zdHmSZ-#YoXhDbbr&R*S_EG!2=Bf@ikw0#`KA$2q{ooLYtH;#j+4i0qECL}s)kXqVq zE1&P)aT|r1nV)*BQrp=vW!K-(;ZQfF>|1j5;c3k@?##fj@8@gOEyLE@u}##uY~a_Z5zwq?a}$Eq{^dZNsqS9d6$*wXE? zC1F%a^ZMwdSHs2+O$!$C7`Tu0G)qyl0%l@Yaj_ zne3t5S{L-s52YTFlxGjb*b7>WT&VO$g3*V?k=K$M)@b_BKfguBR^X+e;r1HpXEB6#o zHwyX!2e4{Vp{hs(f8?*Jn}PaC;*CtKuPV#Xu37b^Osd6sqhgP-|D{`nO`mmk>n z39`8|r+@!q?#+a3E@pP0AHgUYBmHQo%-?;|QQy#TUQcWWgS^l1_G)l_Mb}@(riPoU z1q71uzH5<@Ga*OA8@g7x;P~M~5gGl#7dVX9{FQ4%!wruOYRCdaJcTYj#O!CIPgsGI zg-mK(1F*h~k{Uj^pKo0H+|p{ypE(+TXm1s0<_WyK-m;v9q<%nLwM`r*A0d1Dm1{6E znRR2+OsQ-<0V+)3d2&I)fwZ?6D(KIX`wmj@3OI99V+TcH#b;wQTnj>w(a=1QcAfaW8~o1*Q}Bme}NL7KxsL-yB^ zzl5X?Jl31lbmva!LAgks4Oy3wX`Y8UO%mLtP~74-`KvIP60?F z3<{$1r*LO;;n9lc{s}7*0z^8+P?i5C3~8L9o6nHpSwez|tlt6xG{dbgL6`)|F%9%! z2Sa5ZQVza{3+-CfJB?!#&bq?PjYN)#-wcV3W`c!khtK_=JQQLn{Wxb4SThTiwD zZ2Io+{v6GP-a**YDftcGq2w$vP<+|6rKRO_k*ND^dovj6p_r}2<2gXaQLg3GG9wWs zWWWqx#N0dPMN6OvCY=+CQ<(k(N+o9jJllhKcJpCX^fH*O5Qgf-ix(<54al{O8i}w6 zu+6Q~H;RuOXZeDIiI|b-9veJ7YDUNb2+;I!0Sc1ZLMQt)f&S!SD0|W z8#668V=2moNWqO-l(@%aUyO`{ zWcg8jV4Bw?AchriLF&-Fn|DIJ8NMi$zj$AmivxBBw6RFUzA#rY8sT-!AyzT@ z!D1&S!?pyn^ag3xsQ&l9dGW$mi0$%Sc$`H}qLpNZFoePQp3h>x_e6}dtcZp>8cIfV zbV5-e2`eZpg7|hhCgJBWail`JhbtVn=}FN>j$MM=l(+anlHtooP6pfssJjdPY$KXu zr;ZwGYIet65q5ELf$-_hT?>{jo!IW^b*NQ+S_KeHX4P4z#D_V`e zwohwyM5>|83%R4;4Af-fW~n#ujGI3G#(_VcK6iEI{Hu*2xd8}K`jraL9~K+PVt-C< zhK1W%WY6>W`9LZn)$leKGy*86=OAgx$hT0}QWp)77Fo4-{!tcUgkUU0 z^OBo|XxKr&$)akI+8*mOknN~AIHcQPjizg%Nx~MZcOHHN zj8Y%~4jepqt1C4yb%k+0%L)unQGuW}7#6XA&z?`HzQ+4r!5W>E2}$ffDry0lxApa1 zItC4eqW3Jm$U0(TVgiHm@`hQx0UJ#E+}+*Hidr*r2x8$@sEDTT>ObzE6k@Rstm2?l ziCiLW72+O*ujS=&n!>;N%llCjg)eX+RulA3R-F6x?IVZ>{$>DC-m3RpAcu)%o8Ou% zwJUnh3Z+JLUPRjhd0Y;)+aZ93T|q+YD628^{yyr!$G|hVgdF@rFWBUIs(AQ^SeV?L zxKg_~<_-DiYkS~$qM}P!uK@T~kcVAw4jN%4|JhK&a7Yj_Y zWMwfdyD57%{4fTCL|vyb8Jd7M03apkC(?h!#h>Pm%)W5Qg1NNlY%IcXQ%+Xzeuiml z(gSX4NjtumJJI`k4!A+2fkS{}sO|H?re)VdBpal_wh+R}jC8}Cp?!b=(F@Q+KHfwI zB@71AE>4t?{sjdaVKMLtSYjxbU6|;)be)bzsI)}}=OAGy#UYPu>OUUeXnJTX0tBss z<}!I&VD*tCCaL+aC`Pvr4QMq}sMyURUscSClW{N;YzMN+D<-_Orf);EFS*_0q2O?c zh=Ak@fiuSYu0V`%&YW+Fh+*6;OlO4(O|Bu&?O(m0ogZMf?o|KmH%|jwm%0R5KRJLF z9Th&P^jN05RcFWr20Mz1Ymz{~6c*E}S%CD7{d%=y@cX-U$EQey8#|HOH;_`;e?S>R zpd|enfdpp`x;WcRO|RR0>2kzmnk2^&d99&)MKW5)A@RhJ7(tND;Mh3xFdD%0%&6+3 zQ6)6|&=i~QZZjxrPB+P3v!y5<%V*DnBa1P+T(!OID&WHy0&oFYbW;ux#f#56GfVGn z&i45`D(B?B^hJ$krZ7_}nq>rehU}=S?S3R;q842^0y%+;T5})jtV-A-Dvn2YseSedy9~_AxQwBZTo0v__qaf6PAXOI>TAQ*d+c*}TW;E}Fbl zqZ1<>bfB3SNOz!l>4&8NV7CB@-&Ik#(8nCHz8F(|!XqRNRW!tD#i7n_N;jU(2#bl4 z7}s=$+#R&+VCuSnWRGxE(sG10poB9$xk^?3$uK5v#-T#NnaBHP)-;c{c|bs)Tp*En zS^_H|Nl2jfFJD$<`P3+dB6h*%!=OL~p;#2H$(30s2D&_XK>A^Vki0~s4t^8f6IsTU zo6tcR`-nzg?IYwOkfgaNdNe6i!(UH@0h*Non)iBt7 zwl9H(F34qXq@?(h()-f>YZDHK4t;uNa=U1w<7{7@{WyxS2rGHz8>^Gbr!}QIFV%h3 zTr(VUmQF!n5MkmA5i+~n5*q`=55(HbWVuS=2LIB(;WC_u1{!L^uqn}w6SYM{6uu6U z&P$gqnSRd?WXWfA^g{Bo@}*0Mg3m_4HDewb9Yvx0bpxBxZSXURfQLg%3|;ll2;h?1 z6%%)JbIFhrw%aq&xu%jIc{Hh)di?r&1`~`{D3ej!2jw(DKdiou>I2GG+U7GA0`%-B zw4mrAV+8c-;KCT28Y)b;LXTk%Xrmx_oi4H0 zVXyXeO(-h*_{q%I@%^b|34!Wp9}Ic?s5nlkL8@ZTQLbNHGb3VWzG-aoHVX|*<+Nr7 z#mPwUP!D6+Gkr6MNYdP@YcKvm6gT}xbZggt;LZP|#Nz+*MX5Vs&W%kYsf%%m%{4kt zzkinjjiAtNQ1xXMI>!)rXEjPUVuxuGot)UuB=?rP9k>@8*~Jj!$3P0!eE2hNuTf9; z8p3|Qx;+<|Xr1qSxElkeFX9)^&Mt}LUG0l)4%$U&O`x(xQFXZzUQTy^U}_Vi?6ZXc zE-MXh1x{>7{6p4>IdvJvZO5Sl`v*Y}YFJ21)~%RC)Dc>_k{$TR%Mk&7em5}clO(5J zj2ofj$x2iRWV+PtggIOH#720IZ27gbP<|^nyeFDpD;w-Bo4()thFuba-OrQ<-(7nC zsibXLK)>LcaBn!`K}>Pw-Y?DtHrG%o{2uKOB?v863}XN%t2HccDUJfDa2|K399<%; zBvO*3Xr!%-?M=v6EVn0jF)qTmoCCU+CKq(Yu9V^E!p#A+uo{Pf69$|}!GqqT{cS>F z1X9Ly8&uWQED@IMfdt0HK+kk88yn0aW|PRHwe<`fpwz>KVavi^wzIy-_<l5O|96S9R*;>U5_SM;487WaO4$#Tu`D1 zrmtPY@JyZxMk%XLn>LLWdE%WLG!`-}6~OWkwT&&Y*!{@&uV$o~!}9}-Tppg7^3CL{ ze%{FQn$4zeX%W7-aO+jF87xcVhYcGv$ckFDtqIN^nntECI=^WXlK%iMGs zSenRsC`ZS}%5%EcWnbRt_FeL>{bN(AbyE2D@D#6Vn^qlvrG*O-rsKN4_*OmtQRfp` zZFa0#qw-CqR>1#i0a~Q-Fq0Xd_cNi4ss&X$qO0PNyXCmD)9%ES^Q>93SgJB8tZC8a{#^*15r<9FcHjN9s?l?w))BPbN?{aA z;31T~CSQ0A>%EVPIru_HP_u>mS4I37|7eDBTyxeWUN=n{!LQ&zO#Ej?e#5=jWIS21!$8!@$GbIIIlOmG^beWkIqTr=l@{;boULC!*W9LOrl*qg zAo%ibWW?&COr&bKghQbY^T9uVc-oxf+#3-PMf#_w404ROng0Z(KzFvMXv%a!*^~_# zs->k>dgHycQ7BnEQQuScsPW^1Ar@AQ4sO4?yK$Oj+OSSA1g>sw0pa1{s@2`vVSz8E z4Mkxh>%9TxHl3K_5#rdr^~b^ITRorsV#3__O~(8ut<@8ZtvWkQ=Q#RbioG~g+1YEHLvd!Y)a zcQ>E-*DyyMb}Es>$G%+hY zBk36&8uPOzmDzY5VF>|v;|uDMPFuf%_lH1y96jgQ^Yne&y|FIY?xfUWqEh{L~Fmvtq5Gwx;lN2LzG&N!d?zB7^ej zVcLhZeJMgy!n-@?>CaN2mnc3e-h3uAb z^vqVTFZd|WtISo8J#^r}8lLKAfL|9`%~J{+=nAwJ)|cVEA^;WPBqjrST)8Q=6?zgE zCb7p%OG|BfXv>z7YcAPPKX<>Q&31MWZx`To2C{d0vZgY=B(nZ2*dqIYaDW6MM>@d&3hTwYsS{hFxC8EH#tgsfyj0;nIQQriKegal(&@`~ zD^xbI*Su`zl&$CH=(nuhIUsPW@w+7-)&++AzTr#0tcYL%p-kgz)M)$M3i{~Vm}~_! zQjnzv%NLd6h3q_vzzLb|tz#TdVJ^%cn6SDcwFZNPDOuDN+#H5P_-v`diEGNT{%(cz zO0!K9y%ZUuPD@>i3dyx7C&JAa=#uqq#%KOS6vb$GV;o4lZV>072jhF)t~tayOWS_| zkX?RhLlK2F5cpb_aNKS*t0hILFsDlJ21)ynB0`9XtCkMftr9r*Usf|_%K2UjJb(%& zDc{$r8|<44xOQ;ipvZ8w$nAOwQT@`_#;0B6AHBpY9mfTHyxnHjL-yBo6ECc^6a;gV zB0;IPY}qke@~mG4Ag4W<-|L7#jr^VK0{ z*CnM;ij+janZRdfM^PcTuu?0e6dlWR)@G3EogbG_V!5e?l`qnc@YoZ8IMn%fzDnKm z?{i(PT$2yy`DLg*nwLMd@QS)3dqJq_$U33nuV22Daig*oUH1-V~dHj0R8R-<5m; zUOF3?is%wZIch!l#p)f7XYGEUYG7bsR{rP9rEAX--t6%~^iig^%7sa@3xVM@2d`JL zut*$KVQRMN-%NM;ufJV;Eo?h=%GrR|Th(3f-L5sG&CukPm)wBTT62++#1EQ^8mHHE*lf#FlG;&y{Gc5lu0GNQQls%ohb|o)9 z^l?iYBObz{(Ct1Wu0$P)#c(C~IDL3l0yIwnIYm?|-`@Wu0t&f)I)ss^0>)8!wgJ$@ zBg9D`j8ut<@#Yf*Y6AwFoU}j%IC~9cwkQ&TNZKHE5*_ z#%!1ZrAzqEW;KpC1(taEq34e545h*`GBy_7zO3vTGUTGV;W11zvP6(zo5@`k>Fed9 zk4rt>sX&+g_}oD_K|~)lii1Wi{L|7_ues8A^n{w29Q^FD0A@~`aHNZqNluaMnGqp8 znh;sYdi*zJ@6xve|8=#nxFkYJ)-I9|oQ|v{e~^l-lOL}n+=!D7bDQXg+*#iq!*cNP z`RGT`Rb(Phl$G2hu$L8=UC6%dFmFizG?dC&HpUY}Hi?|NHc|)H)McWV-&5ynDxhj3 z9#Oxqk7xdK=SY1P%GBQ*wmk|^L}nXbhBT#w2Qcb}Ar*9Z*inD&2>tE_*Eei9+zTAG zYqxH~dr&gf5N*1mW55ru0u^)X{{715UmdGTs!M;yXLfz_ywNu9q;bZUX*4FX>LT)- zZFp41s%gF{UkiF~4VC=Wz4-Rp4eHtv4lNRYQQD49u1!K=N;hVDj@Z>B9aBQR=;a3Q@U(VOyblH*N$WR702B1g)+HwGCgkJ)OL*QgNC#hwzVgm7+$6sx*eR(*)96`nKh@IKsZL7flM;#m6 ziF)qP1#q>#5Kr69kNgf;mGZq`&>@K@HqLH*u!Af+M__Hlz7$8MWytJu{c|M*W=R+< zjY|CbiHAydA5%-V((&5h*sSt$_Goh6atpvdzMsi6nZV3_omyD=(zHhp9dedu@%i4# z%~mtpfd=_rQ|&R%x}9$&|*cfp+s%nLm0 zRuYucWrUCV{zf1}hz#l?q7q>!0(a?Z{`S1Ds_;kvp6uE!oVS#NwDQhRwp}d}4VP@B zg3N$wu7_Zqay*EgV6lEjm$u^qmOO1$ZCHZ|cYHamN$b`*5&&pyGpGP*9;F^D;D||- zRNx;RuWQHMBpnjI8w}JE;|;Z(^;2Wr4U{(x8a8w|{9abXqlS>4oj4&vbC3#Z#h827 zu_12XK6XRaXfcJ|0sxy-tvh;s3+@LZ_yWp5A;&QO=H=yO%>&1zr--=666IcM1FebO zW1if(dsomMu{S~AOB)JF{bohnn9*(bnYny!^KkXhfsIE-xQ*gWIX|o#>yq!aW5Ihb z+YRMk;2mUP0wr@Giwg*k>nSSSk%Ni)Rw>pw{)-25B@|0|r=1YHx)gq>Duv#MhSwnU zGWUD@yJ!kvNRNQSkZWEDC3TzTvagwzzZM9cI7!6aHF2VW|HJC2)nq2XbnUdQz*$P^ zkAsUSs-gJ%vgZ00wsRz9c28hiY&nky;^VhqDJ~?Ie7`7tHL(bLe(T=7$9tRy*%A{y zl`X4!rA(zl(McR?Yda7r5cf#z)}Oz8xqZ%Oj#x$S+@QwYz!POXZk)cX4#NSk-?_@` z++x>Dtp|Aho{puJZ5I~OOXJEvz&R&*_W&}?>WlstWhd#kC4O*bU|;l$&L(yVipTzB zf$E?tveR0ftqfeZPEa@=Y%Mk8{3SeOS&PPoyVq4XMz&x!fLLVw`l5>V z-nw^hU0PaN(!A=bN(LolBTHU@PYQU=fb&eRHE?T)?Pxd^h*6l(#&Lul6+pf z=FnPF&*L^tFLr%nG%9jySXgUCi&@?G4C~>P|C3Iz9`{jqYO3D^^Y@1N%Lk9|Rv5a# z@9k!{*(LJ?as?f?qfXGBm^iy{4~;{Oee>7W|efK^I!Itq(Lz z?a1Nn^4J>WScMd}9+^pBZJ-wAn<=YL?9NmWW&Y|3T>+avsc{Lww26$W9*a)1TdHr-Lt0@m;F^%!F?5c1g- zUN6b(<#}%y0CGrD)fx9bIKgE)WDz+i%=5#CKRL2og0Dlg}p0(#lAu5 z!cBeq9tA;=?jH`_?rHL+3?yw7)@D)tz4qos&wBL$)+e1{Y*N+;D=4vp^k1^l{F!sw zs+E*xb}i0!6O%dg0LmWSM9bd2_d`XSy_W(P!Ve5S6V-khkEc8BTH#l~&GfvyFu4><##M~=O;U(+#Vfvy$X>ftpQ%d`Z64>e*fU^(dZOF3d`s_+U%TD- zaCY;>y`FaDkU_{WabSS2ts zt5fn5hhvl9Jx0jk5p{51b%xtDs852*sbytu5tScP4UE&OVn5d~TV|$?Eu-DbyXcpd zWG=S*Uc5JPLH{4d=R)RdG+9*XxG5TlH~Ys2VS?N1RG*KvV&Nu_Gr4a?>K7{H`m~2c zqm3iu^AbEZ9S))1&zU8w^F+Ri|LHFj4f$u@qWnT|Hp%hptL48c+f9_5_Br}2smAx> zw$l8x^rTbH*P$o%OVlqK?-xViFbISLZX3O6UH^yW3u{!RCnF;#tU0as^7^MJ2vM|x zK*0nKp1QRZZ9RJWMot=whPTKe@=#V}(9F=ty|Eugp><7bFnn?7wocYJht)R>x|bO< z=EFb#Ow(LGWa8y3dM++;POU4gSVSs#hRnneM2zmG`uok*r;sGl-yM4NqZmmBrR-)@ zhcC#9>EMAMQ(ujvH&eYdeZH>5aZPx&K=QPn5o~&7)+CT zkf=kcOU28E;0O)5J0!lXm0qhjPI_r+=>^~qfz6RuO0&pg;?%5QSu4J(HDiBxumdS0 z-H8TfZJfP+!!Ur7O$V32U|=I&VLekrE@09!(Kh+Qkl@BSEljGCznfa+mNolXUh$&z zX{O-O!Bq}UlR|E^X}*88wZpynGwLEodUCzf?E(8u`fi!4j5x9M{P>P$kYgh0L9^I~ z+Kl2)T3>gnJT~pwp8oOzsz$yY1&|0;0hW9^8+n#@MDODW2?wf z;dUX;X8&OdbecAAzQZ%0YyGv|yX-Ef)1I$-vhuRgT17?p3yX^RM-1vCBKzk*M8dq& zlaFgh)aDzng>gj>w$rQo*r~c-E8=dv{d-Go(@SF(d^c&S80mZq7zA(~7b6O5R{0*_ zG}2Q@J;Q-*+huXP<;~j}>4qC=Jo1^N-$aw2H4|Z`*l!6StO3?0M~j3YreEa7n6AUE z&0;!^Jn3?=tXW~-njc@@91a9adYokgv)9Wkey=?y&3>i%+&{JMH#}C$zXqnFs%CYT z^C;Ra1pTh#ZceEorkagiG3AC$jl|DhE81nusAf5HTO{vVckSI=WplTCOL}%5U>T6s z^5mto30tzFT{}=&9_%$u5yoZBf zfwZl+hiW>lp6=5JcxhaX)9iNGdt6W3U;cnL&uTdi zoL!?8eysLueHX7tdNjf;h*B^oGWiqj~nl}ik17zjfy&IY8SGzi&B1EXZ}x^ ziNmd!(QTw4rx}zbaAf{X$B%DZ8t>-ptiv+UBrvr@7?PwiLVR!5c&h8z1zsTm3qCU- zuO3Q)05B0@6Ne2SZXRKU;uvS)1{q++GcVg9WjZAfHyfS{K&YP4imiwu#9)b()^V{;HAO&PhNkBLQsg!_rk4PS{E-~6mbLp)%J`yM6t%{ zh|hxezl}_=l4AMLp*D&U$`;be;QMi74!{|~^U3BHH_Nk!>d0x5CMtPI{4?3FIN;CzJ)% zRV%5xC?m7MIkT9F%0`{`qv&oB`{yX>vL3~~sUd}{q$d=+2pnEc{uUwfLNjN1235!LUoK1=FvikEW=CHsNo+ueFvM5XB2HP;>0y4>ZPkCY~trb&nq;?f5g0w6Rlbxk74)j)9`ugf4N8W5* zK2q1;+G7YTVWHn`dL}ne0;+syJPCWfdE}KXvdDciHAT%Pv1cyvrjC?c(8Qiqm z98SARRD&A6U4CP(T^pP$!yI2z9tKSAtj~Vy>;yKNYbfi0o@tf_5`OV-Og?ZLAhIsE zEQ9=%l>p8mx`PZN4Xw^PjO_K5byylU`oLV*Rx*HvvlF!24ACGv6)p!{R>NH))(i99 zEZKtyXnPVZm;+Mu?O8f4T9|{;4^5ahWJn;GvMHn6ripDuv8I#ADgTWgzb+l<_xV>V z&GC50%lF!|$6TgRkz*V4=sUX%_xfO}5r8($SH(XxG?ef9c2QkxNF@EY%!cR^0D74; zf8M-K@bjbZd<8pSN8?nNYl0fArqS|sepGU-Z~Q|mJwHti6=iv^lZ{p^8rp>~_ zVw7=Ps!T!__gfKT{WH=@@&CFuaL~(bD3CoyPhCmV@@wPh$bIjxM^%Iy2Y}YAQTe9 zZ`}CZQP0q@)8lA-u3=5N?CYtKTa)@J`B^?S+v$qcFTxB6?Q^n7+-T5kILIQ9;r=at z_lQAsEhS9SK`IK$+zgNz=8e?R)YKG-0u({m8P9t$D$l=s?hQ8}1noc~PxgDSi_XsQ z$6=}bCTOvmA|rIgd1^d3Fxct3B8Ijso8=A5QY-IX+ESM*hwjW^!tT_R*4o;!*RTH> z?Opk~Zp6oPQGa#3y=v{+wb-vmr!4n5JzPr-@k)Ut3A(ME#m5ikEuK7T;nu`t3MDf; zn^j*_V*ZMlR^c1n4ZnKC>=D1DlU|!EN1Sy^Us&|=<$mB&ioLb=Q)nlY_>x1M(-y+k zBEuJ35@%j^`k^6^LB{VG03>9o%&5V)Bs9p}*GND>eghFx66Es@wgp~%8(HpqbIhvO z9n##x&X4?X`a6zwmfY62OUavle1Fx3g=H;k_{@yd+4Xz8|JrBQl8P3Mu20-hRK2IS zquWZe2>avH(prLo+s>VPJV}B&NT@tBGG_v|-~}9H;VNjLuq9~4X=^%lfcKNV@?xz% zE#nag0&584%6>!;GL7U$)PN|jxm~6v`$PWoOH7{j_MQZvi%7ZSnNP+Gz24n8PyZ3K zFn#Y18Xp-FKz*2j87twXVFy_XEE;#T!x^mf1rg9mynk{^bIQjo0NEPWhcpppMS;ww z#wf3kTch%_OiLOTdF_i%JdHyZv4!}YBW`-knS;k=bar-)EMQN}>jx_D{VH$Yc-X4{ z(Q8?zR<%@=(Qh9<_jvNDaq%L6`)|nex?zQ@X*5KD4G}1#s=xz9z9}p02<2hX(^cZ5 za>7GH{QERx`|L5P>AC*s@8x;TKG!{A$eWX+J;K7jrd|ff|sM zR~H_$R;^kx*HA{Biqua^c>ZKpna4pfvCHU*>cN3&@PLkfW!Dqie zpPmB~P6WV7d_lS}CSx!g=JVcM8B^=x!s38pQvq{M5ye9vaSlBv8i62A zw#W#{_mokm|b1Vn8s0T0u)?%^A@$Fv!u?GV2~$`s(lq$K7m!hGUfRXqPi%=@zX1 zs{&qs^5p!}e^EH}XjE%~i%`72j4&O!(zU z3dl=zh>ngv)~9jn<-7Cp?^@*78=1Yf#+VeLdUkRn@&wqN%CgFR_;AE~)4t0Ev7A_G5fFk>Axc771lYSc{`pRK6*5c56fRUBR}{U zOdI>HYNvm=F``#>1>uK$5qX7c+z@F(KXlD8iM;Un#I?&;uF!#NqMdN|>=JE{6q|OE z>`~v{{e@%kwtANqIesnO6W*y`D-X?^c8@a`IWANic!5_LePUwIS+h##2bQO6hjzCC z2kYs1Zdsdyd||EeS!d55MC-o3;40;^`QPo&|K3Kq)Bn5u zIo6d<=EUT)jejxHwxm=U{Oc3s>}p5@A|`#}KFhyl(9nlav%dh(r1J-$gkCqJ-t>O| zq^D2Nm;=C?_2_?frX7EPM^cdBr>?hG*RE3s`?PDuy1H)~9m%MN;&^zwZRW@}@Nf>R zO@nHV!b;*G-HWKd0mk%@d<@~@pBsEEbZYXlH4p=?4CH&$XuTrpUmnMcA`}9Gz$L-_ zm%3z>QA=m(jzybw3OR=W@iLyC?&lU$du8U+(*t9c$uKY(rbP;ivkb2n&m;yq{KAFB zcT9YOYz_x(DL8Rq;$O2=hz*d|_U4OdH{fq51MfF&M&`!fRSVdHk5MHKdHMF3Qp9ua z6EwJb3@Rs?pV74Y^x$)YV^7_=b4NCE(x|w>W{BXtMmRWjy$+oKAq*!&oTbjeLEdVI z19Glivw@bQCu!TEcP z43K0NZ;Yjm>Vg*+8;UEPyC{3a!KaRV2n}{Fy}wh}dEvtD{5}AjafifHr!G+9-q+C` z{kW}$D(uxdqUr^lu1^b`XA!&yp@t}BN&%R)8`9Hy%xsYJ*3E{NdQf-b2#5A;=o92rFni7%tS%}wC8n6A#JtOhq!X_$o9&|` zB19da1aq4JA}?wh*#u8}4duRBI}E+DL62GfR2y99)^%$D5IkL1suP!b8Bf&x(JCnI z&k@e;G&=;2-8br5M#5}OkL^1zorq|6WP9+9rtJoec5U3=)ktq~==_cL!K3T_-C*Rp zW&UBUO>a+6sBe41PJ8(1%T8`bKc#7SjC6Xxaq``K-f5MJqaltq_nvU~^CR~y9#|J9Pz-~Ctt`vUfFL_c zyx!bz%H1Bkmk9p>{I5X&AreWL3z5osnIo?Y?P0O3T^#&BxNGTg=Qhkg7)u7;OFi?7 z?lp42>Q&6x-iP_|V#|haw2ekQ8m!qroER{J39@~_evyOhMg(${x@mL&iQ9-DXP~;i z75-RX;nUH~EP*m@$ga5JbEPG*$Rs~3&pj3w=ZH}1r}ax(hsb`u3k4PA6Tf1H+6`Lw z?tb4&d9NyMX4Qo9kzc`66jNVg*N9@OdEV>3)xB4#t0_wRKW17hihqWNmZI$Ms#^0W zii)DtYx95lU{v6J@TO^fc1u<&5qZy$tGDGw{l zR{%-&>W>UE{0Vjh~kGN2>u=_wV1|@#Zi8cH&gyURHV4$>i5}&fMqQj?XgB-oarJvu0*e z4(XSBQLO;rUj=)*3(AqV{Ot)j4XL!fg!D7}|4d4*tECulLv0C_&R(-09DiKWQt1AE zJOPZX8qTb$os;IDH377{iIFw|iL)QqQ$|$3HN4A_B3pDoD0^kMS85~#C{vqB2};r2$q5kGUM*p` zKTY<|))lIf0y_E+L(5)rk>MU3r%c%k3KV_#@Jq*istY13sQ>>2I5(e`+7IGkI92j* ztO&^Ce#FMb*@pok^l#p4?q#Ce9p(_C`t6K$^)lGJf$xv2mvlGxSCs00SbtqlCV$M? z^cDd{4hd;>`A)8tsongfg1~7{^8pt(#eYA7?fWG|({B)oe*&XKzq+#g=?>Hr zRUax;e~t$J8#!gl6v)~sUKQ{)chE_-y#lnk1Eu1Tg9lsFm0jcM=}v!v7rHa?bFuMD zFC%MZmJ6JUFO#1-pWLy1dsTt^>hF6lt@f%eQZGkQCs!1gN?#u1>mB2h!wBIf-@m(l ztEQ80X63hxtGS&CIfqFLs|)5TtCo;msYlGQ<=U>zweK={u`j|=&eQJX&vzX3%Tp>o zoCj`nU`ALN9HM^TPesH0m0Q-_Y$bMVNKSj(syUKvL-Gt>IrRr5^IuiC^9mR%tgUq%1OV0s6 zO9!v*efHEUKAnS=yVLV#T1u53)0c*Evi@{Q52#ZC+PaNnG@aQUU(PJ$T&67l;;Ge2 zKRhDB1FJNu$VC#`(7jF$d%WhVOrw^-3(MQq4t=kur)L6fHJFDbB^!f}oY+IL@!o?6 zGYGA|3_tplj->4sKIv{eOA;)hltyCa*++{c!Lmcgj&rU#ou5nestPCYO!>JW?UaOk z;4xpO>?d$}kR0^4T=MQRdXL@wV;K<<(F_O;abjsu)et_#jGO!R^5jFvb&=CYFWO&S z_2)>Ey2FGCkF&?`?6YVFe7zikwgXn5@9U8h-mmiWU72G;X4y$HFd;G6g`pd43&D4V zZr_n2p9#oEf#@eN@dJ?P{;bzNeLBQeoY?K7GAqoiW5-WG8P#v%)yEMJnf2fB%s6M~ z=|ly)FdSSxx+IqliaT~}5g03bEIMxK_i-4L=37CkMF%{13$BqALIZ literal 0 HcmV?d00001 diff --git a/examples/model_compress/comparison_of_pruners/img/performance_comparison_resnet50.png b/examples/model_compress/comparison_of_pruners/img/performance_comparison_resnet50.png new file mode 100644 index 0000000000000000000000000000000000000000..7214a368b02b907d578988e920b82baf87ae1b4f GIT binary patch literal 86829 zcmeFZWmHw`+b_Hbkq$vYLQ3gSMCnvOL1{^8k?xdkkW>jl1O%j18tG021nH1Qy1U_A z)Bp2~^Wl6tXPoyP?}wMM+1m}vwdTCzx_))f*NXBo1h|yAC=`m|@uP>zC=>=Ad@o{S z!B3bvJ`}=#FdZZxt75~K2lk5)__03PG4CIa z)+ivqBKM>}Bj>w^8&A$d|tXO7h=}u$oW- z|9OWXG;HKGB-imVi2uEYhvNUyH~Ou;GSd=9aa$i%T~lN5@+B5KJ9~eg4i#L2>+a&) znC^}aJP8Sj7LC5ffix++64M?Ag(EWDz+EG2YeH{tZ;t2X#MiH1m(uHbz8w4hm^?N- z9RHhM-HlxBGU)^!<4>w=)2plDpFYv8PSjLsL`9^skl{W_5v?izDsjp7;@lJdOta9P zGNf7RyrwmBFJ+IJi%XvQB92pAvH0cF_=JRpU%v=aQ&U~fPb((Y%y}vKg@lY6{4ObL z>E>r>msv}sD(X*Ezv``&iMsKtFt4GZ!EI$~V}mGA(l-!~d|9W~e1J_;fQgC8*1_Sy z#zf85LW=kD-;1M#lsl%s>H2e2F0;kr;s&C-Y)lMjmlC7vd7s_$@bLJgoHMny7Ljn* zL^4-x<(I;4=3UlxT;i+2vFz%G!vzMLTU!k#DV{;*17GMxL}*O9J~nL6b=YnHeY`qc zm|p%>bCpds+b4pG4;^(w*wtsQG20f^ z23K#p_?te7$NT)qMA+kySj24?rRX};yE;rbnJhvj6G`1L@KxQw!h(G+EHt!nx+z%cs~V?bt?Xs$5 zVxtVlE1ilQBcvJ6yt_UMp!*!2Y}7fmdu~qoD(0vni>YY$`rVhtVq>qi$Fc|CF>0mbwH~9mcI_Gkzb&25$Es!5SFcjNFV1^StcIc( zWg_(#f2XpjWa3lP(DbCs5RKJ&>y=vXb7>q2B#Y1Y<-?~x$WTUZV7SDL^cb#=n>+vc zfe05D;a9a>$tvfy%^n%rrC%yJ%C`=Wjvl9Mj8*v8dY&|nj9iB;{Oqe*i@KiYt(DA- z+>7^mUau=078;>;&Ck!9_M}}phb8ejE6b=c00#krOQU-WuhC4)ZIvN8ZN}} z^76WU_wIv0+^YppnmcOQjRdZ2vB1EjD49; znfkw6p@-X-Uhma+04ve=b5@mS<##Vy#o@D50-U z2c_mUjI|*&lOirf=wXYa-`N|RntG~SY*xps$fu{LLqbC(!>)0@dH3#JH)CwE*u^O+ zbdrLyviH4)Mn*klHhR13<5VygO$7!`xT^bYwBGm`%CDa$-iQ8<4joBqvet8_#9B9U zA@nNyZP!hGr^?XLII)>a^`>qms)mScYe$s$D!#5lQ#_;e{rnhE*1f00Q@yWm@I%0^-*HyA@HqX za<;S3&B)73mM$I97)c}ayWGym<9I2n$ZH1%A9_1UYa~tBj~|aLC#tW*6-!0k5Gk-8 z7fj+|WMnL~9N|IgHEfWF&{=lZR~CEHkspRr3tYKONL_!tx3aZfwZU-bj#7KESs#<* z+UV8$_wNr^zq(s{G*2b$bxMKE1$W(PnhMl05v2-yW6M9K1f)z)>NB4v-Zt;gR=;r| z;4rTbvze`1BUoxP8JS;qP6-?6AzZQT*|C%6MD=s+3IX?h4$+g>?B@npvS0b|KQylXh5H zSg>1LTWxJ^#bK0)Nl6*C^Ugksh%2(iW-Die%Ab=m%hTPzuO=vZR1;?({q`-|cP&ns z+F)9*dU&JTwo~G&xms5VsO~q<|4Q6~3lg@Pg+HNu;9AJDBk$a~qsT%AOFW39=6QvE zL2+@PfbK(^KtnI|2QL)u?ChXvO!*USSIZx5&jltVB$Ruel*Yupd&dA}>EWYCk0Muc z2F%|kCv%}=U`QRTk7qbqxuEOm>7}ZD&DibBQoutQ?#%yE)~fS5gT{4sFxk<*4*LO6 z%v({LiE2$0@t;zw=(#@UGL1^doS3*tE{&z7B?b`@9i&V@iN7s@BHpMy$>X3!ZS^UL zEybXTKS{j*lo{IbcA8$D*S{LSszU~d6%$25MU`lDw&Nyzk21p6$>}Om39M%K*ZxT6 zsD?tVTW}MGZFUJ|H&yTZ92R5Lv;DjqfPiA1s(uenPEP6#W*AL32BP~c_GW}xp8fu< zh$b%lYWtxqJKzEsr1(Gn6^aRSb-`;ycBu?VhwHy?P%i9rWdv?X08%5kvc zyk;<4>4D?QAZMFVYL96^Vq)Uv-d^n5Y_UmqKyh)g=~#JT%zdj-UNi)<|7fSUZy66A zfD}rBfc3)GY#Y?l=SX9Z^IGVlNmb5y2>3Chq*_;7fEBLf_Wk>VOQH_2K+wcJ_6F38 zcA-LR$0sI+!3M^m58!v04@c24Ffuxf6QZE7GbD?8vSqZiwBW!7yJONt&6*3_8%E-R zvo1D$a|j7z0L<*J$o$|Xi;MF!!`{!24Ph#008bghl0jamp{3Q{mCIdwD7jGCZw_cB z`*}qX2PZGD>f!ORY|4&1=iJ~WxykOml_BGm%b1whczAeA!=_bQ#vA7uPsQMZpwf`T zf+vOUPu|tlm1f)#S2W2z0)GuD#*Mwb${TK1*;IcE=)Io6@<5l2xs3F`tgNivk}wFz zrjA|&-q2$b0$`EbfR+ADPG;cCGqe}$>T09ALN4n+dcUgYBm6~E3n2x&ODZ2l+|^`H zrh{pJe|}7=p{eSJ~J<)Qqjvy%e^Lh!roW_OX60W63f0|l+Zcqr}bS9YZ1TWlq{O!(H+h_pu0Mgh*4g+d`xjt73dPRO{*9`QaYI-hgY@Z%#MaogUgD z^eaj58FQUv>ThlNeIN;CatdO&e*h+s(9l59I(9s1ji5prED1F=J{1)esvhtvk(^uy zb(O&YAR5_>D}#fB0Ol}0e*6gBkM-p2ot^;AS9MNCHwa6mwS$6CHxOXyX&(ulhzU35%LSdSk+1_bSI$wYwy2yf}z z-p!TY_L*+Qyb7bDynON7S8h%rE7XS}2WDth8+LXM4$%WL9Ssc<`y>7RyhFR+CdE*& z4wo(E5;MmKp-62_xNkI!kJC`yx2%T-fP567gyopaf6v@?N=!+R34yLzVmWe!kdSb3 zFgLM(?s3)G$4D7zX{;M!UL11VKPooAti2GMe^pUxpKc?xwY5boS>MtkwV);Jv>fBe zIXXIu?CNaIVkRc}k0H>t#YSzBD8;Es zG+kX?)6tR~4xgSu=94WM~okQ5)@PB0x~twUVSMCnwh# ze+T34%T93*kLq3C&khS+xC)48LAe-ri=MW2S3pJeSj<+G@wC z@amox`WdAP>irMBZ43z;B6BoN!>&TSPkKR>@W@$uLZ(&-L-fZ-~k zud{9oF3!%)-TGCN?*unOt*53I1G)yHE^O3Y=>BVIANlQOVppmvD`#e-#9ABtFYgf! zWAR*Gv^d~{saDT>{?^GGooDjSCq)*=37ySKWQE|7bN%Rrrip1`VS(sG;`0LLP=qY_ z+ge(dk}iGy^!)kr-*w*JyUPQ3LwPz)aEVj1vluQeE|#MuIFi_`Hum;BrB;8KeueK< zQr%d#1lpP%>MfRT;kmfE@}- zlibEJiU;G(>UjRPJLlDYV35RARAIpAX8?!`iXn`;Y|;xaLOR+)05B6m6*HeCD1Rc4 za9;e4#uh8P;A&=a!5-dl_%MSY#pkznmQuPjY=Y}S1XMwSPRj_*MF4GYU!UFjn7}uU zLiBs){lAx}&e@<6#|tAk)kO!9r!$L)l`0osaC2ks{+A-i&zwPFq8I-Uut_cip4|3QR~&PJZ*|O*%F}UnM3$tvc$;F8^9dnPEU~grKJqKs@O8kpG}>M4;kTXg>C*W6 z4b+o6UZ<|`OOi?N3lY)7DGXEsx5+?PKz7nkbHETlQhdU~!gwt{Mn{v1UYt4v1O_6b z4g2bFt!(nP!~7Hc>pY)-dy^B#;67$yGT^Ukvb#ud2QBg54Gh+o=<5s48 z5*vt@baNd`QUN0-MzV5pq|l14uCK2*c6L62-LL&nS~}G0?D*$Pjb6U=uxmxXxcc+; z#A=GOOiggx*?>NDCh{r3zQu!y0=3{4$Vm*aI$snwR(FMA?<8`A)&wIV4_sd!LC&x( z!hp6e@Z+iNlYcTZsT8pP`zUg+yL*dFG6aOM5SNKpGVvcigaRB2gX$XwirI1}dMqjM zEkyh4`SOf?dTNU93G~nC=xErA{-7pBTgGMmDYFTK4TL8nBLnhPU{q8Tvc`;G*?;kt z=ho_T`Bt(3y=b?A!AS<&7guaeP|Q=Nor^ zcs*x+5>JOT+|{+UKtM)-or!_IUcte^DcS?y5bgu|UL40HT!R#dyMVr9Jl})Ro(9dWadcEG9vTWrMj&!bQtQQ(0monB zhgJ~^l`<5xp2r=`3dtW=R_$Na#Nlo;G?th18^EUY^Y?cc^D;C{15vD(BT%3wL@}iE z{gcY7sxY=^Umop+hK9D=G3C|Qi;o-w&qBnPH02E0MiAR#xb<|Qt3LplZb3sVjW0nm zgrmFZb;tw{U>E4FJcv*ZYk@g*;o7Hx!g8)Jlhi2LCu+rqOjv_MvzQA6&lb2MKGl7E zAo+eEjX@6yK{P(Mz2z1i5e9fq^eX*-?#$Z;07d_A*fam%GmQTKEBe3f&ZY4+0~APA zWcf@>N=oYL>Z%V^yxe6&H`N9-ZU@teY5@>6;P;?rHcC!nU7rAbk0t2=co~8n&_D3N zi8Rc<4xm0l*(vYV&6`uejQyed1L{TgW9Od|GnJ7&3v260khTna(xiU;_;Fi5H>?eW zs6v2?GkI7f*FzrLsr!WB4#7(lGi0xR{``3gdQ$600SDXF;XBVO9dY8~;sQ&P5L6)` za2?@VfR&*~;^tbiT?gb63^;|%<(_OVm>K#2{ea;UfTuXUxEOD7E*s1CIAZhg(3L}{ zk}SpRm=)yUtrl`UinR8Ym!NJyqpz2t^;8{uM$dqukCNOC$^laW3M(GUbfoAOFC_sY zhXU^cX<*@d`=BYW;@1JFF(I#CH!!Do_z}_wl1YC@6flvB)ER{OQuEuQfUR(sY?PO{ z9&O(W4-eNpc^>}ywZvR|ETHy$;kmMc0&zep8QnC3jy_RQMAtaA0)Pwv)u~hS)U9?! z!@~<3VoT>GkpxeK;N{DgEjm{j+JkfoLMCiSv3>z{&M1Mp2fVbtsR<)9Gjnr)zZE=# zPoTO91IF+JR{!uJ=F49nF4@@F01y7AS&Rj+!2CO9M;vE#Ww>ceb29?%Kv51(NVsl3 z`0ce#-I zB{@aTf(w9X)zFj);ieGd6tEmjScWi=%@$y;(`#!402Iorst|!0#Ptd<8E75Q?h6-21P~DVqaVl{T4mM|0FmY@$H(v~d9g)BMdNQ9NapGI z^qb#<6$_)4QQQgR3EpKPw7)OUK=K2@g#~nO;Ify8M@Q2Dzq)SCTw0&3i<+9!ho3Y8 z2m`b0eNwa*bQM6=4=U{E^k2RVa@*@$>u&t-EWq-Tr?La6M;@CEmu;P$gJd%#K{YG@ zU0d^p)%#*Vfd=5xA_fu625gMjSy5&tCc__Z=#ii0 zZFdn3vIK{RhbWXhaQTBtZ(1PwxB2*j-@hk6-d*xX*nK-o_y#*HLIVTDdiI5p1d9(* z{sRIqS)+o3F9Q$?f_?Q^P;H}nC$i)eqzhO&;f^}5pq>DJgz*ZDk#WBf6cl8*)SCga zH7-aig}|;+_3v)UO|7ip0iFf_HXXJqpl#XOS|SEE;p%}$yFpcUv+$~)KXJrgZX(cv z;`()i<^CMNAq?`#!mj}?PXly9Tvf|6Py|x#=i1}>Y;IUuTI!pa1cLyc2804}q=1}! zu7=X^07?}S3xF3lz&Zx0MI01Q5?b0QkWPQLpP%k@_kh{Pp;t={lm=!hF3$6Gs|`Ge z%cYhhe!%t(W}06^1t@o3ql5wU1&}-o=1Vh_%jx;~Ah3YsRwgQ)@aXC3H+Oe2Q6B}J za4(L!y=^xpL=gE)Ef{#LKA;U)dg#Dd3m_$cEJheRLUeR=)C(+H0D>UL2mhIVevQ<1%40LL?BDJ*clB zE8EzWBy`8)(3T~(ow{xo3RV4&AJ}?cN4JDUMBcu6;{!I`)hJrAF1sXk@BtbSC%)K3 z0XT80VpS`0Q6ld9%`TIk!53#6-iUDo#Srl>K&LW*m5D4dKq6D?>yZG>+6VQ@ZRw%x z7J`SFJz(CMBt#B%stwd^Wadx_zP8a!PmyTD(au6sOUotbA59=VZ!Y#QKKa1=SZ%)C zZWd8M5#EZjU*+0^TZPaEI+U|UByX^?5(XT=04NqBpCBF@%=Yx(pV0`nk(CYY&;0^$ z=}1%!5(dmbV4-uNz0mD``hgdz)93(B$EshE0#v&S;|%@mQc~7^kin%=L_Kd?4&Tmc z@YIz_l3tly4<1Br)$4~cuI41RsMhwj4!9Y6$ji$k5E?{k@Bv_8NYvESW(b#0ZK@=_v=5SV*FdU`ta`Zsa0fUO}*^D|NkrU)1!F>DOwQHYoXD(8@a z(DTzy$YH*d1U4H4BDCUaomSM)#8>~6akn`I5~o4iKupP^5?JDpYyjI$AP;l{xRI7H z@&zp~T`4Ik9F$>OB#k6CJtDlczvt#wR#6EgmV|aK1NB+?xB1=a#TQYaIh*ZeE>27paZ0Np6n}Y z>Fa-jZA}KYpV?b_1-rgenATuX#+a=RuDOX$mHiGlL~a za(;gPicLL{511+R$1rdQUw4b1JV6n|YU{0%mzEAfrV*BI7K(0veVm;$k=N`UiVlWS zx58>|0}sTi!WTcUu%0>9<%7urwi?(QMIK;!TtPL1#rv6;mX?-;f+Aa`49FR-k&%(z z{;HWgJk}KnZCW z7`O`fB{(b$52X(xA_9XzbwRKNt(DjXY{H-ID=-y#E|bNA;Cz7drjaS)v$_Ul4DQ_ zF1j!|9PAEImf?dlR4;q(ua5YGUiS(5bJ6T*o{mrsm7Fz10eryw{ITAi2hav7nJ|0y z&%2>lKo7$vB5G?4#IpnKUoz<)D`+A>CR<^4@Nzqcj;zt>`1vV-S07tiScJ(Z2~I`7-}jj#j9OK z;P^RY1%QLYKOj+PKphl4-FyH|Ic;N?=u)j~xmq#rO912JyDN9nwo8sKuDn zn)Tu@AlY5Has^ScR!6_fWmQp_FG33E2VOGwA2a@rBPvj z%j;}U{h^GEoQ^eOF|Dkun5Qe}yg0fc>H%XPMZJRi;>Hjn5>uBX1{drby!$mBK%z}@N@b3`4IqtfrVfHhW^2BoDv9`BMa5Z zVW4sFd!JWNtl`n?9*Hcpj*STqSscLrLpTx$v@hOh{T&8cWZ9h4dvlqf)MKU-g9 zeEwWL7*Frn9>ajkY)OHGbU-D)i*tz5s(aCNfH_c))bhKZ6MY| z&;|?KPLWCOvmMA_LeEEXcvEoEm%hk2gCr;e8HdfiA-!}7f2<;7xMIkHz>Z7f5Mg7J z-5B3(EU@}>AG(U%ctNu8D?i{mX|HzX0gNNbED&)4tzGu^z7Tey?+gQ(%5t)n27wY# z=QAAF#CPiJ>m~oFIghqF4X&G@HYYF6`MQk(5HE2WfiWpq++33fCD#(L1bnp9a{teJ zHv3tTnjHur2EP=v^rNA?Xh_>ZISv5T3n&fZ?ScXoiUzf!?1_d31q7Fnv<{NB0d@r0 z$QCHr$bdpEtFHHcll3%_DdJ9opc5O!KMEk~iKz3SN(g!uhe<*&LgIc%Fv@Rv5zD0oa$$kd1}(?-p1~ z|I|z~>Ee9#H%0}oc3x78xT!9Vc$4Ezqf`Xs{p{;MZxjQvU)evsr1dw*= zyH*73jGN5NM!IN;PgykWDlIFb5O@D0366x$W=V5Ss&8^v--&5+3kO5W^8i zNTbTxDmnSlUl{u};{uTx;S-2y1+img;swCQh_?+ZHMp#duk+V0 zpgjcPH$*6foL9wRX~6(b1J8masE#_Y)$Y9`tzL$h-7UCJ{i!oraynEV9dR9vlw8FfRTrZ*dx}pQ8LFs{Uh2@%BYzegOT#{3<<>o8G_Z7 zsa~6u91z3+;j5+*j3okq4>NhvInAign1!XKSZ`iQ!a5~$ zutH8LuvEdWO9SIkxhi5i3hfb2+M2Vmk8 ziwgoA5TI0mnfyQ*BVPd@K`I1bcJQ+BffpbK8%Hi`Sm43qc+ciE8n?tW(353j%&W1^dixMOkuye|&C z%kAfIgYd8W0op?XxkX5rUB=S#+pfb>@2!z-um?L+#ApGjNdp{vuA|eF`IJf+5X_JL zSY*Jnd6lr~AKIy>6(Ei_l2L@L{^t^a^=WVf0?J#Yycg^{WJ3VedEX1m3^~UE8UmX5 zJn0*dO+SLgw{bs^7!mUT(!rJ>LOqg`^9Oc;RE9Ti-lz|rKrtxz{=EeR8?Z5>^F5i6 z8%=B5A6;>uP4=nwIJ$BHOZj)HmA0lEBp&Q$nsI7YK4)Y!fEyV1669r2HKgL4!QDiB zQ7BNf9-C)NPsc^$pgs^Ty=*rE@Y`)t>0a$?@Ty;|LbEy#Yfq1d|ar< zuxF*yeZj%x7zcR*X);4*LwPjw-6>HkL-`P$!%$LE8m@3)24x^a7_x=+aJRObQxag! zU4fS)CHx90DVjK&kl3H6a&k>jMU2IeQ#QM}t*!O_Ih7JWIujC>T3cIGxZ9)QBYTut z$cnlpv6FankpwDo(gD8p&CNqe7^M*{{&tuBHe&t*O$f-(=SF}kzy@>_IF5hNZCpSL zfnkgFf;Rqs&Fn760bcSH512qDU+XfK~ zL_wO1=L1y{QC48_Vxd654g$0e07MZ^bD$Dtl>oN|fvvyH3E_N+4COf&^4B3DvRPr1 zXhM=c>+GnDgRrXmw`4zP`P1eL*ULpPwFzgowwmhvy3R)G%>53=q4UF~Y7dGxHwJ)~iSU&I z5c(nJN8n8in@RRc1SLO@(9VU73C2A)^j4b}cR!1FO~^NHJ~h!xu-RcI=*dn?M>AKI z?fiITv$?fxJa2+Ue`|v3)f|k|2{4U^fYD&}A>AD`5+V?TFQbqLE zXaRuq{#X|&2}$h3JKDaV|0G;^d*FqPv}Vz1_U{zQ2)H|pd4BZkBs53&3aztMQbm38 zkwl9rE)y5wKAFiief-X|GkE`QJ>BO3cg9H!Wi2~hGgz98X8rXS16c3YQN*Zll)Hj{ z^Ti?A&9F6-4a407aujMjBGSlri6@|zI_O7uCW@D0O0Wm!;8q1l#N$;}^T^%ZDX zX~i}B#}nlei?FR!?LX(-kqTd^(4b)tl zN$puA;wi!CbGR<|pe3q|2ARGBi!4LC9Ygf$v-bKdgOQm$`P!tK1k^#_&L<@n{cne5 zeSy*}Q;si1C&Z=+34E3|v_84~LXC`{_111285>##C|R-19uBtH-gOHZSG%TRXo9XX zr#3vf92y#BPJip%IJv$3(bJakw-*wvJ1@$g#G?{F$-7pX5X~$EH2oOGz3RN?r(~9l z&0I!KP;~O_bsO}oY_eB>C|}r6It5WK-1z8U7ww%~)pK)9t~JGkwsEwFT2$(JU7F&> zk;3i!ZrRb+--vD1g~dgxXg90+CJ)(|NQ`L~Qj^q$SQR7=MBvsHEBTxFJ2d2S#oP)P{+-QJuMr#?D)WT*b~ zHm^AaEncwCX<)TEsw$U*`0sR3hjBM!(_a$QR0flc%Yj3B&CZ2M7M<`&&1Fe!_&1fe z^J>h|Lpsj2?`2A40mQ>+3)YT*CJ6Ne6K2lERMtYhbzfRWt6`x&$+v6w{p#-{y;^D$ ztg@I@(PGR3dC%;8V%iKNLgGgJS2LD|7i1n=gx4<5nNg-ToW(!}&e}oaYbU|;s+nk&Df;am$l7()S3D@& z8anD+m-9~Yj$a5ed99Quy1A|uvdha+>4i)6PM&UnS1OdRBw}?{*JSd$%Q>lferr>p zp4^grYN{U<+Q=oj^D(YVCWB_izt5NT`9GJpZ=aX0aG%-<5+((m zGz|Wni8W^Lc-4M>iwy}ZVAm)9(dT9h^0KgdY!yC#y3I@Q#F9&Vj_eX+Uz=k5lQwV4 z-A?t9j5{pyPt|8T^!ON;&&ymt-XQu*?3#Vop6Fq+u!nDr9M_E97@nq^7gcgg*TYMy z8<8D2cQl-)@x=e^57D}ySU!4kN4T6{y-w*^TR=tg&aiN@ms)OhjX28#*T^7%yxoL< z`a2CDb+_TW2mHF!WfDQs^}Ts72pG738QJ_O00MJ0^__Fsv6XUv+r%3iPOhI3O5^Xd za;B|L7;nT>px|16?-y)2D$ZA}_EL~-l%G=7*3rDZCFT9ej-CE`c8r5STq)ZNQ_RlX zSkq(OTdH4H{8oCl4(Ebb^2awhBBc$H!5eqH&){`J`FII}wrGKp7lo{yp4`t|&M%ho zUZQ_;*v4~wkpy=7qZ|zw-Sx?01|5R+#Et@8I;ptE)naWa9c8%QFXxGUsi-6|rOf!D? zxlc`>_09Oq`o$Fgv^4{x7cm{28+?y!BZ4$szi}sPubwfZ`X+zpDYo(q**|#_(PI$8 z&9SC>ySNj7f^Enp(8Q_D-p>S-;e`M|B?vvAEl^|mMeB{D&U;862P zxoYpquM$ZDqCYUjO;9usvG&Dk!*AKO*=`tLd6M*{u~V6qCb7BzPUDp7R9$=g{k!bS zFcsSkSTDGmk88Da70mGC>5(~OjA;A>j^LFn- zU)v0g=gF0~OqeLCJRQsJFT7YSvZO}gHiMVb>+*3)6T=h*X#6A<=-$2e>FgPoW#+6S z=w1!2dTW}=8yl%`QTmvF&o!D?n#qDrS>wbyMCc-LK4slp@aG{j@lT#WhsUZ~Z4-)H zVDzj0klh|%Ud63y;G!a@$6S49_~wQB?Bt&*mN7e$k?Y69E?X{5e%WHS2X?sOMb#Cq zF+bIw7vMiM5i0{7Mu}G%uW7EiBi`PyXm{vlmPE^u(y0H8cXyc9Bd=vdy%;? zDy1sc*FB#-5vLAPf@+vTbCwM99qBJ z%s&zIcFCw@@1q7>m@F=-5u=CZo}h5INK7v#=6*J<{?Ebt?p`ou3fOZ|L5{f*NdQM6f(@onL7d32TuLE{!_hO4oB zW~&5O381mGJWj`o?Cd(d=ot}@NcOQ~t>fUfX}RD?F?y||nZr)FiKWm+S$y@w=Re!^ zvnifk-ANZS+oKWAOVny~D?;2bBFc7MIn<})AFdka?uAW!r-@oWBf!%Z(hwj!r%3nc z$u{udpqBhF`50|Br3tOh1#geiJ89}|#6t!)u8iu82deF74hs~h&BJTN#ku{Wmq;Y_ zyCae`?41eM1!8~pVslrjx!O6GJq`^E42-x(cz%2-&b8baPq=8!K{}{m<-IzL#Kpp# znZO+htYwjEg0U0YDV=hOAF(M!v(ri1D4x0~svc%E6sPDh#k|^JU}dgH$F?dqii|kW zE8=|#v*-(rTNzdKN)pH{aS6IaM#%1|yy@Vy8l()EtcsQo~QQFyC#*? z#@bf7eo|T<7OWK2(b!rw!k$Eil|CYrxO9#euJU--<>F}HTbavaCK7C)v(Q>b4&Rkp z8HL>ur70A?vqR7_l9+grxrj9=RRKszs0y}3kI-<(sMi z`7Pddhw9SrxG;dj%%WKP_b1LwHXqx3c((gF>aVJ)PYZ}wie0~|9rD9px`0zDBkPB z*H^~do(9JHGd_6|wiHi39Uspp;Mz&BaenvkGhdV>wo;LGcTEc%k;8?E0G#&2gm6=f z4hfE?uUlnzqewfD`y!*T?MYE5K@LnUky7a)RXs~5;`}rOg*zxG0C=D zZ>W%TXD=_^GB&UXKHqtNy@dYVkQMp9D0%GX^5Ge)5*1(5{v{1Q>%UZ!SIrJi1unWL zYTD*pFT7FSI^#)lfy%PM+JQe({T(777z(Ub3EUMkG9qNGjZ6rj{L?tPZTvtw^H*v@ z0w6Zr>yL}(&JllJ2>*l``SNw ze%bN_%@0d=;tK)q(FN_6S1yk&W`KFPTwC8;`}=s?YH_Bcsi~PwuU_r1eLC!x4ru-u zq`NGIVp!J_O8Z@}2z7pOCi0aETG4eC+!m9up1+>8G-hIX zi`6ui7&d27ZkAmvoA>>=Rr4X^||&tc_3do{YQ$j<>I-TA(yL`1D;^@D6ZE zw;r`&S2=mS$>(@1PWI-yUYktK^H1!3IdN~yXNX*%PI%O?@p*kd(+&K)^(@J4U-(ok z#9WYx?%}}SB{TVh^NOEh%5|jM#z82u-}i(tFurOmE>8NJjWeMr)#WVj?Om>BvZ>V# zq!MGWYMwkNpS_S!jlP4HO3Ap;WfUtm5W--4!mGc6gY7r?tIm6nH%pEwBGtlKOIBD` z&G_o;zya#%xY|X%CFT#&Wo(sZrWeN zCBsGeoc$W#6cr(27M3h1i)1ByY1vxSShxAM8+X@bzwhrYl%1u7 zQ30*XLL`SPwS*4~4_}_!&wK6;`H?HVf8uaOio`u{tzn=|X3Aa{YeUYh+$`50{jB}f z16Si|B3F=yzThICkq}QModvkcY;k zeNhyN8t(A{S@Fl5Jq0Bt@*o6o&AncZH#SOw~ElF9dqds5rzsI<|{t<@|j*rFY>HT4GyWRFXZ3PA^JtYWgx5 zL)T@~^S$F3U*DJ;5pvPXs_dk1z#T?3SFn)9B_+`j4;jK&pEEP#w8Y>@5=14XKv0J> zQn&499^&5+rhuHiauVWCfg=sVEUE;GtjGpH`% zDWpR2VKFD>`iDjOrcEs8iDOK`;{#MS^Np)Y+f18&@WhOBrcItJm}uu+=fihhiRO>6 znp($7%c->r4*d}=Hguq!FBO6eH^2@-GHqFSLs3w za~?kmQNTV;I!pPZ=2VSysu>z1b;vr$?D6*SQm=g z`Jp$rwWUKYhHVaf{strmC&DZyMhlb8X8a$6|Hs{vpIu#8sQ$0&3K_z1?gwJM$P+Rk z*AWQE{i{2T6|ZCap)O>iwq}1NV*l*)vr^jXD+qZ1BPUuRsXm9+d8VI8DB~|i_86b9 zSXf?U9xua{8B%oZz8Mk{D_6(cpzA6be{~A`U0hTIC&%=yta6pDAyWZXkaG9d-kubw zXO+%txF#khNJbYq#X@!=Jh5k%#=-zTAh=nG0N4&sn8B%dP6*)u(5`QQX50jc8aS$@ zZ7y_lbo8UU`~ALyy?wiTMgDy$65HFDmBdpQnm>DS`X9mF7@s@UeJ&z)wTl2Y6XVVfb=6KIO^fKw!TiNuC9(8Nd!d_Qb?*6$nF8@o@*mR>2<$R{TWqn z9+#pyLz?%-F27iqw;fYlVZMkj{ZeCsDzpWvC#2vpc2%vBA`=tYGAP^amd}Uqb|j zZm*ZTJRxM`I_wiLQTuB?WemP)=%{+yp$qB$*Q(*mQz-8?w@uR6N3I_u2XotEtITR~ zJ|Y)m-d~CNP)`p@oFWe=f^bE~_VBOviA00?4TqYDV0lA! ztPu=4a0Gq9no++9-o%Cdox8@#mMijKy5MLc)6(}Dt z>K7MJPf!Kb)xwxqSTm5Gfzvut5H&#jYz>W5_q*ELk0!z1`MUcZ4Eb;nsE?12!BavZ zYYUzuF#P9Fy7$Go2Am*%!J|2#$BXzMN(74Rpvgq1se^+ z`OmY!o>4K;sm_jdKaw$!I1gh-jJ&sl2I!LaLesK|tJE-OS1#lmlKNLZ5C+4XhP1!N z!5(1oUY&Sd*JqtJwPh4P_%(mY7Wq~7NAo!&*PQ)F$7||lpJ%kbujEhlw}$lq5g{_aJ>5 zIkfV2-w0ijTZZzC_?-23&Y;r2$E$rwi1+#3@m5+)_F4+N0Peqm=WfOfXjib>O~;gx zKrVgp2mn;YUvkzrhkfb_5neHhX^Rt19>wl=Zu}J+eCUeY{^vR5H6P`FO4dZDh|_cbgEoyj$9I%4l<+M=$p|wcWS|(_>jyM z;H~>;F+`O>>Jjpmqv1H%v0&C9_v2W|qQbG`x+ufn zA(vQkm6Vu`Tq!eZH#3fZcjzUH(}pD3g=T18>5Zg|666(+Xg03oJm(s#@d=c{$nX|u z?0bC~nbsZ$TN1%oxv*nSVkqK5q(Uek79I?wNTwCRc+D|jqeBZST!;{HP4Drhzov!K zXZSZt#!Z+3x}kZIT-Tgw11r6`n`cW>->@#jf%ESz|9SG=1uP}*r|OyhJ`>g~?OXMd zQBSeAaC~!fI8owynfcR;|E`P5jzsX;4}Bsbe(i*`XN(TmevGGoy&nB|B5v_@@B?84 z@}d%AQsOVf62!kpCnQIQ2708l6H6)KP$lxK>skLflBM(=b_6nR4+k;_ndsow-gL~} z=H(556g@l!;KNowf*?4LPe1@xyakvy`D0e|3kyczmcikLfY+~akVGSRs?UzU zBcnqyp!%H$?R8{iD-`=e{RVW%B7B0w3rIKyjy>n>LBT@~+9CM~IBE+~$Yv;^km^>< zyAK|DB$8(XrxeN6**iFF9k1lW^PW_X?m^dv_r~;N2~ErN&$fghM=*FZ6FwV3GbRdfE(ozNL?YGB%FDU$@#UqY6;J%Fo3p50Vh>aQvl=O5J?-H zaD;@(Nn<&*Mtv}3;bq`@*X-^gAE-1i^}m4=9wqzK<8?kjB+$c=rX_#G6^Be?c2JHJBi$`b3B?g|DW%+R&0vO3!&!AeB@)n{kS2^g z3_*e^y2dflIER#y5*M<#X7GFv?;c}hFi8eXzjL7>w&@cl6)LA6ot>&w4J>fxG!O#b z$dm5iX$SRioE`_xt^AnV<|#9Sle&PH#;RQMJI}hHjgE|r^g6?UzOQ`w?>0ZAL4$%K z4yQlisaM}*UqLztLR-6I4g>@Q;O7OWr7^&xzA)f6IUEw9f&T96Q^|==PL71rA23$1 zG+^v=EdQ;^@0z*V80aWy)f6IbypU=`BBkHpxq5Kc6iH*Dki!TNMT$-iaROT%Om*Z; zGC1@|x(;HE5DU_ozyIskudObj4I~PLoDYXF3I@oM3TJEJ`C;mN-q`;>ODkKAi4HJv z3p}U+p5YO`yXy$EX9Nc5|6uRU!+PG^ckgcr$yAhis7whZ^AM6Eg$8BLlu+h`q>>>e z^N?9{RLC6BWGZDXlFUN`kt`x1Dm|~OwbpO{_I`f**~hVu{nvAB$9=E+Ug$e~KA-nB zoY#4t*HU+P6$RC_>8N+P*Fyd$xopzjgt0Z`Quh@rj38Hwm(%H_@2!|rKYX;iM*f!+ z>sq+R#8rRj#6#RWi>vgq1+Fh!{cTv8J$=mIhL_RLdMZgE(G7PcR>)lrh;7*P@4pn= z@BiFinzG9mwc7kaD)L^sv32Q!krYKhtks0K5ZM-JR%Mnrz}jE*4Ki;CbD!O z29J`YAL7L}*5AD5+69g2)4>SIw8vUpJwn zb|<+?h^4$LK3@{%ro;^m8J;m(}+p^&pdK?=e$xSz0Eh|>#4SYop$gKCz%o@R~ z&Emx4w2=nwAdt$2s>|kqj-+=D+uB90gA@u*(+#9`%V3E}EdznIFT>=zZI2dIS{Uqi z`uObP{6{AbZtBpf<{bjF0#)W9NNCZ2ahu;I)xR6_1*0A9nDoOO1jnawrz&yYxPefjPtBdD4yesV9Qv!M z1M>RVp#Jbt&=Rz&xggx%^;{B8nlGhmw^i=$X0EQWb!y(0rZiNmo`rNkFyb$k!e|Rc zIsV`MAX>Z|7hE0)nFFl_|EVNkit&{wL&o}#`uh%Aqsi2s`~C z5z0OO+hFyf?v!sWULmr0P1T13q$!j@WQfu4cg6-iGJZ5xbKUdfN>i;()NLZWo0`s5 zo-*~~rsOd*mM@bl?{eVLi_;HhPBBup3x}3BKRa^su^XW^MTt$HqP68Q+q))PR4Th9 zxRgge@A7#?=|jFIzw9@-+-jw+xV+*+Grn-oRiCCoq>Nr@NWr4)me7U{j6fnjnITix zB2$28*Ofwb=8`2l&`W9?8d?wf^yZDqaP^#c=^a*wX8Yg#L8LiMtiF0Far2QIf%Z==fWIWtv3yN@2N9vqL$K2?LvY;kUym+%~F zu;eekoYcSa)JXoqTnGASPVp#n^I5PwanY$IoK!BG6Jbr^i`kR1LVOds5$}%>(;|>N zd3-o^$^O|PR8v&RQIu~HZt17HnrG@(mOaWI75^tgC6 zH8pi)13(GUdw*a^MLg;8tquAPTI|;_JQTjb5wnP@MY$=}y(=Wva8&E&#~d!B_26`b z!&Vg!O(-fhapisUufME+VC|ft+3k)g%`6x$_s(Hdw_`I89yp*)LR8Eo*}r)+N5oc& zok2kkaq;K=iw3$S91-qTEj~v;cmOF#NMAIIJudo;8Pfnoq~X{H`bM}W@F*bp%tIR_ z)MK&4;9u>YXV5suwbt{AUElfgf&Mpa{3j3a`snl@h{LYk1X zMCt}dFrCvRmL_;$HbpV(5z{ZE-9dH9)@DfUk+^IEaJm36f?vU{y6wGuA)U|0$j$W| zN}k5xI0%7a_gZ5n_v;xnR48_$mxZ`v<#?wFq)fy_DSs5g+R0h?Vp~!``)^AMx-@n2 zvVE-V;(5h&^X+dZ{e4f=sU@*4yp=V#4?^>sw{MjdVt<2q1lSRA-(pMZoy`BCy}k(x z-pI3T{?_)x+9|TmIa^6VKy)(zs!_B(x^`adOM;1b8*x5nwAex|R#sNF6rr$P)19!x zV~h*Cb?H(Ug0SX&m4_Cw&sJ%!dqkVLasG(49zjS`T2LlZE2(Bnd%#o)>QrA zm^!%sO-uC9fMj)QZ@*f~%02o1pcWAbhbCQG+!lG35g3{x6-}n7G;!He_@<-`L+X^5 zpRY6a+7#HX6vK*xV>(|vVXuFdt~L?ATh>E|sv?q{26ivR?`vFX>=_PVYh!F|%t?J` znGmP%l-nqXKV_?uA1s(owaG6{IW;hQ_mx5rX6)-$5K41)vw>TZ5@pu zQ)_GM!rwde<&%ecWIll4sJ;JAx2Pnzxdrsp!X`(GiEivHs(2I`$_hTd!$@N^Pdci~ z{Bzg7eL{E?halw7`X4z5?WwF<^dDC>avIlNEGwy&c43c!`bt>h&^fbzuf2Jt&d!To zwR)&M*U5X{+tq*KuZtt=?ikrhb!$NMN)2<3zX`O-$42Oczk@{PyKUQ_y&pFoe@nIC z!Sbzr@*wzLL(FmpNf?~xYd3G26<)^dW0UTQ>Hm>j<449NScIAFur65o;gwT>LPhD! z=>=P7%v-r3ZcMk}Cu7X)BchHUe~r9kepaXw{J_=AM?^Xky3l~{Jb17k8g#qk_ZR%~ zIn^&KP-7<&2xSHLCA2hTS;n3d!CtinI<$2>*l2FqV%=@)19W>vT? zuNeV#>Tlm3?K$6E*$u{EP!%XQjoM6!Jv*)wFTxK=rFCG?3fF|YPoa)-xMTHnb+e^Z zJaG7g^|$vgul8@ypn-Vl65SVQ>+?B#G}TnOJty_LUvc`r?n;`01%q034%FqbMK?Xm z)pbhT@(&c<=C&HU;q}FHA*}5gU{LUyJ zVr)DU>OL%Z-QK;M4y!6_ilk6<^3)}-!Qq^LPgBe!rBU_NqdLjLr+He6kK?Io9!Bw{ zp_p2a>x1ZKnJ4TU=o5ZO@+ge**oESO&ePe`CA;H=#V@Bk$d_Wu->?3Mt1zE z+Q`6rPGl2t4@VCnF0dl;B4P`*Ulgbdep1zeq$3 z|EfkA;D@{dq1Wf%6(vq^ugkf7QLho(44*TnBg26+p5I)(5agqfb51zj8}nB2QQO{b zNFYK6QS#O6rMG_sF!@ z__#|Fztp)KJVa7Xn}KdBw5MHHRhsqP6kVGNZUqJ5aCQ55SgN9m%W?O3*m>~asVdrY z=gq4*1F>_^dpFy&d-qK=784WC=FPNKF|(bKQhqNlFYwaO20#Ge3>R8xX>|(H($oyV z6sQ2Ax|yx!EhIgGiklpepykbbOzFr6pYrTde0(z=CMTDa=x7Uf#Jxm2y&wYr*h*^W zNN+5UmLcfLPDbn78XF;`nr{ypot|{ow5+t$b8Z{!Zks_StL!GvpC7GiGG^SkeVH(W zhunYs_|hLCcM?~BHG^Aj8)I*|g1)nZvUyl{bbD8P90Of+Jxs;SG| z-9=9P^N;eF(jg@kLH{%ph!gF%QP+Lt+;a#QQ`lrH| z;zzStvv4l4-NE4Cc3`aj6lk$I-@bqMnLC*Fy!^xIlq^x6j_JJT8Y3;ssL})@M~;rZxK$32;irsNuuE_-J*~ z3D;A$S!;Ay<_L&F3>TihuyXpF_s3_~Td&=z2lqD}#o8GZ?5IUWLjxLg1G(z2>nlSZ z@6cAg_wiEq5Ua^?V`J0R#4yCKshlxM=>eA71UTs=l?=B*E z^l*-~IN2I5VcE@&?2qr6)Pw?drkui(wbfgYXX^+MhRrh}<1=8z-V>+L>FY8hkx7q% z!1D7QdA1QV&yAYS=d{8L|5QyMnwtU%Hx^3@s@3OmM4@9Hq8@{=--;1R)#2@B}qf zH`idc1YkEmDBU`;>-tIi>IFLdK`9JwXusyoG)9}GsSS~l9UGcYNOJoZ!uxCg6k}Sc zXWa}98}m#8YkJ+z90n*v?CuFb(>*4<>l`z~*Q;|+C(bxA;8Y7G5f~vwU5bT3*@({-_wt0dm~58MBZL^#yrP?7~Ky&GF#lYjwc^g3x>XkPTG1PBV;-OvfaM~^l% z*BbIbrW*hhAw14v08gy5JcrZ=HJBQ`)IU=Uv!K!4nuQS>m3a)!5x3sQwrO0!|29#2O!ACe(}hVHO){i44*rF>(bic~HT^(shk-!?5Z9JKiSid1HOWbpDY>FvMZMjl4NCc6OeRqE5W74UqwMSSHJ zedgQ5q%-L9r}hzZi}l)hyq4<<@Y3g&$0pq=bi>Xd0xU2pzM5B|DnFMU*QsYun?Mj# z#EDAM?BFFi)x}%={#Tbnb=ibeZ<6k05?s zudPssREChik{0cuL)(eF3bvRc*xl>7n65_ZFyplBy*E!qMD$JR-8r`Bw(14RASklT zfW+^FAqMBtfJOlvpiy_cquRA=SA)M5>JBal zMGp*KWk78I{K8Wb`QrMxo&`oynE3b%1!k_ztP#(< zP=HhMNsL@>HFDg_l2;9hr7c>t5CbSh=!p~cnMXf)`g9hmAhDzr?`0}Gt1luGMjE*x zEKCPw%_fAy^jarw{t%}ooPVFS@9V1h%IoV`v9tk|6Q?9Xz5TI06FV*re`Xp{&w^QE z7S9uIl!(CFz%Lky^I+!@+>C6?MMsOTC&y`QoY$tgN@`fJ(X&AhPd&OEYX{K&1*v@a z^dwI-J)`K6#1vB@P}sO51J+TU$m7XlyGgV}2&<1)BT^J=7jZWRjKLG8%C5qQs?ILP zes!yNG&52IOA&@6hW_$DsEDvB>0D4yAi`@Ys-7gSKJT8`>viYYm_CPmC4{sX=40H< zaQarB$Pt}SBO=Bev~*0`uyI*#%AZU%7@4Ox0>6U0d!*)GmG++ zhx|3MHE5+IyW;Gi_cKS9bXE^pHcfL~%OuCJZV$@eY_eJ@_L6l1OjT~Ar`Huv7oz&JOH0JJ`F$mz|4F#fL~B`XfLZy;N9jNh zQG=}8H70H{W}*26puN#+#__1APO^KT&w@t{5ZwEQgtW&nOQ8sA)!%mPvHN|sDT32L zImAzgM`#wxXNm59&$s>WB5s!7rd*i6j)&B$!n(_tZWB(Lz?J}czmFx4JpSm;(r{tH zisGo+3^dj4#Ua=<(xIgMzMJy_)H{jjvdM zx-F5?=_pUm*YXd-| z69;8y#vD|7ewc7hX^YKXkayd*Z3DuK%f~a}dbZlya6)qx%tOxE6?Bg6aiL(XKHe=N z%aTP$pc-PED%v#Z-{`(={=eZtt9t2R7-s-0fEs2*y#h)F-K(E4Z9CU0=8wl?Mvoqj z*NkNf4Ic^kdG89ERQ=ORrxrOpTEF*XRo2`4I|8ZeCQd%Aq@WYf>)zdrc^kGw_VMS8 zZSSO}Dk%IY1fn{(k2-tqoTd;wjh)pn$>v!boT2P>D*lW^}|W+s+& zz86siB=C=CYuj|<&Ti^twJ()(;_2xrh97ue3V0ne4riz@_$bueI$5a01W)$IcrNxB z{k-tK%<#ZC^jQ>k|EqCQv&btWhn+ir9@n$zb5X~R@yOQNym>RG-1>?i1z|Z8aZeTH?Mm3owYJp9;o! zmgO-u_P_hO?<&S=BY#-+&&E^Y7j7nY=3^bEVq^Sgv2z8ZJ+WB5@bvlf{S|{Y2NXRV z-%Cx|KDTHT5GM$ieY_|aMFtB>wvIYiq37ckb?SUI zML9DJ!DE>Qgc1Q;7<=@Dsp8-~%G_^LYa}vb$ZnJLtLf*ca{|awF-IRznuzVG_~hBgJKt$0)#~j(|MY%)XRYUt zPYo%p#d3s4e*@?^)^evxKka5t0K#Y5!&WJ9lnKRMRMn9qZQ!PK;Fe%8Vz{D;&g>{N zGj`tS@)5#U1}FRZF1a_eFzZo^zwSpj(d`)C4cWtxruW_1Pm)K^o34Hq`=`+}J> z6&M5uG3$@+4qlXBk#W>0dfUq_-I{8+Xlxr=`$Q~PPi!jaUPfXwg=ys=X&Yz|6oE68 z=OC^eqH!lTx4qcoAfJ_`DFzI1C_4-p(v~?3 z#uJ8ghDr+HM*R-rvJ8Wae~P#oQkQI3qJECKOZU0Ft%@jKw zT;*P^t@dUX-I%16&eojktNtEkRYg2Hr4*Af;P}pf=Z)k*)OX3OS{s<|6{$jcK`m(6;kX`C)(Yx)v$Gdsli{o*B9SN z$e$LcDsit%6N7P_L$Sk@>n2VwD_6$bzhjoDt}=X4YbqLrm{(GH_;21kI>x(hojTW_ zKfjtC=M36l*MN8;T5K?b4FYpaX>V5Mb+OcZBX4vuSWo$qb*=bdU+wuFH;VQd{(`vN zO9dozJt0zXCB(xk=nA;JsP7}0Hg!<=@}$x1#~ME(kHaXarI^09YK0UgdU|@PK#!X^ zlzBi|067x}TWgv#y(%hpeY@{{#^uef^|#B#dT2H;Vx8d{<#{O&C#lLkgx$uC63zn1 z*ldCZUfu2KO(N|sQIAUj*$=jXqImq7;4&^-O;};i(8wN8JCzk;u^@R409s5QnSNjf zaKwFa#_bH^CM3qVUe{;yh*LUZ-Xq>BBJ?F%+ZXV$XW(K|8(?4DbYxW%OB-UE*Im~1VY8YUx@#-pKSq{+xV*E#yN4}G^~@GV zEe5)EJO%-Uo#fzjh&H4D($m7RXo@9uZb^qs&AHibiMF?H~ zyf=j?92`dK4#!-oC@saLj|RHqSnDacT_solUL(UNAkA6|O)V{xhNhXw66D?nu^=&I;gz!UVP{I@e3irTK z!(UX_y4~l8(-6Q${?aI3Up(n9zQ$@7C}_|3j}$joPM4Q^5OPku3>^kF0{7GG5?aBbb9sf-K`T9|9Uy1wqr}*6xCIj z#C);c3H3H~Np3O|NFi5-vTd{A}>m6hzoqKm)i1g;m9?DOc?X1G2b=oC0+ z)HOZ&zB+#I;r;vT^aAyrt*xIchd$b;tqRIfQ?p$Y`bG3Ho7Po&VF|X7BijfXMHKQ^tf=_Yr%$soGtBNS>vTAeH&RpcqDI=`aFcno!Q{!4W!IR#GYLJKn|APd^!yiQ z7i%{UW8U>pGrP#YJZD{(k;7vxpb~({IK(r+3WbF1QT|y6gZ~}8AL}l(J9HS)!^LAV zK!RXBAT3Ne{U|y4Ea8-qEdW@cZ!g0>0)rq;2F|0fijOL*xWJ(-U^XmhFqM>gixvUH zsw>Y=TQpyHF^U3>nwyw+Frs~@*p-?cSs%LS{$ZfrY=bm zHW1T`hM8> zY#l4?btx<&V*JNtOk2+yCWQ7xKi*nZpvl@}K|s7Ct_AKUBJ;z`-B zEgzDS>h4_Cb!~-N$JMWXbR&6g?;ugU?>g_nd8aHwqw}7waR0XD1K^qw>I(UTIHj!F zXO2Z^>&EspFzq79!~ZEVP53)(iMk7b074P#HpbezF@wi6N`}v$^G5{rW0TN2E_^tQ zEg2&a{*4yD7{FM?Vg& z7Lzv)>-F1YU3jSFvl3h-0RbHP5f|O?$y*@VnC~SPY42aiMmTDM`45gYr1{<^F0BL_ z>e4g@*$%&$00`0oV0+9?Hp384qi;Dgzo0&Rk?h?X49^_LE16sRnc5bw08gG!7U2Q4 zJREG?sj8_<{l72lq+RD!hjpy>eY`4Z`?6s>Zd^ND_e5lS!Z7ef^g^iFg$I~($dX5( zRb39YVksD@bk39*!f~V2lQ!|x)c)Nkol#JEI=mk>abk1uvs9j-$a8!W9adNJ?>1Yk z)Kz#{aW|6jmAFjvaEskX_7`!8kq4iXc7~t@*+|N>C=38wsIJ%kShsd<;epAc$|V*? zA1{(mthH-vt0~33tlWgxFG>ztUW2wfwrts;tt#7_#G_f#%22Oz=W4Y8*KJb{`|?Pc z80^21m8A|{&9HLQ4CWwM9jTF6sR7|DwY+`j&Ytc5n>NXTGeez4U6Bss^hEQTB&PeUdSafpA@zZ-2){}mEDJ-s~^lLfZZWR zgG{&Vws zRkC_RAX0fQ__QskK}Zjg*>@@JHJ}hMF9-)}aQ+k$9UmK1&w}@qEJ!mc-h|+O>uBMb z9b27h|M$8QaYGkBMhR2EC93DHq9H_QU%i!raf(1?Q zM(;f(ILISF{ltY_K4vNu@n^3*Kb*4raMXBp6K?;l1*enhCX(I{sS+x5xUU<*SWKVv4)Ts1cGtQF- zm~fG^1(ppcc)UBhO4=30t%%kH)ZTJonZRBAMTI>={O-aWW$!CE3%)aZizykb3y%;Z!g(NA zvp9;*BFc?=Ul6}rkTCCe8>4CdS3-Q<86ADrU3k$)Rj=K^=;E6ZX0ln51)VOrH*cE# zy3%oAS285%pJYhP-QJf!h-gDfzu~J?Ybo5vt*!d1g-81|l;7fkz~#{w{=CrNn1Ccq zNfJ)$p+lyc=Ek07sx-&q1tjp#pjWS68Ls$~^umL?{X#AjyE${`Dk~TWimrmIaN@{# z8yf}Wzo2vbKgtubqMpU@;Yv808`1Fwm5OSJ6B-r#n(-_=+dNo4^YI=cA#VbB5vq{S z++m$kPQ^?-9c;PU-d=~u5F()B-aTMx!T6r4V!`KRfrl^v=Lp2Wr7 zj1Z17eZ1w63A>LJy?=iV_?HZxZ3GG~lSDYmb_uMW5F;~~8}q*Rr*)K(DZ7Kj&pT5! zbB|d2U}`jC)Tp`uBl?@~?>RecG()}~@4GHq_G`~$CCx5&fIW)HVN`gdKr3T%diU(< zvAEjo^pz?5eQ80!W266UdJ)4ze#N{KH$UV_>+mBj+Zf2)IHr?PCmu@%FS0V~(Y<_x`Gw455n+KDBtBZ7ds6NZffik_v3r%$cIU zAZwqR_O3}%u}F6c-sexZ)wFUfs-rn9N(y$m{C~SXF?XL&snyJ2TJ(gqm|9{5+I@SQ zf#G&TXI)o$zW{b83*YEeTEo!BLg@IE!Mw`|6BFMHA$#`J!hFxou?wfWmCA76l=H(N zql@b5)EG0%2XJe1m$)a>(NU)?XK` zc3Z-|V6ie=J_1eX8F5k+K0!#^87pC1Tly8{sX=rP^$>1?(I0^<4-A z@1X9_;#iz27VKi13GG<638-so+J9OpOKAlf!F)-)^1+*Z%Jb{KuhKKKA2huCqQZ#{ zWhnzJ85qH)`U}`;!SSC|tm|%QXehhFpdIed6dynV-9TXY>Ws_evT^#*8r7XR@p1Ds ztb|v~J{RUac9WxjcmEHTO}{-Fy5Ung|2-kg?oiOpYS%dnXR|kZ7Kfxh&bXBQtc$1C zKLhEB-Hy+!{olh0hdq<){L2OSPtbx)b5if#9cMX`?hv=QI)n}vwMTLzcz)k^vijeV zA1!#hL7R_ydXCvm>reh0|4_LQ?|SO}&VYd66aO>XcJ}!crf3dt4+9sPs;eMC(q$Y0 z;pFp=AL-%*H>lH&b}|FCoFP`6w6kHSPwxH*N#5QOo(&m|_wB zG0TyZp`In;AjTaDl5LSxddd{E!nJ`CBx|Q%B(5GmC>gQ4V24*< zObkg9H*elll&^X-2h3g26-Z0kfwN;GhRQB6rYRQ;Rh4Cfs%T?nV=>2=X^O+$(Uz8` z%z6apCtpeFKv$Rjg+Fs5LJE4%r~}ln>W`Ezx?^iDu>FQNj#nDos;ba;4y^dGYt?(V zpzq!GWT+E#$l9RFRv*hj#$_Ww|5qNhxXk91TLjwmjK~!VSq(sYH9A67^tTBZb*9P^ z15DGxgh$NvWT<5StUXN#E_d5y3j&QNb8Z!SWQSsg%X#PT4ZXpT>MXXa!UV_EE#C4d zG+e=7SnJhPXbo@PyqQVmqwp078LSB@tDoj;YQ~VC2T?x9ZgroQjuGCHY%dnDB4>B}RcKDo4{z1SDBFU>4!8+G__D*P0@ z@{arOn(*MEyMm|CzF%gd@ry(Kv@&1#p9vn^eD3KVqhJP=cPr~qlw)N6`sLLyrC;?^2PHRvX-sLu3V;3L|afSu| zQYeHU5BdhU)mV|rlP{~7#otyu$bAbmnlh>kT50*Kj)Fl0puUhYICVQS;g89D-|C~x zQn0fez)D&guet;kh|jUN$|K>)$j)|(z8jIF_e7S0t;t$&LX2~TNk~tD1t|j+k!B^7 zL`8IP)QYuWZG!$-eCxmPiGoLa8hbGC*jG|O>6(cPYT>8D(N_@bGc4B$?A2SR=>70z zKMWRE|46xvlE*TA&bQ+5qSGm%C3!YcMlu(rioOZX<&a7FCGc%fjBU7{tO)JY$-Kv8 zk8HlWk!y~g%UTdL+0_oolfzYChLu;YTq$4s{gZLrvC8UNNU@uO8!Dd>C`1Me;tI@14f_&1%(TvTomi$SdD|R^>nbd1Z>` ziU=`q80tRk-TB0O107%fsPej-01jh!l+9?@7$Js*kaPuGhSod^aDkIPJ;i4mH40Z# z*(AA25KQU=gQdC0#~fS_&>>4d5z`*%p>RV{JNgs8vo*j%)%&b0T<}BJ#s< zwJWcMI3pAvnrd<8mx2u#RPiQR+xWDP+VcLVsSI@xjB|W%k*;MCLUeRx$tnAByTA6+ z(h8dP`23p{>$nx3aUP6_x8h^8;AU5{`#*CFnDkWzCNTooAIxw_WxD6tPaVmsCVRINAvs{&aqTL zv{`3qXkr#hbxA$UIo9zWl(M$khvvmH)(Iv*m5cQ0NRLecEQAxlpa~|%@O)7V5jeC5 z4Qj=<1v5$+A@Rt~;a3aBLl-Vyj7kP+)Y*SW-bHW?VU6+lFTcfrVml!;pCt;ydXzeZ zV%D91r*?o=@n$xsBjtW9UQjgq|3e+0RC{GQvp~P=O%B1 ztTme^{AXt?vyfgj5Q+U9s^amJrr4+Wcpsso?Z4!endoTVyxBfWB#A@ zRpKZQhflEhgkPl7L4>g`j(@dl(L$A+LjsQ=yh+7qb9T0QOfnFitPyAQF5?Aa|GlsK zuF(kyuYyb@n>bCUPHhd9YIi)3Tw7Vvbyg#z= z7LJ>eh!~b4lb6;}N;r<|ta>f(zccOp%JG0jRA+cj$Z^DYn)GQoxDB({k*8;*$ihGz z??a=azTK}HLT*VMprKavs9jG`susH4E1=~{B=n1#YPPy-(YMSW3)=6fUk_35&=IiK zdVSFjzt`m5M{`{_ z0fsw$Y7qXSh>SVCXunV2Oh1c1FOzsHuOo@*;E6`j_Gt!{j!` zWz#`xMH|Y>P35s;$4;6wDfPjFrYyj%s}K<@Ree+&=f#;8t+KPT#iCmlJoo5v$}C-f zoJ?PsPHjn=f$$Sb5C0(@r*fBsB^2o0=9gDZTo3a1ctpfmrjJnQhCNAHJB+jC z2k_He!3uDtv19+V+kVbnEJyfh7`oSFzuB{V%|kG~z`QIYOJF87!o@=zl95Fkq4c7X z9enQ!l~kuDBW5|znX|XJywLXGSQBO$bs_1@cX6q)PGxGh?i-}B&F~83L7^8Gw1iOX z7&#PsXUP=;6p*t4ey)M%Z}HCM-pn@WnD=?piGYNiHU&=PoCZQ%<~%VVP+~+Qh#Fw$ z4K~LpnJlaKh!Cc|C*#b#d+nt0V0AZMVH&@GzH5)>R~9JIy#rX*3N~J?yX^t%N5Nhu z%vuPkkkS9J+$ZvgLq3wqK|bJ6uK_IksaGTH1MiSE3h$e)W|@d$$t!o3X|=?8czs-) z0sBGLBX99bMx!e0_@o@VW0+#b*fQ8i0v??A;bKXTI%V2(=Uy!-SvE2^Y{CJG=IC)2 z4HuCeH!vFKR^}HJD8qLU9t(j{t%_w@?WIS)aN&Z@MOO#e66KbVXp^pMSYmLkn4K!z z3L!_KV!Dn`x}caA6n;%Uv$D^GYt1Gex{Oi5cM3pWfRff>VM2P|j1=nNQ7qn}UZsiA_OU0MD`L|7d9 z(jg=5du7g1K1W+Cc$Ta&2KXUHSrRrFs%x}wXkrQng=MC&ah z7M^W<=u>ZQZmOhU7@dlj$k_#t4PJw4xrXzS0neiz#x#7elGvcd&25LI}$Kid_`_Gb}dkpoBYv1tMK#bexP3_!67;(CLf4) zTDu9Y%v$DvP|7*?J4dY83|ZjquA9zwEiT3lQ_8<;WsJL54` zEiJ8O5CTnaRdH&ROpIA}V)G4(##huoo#`PAh-86LDZNu(X)f?@*La({`n`d1JU&6aHb9cXRp{*~o`_t;C0qjmxGNCjc{VbWur;#M4&*DOPV z0kwrBqPO+=#RU`Zv)%Z8!nSSOCXvFDmwjSY3YcXW!sjT&=RZ_>ckDxfc`1-D9nDg>#w8clT=#uEWO+E-m+zYuqyDIRwmEp4hZ~( zPmj^E@Vlvy)f5U@`?R*YVus6nYAqGRH#!+gu2u$ey{Dtgnj4T9@0{IWj(Ps&aoWqTdnRVo z(ZP$7fB#w_u6Z0CUv<$NOF_&6?KI zrS5O-TW4rgTP$7*J6Hv8yB)D=+2GIK2ESyN^ zPY81YsO3o+m;88s^obKz!|u#EXMBgEKkA*nzP@^im->XP+L=^s=ENpZZ;MY6c*KC^ zZyVrlZa?!XD1LNoEOS58{6G7Bo}19pciOTyh0InxKIbWUd1+QJp6l!^8^P38)n2nE zW!;O4-;0kPc^x#q3rr=RFxlbmb}Nip?OJ+)W9?f9bB!`ic+v5q#vSI4)PyDGAFUfX zI=-B?NNvsU`hiCL9JyT9k1fA=XLNCC>yW+2|LCMW@Ku$0bJI8acVr!em~Aj`6i7yb zoD`1;9ElpgLWkBLKaZLdvTI_QF%=})y!Q8ry64`Q6kYE>%)f_9n=7?qoxe{6Q9J*w zGNW6Y%6zrpp`KATKFg{XRW5(8k=(+z{ieBtEVpn;2HpDgvORr=g++O;`8S?BS!4;l ztn}pCpQX-7ce_uGZ=kQgWcO*}tX|IjtgPF8TTAK9edn+W^t$@HBm2mvU0ymo?5~&a zB=yPGNwnm8dkyKI_zi3LfVq>SMl~Et?^cu((^QZ?;rV9l z?j+J>fEW_ete9NpV|nak7JffHM`+KvKy6Yg7tC;n072+*{NP6f9V0dJzCqw^@@ zh0MrWiy6^QUnneM{pi&S7@c0@l~h-(STP3cI?BzGDwFu0!lNfOyMKDok`42z4C&qe zv`M;0BPZ*h1V3l1r+CfDQ!A!34A0Cz?XVnq@M;~i5hH2=Pnua+1emTSR3&egU=?Xy z-;D!ks;Z=87WRuxAv!sbWQu7G@;Kr@w^xkbId_=b@7Fe~d)MFH$uL?=LnBDJZ$k(7 zHWL~rRXN&5?5aJdS4-DE@g_^wPL$ttE$;O7ttBi1S@nrpO;*gHN0bKyeZ)0hbjzY6 z&<_Q0gQTeazCTct;<6t7-{uYgb?pd9(&44g067J4}@k50B^lwOBShK5V7xx$yU zC9^gN{Lz!ocb1(_Onfc7X@H*zN+^6sS%U_A%ev^U->$O95==q{mv#X~|K<0?;R z<6m2m2h)Z6NV{W46(%OKn3z%>ySc-r?Y9)1;RoSz-C(V-Y-$B7R0g7<^YC`hr8(Ge z_;73Jj#+>u(EM+_dX;S9jfO=^97wI}k&p95L_-7u`PTFFsHjn$)z07k()&EMEMke~ zEHW9z%gFX!Oy)G!R()#@r;W{X{<0_Aa18x`0AiF~JNM1>W==+}9#$4Lu;fEiSvoRR zeDfFx!plgbt`wyGU8@I)AQ-f|>a05TU$<^}M%&O!<|UrXKc-*x=%K1%d)Bk+S6>cx zWXT>~{rZBP(Ue0v>wa&}U&|e`K3ueabvnq%d2zjFit_SuwlGfq)odI$u<}Prbx;$( z@^TNq?j8WF3}!^_O@$+zO`gcf`LCYI2V+r`Qs2IPr6_fAaS;Xw4fD{9_sQ<7n{A)5 zM)gNT?(n+3ZhuZOwl~?7(;+v1@BB^u8coQuuP7@JW=r8%h%zqanpXu0!VP_N#_U(aSdIsbFb$*gjt+(AygH{F<*W!C3n|E~8ZQPtPF>uWQ4 zDSJ}o^iXze!jh=WdB>8q+Ebe?trIyXD5IsiXUa&2vRM%=x1LjXcaRZ}PR_mTLn;o1{R}r15qQ5!a=IUD8j9miQ z*iKGsxwAC$iqW|%b!X@Q)~h^xX!E59>IWP2eDpB2>~6zQ*O!-b5WItUo%63}u{!El zy@#GJZTg=#N-NoX_u^{v^g1@DCMIYq*SJ!g+iGv(6FIu2?oG^YgYM;f>Z}eKP>v*y~*ek=JYyP*0Xo- z5%2~y+8(VqG%nm)!_c&g!sK0@w-!57Qjc8@^Y~nAzeOYl%SDJ@5w9CGrTP&3fWH$AhibGm*g*J6AINM0Daq1R7ctF~$ zdo1oU?@=Cg58rS-3tuGCLz8GT$Xyi)!6v?WY^8~ss`73q;7%8QN$A^}0Jp3d(d ztFmDc2wx7en%Jx8jM$`5$N?pEQgteI?%cTsFMp8`p+26>AmuEht$#LH*H$nVlg+4j zUG?nQ(}XDl{5EGlr)l&!z(%30+>vxhSFw@1@uv0oyFjad9n2X80fJkvIq`0SF$QM> zzj9IJR?zdRIn}LGM>Yp%WoMg0e>}rn)#dqr0#mTYprjP0Qf^gHxZBox76UtNOl^O5 zZR&5wl~ynB;Ms`t0QY?XWU&Ia!SoQ@#(O5CSF-#{kt>z`kn)t}ck)@y15HE%Y`})lQ&n%L+`Y6vGO6|WS#*_Z$VzXK(w6e*;auPZBR=~(vxnJm z3nrbJsf5mV9X;c0iXU2=EvC|c)F1)KKYi8=>e{3%0~|C#q0N%l=><-X-fj*9R0byg zI9vVbcat4{0nx?~*_+BHW@7rSN5_qOkErak_`s7so@Xo)TTfhA^)`xcHXElDRzDLG z{Gh?h!fR7iwjk?q9JPv(i%(3f>gV)NtE#pFrh`5}w7Hj8qWyP@)_OH7rTKZ9qFYq_ z^|yYMR+zq2#K!t}Zr@gC5H*_ufWHIVq_$$&Ur&r;#=+Q72xBzr9J4<)rcv(`Xx5`0 zug2fLdv{;vu`vgAc_w=dVcMbrH%QHB(W}JCtdkla=7=p<7H0-=j|Ti*WnUP5cb12M zWtrQ!lshN<%LTBp34ZSOy^sQycZNq~2J3Glw}lhGEZHdO`8ZV_3hCMk4OW|N^jL17 z6hQw!dWA3_WOwlpQRa(wx@Zkr$#B#kybLK=Me+Ve;d1+|4-N-Q9%t&`_1!yu#n|H6 zv$ps}M^3D#h+Hs~dBazYEcI~9iX)TPRr2_Y-2;JpeC}wtN!dB9JCjBn>E1D#K9$!>e=;pTaSaj&uuNi%vG+hAh z_i<`EIpw0|tb-32OUj@_P8+jJ4D=SpRk35Rm;5L$hVT@`J!Lqt@awjiGt#gvSzG&IONQ8nqv?u`bN+|-`=4B3 z@f6~3rfhdOaCLm){%MJh)8Wib8*MiIo6?)&Zl{0ePEYH&C&R7B0^X1(%4f-6C7F}e z_K8$Rh^|N$mOt#67PweUj`sF@O@}d|RO?+Q_3+c}6ng^t=wHP| zjKHv+;=PtaW-?3-bdbm2djf~j7EELQ{P}r&t?YO<05bobT>Q@I&9o;~-@h{YdFOef zsvjoLGJ~WOGrQ>eP_x~V+^9@EO9WsS?f?TCbeI;;U2~#kvzzq$B1p&4M=K4H;)bLo z@8FD4)7_ecT$(rK)e!3vCn&Jg$oX9SgztCm-eq2A`xAbO;!Q}v{OEnJdQ>oc*_C9~ z^s@V>I{WXwv^ixvKBltdd>5;Qe^Bj9FN&C2ouI7QZie?aEhdRuJC8kN0+)_4S-R}m zMQH4Ib&k@KPnnv{oBLzsC6ZiHw-}_u_`qLti-1Xlg7uI|A!9}r4(7U;Q?T3z@sqW- z0mwf^zQWL;k4v@+qqB`<8mZg`@Db4{hfll}6N%vTp1+Em< z9vTSY%g zDbax$U&8A{RYB=$3K%~TKx-^rR6YJwnjgO{S#*OvWRp8?Gb#$n85`Bb@+q9W#^h!q zE&dh?j{sO7UAKPy^=*NMy1KT^6+v@10>jULm6)Karq+aM5$I(4DgeIgJkj$BF~l#c z=I83~`ceDIYDeShPq#{u`pgDEo(*3FWZs0*X!sbUc=-ixAg8dY1E-~T(a{-BhdJ8F z;nWld;eCKIPlIS`L7!J2=ns%rhgf!#PLCnuKD1BGVXj;x@;2l6T+ht3WNWkCua_AM zX1ofTF^197uI)~YO>Z%q+5JJc?8i$R6IC6dYPV_I){iMnhE`@x>I<~l6g5;fyj2zT=u+I~uw?jLO|HmiBoV79N#iwNf-T^;(a@EW+fU)k2N zp%Vgg-Nu^PYu1b#hdPaHwl4~+6Wd*XPc1V?Q3ksBo7lhoY%qx*{g1)qe+)XiYAhK# zy6pYy4;SXtOa|?g0WcJTGguC6qU#xQ60X=q&B-Ge)tqzUEBB7;KXXC425%pTJ>j;s0T|JB^}w9?OwZ@k`* zwY0SCI!d@@W~(z+hol*Ht6Vz9WT=ke^hAY%g}(dUI?A8f@5I%IPW#M-8|B?IR32Qi z9*OV;lv)*xAj*570|**O&oWLl5nj)Ej9t6n+;iLBUHYpap~rI%pJ7W(RTDkFC{xtf z*ycZM(0+Yy?GPsSTJ|JJ`?P*Nd*-cO@7&_vlBa23r>^|+V&5X;ln}$aUS4gEEt2>D zcPuPd-@b}pR)bab9ckUfZX_q?vhL~b1wZAD?id6==Y98Q_Wo~QzSDnw1M#iP{!b%n zyGG(o%al*5cWDSHxs^YF21HP~X659VF~Sphb0>4l_ThT(`eU)w>tOfCuL*0GTMwwc zW+*2LZwS3IJ4M&@BYq&HBV6oMd-UjGe`<;r5Aa0O$TFQkPoEptua|m#{EdgZW#9p| zZtMUMKnAFWyQ<2Vgt{m3Qrqh9cB`+eL~eaficQZi#WOluO>E9o8IiEx?aV1anbW@FN|3Q2sVQ|qD35o>hPRt)gVV1n8{L=o+ z)~_~Q+PVNF`N+|uvIX7JSsFT=tJwAI_V5R+U!D^Yw*{@BWfE8qbTX((Z>{OJ_m(iA zlP^i!?%I3IFNFd~uR zA=!*m6nV3#$Y}_30TS>{>MFF>`{uR%{V}R^Q*UkONm$zwnX{6;*Z7r^QDrsp>iB`k zo#K80fBbxOJo~H8yXFd+SQ8<6JX^}1fUR4*zqc7(uXN2v{-7n1HwrCYf20E9c?s0U z=&HxZrP1nC4viFGRCg~mlACu!C9;7PbQri9On$X8#mty|Z~UC_8g20{rp)X1Pqvje zqxn7qlSYV1GIM|_y!cgO%+{STDuq9u1>GrxQllRg9etU1o6Vyst((X`8M(@vvr*OF z#eZaNb<>&5l4Y?4xxVyNdPeW>SZyd&jgvg82d}e=@!kC^O&5}#<9na{pZSdk5!Cqs+{A%j{ zwQFLG^BY*Socj+<9uf*z|Mpj8Cx1Ub^IgT`-~ZhCwZedtjy-SyH^9W_v?F~ZBgDOS z_V&9$F@HX*+Q`$dCYo19vg|dVHq)`@e$}&Rc`Ph!9f+uI$Bt%U#yoNKo_EfVrIuR$ za2QW&^556cd%wCeV_7Ulx^Gq=y_wr||{;E=uH;kz|kw#<0^sf&fG=$Z<^JZ`XS z0sqGzuliL!w<>J7e`e4!5cE+1A8-cN^L6DKgVBhMl|lhB9&I19+dVx}MkAo3z^@}2 z(27IbYZw83hlX}8>{k4fKHq?EW1Hl;0#s6H$N`A=;D_5Bx;lKd$IsGE0M(s_pjH|I zh=A##EgXk2hx;`YnkapSQewjEhg6+nyhavQk*1j2v%2n4YkS*!C15R#CAL7-6rTaA zF&R{>K3r&>C5Si$TqVwM^eX%3_M-(2J9Vm|@7gt+wWI&2fY7#)@hvQ0b>ue%%=EMS z(Rj;T*Y8(KW$701g=035HKb*O z+*H^hApf9Okv7Roa);iYbIvX>*8NYK8>oK)n>SyBZ}Rozp5-t2Tl87B@Jq%V8K40$ zV7WcM97X|_MEQ&mquQ!zBr%-Tk{#aM>tWGV0%q^r8Twh2X9#s||Jb!}hLj|MCRSGU zdDPN~YF2~y&fc?Y*OzOH#A*No6XZv!H*a>#^<=VFU|B)k9Ti4K zYb`gbi(N+BI<^P1Ma6+6gFj~`MhNU|9lVu$D-<1O7sC;0U2tTMLCKN*n(rb`%H)X1 z$RTyZ^{i7{cpCbS3kj{q=9TMwOiheq{hSDO)>2njpkt?uuBE7PrG()GF31bW0&Uzq^e)~x7$f@( zc)&wXo!Z7>?LZv|jm4BCDq0;e)Bx$S8?2)uq7Y#L!5EU83==~w?@zk{$S4#herj6O z2Jqvmf5flpj}K;ZiuMAMF3l5a@VY`Q;?;Dbn(W+E`m|vR8T{x2AuFJX79qG|$@yK=X@VoMq!q5z(`Ws~eXXZJ%sWz5 zXCp`<_t2~4_ms(#H#7Fv+snga0sCs2=PPC%P|A)dSFEnMTu_aSUf9YZ7z|2nuT6jls7V-CBj^B1E%%z9pN; z<*l}&28Fb-SywUA%)G9Nbj_a000~MagkgPSnA@e5!(1qN;iZiQ^VaMAS1;>R7n-!E zEquzSki{kpzw$R!MB|)hZ~wc$#-hcG8&SKzPl%1xhjgjAPgBW+sv#=mAvAEZJYbwc zT6El*7;7X=dwnY{&C+T=z@OUI9#9<`w;yUob)=OEh_;<6km;xVY8gR7nj@(4+?r7W zXw7XSB3eqYeEN6UT!FSQ&UqoZe*=lobTZ^U9#0Jz%0drC1*`A{XW}4^ik-+X|NT5H zHhA$+M&WACpexFs=hl19DOlcPED*5oo;~A}D<Qd)<^CHOf45ydH4MLKaea|oH=$osn zel#)l%4(YeXX7)+2(Y>>nNzJIfjcD%HbTJ&F0tsaU|P4MVYDxL{qg|u`##mSn50Y9 zLd%N9d}C|td()c7qzrjgv3>d76gjEDl$ymZuC8yd?l5=x@hO=sBkubHJU(mFF>Dl- zJeRzjbN03Ej)1_sKx&XkFj?{{>Dv}-B}(vHkP(739e;}VLs6}NDfK5!8!@#NKqLQ8 z&t*QV(#7tRNxgy_DReCcQA!FbBZ7boUvqMXR;GBsZjb(NoZh3jw7uOvm~VyM!PkP7mft%h0RI&TEmwFE*VS9)2s<`$}oYht1^IL=Q*1^Wt7-@pw2Nvb(MT_{2WMi~b7R}+!X%@QBh z;uELE(VbF6Vl;EJ>LPC){T&7~lDof*Y2(rSIYxe-_9AsRGEn1&N z&+K)6QeBLs14T19BeJ@G%+i{xxDl#x;gzZ5+tTz)Y zK$0eg0m?adNeD@tN0-*& z+w7-1+R%MSi+~PUss$`@gdj_`p?~Hm05-j1Pk7)brzf@Ip%o_yI3cC?jTm(_7R(t$ zlc3s=dB}jlgPi~|6uwk?vhG6ime@q_C^7O)y5u;WS^*|S1Aap?x?rv#(Qoe_t+-z? z+`b+Cfd1oNPcFrrIN`*P*G^8ex3_nDxo&6Rjxn0#+cG1K7J|ELc5zc2KEFSfW^{r~nkvOU;LQ&R<2^$?058$8_`wCJe- zDA4=w)Vf_n+^eww)6DnBWO5y;)MH`+-@sZblUQCY*IOBffgf-#*d@~Q{`ok@@O z2FUcVmW9ZM5h1itVc1e!qr5K}_3da-P;G4XNVn6ju52jxEMIxK`3W7)gE{V(Qi zN(kzA6g$z&%4d>U5n0HhkdG_=!?3PI*h_U@XgSky+BE1eDhlEMqH|dfx>LX)8kTIv zs9WY{k=>`QH#aM^yLC2>Jr}KpdynEm7 zqXU*H^DWoGi6NB>k&Yn{RgNl+*+#_J-Fx(qB*oX)FB^t(LZ)(zt)aX4!g3Wkq0UCr zArL~5h)7KU$$S%Job^YVrv1wW5QZkc?Oh))^Mau)P6bQ^c<^db_eskAT>ix=W*m@i z=W$0=rLmqrKh!SDvIT_*!_xZ3qnHu;F$Kim(rwY#FJG`vTpj1_^_-Do*7yYvj?81&0~DOGC*wW5!;bfM9o^JS#)Ezki(bIs*nki(^oO<|81XETUTia{QuY@7XW9~!&aC!N=Mly4K zMukH$mXpferTJEnH)Z)QHT|uLGraxC&=xD4X3x$#Ik~s}$w|U>*~At1`TcXAtW;sC zK^lOX*011xi(0{{4+PH`L2b=*zBj%hpAwwohPXMjY`K{^>-#2I-BzzYoku1RFQy2Via>5Bfs#7@EEzY!O7(2hqLV z{UUZ3Qlgwjid&L-A_rI@6ljK8QL$-N@0G~Nw9rsy>Dul(4F0+K_*j6qVBp!d?GA7N zJjykOhR@wc-3JF%_I`69!IWA2bX-;-ER!Ksn5$+W5y!h2W7SOTH%!{Z?#7y>6bXOS zdh96O(NQc%!?Vzb+YU@UO3uN2cT4Ozp4c=-AF*k~ef)!h_!vF~1qILXM`;0rF5dSZ z?S?cLMbqWm4+#|q1Yv#M?-bCM5`^iADdfwOS}3s-O?!U)SjHf4g(YFD)zpyG+7vfLV82vSO^Z2ELK6r6pNJ(NlsL+3L+16^!4?j zmRk}w3F->7Cj|xEJiJ5P2XEONVAz|a!J3I=h+)y+IGl$`?+k_=ip*($c5Vd^lvz6} z1XJ&18x))RKm_`Kw4)JW_Pnm@yvM6m>EK&a6wB+XprjFJ_(S`82{90uCGE@tg*O(( zCp@L~LHFU-p~(|AfoCBfw&5n1VGw2^2a*Q0Za}yY&UOyIJLs-h$?6gNIU6=sCSYM; zk}?z30)lfoZ>Dv9EXQ$*keD(w)=glG)U(S~}v>JjGK%YEB1@0rdx z-B&s{=S7o^%~ig&1RU<$$BxbvYl+YuvbwUfi_@`&eb4^etPIlQ#T(4>)#So;049?; zpHGeU?AfycR#>Q{V=99!kl0F(mNDoUgkoRU#N>j-9GK$a(E+g*Z#n^C0ZyP&vKj#T z11&fVck_y}IP8H#ku9bxy16sQ3wwEWzN#uXU{--)ExCMFRdJ!gDgm1cO$ifYAN(bV zA3;{k02ZdnnMq2>E(0E$WJUc9@zU@n6PgA%hz9Z5S6U?vL53j4}Oo)fdU@`hAoa?a_qs6$BcoK zf#l~KI_`XYSFG$0z|y(Sz5nYBj74r8zFTy&)eSI@Rk(L{>prD`<$@Y2t1{=%) zMsBe!z+v{ycq@|_L`7)W081JG03;Fy8aks7AZ>3UE;mTpo*<6{S0OCZYK9b#+5Q&t zX_c!>x89LEbF&Tdf=I2fbZe$q)sNn@FRa6MMBPB>8y5GL;irCFiCJ3Ny(K`v=K@GO z3T)5!02c-fgT-fVL`0A?=d^|us97F9JP|cFb5hj(Ejsi0gu!cfFJP)(eJW>{(w$6 zu678`SY2ZQT+<*rO|Y@B1$jSw!A?uh3VAE9{>+L46 z)Ed)JQkIgLIYeIshLiIjOsRGO(Qt8F4%b7%ZrK;<-oy*Sf_4nUvZM`FpD>c3-Wdme zt)T;;Ch*t0wlDA6E|9ox*mxnY%QjewA<)3I)%iSouwk~@C22(pyB2RQ%_1l;{6X#%E;ZYN;MHJaDx-HCY6)7d zXai5?A}GHo2mqt_x=ykkyN)OlQs0Y;?o4%tC*~rOU^8I<0^=e-OuBY~VoHE0Soa}> z>(Z~qCb;AKk@)`8Fj?qSl@dMulLe?2*!7{s+D4R8?Wk7-91z<{o4vIsC4g%db(V!_}9z_AgwIl8-Z+qh_ z01X&((iEMzDxhSN8V5Rj!0?+F4;tuM%q8q0*5Pw0$VBp(OjC@>W0*-mJ3<925@Q~2 zE|4HYENCUC0-=DLNW(83c3}il!0INq$0?2(L`lVlbn74@!=hB65jkLQLO*e478tbu z#=iiNgw!t4mMa@I*TCx7ttp^ZRfa6jP&;T2f90bw0v0LxIz#>Ovr?hTV*edi9Li)_ z2a8zHhV=m9KIoJVH^x5$QBo)~h^2`E#!-lTF**DausDkN^uUo4j)kPo1ckDE-SUD2 z%gfEoD_}=N(IdD=gGQ%pT)F)<{?bQn)eYjl5`iIuORxTd^lur#zgyq;rGlXu#fqB zSq2xLwjH?0(n#O7W=5#e z+n@gg=ENZP(pn);AN=BK46@N>-TB-TPIvHm7<-&_;Q85c=`NV<0u+6Qaa`SQ^c-cV zt$FzP;w_tSiKwYbNL0TYken}IUt%gU7HNo)z8;mji_Io}{l|HEkf<<2hR>jwS*%IQ z_JkbrbmGzW_IC1jICbh2hT9np8t?_?F&npcPx~Wm#bpRHWLPXxvmM-z(~NimY9W_i zw&9l6N#94k5Av+i3&4|{n8Xi_j1Wf;h0*3h>}E<3eC)Gogi{vkX}Q5%x_uTbzrHd3 zYVSIGXkzSgW5qtD$E*vNEI}F=FUk(#Hv9+5-s)2(*jO--A}s^9JPgRbz>i-CH!pVs zs-`TMFj4FiVd(G+^TIAou>_2|_Vm{-AkD%XUunVAZ<`UoI@HYaVn;>Hal!~Oj{0+#)1LTS^ld8)tzP2$qN0X6gi zjsqLQj=>5S#Q~Yn6C@0rop5>>(!4CV7#utcRfHeJ|16$v`gc36PGa1L&3Tg+#eKDn zLWQ*QR2+Lk>9w(S7SMsr`~f8RnRAv>;18W}j77Q`*s;K$ngzifo}eVUN_5h6(*B-YkKJtL z9^+|04;-&(^o#TIoXarsK6uq)n~t%Z)Y)sM`wJ#T8mC#0-{=sR;KRxE^lLoWP= z;~T}Az7uQ>QXonI_(@#!^kksNWJ9IEAS=u35gPN(oS2+ALyU0rrc|t>Or?Y!AXO<{ z#F&PLNE=gnR7*569)nD#)0w#l<8iaCTm8Y?5rOp~QS4y!!3DHo{F6F1a2b_HLeJOB z46C)P9dUYT-!!wf;tr3x$hI$gHi#QfPYl?%d|~U|Ayx*{Ifm=1dj^Z64bIQ1+Zq=6 z%WPzxEw9~I;jfxvw*$M3`FLvV*CaEpy09MG#N^MY^fa(3N*F4CGS}fi$$GtsBhwk& zPA6D{cBgWi{a*Dj)s4Z>bXR43?ulSOKHJAcLct;Fyya&fXAKj*y|f2>TzDM!dF6&< zzPx!=w_EFal(^iqjI~%SnN%B(Iu2fYduG-AWBS9+V#2w_UOhh#I!u!ZyDssGlR^KH zY|6T2(0NGqlzQ2O`kcbgL-j?MG>`XH9q-=pYp&cxgpE_==-lONB5tOd-L8C9Iv#Y* z@q}5T`N_*>YAXuPolMIIEf!_wfg3?rav3@y-VyIlnB4R%eZik%Dq#n`ISz(OwD)CFbS(Z zYmU0h?R#?F)j!M4=hRmWbn0w=Z*8^y@vJ>n4uRN`|ko*A=0~4}>q5CCEk}+_DxA zNYMTyhs~ZxN|(8&d1{>4?D}CH$GYChFX>-RBCV=IjYZnF?LV1%rn~usoPYM)bI}J6 z{**t>U+2R^$G<++G_bUUmTHaRh~ z_~Sg##*^<~uTY#S0BR+19g$m3w&WbD68dC%<6dX3*%DulO_j>JGFMLrW-v0}G;II$ z{K)_VKDcY${*2OkuPX7w?Q%E`^CMRu zbtx*k{F!t3X|&nI51n9-mzROI1*GlxGOsV)AjescHR^4yz{=$rBY8Y4jpqSV{4FT| z^(Uuntf@@q;Cq|9Dh15@r|svCahIM|?df5Y>{0a1Mwnm6s3ohUhG+jbb9SzVr)J9c z?k{JVn112@Y>Niz*M%-(c`-#M63nl|`(Q=!!UkL>v1Ue5}I zcn9x@4gWsG(C+{KLFGZa+PM+;H4^n>7j}eJG%N06?m?})`Y4-iW3Q0PdcmEGvo3$9 z^5M5&(!mQStWyu_DkUa8ODmZv$X6?JV$NoPsF!sjt!;}Iw9QIn`aStecB8ejW9gpA z%&zb3&u%{xU+1z`gUvei(x+z`8`Af(8~dib2tH>0{`O*VHHDc=gf&cF%NpxAx)=0^ z{n_r2VIA~g-PM$rk&+VC4&8!Z+hsD{i)Vaw`1nSpXXv7XmD(jQFTM{?i{EYU%KsK+ zBRpB#8URa6GY!{|Cl%#9bd?q*{Ml>!WgeTtw%M%T)$`vB`*Kw6UK-rjnY;6D(VfCc z(H5sO-nUvjJdg2m`Mwz1{4^uO;Ky+O64%N14Sv(w9;CiXX;fcvb)i!4AB}FCqYvK) zA5%(XE{PT7%-!LinKzi(Se9F&m(b9ds;6HtXVq-pq%euQ-IX=1yFG^d9=G&YcpUj% z{?b%N|E*L6_QUUhL!X12^J(MPVevuqCCY<5bYXYTw-^ez+2%lc7t7Wvolu;k zZ)>dOVYq0ZG3jpa7a;)zUhg^4cjV_zPF%n-Ggh%Mt*-`b_WLe0p3kv3C#11c|0`FT z_yet(Guy|xk1v!+G`cSCVH1+#BAF}OD1XJyf6|t}jm@s1N~F2RnO(JXQrpw?%g5{e zf}1XHzhaPcqmFr>Nc!j^bwfQ1j)i-qB&t<2OS&SAdL;@k3J>JEE$cMj{cAnzJNx`r zcNW3`ej6Wf@{ulCF*xtoe3m-VAmy+e9&eUUSt>oeway%hClo(heiY~l?v_s~a4)ba z)YPg-uN?0Fb>02o#SxE)PC0Q$Ec5rWEIM!={bq2Xq{z^cO$c5Z;jb(x4#SCMdVe^4@~wxPOKj zFJ1?;E3b~LQ7|>$=`7i~>D~NbrlJo^`Ko4k#GEV)7)DKylp)=VPW`V(%hqS=li#-zE-_7WnH5kv$tZ*r#5z5$l2VT zm(E{KXjr`IV4ZXL-EtO;eQQ(Kwi{eulU$>ac|U211#|Pa%U;?WZyLXL5T5?XdCj%R z9WfR&LfIw4;yeeJK8!KCnWGYMl-KLEm*qtEBdOm5Mkls)W=)*zu|52;PIkG>_7riY zvUToj)Y^Wmy?apLHh*ET>wb@JkNs*BFErz8-eP6G+-foK=1k0&g~=mD1{y-tG~U+c z_u4VT)4km9{>k-q@zLyDt5)Cc5&9|rQqR9NM5S@zv}yO|&C6*EHjOrAS-Zo%HX-fE zT!ngPQN1;K=U&af`&KwGJI?BK!|46#N2Aqb1VR^#uvUGyGS7BkHP;t5wzra9q3h7J zm^(jy;k%2W2Mz3+`Z5)-S>|M|=W5%N@bvak`|B^43Y%Ou=Bgh&&9`zko`y;Be^iyp z4^Tb~*-cI}$$2Dn%&h3_>QuJX649G8E+-r~x$jE61*6bF*N3r4ulvmL_PwX%#1unE zmsaJx(strt4J%C8cjnK8;FaRb_Mz%4IWr>Oi84-e3U4~3euK5+in-Z^LqTyd=Cbmx zod;FihxgU5C|&sANrZX4jp(oS&wpZ15Xxf<-pVM7-6z&OP22AMVe8~)oYC32{t8=q z?0oGv4OGu#)Nfx@(GmH9Q{O3YPfK63=91ChO<$8e4o_3hoh*H-_f?ZOQiG@C^BYs1 z&Jc5^iZ2_*QW{-(#fE?I#f-0$8n#I{<~7~9q33O&#~Y`j_r*L9gM*zfUXQ(eKQPm5 zV_DhniGCgprv;}2IT_D$X5ac?S6d*4lesZGI{BFK9tN!>U9p4R)HhE^d7B&7fGAvjnESm54D{$#9$z|tqB3>jk zy&N%NzI=9c+M`VwEYib5)~ds7jF_q&(M|Hdw$P_fVo1;v0%zfx;8l| zTVi?9s(C}q+pjz+A6=#WJ^J0TUWL(n4RUX1ox3=zKHG5jq5;mk zHX^}XpPw4naDH&g_BH00f4q(N*0SE4mG19Cjf(0wl^^{+)VsYcEq5@zv8y3W=67|S zU%6^u+>I4Ig>HHwd28LZ7>wKF6*7!$mxBQvc|{x%aV?gYO!D`}9O}Bp9qBHxq;|GM zn)IVe?S!?8eQ%r{+{}y5JC@!4`+FDP*XTQZN$Kt){Y<6h=VS`5&5B?h@;!dOAZQ8W z)z@}Z+wm?TngeIYUkI+_3XXLbUC{U}$noL~_Dfl^dpQe|2hXu9PdtmMwD4ity;)!O zivM-jDoUqg=?A{FZC9+w4dnBF|{<^bg;~JLl zLZLT|>P7BaUiVj&FH>KrD;iwp*O=Lue}4SO+9(az<##4_@&B@YVQ*rjeW$ppwWLPZ zR7OC0Nclr^u(f?h>cVwaXz| zZc9pK^*m=r{ObI3lS%x-vyMlcdiuWjalT$cteozFO5K&eYxez~`JL1Kj#N=$W9ZGu zpe5Iw!mQ%kmS#J3jb_GOs`O5ttt-XEo2*)L$Aj^qY`s{x&_sc~|JO=EH_;2?eG+e^ z{Nvsp(K@HF0_k!`jvUdpu~`7p=@KnZ;k&;IdksE3-XfdP*te%<@+@azZ85% ze1zqVQ#kAe?^`!r$TG8-KQg2jT`=83^g`mG7p`Z%iFQYySYS~aYf*uu7`xPpDs9;n zPs7eGTk!jwG|B~*ne8gqD~`qJRH!sb?4EWrUx_jNXr|JHlI3l6_2kiTsa$K8`pO|E+Ttus*UejpjfAuv_WAFsHaSaIiYYh6?qduQ*ofV#?}8p~)I zu8k=&s{;Lh`xS6GI9~TO^$rWY92C&Bh{0HB+Tyu%&gvg&S-<}nk6k%1-E3*Z^D{CE zb6J9q$wS#2G|to*6-D8e9%OL|5BHk{z!C(?-9Ko)4P#sx%)647fciwbymDfOjtRiL@N8~l3%$o`iqs> z&w0+-nENIClr{IN*!c}oS|P*W~WkMJa?a@;GpY*L~a4Hd|W zQc1~j?>LcPK`JIo5)9wvcj0#W`9XtVX5Ee}h2%qGV(iyOAX!K`iSd}6ps44v9H`~| zz&jkU$6p)D4kiLvNy)!+zc#%WR-_-0Ay=FsU(%zFM(e1$T- zmnk$^`eCDEM)80rgn8nK{)JqyQMwnx9SY9+B?x8^Rz>hi47ZWAmi@8oYg-!=1JkFw zW=~)z2FNiMIP-O|IKXt=bmc?Lc3xH{?;ZiW!C(>p*Olk;;vAHg7VG^wlpJBPH^e_n zPTZ=24ra&p&9~m?SVo>^%yJjn4#=lAt$4i-rOuis`=q9(8suP53jOZ!1$@YAdi!=O z8J9s)Mzm&1(v}uo`epl7=|Ct9DKm(hn>$9%W1xNuEo0byld1t$eB|Z|yz$O2bto>6 z_xgc<}96B85gI$4A9nA(Z%B$p@{K2k<-eSuO+V4ardNB)6LGBFA~m)fk9 zxB5R2SGp2+WyfM&KeOE-T`RK*kxDCGBnA?5sNurx1r+nLy+9(7=A*Ew`B>;73W^2v zWaU5!aR*W@2a%#xj3@XLgSuEqhXl$<$^@G-GQm^h3&p>JIfFUf4^a>WEOXYUTxh>i zv~DL$;N{4YjW&~WMMCbhXj$jWKhIlHq9?#R^9_%nN<-#R&D&Eqqoy?#*8_eQSOlPu z=)yKMm$=s;Rwe)3iWTGP>KgO-u^?s2o^E+P-gka3GefMc@CHsXgm?N zVrFIlh$p2JBrer3TcvRuRAa9J-W_G3-{x0xd0n-^T(rkzfp5~wW6=R-EB;Wzn z#T#zjDc)+5 zC*dzOcMN3~Z_iCQ41Z=tAPiGONMYPpF#Df@=3s^pD)4rQY6Ql)_(I+{O)7}K#biuQ z@%b5T6M@EXDZ)#g4rbKbcz^hZQG8+UKm6aR37&zJvgG_y{r(qAc(IBj#87S<=p{>+ zJ=8bbI{A6w#|;He9Z>b5-=_>d$z#XLH;>&s!J_Z$n|bKtv+6H2@Tnt{(&a(gL?^bO4UrWIX&iS+HS|aVa*z2RrnZ|^z`v+n> zF1K6QUvyd@aT*`d%p2v6WADd3V#%%a3jg-OZHwF1>y~V~Q>mOie^B z{D7#m_kLetT=qo*w_#O&2rxP6$hB zPkp2DM>ZeFuSWNSLa%??thI&l_P~K)fs3?Z>9}T!$10o8;upt7@CZrb!X$ZnU#l$O z4GJ!o`yx8J!_es%&){y~&YG=prnL$ZD#JqZNPov~Uq!!NFM#v=vdAMt(X&jA?u!R< z-Hto(YHfQ4MLO_kJZliIfBr8_qHHLA)0I9d&-Wsu4hzXcH?#GRhwO8G>&js0T@sr5 z(3a?K>ixyo1-~k@=S4O7w%;sZ%X-Vh;Nzj+m?DTfDUck&?>suj7CaFdRFb_bR9b|E z!J1a_chZb2nr*mfkAHWgaNw9&<*bqEH5zeWoVsQ+LX8~Y*Esb&W-N}~6v@uj?-_l6 z*?VPYPcc`?N@dP~hyaWf6p!ztNt}Zoo<=0aD@R@IY&TbsQOj{zX-i+xbGV?wiqu*B z`G3d3>%G@&Z%((G0R|q(l@c(khTW6QBd4QBS70myjwlgYNCAP7eZA%#47FDG=Cgo_ z3vGfeR#4%|+chG>Povo~kh1e42- zj0|X!Cfcs9Dkxl1U0xFb1GE*g0waGGB&(06W)`u{<&EIcf<{rW{I2>JQy&|@?XP0*k*>1<}fX7^d?Yzaw8>@ZM+c~hamDeyta zRx7d`!(H4j$1sR<$dAWTb1xk40)v8-I%xzfCKmIti}oBK4Z`KpB9Uv+qEe~pbToSBvJh3py8;=os;|7PqeZNR8Fty5| zwL~T`C+N>~LK2}ZKHfgOY6b(6>+=?}r8w*_AeEHZZaZOi(_qC*wV-*?_2pHOoA`CmvS|2K?_|0{ONfB!*c(!8PIstc*H(xph$vcwx| z-+);0PLvx5Ibnl%b@=aj=${lNsLy=tTcr?WKf7R-2HNXW5RO3%g%kJGBGPlI#E-+w zmVDEeEt>@<5wwicDLEQWdlGOwgR$tUj5r6ex?};0+&B3Y)~_f&C~cU+)rq|US81{WK0Dktmn2*;6hA*3!IAjmxW=&q zJtux6uHh*Zrj++W(g9Q&o)O!O&VB_23crk~2h%Tx{rx*naq;;7yP*0-Vk9S~^T!+d z`uj_ew5X-A7Vhi#<6`u8&#{m|t%!FBBmsDB6~sQp7*Zl`% zwSba?4G#voQldG%#NnRyNtxWKZL{anld+Ss^Ua81u-+bCD4E;XLUUwHOH z!Zr|n3~GJ7 zX45~%7p^Q0Lg=jpZb}HCgTXZ`xXNiG&^J=}7OBUPv^k6J95#O@+5)CCNEvRNXnv*= zgg;^!<^Qf-C)|#PaUM?6x~jChcT?sUipyAVbdUegy_;z%_Uhw|bJEP*VkP?wI;VDl z847_5s6nAOj*@;M$e>&Z;);RT)8N>=MITYSHXh?H`F&*~`*8TdJAfHgGT9049mo1( z!S-FgBqN_0cZe8^AV?Hp3#@ZN!vtHK+h<8YgJ!bhQ{JPp9PqN0I#pFvELNY@1!rw_ zM$a?n9nnumlUE;ECo3Z}mzCx3n=mT?Yi_Qb6--AF)*EV^2--m;x`r)A+WNiZ8T=M3z z^T#wt)jw;EXx8@JX0mNrL+{MMQ|a@z@7}&eHZuzYhH@lEAmc;^0@5whoOdIzHQAyB z#*FbFA$LdIf?9|Wf@H|(0#X3v4odTL*SQ~c71a)L>mI&uF)2U zCr3P6Z1vNj4_2#^y!Ntwa8}se6kF+6^SAH1-l#awXoCMy0yC>wU4)Oc#k|0&y-;Z{ z{|rMchM!=U#zSU7S=ERY2&7Cdu=e%wCCB5h?`iN&9oK4*(rQi&*%N0DBB3&@wVyqC zz2tm(ef7!o&Ephz4fZL;GIsau;aZ!L z2v!vyC6O%?vyOMa<-v*m7%cJEtPWsP38$WJ1KI7(oBPIt+ZxqQy(|7;H}tt~C6AGjmS z$#YS%g24#TV>aVAI}u%mM3myv(!`I$;vs5?8AB09XGUbk4umhS*k?dap6 zh$8RTvk7X*s$9Qf->s<|^PV%x({Ld#OlVpVSPDUmS>!F4&HjH-seo89F<`vO4#qKj z^~|}7Z@|iDI@bbDR8*Q7#)&1Uety0`kg-O#eyqV7`ZwA*KwfG_I04ibCXs3aa3=47 zM^fa+Ed~9jd0)(J5O*uzKj!{ut9^}itYF9XrTY6s*BLMmo z4Oqs3?8&4aA$2AupS|8 zL@}FnO%c^GSFcWo=Olds1T1Eio#KLo4wS14^HV>uU#oB+s0m4$REE>GizhtUYs=f48ic>{A^!;+2)Q8c4g&9$J)6z z{b09yx58jL`BecZVu74$El8p;2voFrgp<$IVI({{nhV|wK-h%3qXdivqDH#8oxbtw zu!^yJ?v*?+w(@~;0&o1*2NC|y9TM-{kpz1i$0_^-_q3jW{v5m+?A63ug9=W~uSP`I z{Z*ZiKG$hm)I__}3)?0(?p4RxfBUa>U-Nc!jh@9LmF~dj zLw#qq7vh~9Mt>rEx&X2mw?H3bU$#?4Rqy8fxTz|r#!pA3qc*+=)Y6d6@8TmCw|jO( zi?Or+KCXCN5btye$mD+TQfmd>%~BIAK255FaUj@Lt8~5u^7FH9(?;>di`gz-v>f$l zfa4q9He6~_dx!sTDt~U>nit%T3k9+E8Q8E7u9g5-mUttNnv4LedbK^Mr{tBDGwz^J z7Q{VhX^zpTyaPWH7~K?sCl4n7Vb3J|$P1PEr#$tTDx@2S}R?acd$V1q+5X4$jA49KfjRceCP|U)2u61L>i`oXVw}+s~L~#RW zEoStdbci51!lLjj%wYkGM|(^NVIQPRfH@b{{GcjeXbqWABGH>&rsLdw1)E%KXauom3< zKuUH6lOAHJ06a#DUIKHMf&t+}S|IDOlR?r5a!;d7ZrtrJP}6L0)H=~ENJ#ZRMGLM9 z+ow}_>*AG42A4cwOw#2 zZ71~A&Q9X<+0jFE{te-fs?YfF?%fwGX|kW7H=RP!P{gGFIrAF)Y;a@CaXcS^{dGQU zYA8D7$!O2&?#~4xI2OqYHN0>+HtZs#*@ZB%OmD^#bZre*mUsSU5`17{pm8T%CVdf{ zX*)Q4P}T(8A1rz~LgW+zRuQpTChyy^&Sy8}++d?z|TlCsFh1lQn!5E=dQ* z4dHVIGM?epI53Z_%mQJn)7cKfa|@`=DQDA7rS(wE9YqJxc@H~q5Wvv%5tQJMhN36O z){a|YtHq1o0uK0#;8Spwu*1wW=V+Lv@Fe7Jb-Up`E5Yc4_g&{b2)Tlw3bdSfLiRZa zKjl71GEc5^q2yy`Zs%ji^vw86u}A|CKN-GlpiVy8_}F!YOFn=8On!?mgQGib^n15p z3{fTh%@ZQz^{qKjr=nIvaMk`8yjlJi-hRv}IyX(*j7RKB{f;U&2C1p>(&FsM%dLa~ z7W#{Ya#oLH%#;@{O!)WvvNU z4g?lQ+Fl6o^Yd%Rn!=Rh$&10HE;ZbzPI2pyt0WfIu79$e))?v+#m#?EI;NH>QS2=K z7FL^?wN#)42*ePpp2sL>%c!RRg-Gmp_;lxIqrHB9yfS)Hepp&o_LR*=BY){YMz-=E zK7F&)%CfTCwZFcrZQT7LJ#Z2GIko-6SSu?|4;`Vw0G5ugw>Qg^$qDzVhCdUFbAPVfTO7>DQ#SUGJk9mJ~v$&CnBtpSMC zz*GQ*n5Q~#*0+4JoN|eAXOoXwy=y0p0Nm4R@Ke+LOr%l=N5xjMqG^N0eO!(2wiFb7 zZ}U&xXQEAfrB1HYt*#r_ZTxWJ{cPBA_)Edy=cx=m#O9ElGuEK%_&{T-OZv&rr`WK( zNbYr{uSGw9v?uR*MMX`ge$qv3_T&Ob02vgZfyt@0?#ieJO9YoF$$4~XG z4nASVpHi*EHMhuiwcVq@dpu%7VlcWR15YhD@<8@xk@kzua+KKw$1LQXLFFOJC>`^l zHg7+qMkC&uqLty8L)9?0)u2;uZ@Cih^#(zQLm_(f4G<=}_E#*fU&nFH5wb>N#RE8` z7(cWlXj9-dwzZ&qql5Iht>H|814}VZ0&G~l{g@L}A8=z6z-*I~KjJJ^n}4MhN;lM@ z9x3aBgn|_Ep#0}Nf59Rf&OvGpf@p>Z6n!ovNy)ku-UGg8P{YCtc_#X6GRvmTo{2$F z93l=V99mF?TGo`Aka$sQCt1TGI~S5)$}%JdJm-92p6{2i=;>I&xrKXucY+n}J#3_p zhHja2-h3Rp_0yn#I*o@+<3c?%JvgqwwV)_*Prw2NgB7e@W)#8#!4%HQS@Q!wXRQnI zPx{iry;R~&vG~@3m(p(9c*&;w>M((Vud;e|>PxI~PHcZeLqmdcDzwTZQWU7$M>k6N zWLiJ$DeZA|j&RW?0@T14*9f(l+8`6;tuxUxAM*VBl_X7^>=NwEK;q&pqpn?J+pUqd z|E~$~2(>jNxq$-a(m`mo@RGzpdHi>NfggI=%tU7=G;6@u9Egn(;&jr#@-T5V!cYIu^HFqk(d$ zPzH6LJ_Ml^iHksb5oc#U=Is2Slh6a{`F1~m3}N1#?O1QzK3s@IcuaNc-xZ+6UpwDh zPr{2NV_4UysvF-a`6t=oS*6zDtUL$NvKKYYhwNcvs>yxUt}gf`4_97KYT%ss za2nbzg&QYk3AbaP zmB&wj6#-L~`Y)4k`-?$APw~@_plH2ooi~T_u^_vRIa?stDz(yb`g(23fmZ(P%7QkY zXZq}U>;VZqxfft$`=67T>kW*yU-G=s0+57qM^hlO}9Ff*A(4fDIt-iUfEz!~gB_A*xDS8rk+Fvj$7SwQ4 zWZ1$Nqm!h6rBtR>y_7Qf8>zAM2~r=)U6Djha0R226&*Clt5mAz2eryU`x9#* zAu-i$ays@pWxkcY@7iY<9Ocug*!eOrB*f1+5kUsg200kV;Lk`7c5B$`rMjF(X^m|= z)$yOXkWvEACl@V__3Ej0e#Ls;%mSiv( zP?^iaxZ;+|QJr|3@7K+&^~|1;a@PINLG{V8Ukp#wp3_m&4FO@2KRY($*n~E)Cf*pz z4G)!gGpFtXQ3kB-Zm-xGw+aV6Jf~q_niRFw=0!up=5d!(8f>&z1JM_|WY`Q7A-vOR zYFH@3RVZb}0&k3OL@~O(JUsHQ96PgxtYzf0^>wC#r8{L|Xopf`4i-@<#v#4{AxSap z6%;fRIEhu7Gu%P`JL%>+kUB9_*^xZ%wQNQY?X$7QasFvSp; zl+*%#%5dM2SiVv02!8jdA-~LFeAJhfwbY=Lj*XLH|HTg(Sp}Sk;ZL7Q6MA z;ZrCc{yYyqQ0)pAiS4m=W6e&W-AZXPoImvu(DEnYOtV7+KuYR~-pPrK_vL&417)=A z;#!*w#GTw>aPqR@?tOu)4$V`rm0C1@hxCl>9;s)g9RRGJ>a-oo1G!%k|2F}pfXwQ- zdOHh5F{wnImR&}*9%?J9*FX|s-K(z(aY7V`H8nktu!#W<{5U`a3Q_F zC%WI7#kamZaVRh@$#Qcie#a1|8|09R7StAp9$ES!kuf73K0N2=+PE^|9e%1%rS6Rf z!6GfR$MQ{A*|TQ|LSHiut@qtOXL!N3yOTYawR&f7ISynTMG1XF&E8x;n>dCO)l>Un1;^|-?VAd+MFid0L7tsc!xGwOmr0^ut|Ng_S zA~w%EAMGXc_nxM_QHjXDL*%e<0c7mZX`N4`| zFqTp9N7(an>=r_oU`kwAs6_^*kc(mm80g%NQqD!6o0Zi~O<0XnBF^yuI0DdchDJsP z!=HkOa{Dw1TvsmwTv&P5G)iyNEcs&fEK)+E1*n0jWqU7HH5?<1Y0t6V6ywu zDXUlSCfR;A9%`DIsFVHnj^VpU^3v;k%&u0yUH|7yn^tT|% z8X|~afG>cE0pm8fD~Uoy?F|cy;-Vr3=9#6CjiX*!TAy=fbQL`OYXJ?18%#%;O>NE%qywtk1>rsX^e^vO z1_Kt*r2ljafhxj*tsmb-nccLfCfMKKvjLUFOwf9Lh33GT19es`T+Bdx(8pjAmICSA zRoJweJ3Be>RoXUiYS1pq-y(vtijkC(T7sKhDW+{%Py>$oERtbesbSJb4EZctUAtDR+_-3bp|^K z;7TLCC4C;=V?Q`PQ(c0)%F5t(`>3}_;F>L#Qn=!l6O5{!{PE%Mhnin(hm$m?HBjOH&>}{#3 zCk7BIjux=7eR-A5jLM;M^G{lMX^)lzgOw5d=)Ty+0>-Pj(fd&tY|B~a#$Y*O++q4& z6bBMG+J+xcpiJw*w$B0GxbfT$7`Ex0KAoR^Xz$)@KlD$7b%Y=#1PY>Jt0{0lc^WhW z4i^r$R(BY{@IORld?_rv5KRY5 z^eYzFUC-GkRf;#esUj3!0x;O&e_>~5hvd0i?W;6b(GOLQ1sE32U)$~HdboV`eKFa~ zw{BekWy-f4<3}ow;B-UGFQ6Q*LqEll(UTOW$R7sm9*z`8(Ug*3G&WN1Q;N8p1} zjN4;uF%;1V?f_yxb4^Fi>_l+e5kT7qKj#b4PK~SXLtodP8XV_;GfAa|bL!KDwsx_e=^aKJ2Bg{*pwf8l0HHu`<%{&l3`6s~e@`v6f8I76? zD3jISnE7Uzb}zSf)K%yC%OW2Qz|D3A=%3iEyspaqGzy8kc+S5gzE6sz{P3XsFQN_z zhg0W(;h}#SKkxswk$LbhQRn}7GQ7ZeOq4crNjbREuyKqv0EtjJ7>5!b)#oaxl< zdF`T6YquNRFn&J>Pb0Xll{|g@dc&?Jz-O7Z#mqzp!a$=o3+IJ*NC+1WXUcCQP#P31 zFW|Na_Dik~!S+C&f2de}`{892kE=318UZ^ma02NdL1m3*#v3m#>UhZyzu*x+`8HNs zUYzyou~U}@Hd^#S3`*J~AOJzjUhq_m!J$azJirbR;NXkzTJk#{xw2=+M|NR5la%rq1(L@WT8neqtmMkS0BwFidw1` zkWH9YGrJMhgSnyc>k!x^K0L`660vwsOK}i%i1&m-$EHGGgpqj2ftyd( zd+!>~4901_L|My?EfnW6vuj@63#8qGzjvxr#Pk-8lc$@51P+OG3SkmP za1I#v=K+`+j#7Dn;71al5MPTJd*GHLLfa%Z5>22Tx^x&^>0_tu|EOiQ-%^$@6!Cim z=|i=kgFq_scq zlX{8a4SuO%>Pt4?Ey5#_Ki*-=IJ{RG;E6vO9^noe4`6)-V{B=-DGz5*$A?GDamp{Q zS3T0~S+WA$L~S_t@gPJ1)#JK0L}!BEZ(h0|XN!=OPtoTc?tQwIT#qMPG#y-5JI8cB zlU#>kLKc9X{U2ZeIHI~l70yF=C)WaJ$bzT${>jn%^9aL%ixsibP&w+bw>e|9E)P`8 z6VL?4S#vwk?$X#M(GAD0Ji?c`nSoRLpydazY#NL-lQ+QeW#;kMxMcX$U;XE#Wn~aj zPrESgg7myASFfH&?22@lK(L1)jD#VhzzZ0Q`lEQ*?QVkb07|nby$gW0 zGTNX(FaW1Py8P_D&uKC{pXBQdV{Ag-1gb8X8dH%lRMlmTv#b~su!J9BVPWc#l2{}f zTcWK>EBPk#cI~hZSUU%=IgWIV4`;Sj1Yobm-9joiKhp0tzI$mUV6LkHjAE z6#K`q?hKE~3HZdxVwSrCzMj59VJC$|M99nyF0-ES{?xJi?bO{{i<@igb}vtL{L?SS zXDM@tSr}sDc}ONfYS^~esPuHF^l`p$mqaG%K$!E-biECC zsH>;Vzwj+}%rV?GUGI~T;f7=DHc%2Ac1v;WK7iO>I^RUmwiXYZP7WeCsy4i-u6~}? zdkX{EW?+n9m4sjRMl9Fq2WCi?rdvnOM8N=gT4Arn(S|j0)NMQuTn2aXPt$f@byPJ=Q2$^-_ZjU#U3k z<8YEI#wG@z=ddnWj<|d2b)S6T1x*&xiSI6=teB^zL7zxa-j+el26*zsUjPtUg0+w+ z)3B5>NpU@3dU|11eHDhE=1e3y!}#vi!NIQ1&XQ-(^uB=a7ACT6)-qQ$SG&U@1zYQm z@yH5UGBdtMg|>8{YoQcw3bX@?7>Ak}2$(V|NQxhvH#zYgDyXlwe?aYIk3*WqRshX2 z!NvxQ|65)p9$YIrfoHfLvpXB+12%32=yBzVsIULM#eDBR4O^Bd!2uGGA(%~>`IyL?)uF&4O-$lDvqwhTVsvDX{6ppik0Q;KuvWx%-v>@q1l9 z9SQsD9wqIpgT|yS#X<_;q^2vpRf&6pCxyO@EFx?7dNjdbcvr!9RJ`I>DhPcgX&TT; z3P}gEEKzNnsK#3a-}*kmhDmt}XpozM8IrLk-9?IeKtv99sL;!TC>mq?WzfpiLw_H{%5m!>aXe%U?PLvdi?y@Cl3K`(x`QNbw$mjk~rDq&( z1=LvBC6;!|`%6bp)um?Xz_=)k(xHOaw;u&Ka?8{o{fhO8h1{aLriMljGv{@cb4=Ac z_sw(ABcOf3KX`wDi21|&JDFW#q3rO~kE>w=;z+qpt|Pybjo)j52uj;L2Huhw^DH`Z z`YsqP;wTAGah%7jjqn5zdk%KpzH=v4RZSlozy5nIGpB!k*aHN@fRaNgGZ$T`^T6q* z`?n_g$3UEkL1;C)8WOzjm-=E2kzPk*hA1*DJjrL9JvLJd(i0$cl+g_u5M~{B>n0IJ zkG0B*T_2C3IcWKlBJ7n3DEw3$8}80P#5NhH0aOpk_Q1A+jrB5rYhFcYR@Mse ze@v0$N2MW*!AaKQIr-`qanOOtA$`f0TjIkuQBdyi4jO-e$PF;6F2tL*$nHV~M>Hai za_~Q9;nL6l(pQkBNE6_nS#GoLF5p>I&YzNXW*E_HnSplz&4B&7kYp zW9J36vf@M0L?M{CC1_BvFbwc-r!ol8Nisa~(orlCG04D220Zc;d?$S8j#$JMn!-f< z3wCU)1qP)zHk+|&qcJ;mWq3rXCwRVh<=o`VTblEwtn4o&8@Igw1LO?Ln@rPTXg(gijy^*e}tR$=%eX67eP$nN#j3samFaJY?CnC1tc_Ntb+|{dd$j2Wf z&RzK246#pKmp?3a`h*k*dYj|pB~SEx3;@@a&(dSw3T8;wMuL<@{Yq#3m{ zwXBMY6DMWD!@uUQ!%p=5?A2~3(?(3;0*Q)R^dW2D_iqcq55R56`4Hvq>jxfjEeJ>@ z^uga`E!#KAwJihnsjg>-*e|Tk_8WfABdVFny=>b(Rt!;USNAf=#}^oH9G<`Gf;;i? z-taN@y$3_fY^P7#jpy8>6@?9x2tD{{EO-={x*Oi-CwTAv$sXUFBVU-m9#ci!x!j^@ zAkc65f7(0qsGRfu|6kcBtyUnG@P;rLx%Y$P0C!nHzw-quN!@GtaIq9 z8?*_n$td|mswA9V_dTY>r>EQcaAyfa#1yYRZXCh&5t*GIK9GV!kxyrAt8IdiW$Y`p_^--NO6dl6l};P6sCBMcBT zVA+)gW3t2R=y0bUjhns`1GZbNI-gS{kNz=@t1ybv=TdS^qg+JDyrt0?S7bnRoEU;0 z22~ltN@DXB)%Lzw6nWUO$IM$=a@eCOGBOz3wN#Pf0h>DUNTO2-@ld)&xbVkhVKCbn zwVN)`_le#})>u&y3wl78xA{X2egddAWP=88l;kt5`2ggyoPo}WF>MGSEPISAk2(Iz zDcz|x(pzx47Nie(VdNke0M0yGYdDPVu2$K1Op?xKE-ee&WVkywHz}3bs6?ykuhpF_ zH378LelYdzp{ysLwA=kO?^efswBc4E$?oXu*wuOH@Z$Bo2A{4w|8_-1sb6=hEQi+K zch3!0Q_(TlWIA7jK>wml3Yw9I`ef-CwD`Zk(EbI+RbQuC!8YIdVHCv+qpxXigN_PN z*1!KTofl_+%0w+UtsA)3`k!gdkL=!JufFe3RK8%Rn z!7sL;c$0}xL{y~KSPi2PHH`b<94~uQky3PAA&q+U6>t703sndUm^q|aRj$J*RjzRL zFmWHJxMsg#1j(aOTFmYQ6ap5C;JCOByzzR|f<%G)C_58@&Pd)a4(m400M$sfPgrAn zcNdPX?bX!8%oih951i#?3Bt*FjMmwDUd{lp*QvBZpk8vFlu|;aUByXm?f3XMO+IZ>)Fv0Ggn+3pn5_fmn6h4geUmx2Re1qfxXtE{gMSsWRl5bxP}b1`@(}@$ z0bcuiqW7~p-H9j|*_e?7W+r5)Q>wiZW*Ew?t&ohcfvt%=={t8k05b0MS>ONXpG5(f z?s6y+VpXV+qVgUYB~-0K&#^yA)yn%`8DKN8N&V?&Na%EF!qZyRSa$TvPh=wqu^o0Bw+gv3(hic1woZ>aik!QRh~Tkk-UC<+PHR^y z+9i-o5w*a)T?9A`;Xnau{50NcQzQEw6-B&iM$roZxQ4%*)j)w-1xdpR^(3we;ZABC zdRsP>Sm~(!d)4c*;q*oyPKHS3C`3uUGGXP#eu(M0{|CT7HaiTF;q@e_q~7W z2tFW5R<7tT^7v@S4pdfF_T^lp*#FELC$>!dG3578lwqv)HsU$xRj_*Fye#!@`MuWV z3HQmqO@gryFPsp_7iq>Ix-ONC9{+XjRk8E7--G~*C#j}KN~b3rujvl%rpkR%Bc3PH zw0Ux?WxG{TkU|zQu+x<)u$e-33CwCB1vXdsaaq#l(Ap-Fn2Szs-@YB0#}2D34qPYt z^q7;>ZF&qgY}YEKwKyvQ?D#pnojrVg)9rtR=zy${tRMQ8C+!6EW8JE|>fr>ZCBEnbsT^!wEvupatg02oUMnR_EwhEg(~kkODOs*K zCt^>jixVCI8!EFNqe+yFrDZVK{Q2YN2n`${q#y>9Wj3G<@5AOFYS1RZrmN~Rk2i3} zngr)8di#1&?5J#E$J803;rG$eW>_qFjC3x6s;o9?p2Aw0Y@~h!Wghmwq%0t zwp6g^vm>&>zGDraC13q2o-iyb&0L7_43%O)h2$_rXn3I=h^Y}qk4S@3a&v)1bs-Oc-vfr|G*|kNmg{y&L_ppXr zATT+^WmnXa_Wd)Y|6*=(tM2y4rjMPqG@qLe^KXsuLF4qbEGaU5T~|R>NgtC#FJSPF zUSQ_T)S{Q`mgS>9P-UB#?X&uMU5SQ@=$BHXF^s5O^?jG0zXP3P;QgTSs5ila*}gVJ=yC*PL9qszjKDaAK4HQ zf#4z}$MnX*DP0sv&pIkPBPQQ-K5fCw4oeiF6+}x6@T2SZ{^3ddL<%&0B_S~I&GYL$ zL|v@Ii1S8&E$?{d50C8E?JT1PSO)G+jAXy% z!szbThF2#HKGosfhlk;ljiVNLG|V-5KC#1ZlhOm-9axwy``3Z{-(V6iyQu(tE?51~ zmeccM#cTnes{2k7D~F>ue`awtKc2ZrT1teCE)G(|PoBt(i7v zD~;2OQ~Q(pq|}cNw4Kr<=4o9g(oC#L{gGY&Z-fr)I|AYy1sNGQi%GImZm(B{2Y zI9vS|PzE)!0d>nY*GD;@T{@SeF?{66i;Nj(-&<)$-Luc>*8TfK@+U5OTmNqNe{O!Z z=pVFw(BJDDSnRqkDXC|&=7!#Yl^24QnttVYQr1yP%Nz{($8h{ymTq^)Spn$8IllR5 zTU#qY;$syQ8^8Mpct4Kontg2W{8d3Kx|}+?`4}Z+UUWiQpML$DRc}0Tz36NUUz6jx zSKl=}$?ZFE)3)x@X72kfOCM#&FbF$imbMjVxTtU)QIIqNB3&!&*dz;4^6~Y^u5?pV zQ(3}r<-?Po*VB^F_YudN^X*IGmHx@9D^zEue^nE}dmAF2?4gGeON0uMi4VHONZK2z zO5hGG1PdmJdEvJpwG(gIf*~v|#3Ll5=(~Z{&)WsnRt%(AU|um&Iq-Y>E4ImBpmbKw z{PTIB$R`obDhu2D`aze@ojWsb+^_y~d0#-cZsWbGj?jp1q_Xs)Cz7#!Ky_o*5L0BX zdein!rfg+|!V}vU!kUM$4*i|9pAbF(9%OG!d_Ny$Dkg!Vz;8Tw5})QpIqfZe(3Iuz zO?l{ALjxuzd_nhMrYE#k>_el=iD_xAyEh9_UGDogXMuK7d5M(Y0fL zRv3wBOYmi(6Mmkf0kKwVRX&5#MkIKm7{~05^6{x@tx{%P*iE=@sk|A0%q2%1LA)F~ zh>VaAQvSHKv?;PDk$fs>k6VbR7UL3DD7ZzADiev5!vA7sT1sEEj%%Q@iT@!?XxLR? z6L{?>6O&w-ol|$A;_{d?NA@puymVM0GZ;r`26!|Yp#E0GWmyUBl<|wvH7^H^AXXj< z^Tu_Rf|~-r1=tLga^T{71OiLYlIyfE65imhnX|RzxLFLP2j^6#y}5H&+gD8?qn?z?xNvkqQdMi9C6gG>yD**^Vl)ac~G4WAc(QJbSuuUztcf8mkBkH`By>hIm} zH!~;uS-po3_feW*nQ9_a;KGYgCtrX)`Fu2gSxK}}ThpeAlad#<2pPKH!P(2{?+4)< z#%P-O6qwCi8|@MCdfdccjP$hC#_+4y6oRB-$^?ZGw;;1i%7&sd%R0slcRYEerDA45 zGlTW35|&i%|E-721tk`3=-V@MUky8*f{@+3%7zscX zw6kd(^=cP~uT?9>BiEAl6IKk(KNHd}ez~XK51Ckn?NGuv<_i4b9^!_IN>qS$7Y&V8 z?2T;lemkT${9*z!6hQ~T`DH~ZI*M-Q=I1QuZt>k70OrX2Mz~l5NuY%F8W7PF$jIk`})xBBJ z`i8bsmMwFRuM&$6r=vqw#)nlOA81hdBgLC|NdZ-h>=P((R4q4c8nunR#W%@}1s)S_ zXBLlMacgM#N3?Wew<4MWT0Q!)Do}mfiv2ibu<9%y1qFqk3j7JeiMEyGo)rJkOiC?=+!576&= z>l?d5o>|N2t$|Z&uw2iuhPrcPkUS=*q!=PCGn(?9VgU^aGk1I3PA{ZkX@VjslFwrVnH9|3&6ZJp^_u9ht5DY-1DiBY4@x++bH$b zJUPE4lJ4J<#{hJ)qCwZjM?)>T-^-VqGvoxm&N`(M9FEz+=-BS#^I!YCd-E>#y8q&u zleV>JVy(#q_JnO&ECto7-0AUYoLep+Xnk|#$K9gZqCRIy+iKM&gzArcI|~>yl(^47 z^e~US*NKb+2ik&RD`fSo09N9!0pXlF_1qv*Mio=6`dVs?xbZqaq>S5{ zW6|)XDXY)q`xf6!PMv6a#MS0;{5M*M%0OmV-dr76GL6wWDu617dU~^aW)k$VK{Jg|bMA zm6@oOavnS|zg9Ovvs>zIy6ldM;Q&+xwFPX4UHRlgtD3H_bBuExOhZju=c|rP&fj&) z+#o76EVZ})`S3LW<>cp~6?M(J(`F|Rc{MX&9<0u335ax%fS|LPf zBUA7^EZS6bKdMOLl;6XU8$CStC%uI#_>oKMBBsz}%j@fh=M3YpBXL%!0!OG>wURx# zT%YZ;=YXWCQowlf+K#T1CjFM`lokUgvF+bmXsQY&OQu`xd>A29C6qKa_Nfz6B5{t1 zEkG}?$ARrJ*CpeZu^^E8x$0W%6}gd~yx>0UNeFr;lFA+k^l(Kth7pdL|42aqS`t%n ze*dE7ki@(l7hdMNTJtv&1^&_ayB2}?@WqRdE4l$Pgs9HnZKu!yAV`15a>`a{wr1VC zbZG?p8*LIQxcuw+W?1iwfnSz*~_GtG%}0EWN$&{m1v^p{ANA+bRB<{>c7$7tM_QCFK25f~Cxn zDrDg{LV$@dYa>rkk;VQ!`R63zV5d_#unN#}h7+@H^`Qt}_zNNhg3mms0C4$!L zePVMmAWP9T#B~zvu@tj%#poc%7>dt1OiXKqA3*(+bd;LOKuZmVqSDVt>>Nw19M>}JJFx4sShvi{Qvnl_d+MD1tcbl^aIkLKR%6Csl6}VVRiTg4-cT3l zYIor%9V^~W9XLh5q>EqRql!Rvv?X~kz#>^BW{V_QM(xmGU5B_8dkrw4!7QOaJonzu zV4~K(&rn$kmPpAp&C~Pkg?JN_hvzH5yj~Rd;9cy4c<-*82mDMWX+v?W{&JSD?>~qS zIu-`227{$kUJH9Z^QF3Q~`#Fb?M&`lo%V+??-pQ_L}WgAE{(1o06o z$Zo9>e#_ENu-DbkuI+}_bRN`)ze_+2*ELkL>7$A{Vz^kvkB#yHRkUWxNFu&ML<>xT ztU)C*$IEW)SR3S&2{3kiv!iOoIPq~#^R%5&o0fRhoV`bBYk%fZ&{m{G;;Ke26rM?0 zg{+k}6|7C%P<9MjqWo<~{P9EF5D}3|jV~;c1bjiF!NR-WE9)`XHT>Gy{zjD+1X;WNt!CoyfL146Km&auIVbczxe2bo`Yj4}%OQU=xjc z-ChoBmo9sm(nxhV^n(bwIYU-`ww5B!z{Fzv@qig@b!WCsTjsX@+-;#v>SwpMF~g|A zkSL-Qh4mCFQ;1?hzetdUWF_l|v$QXuT0;gJ1SgkaX+T36MIj?EKbSAZX(IBJ^bXdU zU>j#3p&te?6s(DgtgF~jA~CX0bJP|0II%(zlnlye%(i~bW$`TRigX9E7#?2dbLux|lY#0-kHs0!jU7 zaH;c>^6&}?h8`}4kWcxALa?nO;}HasA(J_CAE=s zn<(%ZT0;<%K5T)X-!Ke6rQV{o4y@Mj)@@G4lu0nDKG7~dTd@H)MYa!8XC@B#{O*`S z(Ozxz@L^72L8cU2RoLz+jKMFHx1LTf1Cg6MIm z7i3BXB(5&?orD%Xrc4{`rG}O%7y?&x+S;zDjMP2 zsIRJmpZXh?C*pk4%l*_hpWWiMjePfTGjwsiU#f=Lb|so1OK7+0tNh=wN6?2h4psRj z6K*ulhb6UeE$LIoY1N4g2^-q8yr#NterQV0g~lneVoQWQs7~2-+T_iLl{()2T-%Mb z8P0<8M}!>ybj`MHt_Mq&j&I&LFf(c_sw9P_tyNg6KkC5ELGEd98p&u!wQXZs2p@%J z96sF!gJ>V?$&l($dPSYH3dOp%SzOIVLax6J!PkQPy#*mNBc-cxN%NyJ9LOA1^C7?f}A z!t}3r;3FnwS5s+D`rNyX>B01iN1WS6X80~}bN;n7eUR2Y6*Vgno>H5%H!<-aXgzE} zizE$1NS0J`5lQnYv+GChZKCCJ*s&RrwtzBYYV= zbUEM@`0ZzyH`;(znoN_5D<4k8cn09alpybkWK9HA9gdSv>%|=b!1?1c@x|7K}vqy`bVhzI&IrJ+xYALGVGFbpRKeDlfpOz;lqAy z5l&y8G3d|s`dMCQk>3-6`{HC=+-_=kT#4WNnwm~Dc@uhV$LEyI=*b11y9lx|h@4^b6oA}J48e;XNXcVI{T(ko9FipGdK*hby;#nFII3lTt~Ej*iQozOlK#0X`q z`pHmLTOaKSS^Ev1n^JRp2RdfMd9FsB)q!vAmR}%|5B;RrI^fipVVK$8$l}l)qMgKB z8Isz!PtR@26yb#f-rsN;WBG9NZa-Gai%eL?u0^hZ5d>PJUl|+69(Ql#0=omRcCC9= zkT#9H9g&@F`Q>x@A%s7k4(qISwbY^=#Rv>Fl5J%!N)`Qc6JqlU3Jl!e|B<+6!|W}H z20KfOiW8YVLk-SrU7wUKa}ysgE))Kawmve4pfEBrl3E6zf=S#`#745PtF%}oP79=7^KrGTT>yE##_Ztmaa zSjT;TIuW_b zc~gf?Jq8+slk$$(Vx??Kl4*12UM(wa=-}<A*1J)<-u;}{_)lB^E;&fAWi@&7 z*|0=|u2LQ$Hln8ttub(Tte<|iWVE?Z=wL#~BMwV{j+i17U|bzhOGm3#;yE2Uezw{2 zw@ExOb-OLS+i~wbuf*tD@d!W%_N9(Xm{?GG3MHWvRecydZ&+^b@E29EdWnNVJ@vCn zKMxI3{-yBk;FI+49cc4BX*c%iE^G}Q2u~!-7G1P*K^S#IDz3j ztJW=BwnQl~#VN(UxB04iHyEn*2tAZ6_fI@xCI5#A*Q$F+{LsKNj3)#;2~AHCZaF6Y zr(t3}hjO(IG$eDpk>7rwGk6ata3O;}K5iMBH3gBfVfutUj%&CHNEt1{sHj`h#~||4 zVf;(40wh9>cEd5XfdPXs^lhzqJqKc>KtKaVy7eoY;U=uVqL(>cS+Tai@z#=CZ4CLW%q zwBp07Z58#Xk;T%M_gu^1=Wiyn*5J}Hs{!>FQo>N;P7V?#Ve>ow6hsyjZVM||OMk6J%E;vkFF<4<3?e!+L0J@!$P}Qd6HG?{ z6eDFRZ5S3BBGVKdB6#D+G#v3mmLDw|3iB!b2$$_@l|Sbil13KZfNltPg)sbS!|cj> ztCgR%O(U~DJg6-EJlupscjFU^=$jZ z?kn@uPpWLY*6qsfUXy*d+^)2`IX~;ur>ctDz}mWM|NA>@%$~h7tK5Dq<=Lkl*UVOZ z*+jp`fVCM0I;)=P+AC?UtLo(16RDXj0`(|R|MO?J$(4E=V%Ij1Pk$aUDBbZved$UTj;p3f_itL76sI1C3MY|`?n?2iZ)vAVpHJY6}+r~hyKFTNu zy0AEMH4BvM557!*sTj_cH(BrE#fz@YJ-mGR^8BYWijs1w8BJ~N@z&NCld1@R#l^)v zM}tYn^6b4qFh0F`Y@ME-ew4QURY_JuR(<46-UV#14=vM`XBHQK$NQHumkE*42!VZP&u++_qZnEXS7Bgi zNcpo<-?-{&lAVo>jSRm@DnVqu&Dd)M{bfu~P)@_Z%UL~@nq;d(TG|B$)~Qsyd%EEX zu<$YOD>3@z+C%)Kfp^Ds9*h7E0lTBKa~x9sag%P{ySJSt`YCdAvsLv0A6`G$4rJtm z(8#*^9?qG?uY67V-n$y@5v@TV!`cmIPz zDed6FaeS!=k)pvn7_;ro&n!o=fY4|ccOd!PIrBsAS%rKS3Yxc>5RTxDFDfgqkq?8O z><=<((kxiJ-;g0YfNI8e&Nz6mkY98W`5rR34{%!{X<`B&o&$sqa4?^B?8uS%%g*Hk z>ns8`hh5b}b5}^C-rvL|mR};v%Qm}j8C3kNdGmHne28~gOcFtTM&XXp^z6U{@#u`7 z^0`=BH5!cS?!9|Ieqd@%j_sxV{J)EfP5fMj)_3kuI&bE8*;&tDzb=Gi$@*N47s6|L znz5P$&`-*$1(HSO^(@Xz1-Xw8q>DI}qhK zej*77mXLD?Q>38Ut8KNE>UXzRRVemoq-;_sf<2qHQYh>1H(j_1* zb>{Z>p7Wmf&ojn%#`w+{hoR%)@!t22wdb1in%8x$_4BLXcO0SCE^@!qL%Al#lPfKj*czH|O(33CN*P zOeiI}JDRSE8&fWi$w#LzZErPGO1qPOdQEdjeuF-s&P1ry;`z0b#WtFKE8}n$nvD7i zYwp)%PdALic?K-LSqD+4w-su(%1r6?om~1^$6cdh-gI;(9ogX_!xkOgZx4oPk=%4@538cu$knLAJZEI(jmxChbgu1zoR2N z`BBXO4vI2D$^1KrtpyeM@1T3Z|L>j%%WiZ076H8|xz=Y*zn~zj{DOjk)H@h~K|yn? zL)qh=l9G}*L`1wT39blQZm3 zy#s;Zdyg$Zj-^x3M^lr0+HINs>({T+X%8CP+b?C@eRt!7%S^pjgmR{$qGEqmYip}f zS2PFmrSkGy+}zv`>fD_|uLxob*-j5HeH3wEgDa3-zFcl88%!{8EcSlaKu#`j`Y)eB zJ;7@^^eWx3qa)VlXKL_Wj2ig7o38U{fvb|e#)JnD8rOdL z@+I|63zEd7B%^1~f*Kp86UE&IExcL&GFm=)=>>`v7PQ{OcE0Gb?*Dk0~rcC=OFewJ z>T@BFz*w)gO+?KGWz6`g>901h#+LKNHm z?Tw#b#MiAYEni<|S6#^*%~6--v)LRkN1-w`Uf=(C+o(AZ7rPrSAbR`7Bn}E*Qe(m` zYkr&Yo8q2F54LQ-eb&k|G&g^BusLA@cYb$0J~lS^w#_&bD=SuaEKk7OxA^B<7w5!D zqE5|liT=`O-Qtx|EAQAtI&utr&a^&!{P-5`_U+!DW8IeEjT_(K9hIkWJu2j`c3L;< z;*9;QU7+(C{!&~|X=rFr)iE>6%F!wRTwGd879SsfH}1ZY632^M-I#Tn1#NXA{j#a#WpJ&}Ff<&f3+xo(1Z)Ko)?3olRtGwoCO(ns~y%i;K(H&!1^7Ths2c4{x?$!Y}Vcoz@zgo6)uMv|Axa9sguzXPZFq=R7K6 z*x1;ZU0DfP>iuZ4HC^XE!4j#sy_%h34EF(Vn++~qe{t@y=^}DyZDqyDrka>T-?su^ zxciRA2quB(v)>by#JwMdUd6;vJb(VYKTY;hqL_;+d$<|3nxo^*zlU4IVq#)ntE-J+ z=v>cF-55Mpe0~%egu@)LS|6-&v7P+-QkGrKsIOqseSa;eD~5}f?!{=C<-?m6{n95Q zA|h9qnVIbz9Rne0!gqoj&i{7LR+YGH=?~{Vyo49@@2LheRmwLbqvD@5@$O2UA6-or zck}-)bs;`uq=T|MJ$fnbb52+1aflt^Rrsx~jfI=LXXSE{_t{^H9A2yvhh^1k?CkR} z3))ie1lY~D5e6(5GeaOi&5(C@7uBzIP21JGGhu!BdusdeP+`99ol_S*zah%|Z2!@> z)>aBnlaOg?sEwfy@T)A)t ziGG-08&fq@Ey08>LNlHjf|#6KT=)_m2XYn`*Cmf;G3bQtP-$st!peXzwQW+Tj$bW);sP+PiKp@;zk$xSXW=dcnCQNg5 zn8~exFp6zwB+G1P7$EU}%TDpODn2*=p4h~B_{305Ex^19zr^07JdRIZOB9)7l%~i`ge~5;xGgr+7Kep z;%6!A-@Vlg`o8x=$r2vLRwJDEQY7gW!swbNY#VIsp0KwZc$IY*Gco}D3zGY?{Co@ z4~yM)%|dB~cxRkbqT#+7&+zPM>KPM##@@`G!4d#+_=D4Q7aN3qSs zZNM?e{2EU!x_;9qW!UPLPQ`QFDqW&WmoDvX%^2_%Un<4$g<0D&HARoI+gnkGYLnq{ z_*=d(AXqtb@HksLZ~ssJ(@LV)*w__7Fb8aHFnjQ7!{AvOJ3DbzRaNCbn`mlA_Yq;A{J{CTLnGHg+=A-OF7WPs^`?kfzBkbHderp-P?$?B(YIuQ-X@Y8Qn-VG&I!nNMo@Je6%&`J32|{rA(lsv ziF}K{e7UTaB02Ju_t3jQucl&UBrk5D0|RD=mU|;AO8y zBWTRX$S898K{`wpT*&MLF!Pi4pb+2($uJ}H)kf#ecOKbjur3D*|v zs}%3uBbUK>IopVaCY`TSuFO7jdc3T!k!SJYHrA6*{EcdUWYY zdu^~GfST6;5Z5y=PItd!hf}Z&wz4zHu>t~f-#$3_;ayQ&>|0dC2f#Bs$`%TcMSm(9 z;AEy4W#*_(B#qb2F{1hlo(e&%TT5C0E?wk($P&3S<(L03J504cBm`${Vgi{2qApj~ zNAfNS300y`%?Vu-6{Ue}4u5|C{(ba!y-2{Y5#@QZSt)u9y)#DBw(i{WTub$SX=? zxn=zWGonsU#By+JrlBb6H{=B*h(393H(hew`Gp05=+pfZ$a|#J)Ho0@n*!A@rbGkU zEdUieB|Baj`lNK3PSg?SVb;B-+=tnF$zNN)eVg5xZ$H{@A$F=57$2wcIyq1MzU4HCZ~x*7ON_FB-K_M+c)7|+ z;Qaz^lcB8pD?h(v9}t~FhlD1*^#G4~A!8m} zb+54SDN_L}Q%iaZ0~~twwDN_~?o#h~ZMIs;pR-1at!BOUQvBBm4T-pB!AUyL_Ztn7na!$P5M= z3q>blkExa{K0wqkIIO@Kv$okX***aUiYf5=pAyT^EJ87-wNFvgxewVPcO^ACwujMM zj1{N#)jXJ}m_X*)drLBUrG-|0QK7d}#nY=0)Zrr^cXS-ppyz1~O>FTeW zwQ_fnx&x0d?6^Y0AnExy{o(82SFezE`*FEdjOt$q0F~$TZ5I0DlAcG!OKo<|Zn4LcIl*k6Do0O89;jVF1dvlG7c)E zp>;0K&V7}k&HQDA2{(HyUct2>rp794KK4lp9t=9&**enBd4TrCthYJ;$rlnTs%$Ts z!8#8Y1ayq=(^?Y?3VuPCu$ zlW0-ZVN5^(#x*jf$ng6nYs0x2gb@S-;_te;x|$(P?ff)PS@oQO@u!n;Cx=Ac4o#ZD zg8Ra9N90x4qitqXOVX*JdEYEx(ZHgjqT9WLgBmB5xk^!N7Vvxk{DJ`4q*{*_LeIzP zD1nZSZg=+Asm}3!LPLX3TK6^gsBK&vb%yh^EgxANzi_E@YRIAPGh8vs2rJRQsCUzSj}nkDcZcQrJ~hONq{liDU-XzzCdBQRR1H>4y6s1#oIvp>~Pe4BcEkyksa z)BsI2VAxU5I2Q+p76nGHKc3Y(ZAUyz^}upG)465T3yoY-~fQqduj0s#!JoZg29BBqIPoht|Gy(~-QESx}e z^%W7lKM8to!YvHcDJ1`pckkqZm2vtL6d2eDBLNk#85&T6LKC>Ed|?U#lMG+H2!nh( z2Xj{Rq8?g2RR2dOej#&nbEJ6Q}lT&%wMN5;D!ux3M5+*h_<+>&L zz(-#->5xDDwQdejj`4sBk^!6|V85VvbaEn_tNH8CV)x@OW`tI7$(54PrlzJ3g62eK zJsEfo$g)T4MGgG$(vYMBy;5*q9v84tFQFNEK30qd`A8m!4*nOD!p|qa zl1^1SwE(@MUSzbgL4&vJ_ADBbTwnsP_L68(L!FU_BVd|g0L%sWkMZ%g%^)r}^d$UHvN11R zGCA6w>*(lcY!1SYga_#F?kJ#-+T?yDRexR3$-A04>Zg zN5Algiwq1-UUY@gi#@1uwy852?CT>!U^?=gqeVr#vVlPV$+@};1F55ve|VRSjI7w_ z!rO6kd}L*2(zgB>d&cKN0=NqN++kNL^OgRzqot3wB$SlFzzQ`3E^&cd?s*w{d2_DD>R+*c0b##yb^dV!Bzc)pJw(NoH54GtRWl`@(o~N5S}Fjdat+8_|Ozo3IuzH6c-=g07~qW zQdD4ETwHNk896YJWPqv#MDevvO+WShf%cLd^$l?3yB`hwn;d~8z!;I**6qhaWrwEh z5#S7cB=&77ubTj1Jp$TM9^h0^Mn(ouuFb8j^6(zypHJ65F^|oLA%9IQc_b|?B7z%# zkL~pI^c%#G(;w=*2D1r=^Zt(|6@zpcV$K^R=jZ1#ym9!9jEp6&JH~Z?Uj+x_!T+Q^ zxGo1dq_uL!+j`p--O%nXQ?HB2gcC5j=@8D%O-*-z-%&5O(h`6vvibZ0jmNkW7`hwS}oVk4j*bN$jh1T|GSB z_V+(P>JlzYp(M|Q3jRI5Wj9v2gB=OYsxPh(^m z9;1gyaF|I#aq3#w$ZOGYKv2+oJlvcsAo@eo;}Ep<2H}W)raK-#!SK zbXB|W5tlvdrWAMI!$Cts;};Zc0d!qrHz$LMg|&*s2&5CtH4ILz&w*oOkD*1O5qdG% z2t1QYfj8t|w5=y(>Pm(ndYJmkq%}^}xFGGDxYu8DZEbCfANg0bi}ZW1yvWf6Fk=X` z!Ur)IUf^;b+1ok zX8^bY@&Y8SruVEW%AWM5?!e1M9Z8&j^$2+_gUzY4d_#S$B60z6V>Qv>9dZmt{X-L-EDw*N`R0kEyN~`4=Ky%l9pq& z22_-zo`F3$IEXN#Cg-Qe!d`!G78*2!;Zbl#kGTgE(gs6&`h2RIf{2J{duNBEq%_y_ z!%a&v)nsu*inD>7j1SBboQU>8rZNTO_r>@IxbnEFtFA5`6y(={u$iDGH1OJw;rOg; z29StIO-+q}=4MWQo{St@BPwu(F$++M#c&QO3h8x$r$OKt(t4m_V3X%PFV`WAQN{|A z@yp4%?g@>=3ZwxgO;0gPnC2LBY#rW@fFDlo{#iv+L_` z0dr7YZvcKA)TOMm$xIb00O82Ue30LFfYAvk6)pz=&mTzo+Z)0iFW`){`Vgg(#ws zlwe4hA{F6tgcIG2Abkh74+Zlwj3k_28v6b1v%{J7;Z>+VK;?{;SppdPfFdR)hDtVB z?W7_Wy#Z+a=x9ECZealvcmWtR91ac+fY7ZqTlM%*jzDL^1#|#B@GyrD@WK}3WgO5X z11a_vu$UphL=u^mh25Qg(ad43Amep|M-~ILd@g0c0qlGB7*~LXHpf?fl zxIRh@>gBjNpMGs4TwfhtD^y2US69Q0u@XfkrA`ONcl2VE6r7p}a@(s1n&NPFZS8aA zI}x}Z{#N}7FQjeEAQ?0vAt8i?hVu2@q0SBRtRJ)?z$(69CIM0j3r4KUVR_AK8KyzF zw@d-#N6&?bVx4^(aCIJEUkL_@NXzGN$ zPrp_aAb?`?@Oz3>dm!%RvgMfUY<2`@Lp<9s^u7cY;&Hxi74o)y;bsBlq6}eF{!0P} zx#&U0T|(q>gT@)g&A@c6eCPc+GG&2hZx++7b+)CKxHyxtdf%0bz z6JE5N?`Z`=eSQ7oG7Hk@Ha14Kw&74%)7*BKRz`~$A(m5nX)j+6pybg>HEw_2&1_z zaB=~oX3Tr>H?Rr|3m?}w^FV-Y|9EQ4eEm9(wDb1B*Ow;OW0dbDiBf>latFG2t22N3=2+??Nc$&-8L$E+%`snEP3BMS{MCHG@r z=&I)947~5a(4tH)HG@=~qmhY6$)`{7=+PrOaaR%;H;!aUuQ!%MS*y{&mtKRG*%*>I z%FywvCMb-EXob9A$R!Xajqs=fKIfjgHO_}i2U3lxba#2k^D%A{N8uCInq)0)fD!Ny2zUv?<-KI_D{!MN2-brZ8zB>+g@8=j z2;Uz7A#-zv5H+4n33^yFuomwCHs1Sqn-Gv`8_Zg$LIiU1@^gT?MVCjAb`YjiPol_W z2nS&EB$qEgTsgZV$0|oMAk_@{9214y38*fuP^`g)5(t%- z^!DxBEFjq6Fn{Hx1Wa^Ih~lssf8IY-Ab5}6R-RB=%0FomEvJ+$>geOnpG za2lNUx$uGv_3iBB&}p#7ag`-MKVQz?z7)O?#jb{jV!3|3W5xM&-QZ$SX$uHT6cfD2 z{3!h*eF?;Bld+kuofo8$OAcKGA|9{+k$Iwr=p*o>`t;L(YehCdvb6->W4jETM9si{ zPl}Ym2hai##&{11HyqTj*`|QIaTGEkP#!{HzOH;K(5=!ykwE4as@4Sx7sUj1JK6jg z%L(`hH0h1$x~DkIa?s<;!Sz6-t%iGsVvXc$8*h-0P$41?ODi#pb92lv(Lkp^J9vsF z4dR_DyMdiuF`SO)X;zdzwB*R^-du)80}s_aH1zN(BXb~dqLc>*2kDSZ#>)U1;KJkq zf&@S5hC(PcGx&$f+tbZFpx1nwdhn#)vkJuo2W3W`k32HKMIi*%KScygMpNi9d7XJ6 zyuu)XcuNN64LBGyaa+51up1%o5!^ahC`7s0bOCz)GU^-5&|SUCrL7>)+8&n}2i>mw zdSVS9q5C3DmTLX(@^=u^AP~oI{+HjPuDUl~Pyun9sKoKEezKja)tlVbMP4B{PhjD%zBhaUteEzHG!EH(9e+4)D zFaHy1>h=;_x^M1V%+yPO$p9!dqWF7ACP&B_@K1=kfkcz-rGaib!QKfTH1ND=s27u8(;j7|ZMq--VU@f%b4vDwZvvG2GmE~H z#Dtrc{s6uSsQFQVeuroa@5*JhBS_W^yaphUd;>aY=V-?SNt}>`$G7Hx{X%#p8wG`3 zH9_%f%VW<`Ihwf|expgz(Wd+jH8nK{C;qXUD=Rs>Pk7^K?*l|0^{%&@Lr0VW)DgJF zUP2QNNzdxlB+TqpZd;3-ci`XYjEjgs!ND0$SfvyDCoVkIej#)0n+JOsR6&9uj zcDMy1{;eLvgFUZ2uq~PK^MKV1Dk5l}fjR|khzX)n3RGIO)Rp$C5|c@wbjeF>(_wv{P;|ij;fnN3!pwMOwYgM?7$;AG&$CeS;*N|e4 z)L@{un1JsC=>xm_`r7%Mw{K-Z5}KBTHxIYabC}p4{c?4XwQlj&!#h-_Qbd3jA&6Q* zI}MWb`ilS$paY(i_{+0!GSp+p>=8fwVs?L%`*E0VN86Qw&|z*-$0lD zCv*6m9+fyx$@A2tR=k)(A_%0-q@T9Xkvv?up{J|64T)ZS7f#Ru!841$k+8M<`B9`Vp?-_VY0mKK@(u-qeNp`3yZ<&@`d0XC-N-UQ5;u3=U3*E8`Fnj-06O267;d zC_NlMV3{aArh*X;|K1mnVxeO}pgiC&=n;|nkKK*f)B2YE8Qwy=|7ySZrfDCa`OnmM zx0155p5004C^!Kw0KzI1%$Q>IIW=;OpPYz^jt+#mYxH}vN_5K_P2 zoksvH;B9BJj9e802PTl9)R1%KAw4_QM=4$d6}t^?z;Q5~Ud(L!g}c&Nl>@tstZaEp zak7+87{(={RA{iF!zeB(x%1!w2?8xaZlH9O7pKcoSiAsIdkn7tTQ=~KweTzCT=2>ef&K?6ga3PNRf^~mw+Ya%fDGHZc^MrxPZi-C8iswK5fyrOpY$ujFB|&Zq^dzv520}ZW|F{GlN??$_+8qEEq%e5` zlrv?(N(F_1-(l(A<{={8B4`dEWGJ-&I=G4vJs)E5HM~u7x1(Ng0wDc07^aYO-vt(X zyPL1U5U32mSeF3K>fNBV@NH<2LK-ZAP)(_fs8!3VLbHjCyR@G&%P_|Y-kJpIhZ!UATtL9`v)Og25>_BAq)-4 zaRf)VdkfOmQ}OA4L#$jtiGm-N1cnKL@=$|x^RrnXP>%LT4FsIlG(#CAXuyJy18zac zI;mbtm1X5%(p(=cY6p{IKg2S$?z3P5_;G$Nt&aQ-uo);=Y?v64(b0&Lx9s^(4Kbp6 zV9Adb;|xe>Xx@OKViu4-=mLS2U}vthHApphBLpNZIQ00TZ2|TAO6|u&?YuPYymFVk zsi~>C`S}1Sdh^g50GTTb8N9@_lg#t)YWCeYE^eq$o(Dh8k*EbVCYr-_e~kk1GXPFN zkPs06*UK6C=|JrPx1S$0P*(*6NP!mySv#Wdas`xM-D*c%hozn-5b#MD8Hs`WIQsi{ ztxi*q8W>-MG00L)dIQ`$bXkBSF9U%7@$pyHr_M!tKr=|FBbWeKhk+6k1yGC2q}$`# z;0^?!MMg+q&KNwNrxoASq|O2XiA7s@0f+@>eK9;k;Hy{j;F*AT9|Y|+(jWnlx7_WE z7rX>GYTQTzp0x$K1w#{)08rf!By@57-KT6>3+Dd-SlG}dC~)!cnjzsK&N6Y20|LN; zIPfnJT)SsZ?kt@2Wa@+8bzlGEn?q;0A0xc!|6{*@1BONz_>JeFphA8dIRC|iu+}k* zGB~xK$7JBW;TN2*J54u4X@r40mKp%$Gd)6N^*~oUB-`25RfN#v5kPS$05gCxW$dr0 zsCcGJ`|pTN$T8c0{tO;WMqf=ZdIX>TXFWof08t#DD3R_}{15s(U-_f?X>nlOS_!|4*MrGt##XR+RLI6OKKguT+Nfy`X95v8}c`! zeXQ>H1>Pj~>u?Rb6Akp|iQIFK0{1!Qe;Lg!Kb>`_eP1&)Ik7?IdDS53`wfNJ)bd;8 zyzPc;KG?4v+0IAGY)gJ%QA_g87fLm5suVenr+upU9!+y1+}PbjMx2s)2eo43+-CSR z>%$59^>~NLei4m?b`>{6ixXajsrKdzx^}&qjoT}h&NQ3!LPmZ`RD+9_By?G)OZW56v&U337 zmx@cjyy9NHGF)-cTJKAF*la+Pw4T4`D1N$O(BU3~KFaaXkam)gkA;OqNQzu2$!2GG zkWSnApxBw){VAoRm)Y$fIn}m|J>>Glr4G&sy{T>HgY;vzHbFC+R}DD?14w=z*U~?0 zou-ssex?m}V+JXrF(LW3%?Xlpo!2i?IFx#O`3nR1+H4!}cE!gvSfk}RXCAPN9Exa+ zx2rr6tndokl@)ks8K$Najz*VGqPK%m?MP%M%|%^{I6ccQ{Oo{LccXh~|5r-=#masg zD)7&<7ex0`lFu$g0wP2lN{7t9-?s58Oc@Ig9d_*){=xb~K#*d3I5)zannkF}>gSUf-9#l1G! zfX`H8;>h)f4wb5P(WIDo18Zk-7F~xX$y&=({MwBx!%9jai9+~%l&{cH8aAdOUwXX# zhu!-XipwfGEAy#vEk!SSZN0*!F6iK|?nzQr28QO#mk}NwRA0}l&WeUSao=rLntb{~ z5N_HRaXKZO5cV-X{#v*O9-Dc@XAwTFogKx8!ssY6pSNv{x7w|Cx5VTe(bq{ zE#7)~ccSz<-nbI)=6q)UV!0RPi@xpruhh5{22A>}= zVrptKG_Mm#UpGi_5Hlx9$Cb?TJ|}y~){Pde&(80%ymv>_dAZEKZW?Rt1BJO)CErx@ z7llz2dF@pTMQ_=-H#a=l*ZRFDOWxl}!qt6TkMr{22g)~r_;SR?GG6KepMBLFz^qaAULnGB&=ua1)1na~Ebt`Pm6>;IBd>aa{&Hiet$(b3TUDB9(P2^C> zXVj@-Q{6pQVn=hEoXl-UKZio5v1h5FIIo_1U@$u(WP^#*rM=*KC{}L5u0}yubNTgK zzqs=-P-MDmbA>MKu2M(1g(l}(EEDF+GB|YA9w;+?71DxGu7Qf1Ka~2s!lDm$G=7BI z82n&A6KM7p8(s>=2v7VY#(N9T;Xm_~%f&?@_W`x<1HT0VnWa9g8-jlEf+O4BlDM=| zd{YX&ZKByXx)a#+47}-^YN;8kHLqw)Sj*GYJ8bpwOuom+u_%h--MmTq@+B`yd6{K2 zqfo3~S5L%gCN4r~d5MeXimn}b@W`;IsABPc(aFdvF8DxybFnt$ zVtD$q``rzxFn8^C^!itf2S4aIURmm@|Y`^-G!eSbuQ`4OCbC1p)8z;$>nQ}t&N}J}~ch++ci;N=q zQd!;R_#}Px=6&t|(`-qLzy z-R?L|(2(`}7y7Qe_s_f2F>R~9M-B?>cSR*mz=FU2scd}Ft&}&iUDxs*i(9V756?<= ztbZNW<$D9+*miK1L0v7@H9KDpLLlCT0kaRY;1^C)jaxSadj%Y-$>>kt3f;_Qvcf7K z(Z6O;myezsM5wuHo_0^--L8kH@+pxjJv?*jUyUQC%%uRb0ML?Fm{#F;?l|u78Jz> zMR869eeK`dF9RwYE^E9OWQvJDIAn&^gw@$uY29Zh!kSHZTyz`noG0c~os_bY->NnbBO~9V zIlLukp;BV@ZI448Vh4FY6+EQB?93IVmwGmz&#hervpvVo+e8v{(#iwl))*|;g+C34jLOwlU+~7H9e!FZc6RhH zqn0&S{?lxbkYtT2-rfzmpRGC`Uxzo2t;+Cs>)m6`2+PXK)O9XWD+)V%hEEZx9?G26 zpFYvN$ji?+F5-0YjrG9E2}W{X?#%qT!ec3Cl(j&Cuf@8E3e;HR1+ z)+gU4FQ&K^m$c9+;p=N|IvVE#C$@cw7g&t(@jp|p1V(%OD-p~wDMTmuXXB>mD;Sr; zXV3zX@d-UXB8n|2(D>}sF%{{!+xh!LZZug7eW&$=?^KK5w6$;b7KN2w@b3fzk)<~( z3Q>lo6Ifa*MueYm%o(NbU&0xBC=M}Ur3Ki7uX-=^K1B6b29}sRHQxF_uMUsaKZf z8=H88ZS0klt3_TOOS0xCzfaSL4roLUo1Wogt>K+n-%*+p=dipOJBvF>BJMT!y`w0? zPAU`K^8uH$0{X*7&{#nGdsR?i^W>32cmF+?SK0>cDa$N&D-HVmj~*MfuM%a75^zb0 zn8&|2&F!CaeWa=A75GRy`6zb^4M)&2&6|mDb$}~L%pVnzoGzzqAKXM_;~hv6a>Rws z<^9xGp=R>F5{5#p&lCqBpHEEY{=xqM4s3VW?&my2b{3%^G$jHqhF8c;Kr)>-pV zNknt$35;nEqXe>w4BS$?P{u^)pEq25F)@Ej-4CkFptu$x&01+D|mQw?-m`)oQjvol2( zB@S6EC-SsTrjv?dKlhi%a;Oa{dX$Bx*!LvM{o|Rx{vQPtYJK>XaQUv(SkwsJlD11p z(;s?Ft?w6w_DWK`qqW0hi(7Ftj}~?`Ox)Pq>=>rzw=(b}4*5TQ`Y64BBhPGg+X}sQ zH>pWg$wjeWeuz0=EdTlXlh{vqjSZCt8JNl7No2w)3rnc2X+$9CU?%^o& zB7W6s3@c{8Dmf-u#RsF_RAC~CLGh>VZ5cfa4GQ-#>^^H(>~j-t=w61N&+V*(>6JkY%!i!Dt`<%IHVd=( zzt7t3T|cr)V;MG-kIW3UC>yaQqoFiqp4_J3Zc}-;s9dP}mAr#Uu2fUlahXJ0_lciE z5yKr`u8-2u(3o~L*iTQS{rOa@#yWO2m4#+&=5?JmMKNbP2|FqwCHuNqmHzdS>m4fZ ziZF8H$st@6;yCr}M9y!lRS)y5b3!MoF479%H=6DQ6L?MLD07$4ZgqAp<8grV8meD*C}|T({k$ISW(kf7YIHv80m3o4?5dI}w z{mZj;^th;91+*`@%;j}>LEQo$I#cf8NVA}whpOGJ)Np4SDDo5XM0(eJs7 z`@~OW)N~OBBFBQ-H$ofAs%Xmht?}S56SwDv+se0dM~fRBc9c};Q`ml^A?z=II zf%+`^=z8bcI&i8v>(7W$i43vG8R+IzR!lYL8N6m!=CG^?n^rVdIs4wXVxU#lPliY*;f%U?(x zezYED;&o`GA;UJQ_4A970Li~y6W~)T?(?x%#0IS_zvfJO-m0A%Fln^yBQ!>ODQu*m z{7MHaG4T=_?k7}r_NR0D5b>as87bmss%B*1r^P*Slav*PiI|R)`-Z|cq!%cvKh-ub z9z0YZ&Ab0(B<$Z^;=o;Yo}8?{DrwTz_>i{aBP83`5)L1&ylY`#imClq8Bn2hj>%Z! zH6N3ZPeP`zgYSs-dmip~pNlW&kSjy3xELc6n04xa+B>5-Q5VzC^zf1Ui2n`HsQ`+Z{(G`` z+xPE2EuU8|PU&4uO=j;PCu8+P!?DcOQhuH!MOebecs275x}o)HWslYq0!8FjbGDDj zL@kjCIs6qi3ouI4pIRz3}Kzh|=BxE-_`hbd}IbLV;}Y5R@KlW)Z1 zl79bW;M=O1$8YBtTeApOxWREvMq2UO__I6+#J8M47?BQGhoQkY4_aromu=1W2T$rI zFkXl~I+gNpVW8Cdd5(4})>BOU(cia?llM9&(~BySthCstv0qbEiLJ}`+e@8o@rJeJ@cbFWCArI>$LlYHc*~k9J?&KwV~kxofG1U+DHcV*|X}+HlH?8!`0gS6=VmydsJW$B0HEhDI>ZvdH35Y;5d* zE0z<5)y>Yp+L8eaTKI3?3`|O*2Qok62VxDr@aoNNGNNoRPatB#DGx=kDACvs2Ho3Kv;y1q%sA=#JN|hn8iBEw?H6$klerq#M48 z8n2|@-=JLCNg+ZWqYgWgDY~?0fEHP@wY|6Z4HV0mXW+p=q!G}XhS$_!y*ALQ;B^Tp zeGW`9n2sh@JXGHRv)KgpAW$y(Z`~?5x?da2zI-TMZ7zIGl<(Mlr>%*I^$MJQ;GD zvcrMP<>O%v(#jDjDdFG)@jO3X=3E4u8iTR1v9fqJ$Pvh<0?0ZsWIqE~o!tgB?RF=7 zO0;`YZLRpi;v&c47g!RsjLq3NIYq@kiPQ0i`O)=aR|9bd2`*{9Tn4lK=XfPyL`(x1 z8{<}@n#hd!9Q46AdKUO^Fvu5IRA_TlR#dbBv#Km^4x5!6!O|o|Nx<<(fuIOZwOm$# zR#*drC>1d@9zjZbeoJR)C}UJ_Pz}zsH*TJLh5y(*PjA~@WJZ-UT>2At=bDC|l{N`x zc`3`lz`)U3UQw}GFAZYBMuaO^3!V*mORe?F#+#7?FW~JXhz$lM4ek}#ze5la7Z4+o zqvNoHgqN4s%6gpm&zdZf*K!um%nMifk+UhP6kqy;>F(~1#3bmCnq6SL+y((T&J|?J zFrD|zly8}F(1pLJAV=qrz_m%~=t5zIb`va_q!)E;goRZhU|B;%78E}$3`Q0?RIzA- z=wD*Ls0@-tDntvYy@(!yZ0-=!YJ7hh4~ZxBcTEtjpoX~at&k!HDA?=+R23tTckJx! zP^eE?S)cb@dy*w+St4-|?FsZmL)ef71`FZz2Zwny$9e@<73)t)RC#y?KGhXuwfw6M zJdKCG!%Vkq^j05=T_&Z9!8`nYup->nmUMAtx`Z4oe z%Ns!)q*~;cNi?p;aoBkq+<4J`Mu_@Ke<;+qRfAD=FKw>l-zZefUvt!mJa!@SzBs?Z zAl0GHAyu<{`$dw6?_qmoV0=tua5}=Lr;mr>;^gw*9(pCi+t@Ehm4tm;<|?1{wbNrN z)${kLAYI=Qw_fj;2s9!cd45k?F-T4=G}+{7T!Q>RwAij80VY@UHmd($ipm-1NBcJf=~gY6|NVSOgtL6UeeXY8 z5dAMXXvM%EHTjn3R0+AhCe{bv{svwJ^z|C_xJqkLGUR%L8uz4gB6P9o^mLDwS6YKk zkR#<{?xnAy6dnGU+SAo1`*~K;Ia_+n?qAE?Zk!hrzBqB8bjzdqEfwB=;pWyNH}l|2 z6Y^3bwGGk#VAn+a!JMzu^`*9_8+nRbrHEOgD4opI97CH|a#~c-Kj1+SY0AJ>L7Td2 zhYW@yK0lgUK>5ou-=rYsaZ?8Wy1tnRw4?i0fm^-3FV!BQ)6WT@8tcOf%`jMz`z>g~ zTp(m4U1vJdGyl5v)9-?Y)}$cl%I{TWCQcnJ70n}A1%*#lO#e=uhnyPMAC;8vL~UO5 z)}z`f=JhKkdQ>OyS18W~uix5vaQ(9Y=f6=g(+xf({Bin(jnZ6P zF2KK;)kQwD&Uaf~=GI*#0m_Ys!ng04&Jfjk6g24<@dKaPYdhXR`cld0{DR*cAu`IJ2CaD|=6H4^d{NKJ(F#!@xn6o}y z#!fCl+}#)s;6%eEPFRxn)i~hSW%2TV4(U5MWWw|(X>YrgpcA!Zlx&Jb5 zlP@c^%3SS7B41Gj!izOhF;Lrismtqi7^cnhaSMLr>nww}0rnvGFE zv-y>#&4>T&d8J-%g#MqcuKu??=x!ca#KOXYvW+cR#O& z4uWH3T-+rvu7%QxkijBO9+#)!xefqb^GBgx9=8KHAz>8(2^m?s_Qfkk??%|J2HB8h zWo@lq`xM*(h_nlP_uRXG-yrXwcKBJ;Q$J6*U=8ghEMbNH9bRrtOCSpri5bMD!7%-A z2_`s@eJ;)pxyqydQ;reW^Z%_J!}CZWP9(4;5QA3&N#Ef1gCd%#&(6n33|Hl(Me& zeeg>Jsr2BB@k0zXP$G~mr;c{!5z_+b-m$K#V0ZyP)Wu&RWhlAt!7_d4j$c*_Q#u(8 zfDAXKwJ1SBRt#kXD-)G`@R|+>o#j{}yXu*6 zte(lkHa_qMUcseFjZ3IOD{ceNX9;Y1qH#F_%;YS>!jG#`Vq&nw#A@s(IUtp@`7?nf z^O7qp;L4I^PQL-}!j`tStl?$J&(sCV(#X0B@Yei-_X#W6sm^y(E>a&A>GuzC&{-}m zECj;x?=*1uKC6I@0scpk|MugDHjjz3lT#}I6F^7A5)u;S4tx9iX+=c_(-YTN(se#x zKSnkt3JncCvXj+p!SVM^Or$e4HAP9sUo~6Vx70!}g-v8&7me8q*nSH{?}x_ppvJ=< z8Zke^l9L&bXh&8$z%D6I7$_)$09nDvLV%XQaxbu>J!^JY|G@)Bu-FftB(UL~JZ~IV z)W7YBMCn*KgVNHNeVh4`!JODW_gG6BrV|+7<9-Yb4Yh#3<*E_0;}eb&5llC zbp*_YR@fl~4FzlE6krRhXf9(I93t3f1X-zq>{SE1<8bTOQiBtZq6TFBK|FZ}NqE4U zbhxB-_Xf5s@7zJV9Ko5Yh1&tUt1-tYBL?QY=nGiQ+tJzSmia9s1QsrUw?7cJV?efA zd-3AMSfwo^m^M%-@WOQ^6QN+)PRoC}ZNS@_ekTAMxj3@?RYXWZPENJ&1O-A$0yt6*x$P3zDn9F1XP8Pu<;ssbNuFf5%TARxF&PedA% zmOt}IvPdgYyY3RKH^2gHWwi(<>CF13M98b3+iKl*XMg|JFLXdQ!h@voWd5aQ%*u*I zh+MjTLEsW>8U?MwBO^_@5tY1PL-?IIj?W+%mIf*x!#YOcIcPm|67GWo9s0d(6 z|H8r-6p)c``DwE#5 zZY)rJEANsQF1rClGS2Q2ny~=CdD!2QAbhl{+TyTaD0;Be1iYEKnmUCJ>tH;RS0`1! zpQa^S4Bq2Fa9#b|%mE^SfRJzp{D5iT=z~qhO0rt=)OkONHy!a%k&wIP4%?i@;(v{3 zj(Y~zp*9wPNgUL9_a#0RW$dE?D+xC@Hyi>2IdH`p8XEe7;cgDL0nlXNqtSK*a}_vP z?id&_!saUvX#ZY|>Lh`kK^z<$h6|ojfzj=smkKth;#jX0#_R@jI;?cL3!Do)y&)`f zfQhWmS`Wr7WXTEtjT`>3@fb8f4-5Yf_TD_M=DmIUUMVD`B4ad|5}D^AG>~jmN+e~< zkU7H^l~5Xt(K1h^Xh6ysWk`w4ks*YV$XG&3Jnu8E-#tC|ANT8d{=E0=dR^DP_Reap z@A`bs;W&@uID6e}wDCFW$Te%$yutPiiLJ0cvENF)f4>1&Zpg@yx7x4hs5pi}Q}?+~ z8}oDMKQ7T<-9M_2Rsy4vi+Ap5L6YUIr%|xF_VJC#A*pC_QH4&)`4pQdo>2f)FQ_SB zCg^pVkv~JFUu}yp+6`;9M-RK0HygUvV?D4yX6Nae$0n}y(6JBm?l9KY^R{(s^u>Lb ze0(Yfd@msaiF6Fl7aAn}lgkiYB0U}o=PEnr2az-Zy8sF11afnfHdrTBX1$AtvjunL z%+g{bvx@iK;B;#u&9rUNt4kM37c~)&iO^H{N00(Def9~56wzDY_U(;fu15sw9&In!I1LuL5XI3D#vC=J&qkNYCDT5-pK-7^^j zwSo=%q915QUqP+YK%q`+nC+ReCt>EQ%j;62=AP>q={Q^u6lfK4klHZW8^Hl|ExyvQ zgO=JLt2Qay z_ky%&jt+JB`g|*^VttG{#x>N@F;Zx4-n==?eae+}+W3CB{aCo8&%%RfsZ1nJ?%{l? zEBx`#s=dlU9nN+AAsbs9(YcM+p<46i%?mE*LSGZAY}A!e%J{?29~Sw#-W%fFqo$ey zJ=$6<84P;#(1kf^l0m+R36J(TXm+G%isOCLr#?URs;Ql5@a&5Fr6x2+OBxP&^DOcF zK?~yl<{LXyJ2Aqmo(L8r3bgG{{_6_&;?hn^2zGP%DB&=wcZasT83=R(Fz#mWiM@x-|f>GS% z?Cg_IzYlh}M<()obyd6c?Y){pho$^^`t<4i!duO0vEiGI2z1J~PV1{)l-u~ylTP6m zZVfw}HGb{2ANwYqj;O!tY_C<8haFp%(hO&$2K;WL|KmqMg=3h%`tRiv9gQvoE9Aza3d!Zl~^xQE)omhP_K z9nH2mCG|qCD9XpqoozpCPf79lG1o78>z&xPlq=IG18batt4m-%hWQd1!W?<)7quh(J-4&n5+naWCr6uTm4iXG;i zp{3*5kVclJ6%Wt4g=S}ddFf1Hp;A6eO$QM>v+`Gih#bqfyk(gFZ1lJ!Yp7HfeSF#kw+GR{(kR3gv%e;Q;uN>o>U0+rEZZm^ky&AyYlni94^$s6^K3GIQ5S`_U!GUArdCSh`OOqT zCyVycxXrY+vC$Ag3&I_+w3mogEzriTsnaBaCNR3`%$xV3D6XBIcY4+ufFB4l+98+i zvgbSfs~BaoGy0IC#dDRTAE$QWr5u|Q>gWh3;!2{Wrb6M>8wT^XovoK$o-v}DQNy(_ zQTU=i@!UAfr=>Ed!p=X1hS4Dj$%`mUMg$JFY-amRj*$CPIh+o zqpvZq@5Z*+l!aLPCB4s1O(sFn=4q2(T<=tS%lf>A9XoVrv^MTz4jtuXj2uMk>x~5* z&ℜ3-mn{c*;cB_3Z=AsmTR;J+tWT=92N{5k2ikkG{*LwT`mizH_J8>WLpu7__gH zrexp@++I5KJ&AJo@#7{qL|=M*e7d-LDF(T^w!o6l4pm;`#*Oz2whTJ1{9rkBs6{KC zm$Y#xl>8`W&o23ZjK2=fQThPhpxEjByU;CL4)p)D=il5*2zWaqQ58QgoLfZutM>Bw zwjX`__7yi5ktmCc5Kl;P5fJl5VPVbMwQEbcDxy&eNQcQDq~>+&*H6MdiBMuhkDw5G zKDXmyTAJFyg9inm#kj=n_46&k!L?xdzR1g4Wwo}Uq=!dXeJTuyy1X1B#vYOO?U13Z zrlpG8{fXWIN!H}6n+yqZ5=h~WI~GlyHf=IiL((k*2#8xEB<*2d$o;<Bt= zApb)t8~%W2eY7O~G9tPR+H$cU6~w$_$BqQ+^nt&B9`L+H=kyu-IMElwq;HHnEA}Dl z;e)^U{(QwyOgeKczQf)Ic@QTQu`a}1O;_0V_S5kZ5U1_gvpa~D60I_idyt7T4_u+Q zS4k@#BjTu)KYx0okQHftZf@=w?;l-9?lW5t&At7WQ#6fY^pc7=G5GJSqTu7O&J+%s zcGqv+8iJm6KK2GOhCyrn=GnPk9M$ZP2i&i3J%c_BPxN5K5olXgjJ;3iFlE?Cf|>}r zflYuA)^mNnPf*_OL*gR-$s^Ei)X3@R_P2EnW@v|6x57+V9n)d=cXz^9L#&T~``ML= z>}Z7n->ygB2Hf|lP)r(8)_0^9NKTz&I@Unl&*05l-Cr)p_Ulyh4n{RJo{IjYf%+<2 zTU!I7j0Q!0W`46?bH(_|;nok73ITZ=o1*8`B@{*37t*<}8rz{EA4KVn*GzqdIvJ6Y zON)jNv-FEy^ZDcr7cE{KK!MQyddZhB0cg}}qxI3{hmfb3D70`hIDEMN|58LND|o`I z;^N|v&4;9sH=Q&C-ECI|A-6#8jdUg09#Te4F)@eZip2 zi9Xc_Yl)egI+xCT?%bm`dk_z6Q2*Cg3;+CS9KatgT1yKa&Du$({gFIofpfO%W(>9X zhFapdOm3!H8%)K-yy^~K%p(E`R}Sr7-%u$n_51lVb)Vy7Ds!oV=t?@zxMQHeU_i_j za6fGSbO~j&NIr2_?0`=>gw(PJpOxLob%RJTyg^Hn@0%;68~{>YDBf=cxrSc z=P7d_z0KvTSJk<@Chj;8m=xmQ;M#f(X~w|NaIky82%Hv_3$f=6-M(GDe*OA{Oa3F$ zA;BwbY|O=@Y;u~q`=~UOfvcIu6#M|NDJ#7<>Mq8iVyjF);qF^Q(HTKDf3$Jarjvh` z6>sl3rzr;Ls{7|RenvsB+!v411`y}|vO%~l?LXyK+T;qx(ew^LTaLL~@V!MSZR0Io z8i|R^W+su_0Dn3ArsRiW7k|F&B;~~k>!f}IVLfcd6&0?;N=`^LwXD_FV1@8=Fq*t z;UM$yL41+Y8lOCCRj-g!ZV4N%zkpJF2G-> z9%}&SCZXsP=-|kaBbh0qP^xon)6l=a%*(50VPPSj&{r9)zfI9SdBE0qB8nlfoRfXi z3l%A4^21&<;n&x#j}8K+0u6U~yUetN=Ryt9yeU^XD7qx< zZ*MY_nWB98bRxBICsBSlu+hw%Mtv905TROIF24I$z}>`yxP z?0MzPf_zzVAbAi>A_M!KOhw6N5(z@X&GYA{_zdGxm483)24$@^E zoAt1xd$m4OuWr&=z6@wAfP;~T97#cdprG=^-9Uh=JQ7DxaPTGM@os0$ z8`)`tLIa38#fN;W&4GqEvued7|{06%5K-s6LRuMod>PN0~c zNVFjDcB4QG1n;Y|=lY&Hd{~V-js3(w1=CCe+WyrPVm(Srdah)k7BZ}PK`_%M>xhSIxt zp{Nu67plIFU-YPg^#=OtVLl+XX(EtCc_z=1I4-Ayh$JR0g52>3u1Lu??S_=y;xr=Y zGo_7q{)){H*J8zo0KXY&5Dy(Xa9Mzv=B8q3OYmWwx z*62N3e1lkuMEbZb|@~W5b zn`PrJ#8*vpcKV}9T!#wu1!s8(Nm@lAp6);$xk0~YRP`y@ivSB41Q;--!QM%?;jeiH zhtW|oN)n%>S%0R2Mp?0jW78{f*%cF7R>D2 zwdw_{otUjtj;+gGfNrTm`OzAR-Mtr717hsGtTcMffvN$j5o52JVq#|;X#WH z?tbpCSR^n=IZ`z<^@zfPR}2(dw-68O;#>VbT?9@BxI?XO7uQlJ`2MMg@Vk7yl*L04 zLl2+50=N+I=HU~|cMMp5s%wpev2~cy2%CeRHTJ|TExX4jJk-a8vaCTB`8GR#dek%#e5}h z2k{S#zke+)RVWy6p;dKkafs=gb*pL$@j|VssV4?R6v+hMV5)+j+=teC$J^Q}CQNu{ zvN55R)q;0#W<2}%N@G)o{NIIRc@l+;?=z5n-KJxuwSl@tm9QppFco!+;0MoZw z$@-Jov)6G2^C{mm7p`445`nfVvGmQmcVc3%P_PMzBm_Z4`hMxz1m73FQ2yhDDK$|L zGuRb(+)4aF1e5l`^L~^Rkk9NsR#a5vlWkIg(dzONl*vZa`0kCDTIIFLYUmv5ClgIE zxZxseBf-VaxMlb54prZUoSU&mZak8w(J@1jHvBPjDK@(`oOh!py^M zqYp>)p(+EnGBh)5LAYIBupKJTCf4pydN6C4LQ*bcJ4skZ`)+e1a%|eKnFxJHoie0x zueo|i$|EJ-1exryJiL1U$MxFDmG;3I)m4CC)pZu!{5cm<`z5WlzneGxMu)tuZ+v9X z-o1N0=MF2L#94oqO4B-@Au6#>)%c&r-#$HOBBkD8>@kHxcw92m#9AuKwIy$k3Jk=p z`9z#bJJi$E^quM58V`gtT9Mk|MY7ZZ_&bfP`gz6g?<6orH&F$LhpV=;h$8m`1^t{2HZBuh3I>LewIILPNFjzQ)1RkItF(EOi9+EZnzZ11fgey|hy^RDc%O z_tj(Srs(U2lft_8QdM9KU`GK+OpD;#D409i&F#X|Ynw2xXwiS^e*1>fX!bN|%5RBI za#K3kQq3F#=6ag4yb;w!_M+YdX=OxoqsTYs%8H6A8RJ;Gxa`i{N(D+WEEZv;ZG|81 z#`5=UEf^K$@{nO1VVGX+3JF(sLdbLtLp9!k_|t??u;C=E$t|fg8rQg*vCEIxzRLz;21|Sl zPxN83FCNf93M)ZNRE%AcGA~~~koD`rlca^ytAe2jgFD z$B3s-U1a|8hXATOcizLlOu?Vd;TJN-E42)@b=$rBuDm^q=3t@lv~n9%4rsbRT-m_% zdDnZ3j4dq-BEDK2GZFWq%p0eUQw+F@yu_Y?GsWxuuz7fm zDxAeznN_sN$eaHw<#NpOT6UpCb3=|XL&y~c?;sFdym8~7qy@FiB)w-X`Dak(=e#tt z!CE%E-+yO=0YF)1oP7c^dVzjZvXR7c0IDmb zkfrY;;5TlJnfeI#ZoPU10__oQ7!n=2HXb5hYywD09m*ikdxe-%X7<>*&ODqS#uas(M+?^ayvuhWs!PDU0?svV(uCP(3qn7)*5Lt- zkft=tuRndNr@)`HHkoMB?Ab1PC#lu2XN#8)+Tz)DcA*N*PmoHzGEp=p_ zi)_jOA;(dwH)Ur4*qs15d)lg3d*#`umCnunU3E9^vL1h|roY4023>z{Ep292_El9) zug=B0Cl2h?qFy57Oe|OS6V@U(=q~*L*A~=ld91o9enzU z9cN*HTkp@1XV#V9S>3vIqrY%&H=u^ZT}*dm2a+_k3UL}EK{A6AdvvM% zIFMK{t%d{8h^MfMf=uy3cnl!U1cd6%_hemI*rR)QOR|&AV@}4g*^j$oB+E!EPdP=C z(Sl48;cD+@%Nw2`vaAg=7n1>$AhFYaJ`Cx8@I;!0-+p1%Mmxw1$zJ34&oa&Vi`(AG z9Cg4@5^- z!?cli3xxa9CU{r$sVUV;*666IET8wuJ&&pB6XWFrEfs2tzC9uuwPKYRJ%AaI;u`sZ_t(1jlTblKEhzzb;nHV3%TF?y9;ARg(M^_NZMl9zI}UG1hHYP^SqRn z_DcCAdWL+oZo6`AX^2vqeY=wTYk9(%dz0^Pd(f#v;IyrtkQ$u!zpq?cT>HS}6a&Xr z_yaR1HW^R2fLk;`{8+##ut`n@=b%DWJsJ6OaUV1r;G}}#P-e&gnKvmI10akQ%SLh2 zTFu1m2KL7?a1VQTKAI-H_rlk8*vYYuDXt)a@M4Uen(YkQ72ResU;CCf*4u;l=rv4I z^jrUM>lEZNCbKvsc*6!ZwghIrX4Me~Cy#lnAZ);bo@AW3uy?*RppN>0dFH@LIj82Vr62{fI^i;BhSA)J zoz|_Ad+!hJ-iGr=G|FsUr^m~5$AqoRI|arch#xZ9&?u|Tf6>qGy{kLg#!O{)`HOb2X!JY0p7cd9jqMDJd9 zb^wr(SObxVVou1eK@9$6CjfAcIhC2zy!75C@5pn5TIpc->S(%##*xw8rMq|Sj`-f4 z2J$N1eV;x_nP#<(VtZFHlAQ;Unr_!zVdS>gt@VfREZkr!WwMJMUzrJ@_~HO28%Fre zvHkxg-H~cn<5f#nxSDhFGDClbCGAHSmVzkXHEz}n;7JvjVoW=l+rX5R6lWI@Cn;Km zCkk&82vYoTA;q@Ng_JtjYGBC2HtN-;cHMF1=qQKLRgC@E^r;$6e216U)&E(pQcbhQ ziy1k0O_Da8KW|3=>(CRehv$u3w=`vT$^1o@-zhG_Mv;nKOoFMhWv!D~O*6LlE>3Sm zF|aRlG5R?M_p4YCneJWD)B{H9qxlrT6F5yS7xkW6N^M55PNB_ zDP+hAN3(8`rwnz1!zv(z$+Pe+f7`Zq-@Zbkkt{|4QZ7~-EFS+a-@mT|1TM${&QP{3 zHgZJ(P=EqhB3fNC9EI!R-x@(DYKS^5!Z~_rUYB>TIy}97zMjJIfvSBAcy*b3*U;OV zGm4YCB4KWE;4dQrnB~-|tvo$F6%8~`7GV&^Vp6xaIes3aM9w4wU=nV1l1#j}K7$HL zQR3*!z%`EnBbZ`vNf}XJtPBrNIM)zZ%mN=dRyN3<7UhzIqxNlp+5&+_#&hxBK1OrW zMMDwYu$^hKmVv|D#jNqG9pe!D)Qq)+I`1eSHOHTteF@08w&Dy%g2e+D_HftauMfs^_Y)Gmnaql-GiMzb$MwfvU5)BI z_7|^TuK~B6!Xhk5Ly8xm_fnbD`$LUgLr>;*~J1Z@Ld@{ zD)LCHPVvIF-^pS7mYciwQ~vpkl=sWE!jBd1kFCGah}H@-e2px;Yut!iWe1Cw<+y zS104 z;4o+Tc*Td;Alk+nLyL+6*09lYupbvu{C7A2J||E*#urUYt2| z#xotm*KTCUmh=StjLEJ&dnP317%jU;*(wJQsn(lk$9gU6`!(&?t4b4#gxGVvBKtE` z^84QcV#aD5l-gY`re;aWSf*M+G4XT0dY-C%*^1zCQX};xfc0>HM%bVB}A)t zhkd}Zm4Hyv{@;ESvT3+p2XplV3}9m@e#5&;Dvd22x`x_j-RP7RgxxQM7DKj=-T&RV zzP8zao~tb&Mjm<^5&r{C`wRd9-HDC)^X~dy};6`gD;e z7|~VmZ@wF_8M}7RiA9O; zW7>6i{OOUZ+B(_3RU)o8QSc8xGn&uGFwQaS7upoLPV($!i0HXK#QE4n75caZKT6z% zxxiVpqLd6?Tw6mUwohM2TBoKdW_IJMp2FSXbhdQ?e${K$YIkxZgtxPc-pU*kS8zl2 zJW<=JDysIrGT64Y-IUli{0z!bG504AT42AeHrTe$-q6sMz^s>{K!aur?=$BsJBfmU zeJ{c)1U@NyfBg2QJ$sJy|4ZkAo7;>z$r@xfqJ^OXDk!Ev&L@jxHeTD-qD`ANbX_ai z6)x0r+Pq0bIK{6IeV{DKo_VAXhU(_QDQ1LOd>Hq7y0y|_WK>rsjwl4~!0;j74=vaY zlyCrv1^gGZomcSF#st=)AnL1j!GE{_#j<8y25!_`3?y+8{citzZ&3I3qGhr(-Msuu z!&jsBMu%P7*MIQGo8ujqO-LBKjfRfsUZ`Iav-XLl?@ ztMo$JROl_jmqTSc39+g|fka5m@}XZiU3pVj=u|X)^2CYzv+fCkMtLL*#-O!pm!GWn zZG|`pQy|GwH4&pg76@VcR!tm5_G(f|2(mz{FZHT0g)!>1jatup7|en=8Y#UNLwfAA zyr)8DyBlZs>UG}S7na6}`!1D#xjm`;QpMFAfP;VzD>no;xV8M-prLLTGLJq7P2*Lk z@If(@9Nh;hc~?0|aAN?U zRG+${vigAK6(;LnLR*t@T^IU%WeI6nx}`enX*r3g5KBC+nJ=K-siLSek~qN2qcW=X zmm|V>1904Z51%d-xG`!Et@C8mSZ91I`x4p>wp>G9zLr{Agii{aIc>q>`S_+^a;LLB zkMg1=qZFjTfB75-`hR@!;EovZzZ{)#Y-j&rQBOTeruP1n<5<9p$P^e3)b<{;w%R1olj3438z!GJ zF9qXM!-4nM{cj>bppTkN`s6F&KSIV3`(LWTYJQ$(*5lgGjf#vEy_66Ugb>AE7a7?5 zmHU6*oa3d;PUhpU0fdW;j5L69!XpaUaO=WGd#hn17tnfnd>Lo6bN^8~^M)Hq_qJt^ zsS(E)9AA$&wYhmU0-rr8-@^z{g@(w;XuyE6z*0|VJ^b9gsPjK(|6N`1s7t4T2|A{* zVF){!^Kw;WX$W5$WQpN1tV4M_xZ}3CFl_w8`Q4dQ!>@`jDdAGY7Qm@^U-yQKmAFj= zU`y?1cf3K!L&bJ%a=M1Q{V@m!CmY+$Is~#9b!BW?R#xJTiuS?m6wCD9Mjix1-WFFE z`o#OEN%is7-eg_&X2?nX<$@S{7*6Q7T)8hh23DYewDA0tLZYf#1i`4h;Pe;vstE|F zT+Gl|mO_iqJ4hSojf!viif`rlMNo1tmP6VbJeil}sqwe*J(F z4kTCL!Gweap}m6WDBCY#i5aL|Lqgc)|JZ~h^|3b>4{yG8rUQmux@fbh%==K1M{aT- znQuZwc_9G~bgz4l9&7JsP#qS(x!Y2NDMU9}$pcH{4gy2(Ev+NIH>q;XFiAwh5#7Nw zmR&GH?v@-vu8zj4mn}FlNQ}8f6khP<^$4j$pm<=NEh|KE`Q07&n;0T%SF+MSQeVkh zC_WU2YZbvZLbQWE>0WPwo6)&oo1z_rNQsTqIX9ExyZ6d$B=aKx#67n6oRS^96rW0kVWSzCRHqbb%kHj2dt0)$YwF<-Uix%^T?;(Dw2Br|*)fKw|o>_MZMTUVmKG)<%q4 zg?sZhH|>;~e(J~X#O7pKmTM29Lq-pAW;_7KA)eHH4jAYM4Zo9dE-f#2zo2oG;RL(K9 zJeruy$`5^MGzoLAFJ^udS_gCBtbC&1C%tZn9{TdLY^sjCpRr*6{Hi^BKD(3I_<0#$ z2Gc3q{Cnlst5e6HH3dyuMvT&}zo|bM@}SY9yFGlofzazR-75D42&;fzjAhN?ebqi3 zci@030bhyz%-srO7#*S@4A9Vpb*GB(2hvnNA{2!mGYx>XAvf(_w;7tnu(<}R=?#Q+!zfR}Ns`(Y^9A<{gG=Cyg7&+qv z0NX<&o69B&CYgV^=n^p*+uG!d`!?H2Y@3{8WJXbNyVog7QVBaiv3?Ap!G4S?FZAMm z2;l*cCTiX*R{W?Wm*43_RV0THI~w7Dvzo}E90ukP`Hth`H*&VvHPY^OW8nia4v3&^ zpxBvnw314MQT1S+FtX!~gt;jcRQ%jywV!TbW(nx{K=rVN4w7& zO3w@ms44-AEWMX3Atvj>9R|{F3-ez{3G9D@Z@y#JPoQstP7UB6k*^X; zsG)Yn<#LZ@eo4_HMKK2afs4KwXRu`U*o?bILc7P3M&yzwJ|TU0^Z0mM+5F8TmbQKf zZ4<&Y$LJ*E!_4T};wDb&yo;gy3(`hZCERM+I>0#-+1yXm*@(TC&bq*f^Jc?s6xib@>S%~ zS!tQ8!?ACx2m(B@t~fpWv5_JOn$zKabse?j`d%v~l{j+mUicPMa*rlpX!Wr}S zUl2JC{G>;=L#D=;4_<>QSMM_QlAttX$Ph4hcE#w}+m{UNRw{Q!T?NSV?fVUk z{G=S?5GyXi%9CYU$|dvWPN(^W2Ee&I-jf|8>(rVxHT<SlTSwd%YhizK2PDPfy?C9z~UlyBl zM=mbmXYbsR%&5GMh7I@Qy6Jm3Aw(G2pctYRmR0~saxUP1F=;Iu2%?05C_ z?R)mDCq+)lP4-I4DQLC$$NEtvRhBQME0>mc)?L0L*+1j^XvfUkA2y_){PMjx{Z3Km zg!|h}_o6o6`exn=h;^|pZab8iaBu5$`TF(tx68q72LUb%d9dz!r)v*+dB>s|vt||c zZDt4D3g`h_NQJ9#?f55NMYi;rxX+s#55iA_&;y@?NP9C`yfNY2Da(1oU05lJy6XU6 zsn^F&H(ZkA*mr3_IH0PGo^Y}-@UAKR8$7ofZBF$Jv6#(qd z!fJxy&+QfG^K=AXtmnTnSTpgsTIss58IR*l*8F@Gu+=Bd{I+%Q`wl9q_p*}e&gCe} zoH|1o=z2{6l1U-HL?^7_mPWJjTFYO3<(k_ZZ|IfJ5GT221% zprY?EeaoKz{uL;=9Tb!TS;beSnAddsxH<1$ve!a+yI3?_L@~(gesY-}ojZ+NBW_{W&$9EWTeXpMP{pU3QTco_+A9#aMm*;p(l4GxQ&5TnA*N zPQA?TFp<~@eFHSSF+6qH*mjfiWrD@yT!X3xc;@AsH`_lwuJ32M2HUZ*|X+cp3P?O zRfU@)P8B?;{!T8)lZ0^&Ph$9FXN7{u_vrZadRXL1G541-y!GR)aodxOYc=aRTL@A+ zd(H_VVNW3-@kcT5_}K6J0CI(J9dY@wfbK31bhMhfG4+_+?e4o+gZReQBw0Xr9t{cL zjP>+2)|Cw+I z4H1QNJc+U4B-BTIAxHa-*4F*ckA$#=21GBrc@3WzOM6sV32I;ak4e=3!Qbd+Qmcf= z!~}z|HAgq;WhWeetyjVLOrBdVzQN@OG5Fi%|Je zj9buBt0<%eg(7WtYW6rdrV&(*I7e+Ozt(5gd~8$hvtCfF{(wbvbaeh>ZbC&#HG&V1 zGKZR{4t4uw5b}v09vUca#lOm2J!S)T1Qd_87y@$4CUQs>eq8AYn4l!a)s*e5x(lLr ziA{50Ym+eAAi1-2DV6lZMp~dA%1!~7I4yx89{veu!vpV-=jVP>orFOM=ZKTClciY|HmI} zksNVXCTq}aDII*iw6@yUxoYySf2{5I{Ls)}iT5)e6U$s&Ttr1nes%fzgym`}pTEALwReoW+yv=5)a(N{4nz;m2baHks7# zf|@(95Q-2WIz^jZY>pDnH{-D@WD+U3rQrYtVnbyoBE&K8AD+sde=A(6;1O4;2&0!a z*P%!V)ilo(5*O*RQG02C_(A~xt5}UJon!`10mW8YTVdd!+Lc@jrxz5Q=>8(t+s zH6J~C^waY`<4r1PK>jRu<&SW_Cqvh&e&rkNVp$T(Hx!x3ZN*uvJz?U1tfXlphbpwt zrPx14ti!sj(5$gD1Pl(ELBusp6Z|`CuhMK$#UK zen~r^@Y6Ey+kVqNDrzQ#`vE=dci6{R(l12$dOF0rPAET3eqz=p(MFjFRav`g*y55g;k>sON19Ht8pkdjw08|oc0`3q2*)5$56giI8v6PbwaXcpt8f9@ zHs2N=ZYa%d`zv`HM(GbUU0jof+%jNcL783uWS{yMs%KPdG!`Wq+rR!HV_q31y<>q!J($?71 zbe6qwgR8f0`ObPjFLRv6)|Lzz1jyu^d1jwHaYF5LQQr{WTRtuif9s`noMy@T5N_Y( zru36^0X3*4DlxtrD-Hn0iMBEJt*OTIX;`z)1BC(<3@$31@XF;;W%&|v(n6mX zz7y#A=2w<44l^C!5b(y{OG=fmvOQX`Oh9y*f>u*{qg5R3e7+knweUwF zT;mQ>Jk$kpgp4p)GblcSg!ao9FDgVoAcSWk^9z|bQo4$Z1x@+CdsmAq zN=m5OYk2A{q$dQ4cUXfbiZ-e_SwJbp-%qN3X1bRVHO+s0qsh~=i>0kglUg!zRdZ4LJJZeM7-hr5cspt%=8v-;HzXBvK2U+{Eo?$BxJSZi~F0b+5M6CNn%d z@_2E_KBtmJ7zS@%bhA3G%~Suw1rX&RFK~eKB_?K2;f0*}6ChbKIulQE#-h-bVY|p) z=F!EwIBS90cuXa=7IJx;&b%*X0PY_;VwkO3L-=W$Dc2cwXVcrQ6vUqVQENV z4MM?AHx(CXYJL?VWhM>)lDEt;UX-Zyf6T_P11a0HcVvFmYduQv@RA81ic!GGnYp5GPyIJR2%lV|KV2EFl#jES-=Trhgl*eiO@ zIXOj|fkDx>l(u`#{z=c(@AR^`(m`3W@Z6LIAAb6HcRA7bpAKKM8!#);O}n2n{r^83+ElmiaWAMjIUoACIe6<6z$+(I{D)67%mjuXcARjo{!vP2lR4 zF`A3&x<0FTl616(iTAV(VQo8wN8R|{bG7s+mR6V4{1uAV1$7oG-!?Ul@)z@mpy*bg zizYKjX;S=pi{+<*zxHhHSXKOWwKZ|uV>fcd2%q62jOHj~xM8*BWkm+W* zb?^0*0hZBbJhlJ)&H8C{#rw~$Hu!oEAPVGte^k1o*SyRB^9xkHhmc&|PUc{9{PlQ# zt%m>Gr}Niem3&_ToPscsUX0Rtbi1Ktzl@7q26&aIA(04R(>Pptk(5)q zTx-COFFs1WDup0WAO)2s5OEqeacanz0UPuOZlD2?$H=UNLSxyiUC^(>q2a;Iw@MGu zOa-KF%NyoLFqcQBX7oW1Ler18{4}uc?$j~G}g_o(1dP6R;xlKl1<3~XH$S( zjGd0|z7cvoK6>Fv9C^gM4W6j<1u`sRb|B7ayktE3GP0Oz`!R=&9V!{l%b*&dZZIrf zfea{;WKb8S7gNsjl*h)C5NluVupjLrZh{-JTCv~YP#MzKsSiko2Fxbe2N2buiO(W6 zpHjBFw`YKN*nUiujV@!=vi|QX%F3iej0^KxzI-i`!n5}$d%HDQd^9&~$BuRsrfH=7 z;VqND+bno3RyibUYas`f0+%mh(d>!K-ktbqp*Ksp8@hLI$DvC6hx$gxa-3UAzg8CEuBAzU8*N;ZhI=9(hM*&_*v`vMi)lbye68-DEqkx8<>0Yewln)#r{q(zU7iK6Jr?8i5_>%uBqYq zUi@Zk4aHDXTUWRB(4bH2^KLj2jby+@Un-tQ(n|;djgbYsRL2d+vLkvg{3;|zPKr>f z{Iy88Ro*mA=B{g()NFa-`&WJ%mXwU*MuevP7NQ`>4pw_~zp(GO=u>3cPO-G=LdZy~ zRHy50nXX39u5IY4Oc`FeaoB>5J{XBTN_4MH?%5v~H{SVWijzh^CqBB2xUiBE2aBZC z>Xu zOcR)Vdsp`my_de7OLjg?&V2iO&f~=D6hLA&CRoklz%Q4LvV?|%ItWt3m+ ztn%aPxWA+N(}8%9B%QB&ebSsc6C%s?*Yfx_NUJVe$Z^$+rB~WH>3kz@3hk^IYI1!# zDcz?}S5=%iaYCzk^SW)@9?GigJR;=F*+fRI_A&k>E{E3yHBqZ`(Q|TgqM9B*+hF=p z&etj`VQI}2ew<*#)~%~ksHQOt+#h~Pd1cVAPwyCxK(}@p?eGZ+>Zx$fytpEol^@SZ z55?pJTCV{eFKCkL=OUj8>MEwDrs5SNq{M|`rRVedeHl7({CH`ds+=6WKvjSNwol*1 zMCDcN3R-6GcPlz@Y!t_kI1-h31A2aefkS5o{%|EWJI2^o$CtDxjgE}h4|%5 zmkxs1-!At1Lqd?0puEj5%Obp?#}qDe(}(m2RHTI=Hbdl?ewNGj#k|e?ck^8WO#lX) zh(dH_+(`MR1pp+(E?$}+jq(iKK@6`&Z6Ovy(rwHChDSxUhEgAQ)MC>JyFyq>UgjXW zL>~*r-orP-Y+l@a)oi}WdF5drS2^BKI_py*Q*05v*JpdkWV#n25y{?`qP}X630RDY z{HZrM$ZP3|ba@^+kZ@D6sGbmLAY5sjciRE3R96UBQT`90TWfzU@o$RY=-5{R;4;wf zgiyELQI{kmLV5czn<#`6foC-b_q5s*$0MoDmsrd;^4R3~fdF*8^e!UfuBDASm~?1k zurEje42F)i=ggU7$DBoU6*KoSFO?~&O(o3E5wPCX+0Mf8ObwkR}E_-$;%un5}G^wvo`3Y{)-a81#(PlTGfY70t?K<>G3$i)wrgr1T z#=Cd#CjB3!#Ho)yMw>={jK!;xT6OE*{F2gk`rKvTKMF_FF$d0QFesV^@9yP`7X}jJ zf2caGj<}^fbZGj@uLGy0dre4TWryxobM?IkWapQWk=N&@e=C{)Drx?Y|0bfInLuctoXzzc zHmrfV6QOE!zXnFjM03j}St$=Tu2T4zSjd$~;dH*dC{AZX={?cEM5|N!$RtVNtY_h2 z+ZpjpKD|CTcrZzC(!6=~7B2ka{qE%4XHA8d0F{oUC7WMp&<;MzKR^7&r}~C!_?ZI# z-yIqScNpK1HZb8f7ybg36H+A6ay1liX^vr$%)9`W1C=K#1JHXus3D`wQ3qR=l?jpF zWEBDaBBeLe_-m&}Qd=G^uojBv>eZ`HeBj;+mxPFjuh=U5D*tdz{*m1@?wMhOaq~#v z*t=Bzrk4|44xg+T@p8=1jrA2A`c)KH0=na1BxGpxP*Ig)>H!?y8Sh5v?obAKwGict zsD-GDE$B)q&mc(qop1}c#e_fx%Ypce@!rlagx4)}DS}csnL;=fCps80as@bDjoA*5 zQt)S9H0H$Z9r1k@z7}aOAc(?GCWSC*WEO;>qo6JTZ!r^Y?FiwqM8LH`G+;qJ61r)V zA7B#3%DOTu`1;j~vBLe4>MI#z9hjURB0mNWe~ju)5x+GRUv(7)^cxKJ2y?$5ESYyD z|K&>+)CK}5{J-M#KW??v)^$VOE*x{wZHTVY$;~H#=$B75<@LY~v78Jbk(TdhS-G_8 zi>gM<6GjNuLkt3Rh!EedZrv`66W7wu6J>GYuPy!jLIr)UL%B5OWI1_&& z{9qaY`wR4$fJ)ZCw`_npGku?rfc#7m&Rb9K&X>TXNaclN#AuMm+0V~6CzfBL3FxvQ z&yxQ;A?hZ~uldAo$Z``2y$V;KE#kEzH8yvUVeam*I>{XoN>J$v*>(Y@Pok^CH~bNP z?}GP%077MTi6v}K#RTBxp?$uo&_TpIOsG`?aJml#$79;9F8cb;L8xc6Ogxt!>-30+C3yhq2?) za+wTq9hQDO5LpYlXBZRTwal0_4E-rr&kL_d%Up_Ci{xKRn*Je?gl8?ewz)Niu&z;^ zC_+P(%GS!HGiQPrsaroG3a2v5;8$Ax2b>1MZJ&MZ>Qya9qm4M4&`pHMUd0(n`1D_# z5q~X=V^W_zt2~my)o9OAjL@J#`DdL&PqYFrKYTFt z*UIBUFe`G}ZRX<~^$E^!EkwaeUrc#BeCT9zj-wcJzY7%$E-uwuG}G2?uApsPVf{oX zJqmw$7tfH`)p>nkSkBw zjHBfO)T8tc0J40VmfM7X(-eng4b$GT94jgtCgXS#9UeohB6}%bvRs z8mRee@=Yh1|7;~QSCciI!&KPG^y)WGz8Y#_p=z$KOx#^CB!2nA;1RbbMC5k3{=;^{ zrrqb}Q@+Zy0CA7&(Y}z#fs$SS}2XaY$u1N$e zy0zWeQ8mFqA|~$ym}FWTbzGwly_%@vmG$1W{_&(aH(x+b6$Q76_U|HzMxlTal}gV} ziJ~w)l6N5;rnz7VWbLpk)%?gVGB36IcpDHIFgh#7dhq_;IV#QQ$xJvk)_IVMN+ZYU8C{mEjSougbL3`NGzZ# z>OiDINr6ZUVASE*1fjYfIkKKWG6nGV4931}d?~(F439xj5E(=WoQ$!k*bIoEk;;T> zvM#|;7}t-F*QE0a*t^#?Jf}c5U~m!B=?!b~iv*9CIVZTjd?Gub&$ zT>&#i2$N#yf%&6kI*y$?ZA0fCu>4k!?p+fPOEdtIy02)SdNx^VxbRjF4Jlxgb z3h22QiORd6!<6a|+s5^bB(_~$@TxJL4-J;gy7|N$4Rw%FQG;Y^$~X(bb&i9jcxy!9;iNySs(T%)2dMJ-eb-LhT>!^1=0Y5F^OHi+0i zoy)^t(Q3v2iVy}U`9Hhv{_o%X->30^kHr7^EqHUFp`KnH$j4Gv+fo-ZrL9SnGA%Ba z@bdO;E!5+LgbowVo=D3u_|crLI0cJK%Y}BZwWtRktAyf!qJ^YWSvvE_uUVCm1pG1)D`1p03Iee7Wt|e$6p37p|D`x-$92$;3z)E_JJ$Rgu*3*!Z&Xit*bMg zpzf&BLq&D%BSxq~I>XTeX1vHB=hJe<6I&RyOgNWr2~OAvA3y58@@6jp^Fe{LH^9u!|iWFriI{m znOeXW%!qkPxQ)^T#>#}FUddP_*U;1xdjt7UR6!W%w8o#%AX_wSVmi*p;a|zDR<AOk8CAD;|x|8lo!e4g9Hq(T;pXNPeH@BZjmL7P^L3hbSJyRhP$+f}Sl&=1*s_HR zVM%P-{Ng+-DGp#B2Eqdp>MPAtICWK)PNZ-Efn1o3dCyGXm9eDXpsw~Rj1 zN6N|PY4c~rAOQ<5lSS(+Eu%(iY5Sy6he01>IfUF)9J%#7cj^2eEz#+<9DEMIg>)%?mJp!sW=Q~K9Zq(V?a7F^Gt2e*N_!nAoS#kq5)A zef8uvE)&%i;x%0@BMvq_&t&tdwj*}7QUpQLlZVG3rxlwu4EVhM1Z0>pQRr#FIlEB4 z{@ZV?QH1@K)?+P8hshd}Fq_siXu^nv13HC63usqOAyqZe_1MXiRs^rot4Uk)!Jthl z7c&sJIPbaJzp#>J1js!c-R}GM??^XH&`=(*&n2FCS1^;=lX)sGuD#*~^0BLqzsgJg zAJX1D9Lu(C7r#vzG7l-rR3s`TP0CE9!AuDmE2)HvL})~lLbD-LhKxlibCNVj8A_rI zkvUQ_WbKFNeZFmd>$m>+t+m>o_u1Y@-0u6j&g(pnV?XvG=k^*5p_p)95#*m@PRa^O zJJ8lXVQ32#?)S}{3{WS^K7QPuSWQsc%a@x%rIjF+4#IYi`(W{{4&5qtOYmp@+ z`;QT63FpYwDy$vibc2B2@#zhQxRB5h4TY$>7)Ib;iLSA9pl2LihXQ!ppS=5jZ_y53 zU(HQ}ofPvp7&TB04E-*3LfFb!7}z+@;lnuMQ3L*A6SS!|IJovS;9q}1T--pyeh?-I z00;Rug19pa`T~H1!Ru}cjd1FE3ie}Yf^}#ZfR6dJ#5&UPjjffmX|a(oGj72SZ1VBV zpD#D8{s2x$=)(X!9w#Fs5VP?%pkC-&Nc6>E;M`T{d;R*g*M)SbDoqD~tjI%FU6jq7 z`E3O1$^371aFx*xwizBv!j0Di5(@0(AIDd?;gZWmRuqIP|IzgT7G?)NLAx1=h|ymb zl;p8_AZ3>f)aG}tl|O$w0RpRx5?{=)t0J?QHSv^@(=j$6+EuW@B%3CNJ_rB@m*80< z5)rEodMnaCM%oJsqzGkBAJb6x9}>M+K!7cJL|6`3Mvu2lTzw+{aQ({FtMvgHW(=X5 z-{>UzaGNMB$Jt0B0pR*m z;|3g03*=**-0l}}ifM__h7F#6_yGkdEJ7dVDe-{F3E!epoNa?3cdyT6S?kft3I-8I z^qDr}gb6&c?|6oFgG|DwIBLkwf{gyJ!}t-~T{Zv$LA6;jajoQvbjoVeEwu0Kfc9XA z6S=2_7-q1d;)9CcNcQqaw7J+@TK?o^oWAU(xDxx97a+)B{g{ih{((2LM10~Nc?m$QQf7}!vJ7tQttv6YD^Uc90VjT+c) z4o#0d$W_<;mm08w7XRP#;7J3GcCfeDbGVCwB&d&>9>AdH6eUCxmSdK{LB1vA*N!Vl zEM2-bScC_00Fh1w;EZpNXZ!@3d4q=QbZ}y^mZDqC@*c1wt&E(UoUETpzCZlU5GlHx zP;p#Z^7;eLd+OrFt=Z#%_f*M!8b9UOS{{;YnJyN+{^$t|4~bEiY}zsP;mL4Ta@8-s zFD)y?$BR6frE3P+_zFu*;dNsxM#*XdQ{Ub@PnVU2eS9jGl zRk3NAZ4~%u&Y`Ebopsl@iNBh1XKpym+jVf_c!t~3Tf5&JedEu|qW1gf)%SvCNt;WF zhK|meJ_s&0IAE{Ezv`66vHvVa- zCP;fg!hrnFXdR5c;HxNroKaFEV71Qpz2XdPPmxQq*)ed7e5b3r8 zJ0c?ysHsUt1s409rs~-!6bCUV=!y6ZwL1ppj6h5YXb?IIWhyzwkTV9(A=+r*-e&T( z{J0Lh6$CKgC;JKH1_OAf5UOJXg?I1Qt`NAqmNRboR5zv7!+hDdjop>i`2$)QZl0P_Iy0Py3SwW#N-*VK|nlD5faZ6v-On=8w zLYW|-P5Qi{?m26M#G-jeI=Z?lr{7MpMBBkXH`_{9;aO-R#Ao2aCLTy>FjnXayRYfA!U3w&UsHWmgm*p5eyot-Qj%vzsfwEcMMr z57iM~xdZC9NxHh?U%B3dZeegR%)=U!>`%TP?yA4%aN(ju=y39@XBsQQ{C>%aJ@n8& zw{_(@KBorvuJPM_!okI!$Daqin)-6&hsJ2aOPTebAH>u+<>?-t^h!EA6SpSyivFJi z`|g;Y{;XfEwZDxKVq@Yi-jH-)^VR*vO+&_eR$e`8!0U9wQ6w(3!s%h5+P(F0(k4x3 zq;8#Dy^c5i$MPPXliL#)SG>{lx?{O$`C$6*uU~t1E`07fFu*J+DsEF~%Md8xe<4uV zpkmZGS93)C(3v++Yk8jCY8VPTG&W;y)4sQ@S|2CRU6cAfbgK5_pBA5iCaoVD+p`YT z*tn~mDH3<-`JLui*b!c=b)}`|(}>1s?F;tbB5eG6n*4#AwtEjAGMjI6%2Ppgs`e9a z$z77Z>f_vhbAb^#9ce;7JH_5%jvciMs58{#b07h z$7r_NUy^%rM9VSp^X!Z&Zyj?5kCl5pBfBS8|9N@3X*qAFTEMfZ_J00yKL+zD_JE`OV+ziXMM)S>e@~hfgDu!Gic>fg;YU+3-=Y4kYOKZfc4+{jf+={zZ zR=qlWQIvO*Z}mG`11v;>;5^ycS1RWfV`+|MGV2NJ*O+F*U(*Iy2L9o{0Zy!=BgqS5Ka?xM!sx z@OH)Gd7m=woz(RZ-|qP0*{qkgKZja68|C-+rg06#M``nlvv`Ca{UcIu7T&Ylz#^bC z)$`SmMyK`j>UAM}E9t^0OdhjX#kuH~U)T6COS7>vOGR3Wm{JOb7I^9v&M7jxj3s+e zwtt3iu1D$D2h0X{%FI8mVwB)t6E?!UksRE-N`YdA48Gq9n{{|{pC0(Sr!6n7>VlhB z`JT`X9w|9ITqeTO9>J&0?kBEMdcvl)c+MHCqEbscy$cuIqL&)%c*q4WwEI!>84IFR zFK&8%sXO9fZ&TscONyZ~zY-?6LaIC0?3_THa*DKt8lOLpp3|zmn^Ge7miHMwiyRe4 zW%^_F!M)yG+GC?}FD}ljv@4caeKjECW3yA0@S_XIMNVj&Bn|}p88chjQ93+Q{q>f_ zg&jM)G9T9d=^g&yad`OxzR}CIKGm%@wW&vmd0A7K)u`ohJXYUZ zB112JUq2*LF8S*1xYyOHL!wUUo)ZP(qsH`*-NTBS_@ilZ`A0aPrgzsBU316U+}HB6 z5Ww9>+!`KP6eF0 zF+W|^|H2Yn#(>7ZlFIEC?lujwUMoYHCV8HJ{$%v!=`l&|C5js#r1|bw*t>7Rw`See zf#s^>*IVbl+$&Zua537B@j%=qKyq?kkZtt6`)$$H>X&|BEVz4g=cj=kE}bSO+Qzpw zq^qx8J@li;YPZKav+)Cef7n_b{H`k(9sOZOf7|o6zkh}wiOyp=d)N1=|3uHtK85kW z;;B>RnJEVQ-p;yH$gVYSI6!se>fXM3v6+t-m%MYD>0##2cS2rl@^@hzp29ToeJ`Vm zU%EL7xP9p~E0R`{7Z$P8(Ahiet=F==Lr4G1bG;uTe{V;v-V)n)2k(FrwjL7winYdnqR&cMu`C-HjH-%Q^~US|fRlBm9LMKe zp3`xGlf0`&t3J+YX#XJ}#crCQpzM@jVP)d<(pcxUQjhwTLU?gSC|kvUFn+pJR*uj8 z(eX3>m;BG`KIz@N@&otERm^xixAGW%>$pBN_oZ81`rZ0*4ex!=YSO1lO9$%9iudn{ z8mV}2#O(r`(W~cHXLopg0c+J=gHAOBRYi_%qA zrRVn6>VcN__~hhv|NVate82V9?|YY4fz0?!_5+hFq6wvIJ9!yjbnRD#3h#aWbH5pH zONxU3g`dXT1zYF!J0)(?c|8@jDfoTZSN@y3m5*}pn3E?M(<8(A{QH%jTP=N4cOWa$ec$z-xr*yuP*_?aNaSZBwa$=$|L6-5^w%0W6S znH|u%cibyO(5z|Wf_OZDS2qd=_1q2G(8v9zc5#c@jxmYkKP^iBLN}FSR4?8Z(yz&A z%Ci!)SbzC#6Zgt@PDX26ufCRPSG_l*{o^k8;0E2p168?_YlJzMYv>$aEwm!D<(aj+ zTYL*o`VzC1p**YPtyQJnJM6c$SUGCiEg$RX=yA_VsavU@m+~fC+4RO-o#sD`Ul|9& zEWEl*a+4!Ug4n{eN&*Y~;&;9u>a1}uu=>18_@&2iUc;jMdEsss`55&}ZleB+duNgN z(|@%>s>a3yMsmfM zv}L|+kMlOy%by9Fq4#Uiww5Wg;b!}jeMdq|eHGpI;fYafYbs&oGu<@K@+ds@=#FqV z>l-Y*py$5$()KWxfXviP-|fA{{=_BB2)OS|2At!C|Z z;rWwFo5RpLt4~grX&H|ng&W90{IOX3V`LI*uAtH6D$yeh&)={0&gab64cXiyeN-Xx zuR~nofTB;m-RYlO_8d>0;tqez<#?O~(2+&Ym;9>e zJ}2YFXO)n%%g?O({Nhz$RGMNt*HcdI(?T~R#};0>ntJZV`EOF;m2Zof*k!0sdqx*< zP0Zn01Brdi?Xo*vPlo8PwAy33=v=l}jZLzJA-7+Tzt8BHmBu4OxhvF9su?c$#v52U z+q$#EUaNkKkEkM3?5p?rFJElEXIZ$=yRRhO?Yq2!d|+YPj{Q#;pndag&t^Zhd25R0 z-fvv;2^HIWYzF&+nI^5`cK-11Ejr9Q^NLg0!k0?hucqwVZ6K?tV!P(2Mw7uNCS&EF z-9s-l7!0F}ey{f#*%AddXTfI8%C0JP;jG1x$z?med0REV1sr@bD(UymMIib>WAz$# z|7EW>&ajsXEc@VbJ%2!BPkR9W!*f$X3?>;}P17G%>fdWUY*de%-z{+I?5_w3#}(`n zz(IaKqgRJ%Q?iuTdKSe?dklmhN{0qYF|OtGfBCNJ>s;>D=U3C@C1}0!W>DdxV^g*= zc~>N^B{|Id+V!1H^9g(VKz#B1te4@3g%;xy)y#+4#2nl2I!u|(Z%#k3X=RF7Ajx=J z{yF1~Ba-J!)!#6T(`6lZ9PUXO`CC12hsU9wfP;fiatGbTs~onGk}S7;W0&(G(~!Vu zhwoYYAJ1OZtf4KVbd*w8en)$;g{B&VXtZT_oQz7EHO2o z^}*Q^+6dm$)#c;cs(tsd#{uz!3oX(f#xD?fp;ugBQ$BK>uT)*_l?Z(xfg7C1G%H6X zejcAKwompXYoF6Zg;b(-h}w%|gBBB3kFFl$ zDLdKeJ8hp1L)1`Cj_=DNe1d1h^i2O=^OcYY>j>*jcy?S<4tajkbH1#^e<2A~@*~qP z`V>rS*K(JrXUFuL-xsWoI3izHXJlYr<=AVZB8vKjrNviLqUi6+nY5JzOYFUUYOC1q zfur$XtQXYkEoCsYYGd{!G`&hY^7+3%o9V6eY{s1xJ*H_8uj6E;5}_V+KB4}4$Jo-m zM}`NtEIla|%Fm{C-&V-`iDA#RnJi6=3H66omuKKM`V5Eq3$(H=V_#xHRpK!Rt-i< z7~V8OT?}5*_V>NB&p3sxl2w3ZXsAX>N%BcwZ&ukewi}+D!lrtOBPu*S9QsB|j7Bx7 ze|v?OW^`xbe9wDnDrAA@O?j&I_4VYwi3x#X({Zq@NzYGU9dJ3!5L+=4mJ6+4<1Bh4 z_VRVMl)-@szBz$GM~{B+f@EyFvr>Q&c(Ns3 zPpiN1F;@(|?e4X5QZs0*N#H8r`Ag80Q6dPY>N?ieH=Fw~NIykMMHEO(4krgbtJ1e4 zWlP1QJvY{rnSGY#slU>erUq}-kE{@0IokL@a&)mxp?_>vwpM?7Z{r5T=_h27mN3ua z!#{(2K4i}ad+{vhBd%2>iUvy=!yMz?x5n&6py7a<@ z>`E~@ zaCCC|0du*%k-vcrW6<&bi4o}5gYcmsHUef~EW}fXn_OF@mECfm+ht|i!n0=WZHA?M z!1h6@5Wo8;K?V5`~Ff%!zNny9w0UQ~`Lc8%eySWea?C&?G8(*oN#jeYTSsgs5qnak8{IOQ7!m;!9*FX!p@a|U=F*qDzWIXFikYJFU(VSfQXCw=JXU$n zo_=pyA6VjuqL29!2>Q8I7;pV37c%Plyih}9=fW581LxNhxZP<)-)w$DO=<>Np+VfE ziLhx5q$qe7b5{qjuNi@$n}W-_zf%AMV`XcSx*Jz2n>)9hO&=Jt$Bov@f$x;y+fC|D%Swmjl5%;>ao{bIitLV!LSJ_4l;XU7HgXNn)1U0F1vhLa43%5^FVuB zL*|>-BfE+$|NKu-z*dR8U7;80m6U;X`J%L!5o>t~QBqs|7AwS98jQ4&BSLh*ske z@>}PqITbl7+skr3w~sB&@zWK)s!r(xTeg6Zk3ZYxcuRdFUfc_;*O6iwtZ2JD>hZ(< zQMxMcy3EXW&cI-=Ll^l0g1||)= zF4Bq(Ud-mcKfEH0-(6)SOaIpM+fp3IyBZ!zw&t|*_X_Ome@ov}sf}gLgY=%>wtGjF zsF2q>wgL)T{JDAtA=uO3^M5-X`F~lO|13xPtpC^Znu<8VFlYng>mUAa4)VVsp%Zxs zs!R^h_o3HugcxDN1zI%0T@JfUI1My48AMnEg-QbSkxUC)P|=>j^Jfsl^Z7`9#Yj>Q z0dw$$PVyd)i<4PDQfo$2Na(DHRs`Yo42=JBL#gr&uDrq#Cm;Haw`?1J@mY)voiGTM zU%Xfs+66GZPC~z9{v-oT7LpBt@D!P}7?K-sg)3ktb~tYelztL(O%nudmS&m|m;ZEl zxDW!PzI1hMi&v#zlH8wFXnAZ_9gCo_@F|S^rhQK^I0r+DI8zTGV}qRR85nx0=0^XV z$2E72B-4k8^np{$_xDEx3X-=X#6hrQmj;gxM%96aiOSQ5G2(Hpdf+F4UEqcDH{RH# z2==3R3sOvABD);4fz2`M^XSCG$5NmCX$JSfw_pB}ahYKvJbJHL=0p09w=>f31@YU& zK)@8~&*+GN%$Y{o7(9-CIm!34|33KEem`sUS8AA`vye|QR}dBzy|s>8=AF*cYdSb| z)^VTPb(yo0{DF^8`jR>be1z*5kkjSF^swzDpLz*})ZDGG_1XMQBhr>ze3-`Uw8H6AtWTW+>R8 zzX&QlB3XA)PgaVqC?>*m%gOhIr~)8-@r5KfCv1m^@CP}hU}P;)-`3yH3ku8oPtY1A zzE?C&h0BWS94|Qi;ewc$o3ozQzH|Q)SZJV@r5?M981XQLE$Vo$F!y>{!DFH}KU8LD zco;v;#t^x9{m#w%$*j)Mz<^GtV&^|%LCTwfRUm8-z3}u1_^bGIGGJ{$H(C<`Dxnhy zc)q{CH|4KD5+HZ3b^1mh%BR4a@7|)0RbLZ_@$!qaw{G2{!7K}ETt-Bb&##|9+YjDT zwZ{_yW4Odmc-?}bD#geHs7F;Or3=I%qSas$4GT1)zOyi(v*kp<%InZ=un{i-?P?j6 zVFiD_-GZSf&9tGpuy_)VXZxJ7YA2YF^{+1To46PT7@a$J?nYoO(>X&;kbxMO+pkBq zvUSJQehrG&gHECZ z>?X1>2b=Z0m*;ft)*G7qw{5`nb53$+Aer@$lPaF{I~K%>aYBvt$md7g4Vg5t48<+F z&iUlrhkfKftpLWj!3xGT1B##Aa(A0lGi}_mMGv?6aMk3*1h;xAbs*gpaHWG}I`T$t`aUUu{{t5;G3+Y_3*pnEBXLfFZlOasj<8H*K z?c|D7D7UH9HeoZ`Yh{qw^|?yOQh93;+D*}Kub2`{?SJk2T>b4ei+Spm{jYG`C7UZi zG4ZsyxyX7P+}L^(b6xYD+iLmmk;W3W9+3pF6iE4kZ6cPe(zEP?@}H(zZN1xETK@X( z^20rIPG;f=Z#$$Qc~wu>+}$9tgk3UXfw;@-TUMra`e`Gm;$`431!){2iZ-C;Cl}w> zorzCF!6G4DEF7=CLaU*w69cjmG#Ouzvj^)c62@m13~`YS#>b=y{>60V;1URHsxx3} zz-X184Y|)rd<}FeQV2nerSRP7XI6GHt1cwUpk#+b)j7&@3it%A~`nZv1ryJYQQgXeD=6MyB7Ii8@PSUHh zdtlD;6O_sX0Uog<&*5g1Wg6-Lm{?tXa|FGJE)rXy5Q#UdhMtnLk+5IztnvLRPwds0 z<6|lB+LRzA!sAEYSmN#lL1Z|F2u>xKN)}>RWsJ^>&|6PVe=K=M>NO>su3b;K~{_=itz= zwZhtgh^dWMtGqMhnd6d;BPP~)AL#2*k%H+$#LJFW5S&z8^z`%`p$;qu3nv7JCma&^ znSv2n6kcD19{A4expVIw9q8zgJ&yNs^EbGmAc%>u^N1xZ>S-=~g6KTw*T=xy_5c^_ z=I_0^*3iNVTcumOEc-^U2&0%8ZSt8af1yd-@)1``kKt%)dL^kBo*bgUd{X^v!K+;a zfqE!@_Wo;ZAi9iJ5MA|5grC6;AsA2n`_N9T+Bn3W`B}jws~r3Mobxf~GvivU%pNu@ zzO46}i762L9!DspNVjNWa=Cst4vdv%<|FS*zHHbw%b(NAT~zZ_@TxMA+ak9#`B|}e z95ruk_(q;lKX0gsU)PY7S6~(nRqgr1AfYf$!FV)RHJ8+@aPv0hYrbCa-nfsZy-D{apZg2UF8m|XM(I+ z;SY#O=z~`NM~%KDiehGpWrNcRtm;i&3q|JX^Pl3nv?H>|G}6l~t;i{RN0xwd7-FU}FVox=x`8yHyzy zHSlDrL*VxoZmJ1MN%g63Qj@C=yVj+O{GAv!N}sGMc5ld-2X*QtylRwaXXcf}oMhGo zz+q?yWN7}AUx`o+qTo_(wL;%Y1D%cEtm_;*U+^#jV;0PeVUx@GeKI&%{oU%XHZA3c z=HsL!*Sx`BpWK~L)I5R(4*BvS%^u4+$@*|1&g5;64od$WnEJ~HMJ!qKdq6uIWp}C@ zv7MTr^dZCS1lHYj>}tN(?h8F>si`*CaLgk@gA+yUQ&EPzwlb=xoc+D@Zrxx(rH0o4BnS(>i*<- z6fmU@W@ReX^Qc2S0w4pWMrzaDyz^Dtk`1_7INzPS3t z0wfn3s5w72-4VFVxqWkwX`(4YfNbxdiiS%EnDegRlDqT0J_W8Dlj}|i`n0Gdzps}K zk6HX>g9VgV1&0V;6JY`L8Q4CTuH5u9IQu%QF0vKyBtmIfClNkop!{LnB(zWpwTumF zGMrq=`A?-}R@_hC*o7ss23VhC|1t?)*MRk1lgm`ty0nxG8YShpL#dTn9HsZU5%k|n z{|236H&f-)ZPmS_?5E18c4v*yJM-}JqJPkN8D&9KjAOt0W-`civHoUjqLHjcUCq!b zKG_>0Z+(OF3l{I4(~Q&Gtq@L8MB#nl-Xa07q1E0#cxAp^tT8Qx z$i1`tQ!*eFrKFaXjSrKPIp8W!(bu3`lGOuNj1x|Ec`(CGqOMl7v`C(i!XfaXu@O2N z4hl@qe=76N!9-08+ncJ3k=lqbx`SnhyUpP)e^Xikm%Huy3idEVB-|a=Kpw1dieX-g zBHAA+9m(`l1|(_p1c$w|ltV*9L6ree!lxWPq+_OdIGS1En_!=@ z`8X2GIvuf+E3wE(9|fbSHS5;F8TDOzc7(#xrDtrGuKx5R6t`^pfe%QZA2Sm zzmbONtYQ*Q=n`ysXc_5p<1u7nkdNIZ3AHa!z@sOl1{AskP=P>+C4m%05W}=s2!!&tohLN6~$d$Rqw}=ru@c@ z2ChTPy)*ts3pjm*j({8mAu6*+Jq6XdPOexaWHCNH=?~u`y^Wf&`SV%u6~N21F~UV) z5GC?_G@vMtt-X6`egSs#`~?ei@7m?tWE>qFTsKf~k~UvhmOg{E2$>IH&xXJ2Cbt+o zHD5?ZkIfZ%ofk#ze3-~?1!evZ$S~2@THmdi1aV(mF8nL`oVK1*%7QNnk}xdYZV{au z6$m-@pcz2xh%4?}0^&Z3qcWh{_Bb8D@s7g-33p_?-3>s0sGcP|MT*Cib47H)Y>8vN z%WuOv9}c%Zzdw7YeGW&RkUl5wDxxMa_n8mhIH-uQ9h3kcCm_g6f}{xeDAQO7sPP~u zGYq5CfBk%KG=IekA@r&n;0s4%7jU{=Z9DOi+AumO$1#TUa%cMW?*dZR{963FA-XbL zCt^pq`1nMH1O;Esvr7cjqswr_C?Q`58}WQad3k;%n@3W!goSG#XXJb73GnNkj%#np z_qxDuFB|RC%87W~4%E$#9v<0U{ryWf=}qdM$gRfK^DJk}1t|55xp{f( z8+SpZ6lbm=^8VdB6&K`hh%yBu-+~|2?*4uI%9E3mx!PX7el61Bng582!0o7IKY+3A z0Xqu(l2(iY@b2JBvg;PA`%p$XYKAQzai^97PTbQBxXKm^3gNoFxk>xbj8kUox5Aj1 zm^GL>rC-M}wlEL2x^Bgo`P9Jft_cb6$Xi*$%nw`f`H z?Ykd1Je?sB_y_>s$#LZ38QNq!p0=Eahor7iPFdNsw;?k|J~VC=>kDZH7h3^T<>t9I zK1dk_Dia*HZOH5D%MM3OQX-u)-4(|CU5=&yTAa$hVC~vDu*uN^D<9`F#o(b}nv0$g zGF6r7{e8poW2<6~8G3rMQ}r&@>$FF_ZEe}>>w-kq-ayDp_++@K+14R&?B3kgP~Bm zKT@U>L@1CLRY|Vp2kx0O-#0FV1Hm<=ozx9OHb;(CaF}Q+4xo&9vJXx+iC0k*AT157 z4^_sEFzd-I{3QmXhYO@Vz;}t*u`pndHG_T}Jw90;;Hj#>o#Tylp8Xk*XncMVHeu>_ z5BqAOv^v~;&F(22)!M&(D?_e|*H-*3x`F!8YE-$H=tNd*x+Hh*=BfV9&fU`|Yj;z@ z(J47eT&;l~WlqYZK$j1r%L7MO|KEbe9pX+x1Bz zaI5WDx6-=v8~ItI9Jb(mOt-WWJ9actr_?Z|ZZ9@aM^vGzpRTTYvl;?2;37zn5+$q_ z2rQfdFOX{%M07rAduit&f4NblkWsWZz{_%bZfG7>p&n&}WE6O%aZeK@qWej*18Kujd86-er~fv;lNrEG%s;c7UkIY9cC*7#0Pp6QB#%DmSRmPwR#Gy$@BSxy z3b_!SH*BC})J6f0K7FMN1qC5|9%EYMU=&Z}>}ozyG&(di#1C^cxSl~PvPWHU>Cy_o zp(-P_6}aIk!$ojw46|G|T#Ql~z}qRD32>o%VCQ1h&&|nMiuQ**>>%`Vz)5nmno@f3 za2X@&D#7YPpoE(n=HZ^ljq)&Q8!}U9kMXG1c=xx(rR4((RJ%!hP&hA<&&sHANh~3 zBFGkk`1A&^YPGZQ?qFgdaf1W*qu_V*F{<(bSl+vDdKGb-cp#~I4V|c%aPj(XZ^3Z2>GX<6sP~O)2tmkQ-SBUH}!-(EZLMoDo7p z`cG;t@ryJe;3Q2HdVb0p2lhcV1^ilqkzP3$o?)m`Zf;vs=Z>K50n+c9uo6DL2JlCPlU z=e+*k4$3oX<_0*#sgfJ)z|>BhgrNwyq{{}eE!l);1BPFZnwxZQ%kDPmY@}9;PD!}n zz*4=0MBQ5aUj<=QHH9%V4!YPCZz!09YOSfD+;p zKZPqm4bqehI%WxjPMS;gCp%e+(O;^_o2B=JTEZytyK$VCgq&&{=}=o`R(OP1c|^7)Ym3o79(2=ct;Eh?6($;iBXCcOnns9P3sHn#H zs{}^Nq3EP=*74AOvu96oz7BrgS02`bT&NxE$3J5D@dLMxI*1ceUr<`9e&RS0JFGBI z4A<8QwK0u=e0z2xQ60M23rChh@c6D=D$g%j)=Xb#%bz@H~-o>Wqh< zF!;k4(vD^t%QG0j19`+$l84t=D2T9XF&M~cB0$JE#hhwnod0~TxnBQr0pJ)IgcD@@ zKoNo=!d`Y9mWWqR^w_jumUs1a35Qw3`-@u=f9VOG1D@*Ug7C=^Id{p z!^gm&v>e15xS8?5EdxKsiQVg6?~X>V95|Fv>jpDuEF2aFCX$47V(9<=U&))vR2$^gYMG2+7)pLplM{-DB(@-Q*@TvV;{{Ena!=#=D}x_MSt^DXeBOdoQ|bMAwYx%QNHu&$#j6c=Haw?}^u>BLA*a=|#P_ zdQE0Rg!Y}cua+!Yx48$hc*=Z1j6E;9QHCt%Q?irsXxs2AYMm&<*xdY-d(od6==)ar zoX6Yq8-BeS#}#Z})06ygibhu4z4YYCIx3M276?Ar_fZhkwx_gr!EduJ*+WR})nXZ@ zweattPYy_%zJ-WM&dv!yL9lP629;jOT0AD0(Y!QpTapbwLsE1H_n!Ba`?4)|VMn)C zIFn*|6i5J7ngeKJey(bT-y2##zDUE<*c#z%!-BNmn#QDwgK4Q; zTwL#NX4aswT_7hXCVN0;lY;=K0jh0Oygz_7)1sjv4@MI+a;+~QAQUc`B`^hOasZFH zF7%FF*jV#869bRg+nY)z;(K$&uv4NvycyDwh~j$Xlx?1V{5|YT6rTke76rKh%sG^o zegFP@U!}_iQlW8{+tIAOIo`1_ZRQC^uJ!(iMn)H<#34u)HRM%7FfCTq;ERPZ zUp{-cJTo&=JBY1WHcW)3a;dA#-vZrjaUObn{a&|w^CE(pOHzPG{E#l@n} zxf1yyAj4U^z}B`dH9pTnXnW|`v9yG%)Z=41@)a8zl>yTntIH-4zfMFnpIdD)ppP09 z_MN$Q@K2z05SW(ICd#0k7#QqcLfJj1;}MRB$`>z6DT)(i2o7~A)ILB4$P)<0$RU`f zk|8Vc?XcD_cs32u@q+cL5YcXs<6vh&GB{B^s`T1W{pJ9B^N;53fm+T00 zqJy0=h>Pa{AOnFWV)U`4bCUd_AoGP;)?ZqgmO zjN|^T!7i`=Mu()qaPavq>=85>0R}@}<~?IaZWo2(AuG+PwCCnL`iet9&?F`F_|A`Z zX=>+gvVaEmGkLaP>xCqVUwekHhf`~xI_sR|2@7$g)z&%}0rrHW&B4fDpD^2C)k>}z z`--B@LYjm5g#YcsmS&h!4O|bLO%cg3+hC&^19G?MV5DjXcS)rG8v0it67jIplM9WY z0-@_`DQh`t!pT0_r!TjGfA;KPOm`r-W`|9I@}okbOu20p{xzZ&CsA0Hokyq<;XRy=cLin>BxuWjnA1Oi>?H z6&4g6lF&9MyH_w3<^feR>u4hv5kQF4;Q={LkOZa21GB z1LrM^i~yW)SYb5<=#AXwAtjzB;~-&;z}e}nxkBCdJ{VZN`!2#n6EoDwl<9X@$`UNJRN${(|pcllCn zX?=B60XKBS^56PnDZ?I*Y@OJf&ANP&|L`dtf#cB7hmcNG#}0t1;HtvH08LYR{`{w} zOlbcpL=pDg|4~1|ZpdPU2iPpw%s?rt1RxZ{hxm_jva)O^q`C@_Y|YE)s!taOB7o4( zyT3o67}4Ig?Rw1{7aEn$ZueXZlc5u2!5AA`GF_H<@EWj@kwd|c3UObJb--*+kHdbx zW|Zkhm}$DMjpf`EYA5c5<**|@r&&Pn%b07d$-0uwMmiu!sp7!6NYU*$I${0r3>C5m z^0LUw~>ulXXY6HsUE(R-Nu-W1@hUO&uDj!al_WUdcHT@OVLPTo8B zf$E@i348JCy|aS&>dT4X!?~KIMvkuizI*-s;W_7HK!L$%Q zBOFK(D}h@#5-!Ll_<{T1S1-`_Gb7RLBlkJvGa%p2B-2g}iE~8j0YN?TnUI9UMU?4U z@R|hdi15@3u$dQ}b4OEz46^WgeDX_xcO2nOw`|p_GsuXc$iPpX`&;Ew71pq5Gwc>f z1T2^eTFL2l@b?Q_`N7ur5AHSH`@(1e^BfgK+oy$DdbteT8BYCas=LBH5;2O z8e3rn=X#WUFRRFWoNOqrUcHL9xzytokXU-45afRDKciG&#%I&g+-^MIV!pjle8AG& zA=Gd^t1g>9utq)A;+Z~&hM!GIHZ$IwWo#)_P+7nrFKsGV(N~LpsE3>D#}_}*34LjA zH{W**(GbY4ISp6eUgdNc5W!-K4fW-frAA{wYX*y_Ci(u0V+42|mNUBWgyq`5lOqVr z7web@BjaoPcYqeN??`+C>e=PTe(hrf(j*NX7dnJ8RDT#=?7mUH#r)%PrlUrj(pkU9 z?BpbLbt5mRjNf5MF_H^}?n{AnLC{nUa8;l)cPZ-gA(%OK)&aNlYoEh>cgd3U63#J@ zMJ`7~EJ)vBZ9U6ZP*O~cOHEA;<9{i+*n?Eb}Kjczq|2FMNEYFBH$TmaXkPY<){rnyr3Yn3x3d4;4pd+U2pu zYVH7zHRjp+@WOw7^H}T{`?e3q@9dj#Y9oh0&AN<|Uj3rbTWlMRjLxDmL^wGtNM_(O zD(~M3<7A9EbCfZAnSek5N#B0)H-v3%KbtPpk&}~qbfoX*y~P-gBWFzZK->N{EMcGB z?~fVYF^;`=RePolmNJM=oGP!V?#7=&wV9yXM(93(lU++Acgixp}XcQn!FW z;ZxAY@6?*tzAby8r$=+bKeZ}Bd}9`egmrb@FZ>RiMhq^$``1w>@n1(7!+#xRBK~!h zS@ds4g{jzo7{&=I|1okgaGf@CG06rq43QLEi_sH??>DouB6B~=eL;=6T^!{hP2)gm zxq}1)vhDoyM;f8KsY1fSAYn2cJ2uX|oO*A#qQPUBt-Dic*REYKme$1xCjSDPK%v_7 zFSgL!YR9{^d0hxLNl(`nl+-XI%YosQX%F^a#V%qYN+2IDY0q8vbAKf2IaEVX$Q zhz5HOn2!jINV+Ep7ddc3XsBgbgpmf!Br`Q_Ks_QpF_6Uz7(9UK!$j}{PJ}zpFGod9 z=IQ*XKxoj57b#hPLz+dVFM&e#?Si=zSAUB8_J8LrW;q^QFmL`0dLGpPED=K zy8$0a_#6|cfJ%pnL6|8zyKKa!6TXQw@IV-ZVV#jiV*;>X&T;G>%iv$X??aZB!V{#D zqN2cmhcSCB66Qjz9Jy)&LcG3WH!smwP-kxZ6o70f1^R~1 z(+Zf^eQdr|s`B(5x6oEvD6!(woAMT;zcT_oZ!s<6!Exf6g`T$c#aBi6j9@SRiu38$k|aurHi$kj4k*0oA3*Z3zG7EYRD^3>aryJx*RGMpEKx*JGE(D(E{TId_yf8c z)b=wTJ$dpJR~g&@dhbRWlMCE-6D^=IR}qCO+%89^-+M}75mJr?EL5aff7!L1`>)!- z@mp+x&?P^z?0$G4>Olrv2`4o))*@O;3jrdqb3BQoVdJNrQ61y}2&f6>UM%1QbC?Xy z233wa4!GBSh3g{v0Cq%C*~Z#o*a427)G1925^#zV9u0>~gTt6i{N3<;pMwLFzrVlh zXDl~ZR51cEo@qt;AEhSji@}5p(zF*<%?|?NQX!2*M<9FfYbo2S^umiEJrg$r@GCEGq(Ra;k=xE~P$#Hs{UD$x^}g0tt$`HBUg_iOc`E@Qfz=w^`s z{b=8e4i_9YXarTv&CSWpb-UwF&t4@n6O%<@KZTEYjlF!%RA7*Rp$tQF%p30R2*td3 zFFGk{SN5b(;eZci{+Y-QYK`z^Wld_=)*SQ5xuwA}w6xM_D*|(Neo64h{!Kxl*fe&e zu1d2=JA>}H4O{jzyW@#*am?__$Lfe2)om#+CkOefY6@$lrSF*kSP7sK)eFU+0w;1l zm!E$$rBTMW<=_l(RgwkakK5()dFJOOPuecEYdBwXTy$(zf(j6sleqd$s0OJ!BhET( zMbRutfrpBnPeI{yfP@$bJ~-R2{_z*WH0lJrVaevTT4QQ1Fcz56D_Dr*ngF8aqRczc zJctE@Duz6baVs(QG)r18WFdR|pC^F!|Cf(NfBpJ&?sT7MT%O&2#z6h~$}}`sXK0I4 zSC;@>cM65m4Bol)av-EWEh&k8JNY0t_hm;?AFOp9Fzp*fMLLi!ysc%h3`MlpW%mm3RMt`g1=cH~5>6qz`$q1q z@N4sL32gtC!`(lt-=$u?W<2>(i&b=Hp_<(Zbb96?9o4~y9`;@wt@Tk^>1tgAnXbsW z>6IoDn-q6uR?-p$Rc@KMhE_Fjp77^?J(e)}`ad||QX}9i4Gn5lDDTN&!w~vC0wOPw zaEN<@m47wz>=|G@#u1OA8f3XK$a?$sZ3*homNoScI|c^$^!1-Jci{BG&Bs58m!OlJAqbJtKddB?lwpYY&Gz)bESh!H>g zK8eE9j%@#Crr&}+ZF^Tcs51(9+*+p(?YIWeNl(HV859w7=O`4D#2$ZJUr#QKh-1Hm zvzMWXsSp_=!cEWQ=0>0G7%iURo^t}@H2lNEuU(mUHz|pn>geZi3}NnP6FY@bN-RKf zD$?;NS+{lvKMC*`e-xYPXf(%iNt7 z78Znd0B9*g8OOCx!@GYjh5|&jCMcNtQ``x%>0^|j8a08M!*v$Xi5h>S*!+<1wGeG?gWU^OdxaUgL3wFtVap^*%r%%Afix0 z^NEuuX#h+^PMp3B20-1-&=E-b4F(CvW3qMGDGs;b)_2s}VvryN0b^URU_rz_`>A4( z!^q^2B0j*ZS%#gGT#y-oV9&!-vjk%h%0mPe_5*uG36i7hDH8&9#6kQ*JBbYgD!HM& z;Q9m`$_;#6(KKgj?NGFHZ2h&JWwD)S4O*UvPA#_P7V%fxcKWoIp4t7Tv4Mt6Nt}Ze z+zf_PWuE_d-^1C;&Kz=^uQM39N6%->+93Xe47!l$lF){3^ zPv+nVB>!9~%&}0Suv4v$9*xOoxs#*2yOP_&Vs{2br@)K+$M?H2j(}oJkK#PT`LrZg z2&VDiS`qq%X$}^`$(uK=gJ$o%is=*R`CqZ3ga26QYcY!(6B4#kK$&>ATt-sJBadJ+KGn84lQiSX9;dXVO zz+y6(HoI&1$vkb&d=U3y0RiM!5TD>sf72X{pRY!uOZ&)%>FHUi0H5NW;3g=^VL8gw! zgPN@EM}aEPTobLEEBImD8FqzSZf-p?pPm5Iqp=YrNy#ZGD4=+NgH@nUrU45hte<^R zf5;BV&~*P{-UVVQgPZMu{4)kuGr!%ob+SiqHV|lsz%ft@T5&LLdwgh-x%uA#a|MkT zM{k6N8U~GT=X!!>*S1h|J(9*4cIWqJ+7&uG-NMhHUXR0W_w1L^&!w&0)5S;J+qaq* zmOBmhma(nBI4U$|``Os(Rae8nwllMo>?rsBd$>2_;E^L0SOyT+n6)1l6B7g567nuq zRuWm8=7hNeM#hpY0Yj1tD~}ldc$4oIKcmq4uEb*wly;6AMCn_GTx8`bbtbOS6`=Oz z<=@hP2xdRGnhkI0kCP~1+)WV@3ety%eRBdLQ?Soc zYk-GBHgXH@FOVc)>Gytq_e0;JJ~iQ$auGjD-leEx$@~DGrWHgG1ty4Uv+^pq*;2UE z#BG7eDVU)&*e@?Hr!Qif6NqxFHV>(9SeRKTylsk)&jX@T4zk2HsUwFEA1426P#lRw z_XA`+igGd4c0^(drUux4$b#Por~Qr#7J%2XF?|C4o)uf%S)>7*TUczWtH3Dw2bv}a zJajbN1jZ&HA;F7+UM5^N&ikLFYK2@dIA1PZvKP*&AZebW8;C-g_?(zVv8`CKBIQlN z(cy#e3BbVcDGDy=YzRfd+O~@IC#ukarfrWus$sGLk7kewv;cDv`@0r+cIrS^S1=I( z5QK2>;K3456K485XQ6x|a0`QmSnL-#;wmtcOz2~o!Hwuob{kt;2S-O4+Zn%-NY6xC zQ0%W%9b=5UqgU4TbP7sVeEAHF(y4lf2kK70A8gz_JTBJ)1uSN9{pda&V(tJ;92V}u zpwG^pJ=^>cm$lC?T6@!`p7c+fFkv1X%{1upK62V2M%{(qp-vQ~9!Q{- zOFzpfY({6le&?}c1z}-SKq~jXKO4d>U}1Rs;WRie^Plc~fDI{QcE|O8^4d?=6S}(S z))Qu<0gODL4isX}#kp%uoj)VifQ{NpL?fW*MwPz=U{HA&DljThQ0r^UL>wtn310>ByB02N{T z->Pu`;X{3t-!uI)M$=G}P{ci0IZnW~WxK;K9iA_f`nYOrd4aC^@zL4ML(k@fX*;{P zNMm^d;0llQ1{?$)115MFkYRCeKzkT^Z+PO57Y@povuLIfmwKMlK$>)cxD)WP-(+cr zb^xZjnY#N)U4_D*x2dTq^xl1xqiAjoQIY@I@oWnfg+DX!fhl`r2h1bn6<<;7jXeq~ zv?m{2e*tqR8Kd=qmZHh}Aur0y0D256rZZ$*HBaPM zt`q^Nx&iH8Y)T5JnVA_@mTYVP?|-=f#NTCvUAb}srHIwR2)0G@U43AFjvHx`ZoP6n z#$q<(3Y*BY1mn4v1-5ACA&J31$l<_rlnHt*vm3v4B1MJ5w9(APf|G{rCM*&XMb)p+=jTrLDOLW2>A8WT{V;+m>ZxNS4D zv$GRX1G`a1*FnS0Mn-(N*-F>)I^pV!a)e4M@Ii^<00;12+CA`wzP5>5HZn5O;?898 zrR#yh4u~74$u9ZNqYZaJJ5Q|?NNFq#bbf2_G)mX)#Bo5uRG2WA;7$OwehM^Z5PF$V zRaSKOWf`{(O8>@Gf2|6M@Eqtt1xHOK)JV)AWRjCR`kV6q(cYPd^_;JL{HG=oLzWU{ zDPyY{6^3k;6e=@Xi6Kj6WUGvflS(KOG9huspj8TyHI%g?yJ@6|lx)?cMk-0q>(0zM z&w0){&$*uEkLQo4>pI7Ej>_-%`+mQl`+dKc`zCbr8dX*4hncH=Ia&CWgl3Xw#gQ z97e{HEw#o>+K8L>;k2 z`%+Tcb0f0Tcoxes0C$nDJ?k46-#^?!_W0%I2CK*1UdAmSzh|m-R!7A^*R!EAqMs|z zqbj3>als2~+ zDB}X?e7r2O%@o+Q58lX~}9 z#IrvZB*&njlG524V{VV^&C#e(7g4)AoK)lyw$_EwmWA=>cIy@6zn`IzJicdV{`_L= zTi0*7fEOt=^E^G*^O=Tmw1yIWned=RD2>0WZZL32Qz4$2_zEsM+ci1B;#1L0TSEzZ z7G`EziFYwIu;I8=C$Tfr!E;7 zRN`3UxLM22QGR6`E>r$8L&FrMMK^X0cchP=qFITO?e41xg}t&Zo3wjW2RKnl9O`zNDlUU!<11*UH5Be=@bAHye-I+QB(g(8%qqpItA_9mBTgU&GBu%>|0`ho`~A5reM50#Gj=SmkL zu!})l^qY^qsS8AV-oC6$b`8|B{V;Yq9>>oO46g2bp1Gd;hD&45Vv!eRWm1&|Rt*{y zw!7;HxV2z)8LlwoOzggW>i}eg1Y0Y-U*7FQB5&M@$H`@gLaOcBtvz}2q|-B|?H3iz z!iJvnDr1x;R5LQ8p?A4qp7G$p%8LD8PoJ*CDb;Vo_c;GJX~xW*`%HUtX0mBrk+M+X z#aA}VA^|%`$2CWf9z9kFL%^~km4clNk$}9C^@*4pWTOfr7LFZD=X@|SHXh(J-Oyn1 z=dwAHMdRB0U=fsp-W*jqzrxpG$26-}<`nrpQ9o{7SNHnIQ@dXmKU!bbx~Vv~a#&IB z!5&V(;<8l$z#6>#5@b;+3Hbj^r1ro8_k-G-==IYEiJV!)7RIB6DjEb zYs*qGJj>(CLPQZVA;O*j4eKx+h81$n+tct81K;}7LS@5&!<6lMg_TD{|N*@TVh>)K}zT@}4P{BALu3+#6W)~;K>etr3e9Kh2qvz0xdkVEd7v4=rA z0{{9YiHTr`d zm@FVl*7JZP4!EXwLm!MPfGb^DMFQpm?jAPyq-OFbcq+A@fBHdjOZ~of%jd~OmFf7+ zg@2-c*^bimN`{ofIdBO}uR1aXZgJT-hc46SUCgt*=0PHhABJLOl-+IkrUn>|tr;P0 zFqZ6f9DThBr@(BF$3zF^ul8Atlg~%BrsMo>NiR$jjMvtk9JfX@0fKVg?Yo`FzW*k+ zVMmsoqhaCIY1TBX(MSjqn79pNL2p@?E z)8U}(sov7ZmVjLlILeKDQtAjnDICcZeihnb2$s$YQ6Q-N!hf@F=?{ehDn?Y zg1Z&pr1TPl$2wJYj`^(1@GTBBB$x1rP!QiwUaA+P|BQMj&g(Q7F(k>d`C4_bUoSR#3|F$Z zg9!_Fa8E3JbT;k$*8QPjzl*armm%D7W@&V9>Ad8uHbywU{tJTbKVZiDEO}q&!zqpX zGB&s2T|)y~|Ee~+hM*ycVaQ3h>lw{c`m?%+vzT3kaR316g7?R z-R#$Yq^LW}8X+MuBzi%6>0q;mnFz8D5Lr)}cOBNmK+=hOP##EVu>O6*?_wr%urZTF z7>JCjybhGZ>rxgZ(#_HSQMP47Se9%WN$j{{si8{#)LYlm_ds_GxED9EY>5FNd459( zf~{cgH9!XB>9`8)y|NfDZc@c>wdZI#I4G3Cll1Lyrkwynn{#hcp_pJXTh0LrCLTvMBhr? zj3i>`&H)v9zcbj}%WEPcI)H1!xsS%G>i%-%@Zs~afu^ji4X;@4TZK2CM@B_lxPM=Z zHfub4m4qUhI+5-sn1|!{ooD2|Iw_=mJ9YZB#mt#KQL^RS1zh&gE@8DYYIJAmha}2d z`Dip|(3mSbsGfvUFg-|UlXc(FroiE_1C(Ir@&_3?(%ctGm?C`tsiCtXamV**Rf1jpe;oo zRzeWiQXw2O_?v56L|=5Cd7m;Y?Aj}T_->dab0CdmnOLYKo+3UN*>@P;W3>KH*T03t z5Cc%d+YkUX;oc2+lw}8^O@YK?Af1cpR%KR8+}9p+OJDc zd2+jv!EHlRA913Ih?ej&8THS!r922T3_}(I@R~YOMihyi4}1n~5Or#$Uwc)*Z}BCz zR**YC7=xQNX3$%zpR75n!VfTL!Qc_@Ke{}tk9HZkB=L&>F4wsgnhBiF((-_Q5n-($ zFn~~O5Dev7%QC{`bL6wDsy$__k|mec! zL9R*_GEoLo><4(-=Ax-`>^rIIT4En?}Mo;4{*p@N4;UbPJ zrfph#@b=Iu-`@435tjDV!x(B-_I2h5((84h5Ei517gOn!f^gXDzuCe0Xk}+-7yY1m zb1B7zcLYmhNZRrX{w9;9)sF5HQ-19itPT=U3X%@Q;%wbiC3OJt)>kwId_z`PR+3- z@dUsD$Ss_mH54K%Ggo`2P_FKKgYRcF5`U&r;TFAs1Ty1&Jldvi4yAZnjxJwjv6MtfYinb`5!@2_6=ziOI<&Tt_=Je(&Lvtrw3NsGqhx?JP=-%9hZJuax`W zo;l5YJqt>S_sGNU=1Z0qjs1{u--g7C1@*fx>t) zQG^R9XyXYJ1z?3|agUL!G&4fw5?dj+-PVHPOP;Di*Wvo2%B{)Eii@c(&S{8RIXR?p zmzwt0_W{+}&v>B6aD2GeRBt^J^R8bGhOET9XI2hVw%*s2*?6X_^vm{Xsf!<}-B?jG z{x{dg*+0i@atf@!AHAe(FlUB9K-nMmhps|Lxf$#vw6|=kAD7jX-oD>wpIs@qgQ!79 z>VVuc!jVab7&(LRGo_FSSu2H%1H|dr4;r`4rY{%>00tLjnv*?#5&b9o*0$x|d~oI5 zk%m%VHbyG-duYh+V2MrIv$Gh^=gRh|Z`Bx>V#GWbDaSxo=K5?nc90KUf`L5yB9#*Q z$$2Nz{qf_z(O1XMuI{ua%B%m%n&)pu2-afdq&FX*Qnm-cFzLjLMkei3pSR4*Ju-Ou z*~X&cigl)rbEALCGg6r6rj%8^xmvR1r7Uz_UiVtpwEk(QW-Vj0={r?gig^ULA;DxV z;&g`A%A+~DPn+dcP6>abNet(f#q-li?692v67(0ZHLB#x>Yw26WMmC#wl?BvhKVSG zPe7_@1$7;Q{9r--5U?QO9&+j>AQBm6gHNf7q{rDBEvXt3`-8a>yT71Q=zpst4f5{^ z!zIP|>6H#N9%34ife7BuZ+4eH1S&I>#n4<*l#pK{Y4l7t!`>Gk)l3FhT2^k* zOmB#t*8RJA^A?m`UD-Y-US=BcoNJuQ=&5MgRx5}b4Og-<2gXGR&Txv38;=2-E)DJS z09LewblMm2R{@jM@wd9Uxh*Zf-+{A>na3A-P4SK57G;`=Y!E~qv*5=;`fY?#lV^;1 z0SX1&P!>G0X!Ue;C4dDJ?;-`%^<%%bT$yk#k|?=?_5>7CE0gXAaG|{^fU?h-%djpz zeKftxYiby%OxA!(K80V3n5-<_5U5E)q0vc6VMMTGHU|7f3-~JuArp+MJs3U~gDiZC z%+C_sDh(S>g#NxGk;Be&6UY=DK7mVwYVlp{Y7yPh5pQO}U%Z*r)5zr`Y67-@Bo#ti z+iuU9R&!#t<4Hd|XSc;)SypbBgLU}uu2`mNEA)>p>se^K%JujwyPt3OKRWbflkmrM zG5`(_t92GOP7f;$Zsk4d;bpPUWPJosd7-w_(j z|BtBF+FAWX+tMk`s717mb)H?PKcy+r_0FsGZ1dpRaqpU^W+tb9)308lvUJlBgAqiK z^z`&Uk-7G{L!9meOmutE&C|lRHnT)Y``8ex^!7=;OYVG^xW7xIJJXJ1wlHlb>4kb| zl1|@VPM*#wu~YU=w4E};W&-D8=!?QjBYly0wuRmb)b*>asy*i4Y@~Bo{oX6;=Jo6T zUwx&QG_U^LzLY?NQIVfE%gAw`01L!^j>|r%%i~VmKKds5dVV|AWgECGo{WgFKk|6l zB(HGsXGCnl8KM-lEm-}|nX4#F?*igeS>6yDI>8#2`}+0kvPYfvX9_Bsli-P){@VsO zdp~UwAdoug==ph7^<64&TJImZQJ1&BGV-$~TQ4Lz1C!?Q&iWHS{Hfcy1Fvu30$(Q! zn-!E{*9#G1f3^4NVHVhS?AS|T?P47qR1>@^H)4Tmc0cFkO?egN7*D?k*?-dOF-ziC z-F+AE*n8Ktp4q3~`)rI{Q!@R7Nmh=%M)p+IRp?i-_$KY_pA@$BT0OW|B_j+qa*@O@ zz8hv_lsL$%UFD$g@9ImmqRhgNENT8FF2j=wSmRw%VYKLPG(I%F*jJU6l{4>*vCDLH zcD8~S+MAG-T%FS?JB@#sf8S-e$<5NuZU=O9zV*}a`h9wJ{1bK|%h$hfK~?W|Ar#9h z*J!**q!*sh;Ah@z-@5e}CJi7-t*6DC&Bz9uuw&KwH?&%*sEmoW>+FJ{nQ%GOU=NE} zV_`VO%=a<6GxN+XI1;J5Gy?2KSEJ*J?WlL~tj)+Qdc%+m>RiTwIDQnSH=6&z8IH#X za0g`t&ZzWwLsdZ9(?{+b>ocbNaj1;t$c;1Xy71?OPh}j{oZ2U4I~09;Rfha$nk5Nu zqE2$|$j||d6*7;9mUNrWpGxfuj9a?;8!V)oS$GGp@SNPpN#n+~B#{(G5sXRNickR=)gomny`6&N7J8OIA(Xh=*K_-}mB7EyI-Y>Hml;uBjpf;SV#HK7i0;e$VMmLDm>s0gjd#^!R zvcsOXo+z;tW$w9Xs{ARxnxsJ%fnJ?!rrjh6NgXlhpOln|ag`QeFquS|czi!UKN`7F z;(b=@CJ#{6H4>vf{z(vz)xDC2>K_eWYW)^ae?!6mbb2UXmqo>-8A9+(ynWigyQ8Vt zsJ1~ST)7vbu9EX8!8CI5b)laqowjA5cbM(L!eq$6%;_cmk2}BpZoq+Ek2cczmY0{u z)j<3gQBGY>EbPk24VP6B&h4b4K>+w*<;lBO%F7t=@n!IOWh(>@8Vgn*qlkDju10UG zdPaEbqP7zvr^Jr?6uw5YZmF&a&->Kti744RH(#vx`kX(E+FiZpSmcoy*K70V&rd|j zSFv)gIlj2?&ncxHWlSmJ!CLMYIj8QQ_MaIzIKFGi6rM(RHoeRESsHF(^TZU6oYr7b zR_!~nen*g#55^gx7dV97OND~tLPj?r7$lfC!W-z7ovY+|p4lhZoMAj1>N&kH*G{%i zDdqQ3n$uUuB)tQ!)=wL_R$WQnJiL*}aTo61tv=rHIv{N+Kiy`*n#ybO{esWqb!tvzuIhyj*&jPvuceILVKf5`Y+LQ26W<<7fVOFk>F>z$syWgr z%b(F`@I0~h_Um5XG^o&OXlU5LO|k*Sj^QSTrmRDd@8HDHf2FP;!f`I-tNfKhY$c)m z4@uV27UHdl$KAcdsRB>i$?N%7Sv&X` zh5^lmBw*0V=kN>$GA9*JwB_>Ur_@)cxaoHz=W-dE$5I|^ekU%c?W4nPqo@&?zVfQ_ zVVWa1lYDLxx86N*o4IG!a{M1L+!L5H$C7;uU0is2J+1r{^V~?=54~s3oKc;9FFSiY z?!qf?*N2655(Pto}nGLs!LSkNw@@`qi!rnkG3u}7HKh>f(!oz<*X zm4aJY#w-NHhQcOl0dTVwAXW;d0B6#nVjKHqMcN{Kc0;@#>EMMJcx!C$m|F9{7M62& zJ_r?0k8qnQo?@hex{%3}7{6y`MP!_lf78ANyJjhT-2BrCSOneWJ~Xu%u=St;X-Rl>8V>E#;=z5Tz@U`Bf620o1}+ zpO?+xh|rw8vmvij1E3*i&7U7eK`KT!7gm(>S)ri*&&yBk!vl)ZF&KdKgd?DI*>%e| zfpye&F~dsR4Vad9BFn|;%ajzjQrhwS((UN+a;YCePf(p&!l9{!XUuOlKY571hFN9j z*$|+%u#8JBat%kBX4<*ar$-=yQ<65})e|E%B$J5vjLQ+yxqs)d>QSHQB^47klOj_7 zFs+k#LikW5v&8Y-8<&}479Kd#iPv`x?}XW37#q~4!yG`ohF5mUFBAfqkNzGmDZ>h$ zO^jhfVLxtf1w#*JzIe;$VdLvs179q3;)F?)##0w%xE@|HDWSxXGEdlAIyne;sjkE% zgC8QzR({X{m$Xt)^V*Al51XC%u{e`T2>naM6uhh&sP546;~n(x5+UCd2h2>rRZ?0# zrOBI*Mhk2%-XRQYVgwP3*ZB=UJ}LZ$kVUPLOV=0m5pf?T7M z+Y8i8T?C5$dqSqD=Sn?4H_jKHDl7Vk5dfBXKT$W7u3CTaBgB=KU1wS(9)G94+y!eT zlfW~ma)QySGB-hM)bhgz9>Q=#eW+~N?9XqaY_|l(v}@I>6^jbur_bpZlT?B_&)F+A zNzsem4cc26m46}f6-lU%pEPDXk(fWYQdwdR;q<>m5=^iC!JOvXF`k5D-$tKXGdV|9 zu>QQqm|XX`>BBmR8<~s4tl5@KJ&}#`LUO=H_j!f>iAmzRIq3D{xCn1t$Ue!mx{7~VZKIEZQ{ z(cndkD%3_EO0v`qJM9t`8v4kV_|ft=FYKcq_bBJ3pxG&vBI=PWQBsIq27GGC%h_!j zpoTFTSITV4MC!F}V12=zFS=-d`0bXNh~nG|SZ$PX7m3b^31?N9n%PLae%C+BSY$6= zyAcQfC!N|~zUhA@fcV#wU+)jE;~#eY%H2yoxqJU$u$6)@Ypbs< J4_VAw{cq}x%@P0r literal 0 HcmV?d00001 diff --git a/examples/model_compress/models/cifar10/resnet.py b/examples/model_compress/models/cifar10/resnet.py new file mode 100644 index 0000000000..386ff8321c --- /dev/null +++ b/examples/model_compress/models/cifar10/resnet.py @@ -0,0 +1,115 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, in_planes, planes, stride=1): + super(BasicBlock, self).__init__() + self.conv1 = nn.Conv2d( + in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, + stride=1, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion*planes: + self.shortcut = nn.Sequential( + nn.Conv2d(in_planes, self.expansion*planes, + kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(self.expansion*planes) + ) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = self.bn2(self.conv2(out)) + out += self.shortcut(x) + out = F.relu(out) + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, in_planes, planes, stride=1): + super(Bottleneck, self).__init__() + self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, + stride=stride, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.conv3 = nn.Conv2d(planes, self.expansion * + planes, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d(self.expansion*planes) + + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion*planes: + self.shortcut = nn.Sequential( + nn.Conv2d(in_planes, self.expansion*planes, + kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(self.expansion*planes) + ) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = F.relu(self.bn2(self.conv2(out))) + out = self.bn3(self.conv3(out)) + out += self.shortcut(x) + out = F.relu(out) + return out + + +class ResNet(nn.Module): + def __init__(self, block, num_blocks, num_classes=10): + super(ResNet, self).__init__() + self.in_planes = 64 + # this layer is different from torchvision.resnet18() since this model adopted for Cifar10 + self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(64) + self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) + self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2) + self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2) + self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) + self.linear = nn.Linear(512*block.expansion, num_classes) + + def _make_layer(self, block, planes, num_blocks, stride): + strides = [stride] + [1]*(num_blocks-1) + layers = [] + for stride in strides: + layers.append(block(self.in_planes, planes, stride)) + self.in_planes = planes * block.expansion + return nn.Sequential(*layers) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = self.layer1(out) + out = self.layer2(out) + out = self.layer3(out) + out = self.layer4(out) + out = F.avg_pool2d(out, 4) + out = out.view(out.size(0), -1) + out = self.linear(out) + return out + + +def ResNet18(): + return ResNet(BasicBlock, [2, 2, 2, 2]) + + +def ResNet34(): + return ResNet(BasicBlock, [3, 4, 6, 3]) + + +def ResNet50(): + return ResNet(Bottleneck, [3, 4, 6, 3]) + + +def ResNet101(): + return ResNet(Bottleneck, [3, 4, 23, 3]) + + +def ResNet152(): + return ResNet(Bottleneck, [3, 8, 36, 3]) From 995f625963a9e6bf76033e1cc8e7dcd4df3dbf65 Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Tue, 11 Aug 2020 18:06:41 +0800 Subject: [PATCH 09/28] Enable gpu scheduler in AML mode (#2769) --- docs/en_US/TrainingService/AMLMode.md | 18 +++++++++++------- .../rest_server/restValidationSchemas.ts | 6 ++++-- .../training_service/reusable/aml/amlConfig.ts | 13 +++++++++---- .../environments/amlEnvironmentService.ts | 4 +++- tools/nni_cmd/config_schema.py | 5 ++++- 5 files changed, 31 insertions(+), 15 deletions(-) diff --git a/docs/en_US/TrainingService/AMLMode.md b/docs/en_US/TrainingService/AMLMode.md index 0907f29a02..b365375b1b 100644 --- a/docs/en_US/TrainingService/AMLMode.md +++ b/docs/en_US/TrainingService/AMLMode.md @@ -49,30 +49,34 @@ tuner: trial: command: python3 mnist.py codeDir: . - computeTarget: ${replace_to_your_computeTarget} image: msranni/nni + gpuNum: 1 amlConfig: subscriptionId: ${replace_to_your_subscriptionId} resourceGroup: ${replace_to_your_resourceGroup} workspaceName: ${replace_to_your_workspaceName} - + computeTarget: ${replace_to_your_computeTarget} ``` Note: You should set `trainingServicePlatform: aml` in NNI config YAML file if you want to start experiment in aml mode. Compared with [LocalMode](LocalMode.md) trial configuration in aml mode have these additional keys: -* computeTarget - * required key. The compute cluster name you want to use in your AML workspace. See Step 6. * image * required key. The docker image name used in job. The image `msranni/nni` of this example only support GPU computeTargets. amlConfig: * subscriptionId - * the subscriptionId of your account + * required key, the subscriptionId of your account * resourceGroup - * the resourceGroup of your account + * required key, the resourceGroup of your account * workspaceName - * the workspaceName of your account + * required key, the workspaceName of your account +* computeTarget + * required key, the compute cluster name you want to use in your AML workspace. See Step 6. +* maxTrialNumPerGpu + * optional key, used to specify the max concurrency trial number on a GPU device. +* useActiveGpu + * optional key, used to specify whether to use a GPU if there is another process. By default, NNI will use the GPU only if there is no other active process in the GPU. The required information of amlConfig could be found in the downloaded `config.json` in Step 5. diff --git a/src/nni_manager/rest_server/restValidationSchemas.ts b/src/nni_manager/rest_server/restValidationSchemas.ts index b845dcc30e..cb1a1282e7 100644 --- a/src/nni_manager/rest_server/restValidationSchemas.ts +++ b/src/nni_manager/rest_server/restValidationSchemas.ts @@ -39,7 +39,6 @@ export namespace ValidationSchemas { nniManagerNFSMountPath: joi.string().min(1), containerNFSMountPath: joi.string().min(1), paiConfigPath: joi.string(), - computeTarget: joi.string(), nodeCount: joi.number(), paiStorageConfigName: joi.string().min(1), nasMode: joi.string().valid('classic_mode', 'enas_mode', 'oneshot_mode', 'darts_mode'), @@ -159,7 +158,10 @@ export namespace ValidationSchemas { aml_config: joi.object({ // eslint-disable-line @typescript-eslint/camelcase subscriptionId: joi.string().min(1), resourceGroup: joi.string().min(1), - workspaceName: joi.string().min(1) + workspaceName: joi.string().min(1), + computeTarget: joi.string().min(1), + maxTrialNumPerGpu: joi.number(), + useActiveGpu: joi.boolean() }), nni_manager_ip: joi.object({ // eslint-disable-line @typescript-eslint/camelcase nniManagerIp: joi.string().min(1) diff --git a/src/nni_manager/training_service/reusable/aml/amlConfig.ts b/src/nni_manager/training_service/reusable/aml/amlConfig.ts index dd8c2345d4..eceea3f6db 100644 --- a/src/nni_manager/training_service/reusable/aml/amlConfig.ts +++ b/src/nni_manager/training_service/reusable/aml/amlConfig.ts @@ -11,11 +11,18 @@ export class AMLClusterConfig { public readonly subscriptionId: string; public readonly resourceGroup: string; public readonly workspaceName: string; + public readonly computeTarget: string; + public useActiveGpu?: boolean; + public maxTrialNumPerGpu?: number; - constructor(subscriptionId: string, resourceGroup: string, workspaceName: string) { + constructor(subscriptionId: string, resourceGroup: string, workspaceName: string, computeTarget: string, + useActiveGpu?: boolean, maxTrialNumPerGpu?: number) { this.subscriptionId = subscriptionId; this.resourceGroup = resourceGroup; this.workspaceName = workspaceName; + this.computeTarget = computeTarget; + this.useActiveGpu = useActiveGpu; + this.maxTrialNumPerGpu = maxTrialNumPerGpu; } } @@ -23,14 +30,12 @@ export class AMLTrialConfig extends TrialConfig { public readonly image: string; public readonly command: string; public readonly codeDir: string; - public readonly computeTarget: string; - constructor(codeDir: string, command: string, image: string, computeTarget: string) { + constructor(codeDir: string, command: string, image: string) { super("", codeDir, 0); this.codeDir = codeDir; this.command = command; this.image = image; - this.computeTarget = computeTarget; } } diff --git a/src/nni_manager/training_service/reusable/environments/amlEnvironmentService.ts b/src/nni_manager/training_service/reusable/environments/amlEnvironmentService.ts index aefd94fbb5..7a26d0e21c 100644 --- a/src/nni_manager/training_service/reusable/environments/amlEnvironmentService.ts +++ b/src/nni_manager/training_service/reusable/environments/amlEnvironmentService.ts @@ -112,13 +112,15 @@ export class AMLEnvironmentService extends EnvironmentService { const amlEnvironment: AMLEnvironmentInformation = environment as AMLEnvironmentInformation; const environmentLocalTempFolder = path.join(this.experimentRootDir, this.experimentId, "environment-temp"); environment.command = `import os\nos.system('${amlEnvironment.command}')`; + environment.useActiveGpu = this.amlClusterConfig.useActiveGpu; + environment.maxTrialNumberPerGpu = this.amlClusterConfig.maxTrialNumPerGpu; await fs.promises.writeFile(path.join(environmentLocalTempFolder, 'nni_script.py'), amlEnvironment.command, { encoding: 'utf8' }); const amlClient = new AMLClient( this.amlClusterConfig.subscriptionId, this.amlClusterConfig.resourceGroup, this.amlClusterConfig.workspaceName, this.experimentId, - this.amlTrialConfig.computeTarget, + this.amlClusterConfig.computeTarget, this.amlTrialConfig.image, 'nni_script.py', environmentLocalTempFolder diff --git a/tools/nni_cmd/config_schema.py b/tools/nni_cmd/config_schema.py index 631abcf5cc..f6aee3a450 100644 --- a/tools/nni_cmd/config_schema.py +++ b/tools/nni_cmd/config_schema.py @@ -245,7 +245,7 @@ def validate(self, data): 'codeDir': setPathCheck('codeDir'), 'command': setType('command', str), 'image': setType('image', str), - 'computeTarget': setType('computeTarget', str) + Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999), } } @@ -254,6 +254,9 @@ def validate(self, data): 'subscriptionId': setType('subscriptionId', str), 'resourceGroup': setType('resourceGroup', str), 'workspaceName': setType('workspaceName', str), + 'computeTarget': setType('computeTarget', str), + Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int), + Optional('useActiveGpu'): setType('useActiveGpu', bool), } } From 59b76c2ec6e66d041fdd23826d8d36a6f0fb2767 Mon Sep 17 00:00:00 2001 From: Lijiaoa <61399850+Lijiaoa@users.noreply.github.com> Date: Tue, 11 Aug 2020 18:09:10 +0800 Subject: [PATCH 10/28] Some little style (#2762) Co-authored-by: Lijiao <15910218274@163.com> --- src/webui/src/components/Modals/Compare.tsx | 15 ++++++++--- src/webui/src/components/Overview.tsx | 27 ++++++++++--------- .../src/components/trial-detail/TableList.tsx | 1 + src/webui/src/static/function.ts | 3 ++- src/webui/src/static/style/compare.scss | 7 +++-- src/webui/src/static/style/overview.scss | 2 +- src/webui/src/static/style/succTable.scss | 2 +- src/webui/src/static/style/table.scss | 4 +-- 8 files changed, 38 insertions(+), 23 deletions(-) diff --git a/src/webui/src/components/Modals/Compare.tsx b/src/webui/src/components/Modals/Compare.tsx index ac73f860bc..44f7afd0fb 100644 --- a/src/webui/src/components/Modals/Compare.tsx +++ b/src/webui/src/components/Modals/Compare.tsx @@ -1,11 +1,17 @@ import * as React from 'react'; -import { Stack, Modal, IconButton } from 'office-ui-fabric-react'; +import { Stack, Modal, IconButton, IDragOptions, ContextualMenu } from 'office-ui-fabric-react'; import ReactEcharts from 'echarts-for-react'; import IntermediateVal from '../public-child/IntermediateVal'; import { TRIALS } from '../../static/datamodel'; +import { TableRecord, Intermedia, TooltipForIntermediate } from '../../static/interface'; import { contentStyles, iconButtonStyles } from '../Buttons/ModalTheme'; import '../../static/style/compare.scss'; -import { TableRecord, Intermedia, TooltipForIntermediate } from '../../static/interface'; + +const dragOptions: IDragOptions = { + moveMenuItemText: 'Move', + closeMenuItemText: 'Close', + menu: ContextualMenu +}; // the modal of trial compare interface CompareProps { @@ -79,7 +85,8 @@ class Compare extends React.Component { containLabel: true }, legend: { - data: idsList + // more than 10 trials will hide legend + data: idsList.length > 10 ? null : idsList }, xAxis: { type: 'category', @@ -209,6 +216,8 @@ class Compare extends React.Component { isOpen={true} containerClassName={contentStyles.container} className="compare-modal" + allowTouchBodyScroll={true} + dragOptions={dragOptions} >

diff --git a/src/webui/src/components/Overview.tsx b/src/webui/src/components/Overview.tsx index 54e5f5ba5c..b2e00efae7 100644 --- a/src/webui/src/components/Overview.tsx +++ b/src/webui/src/components/Overview.tsx @@ -12,6 +12,18 @@ import TrialInfo from './overview/TrialProfile'; import '../static/style/overview.scss'; import '../static/style/logPath.scss'; +const stackTokens: IStackTokens = { + childrenGap: 30, +}; + +const entriesOption = [ + { key: '10', text: 'Display top 10 trials' }, + { key: '20', text: 'Display top 20 trials' }, + { key: '30', text: 'Display top 30 trials' }, + { key: '50', text: 'Display top 50 trials' }, + { key: '100', text: 'Display top 100 trials' } +]; + interface OverviewProps { experimentUpdateBroadcast: number; trialsUpdateBroadcast: number; @@ -70,17 +82,6 @@ class Overview extends React.Component { const titleMaxbgcolor = (metricGraphMode === 'max' ? '#333' : '#b3b3b3'); const titleMinbgcolor = (metricGraphMode === 'min' ? '#333' : '#b3b3b3'); - const stackTokens: IStackTokens = { - childrenGap: 30, - }; - - const entriesOption = [ - { key: '10', text: 'Display top 10 trials' }, - { key: '20', text: 'Display top 20 trials' }, - { key: '30', text: 'Display top 30 trials' }, - { key: '50', text: 'Display top 50 trials' }, - { key: '100', text: 'Display top 100 trials' } - ]; return (
{/* status and experiment block */} @@ -123,7 +124,7 @@ class Overview extends React.Component { - +
{ >
-
+
{ style={{ width: 0.5 * modalIntermediateWidth, height: 0.7 * modalIntermediateHeight, + maxHeight: 534, padding: 20 }} theme="my_theme" diff --git a/src/webui/src/static/function.ts b/src/webui/src/static/function.ts index 485585e224..221d4da0e2 100644 --- a/src/webui/src/static/function.ts +++ b/src/webui/src/static/function.ts @@ -131,7 +131,8 @@ const intermediateGraphOption = (intermediateArr: number[], id: string): any => yAxis: { name: 'Default metric', type: 'value', - data: intermediateArr + data: intermediateArr, + scale: true }, series: [{ symbolSize: 6, diff --git a/src/webui/src/static/style/compare.scss b/src/webui/src/static/style/compare.scss index ba45ccac98..37f70a49c4 100644 --- a/src/webui/src/static/style/compare.scss +++ b/src/webui/src/static/style/compare.scss @@ -1,14 +1,17 @@ .compare-modal{ /* decide modal size */ .ms-Dialog-main{ - max-width: 70%; + width: 50%; + overflow: hidden; } /* compare-md: table style */ &-table{ width: 92%; - table-layout: fixed; margin: 0 auto; + margin-bottom: 20px; + border: 1px solid transparent; + overflow: auto; color: #333; tr{ line-height: 30px; diff --git a/src/webui/src/static/style/overview.scss b/src/webui/src/static/style/overview.scss index f636424fdd..162c878e5c 100644 --- a/src/webui/src/static/style/overview.scss +++ b/src/webui/src/static/style/overview.scss @@ -12,7 +12,7 @@ padding: 15px 20px; height: 100%; min-width: 500px; - overflow-y: scroll; + overflow-y: auto; } .padItem{ diff --git a/src/webui/src/static/style/succTable.scss b/src/webui/src/static/style/succTable.scss index 3e2dbdfa86..05b37035bd 100644 --- a/src/webui/src/static/style/succTable.scss +++ b/src/webui/src/static/style/succTable.scss @@ -1,6 +1,6 @@ #succTable{ height: 404px; - overflow-y: scroll; + overflow: auto; position: relative; .succTable-tooltip{ position: absolute; diff --git a/src/webui/src/static/style/table.scss b/src/webui/src/static/style/table.scss index d8f57dd424..25ca54e3e0 100644 --- a/src/webui/src/static/style/table.scss +++ b/src/webui/src/static/style/table.scss @@ -7,7 +7,7 @@ height: 324px; overflow: hidden; #succeTable .commonTableStyle{ - overflow-y: scroll; + overflow-y: auto; } } @@ -55,5 +55,5 @@ } .columns-height{ max-height: 335px; - overflow-y: scroll; + overflow-y: auto; } From 8961d7a5d3663bf1301317b61a75ca4f9401ce97 Mon Sep 17 00:00:00 2001 From: Ningxin Zheng <49771382+zheng-ningxin@users.noreply.github.com> Date: Tue, 11 Aug 2020 19:33:30 +0800 Subject: [PATCH 11/28] Sensitivity pruner (#2684) --- docs/en_US/Compressor/Pruner.md | 33 ++ .../nni/compression/torch/pruning/__init__.py | 1 + .../torch/pruning/sensitivity_pruner.py | 397 ++++++++++++++++++ .../torch/utils/sensitivity_analysis.py | 11 +- 4 files changed, 433 insertions(+), 9 deletions(-) create mode 100644 src/sdk/pynni/nni/compression/torch/pruning/sensitivity_pruner.py diff --git a/docs/en_US/Compressor/Pruner.md b/docs/en_US/Compressor/Pruner.md index 824d8625b3..e059834eca 100644 --- a/docs/en_US/Compressor/Pruner.md +++ b/docs/en_US/Compressor/Pruner.md @@ -20,6 +20,7 @@ We provide several pruning algorithms that support fine-grained weight pruning a * [NetAdapt Pruner](#netadapt-pruner) * [SimulatedAnnealing Pruner](#simulatedannealing-pruner) * [AutoCompress Pruner](#autocompress-pruner) +* [Sensitivity Pruner](#sensitivity-pruner) **Others** * [ADMM Pruner](#admm-pruner) @@ -588,3 +589,35 @@ We try to reproduce the experiment result of the fully connected network on MNIS ![](../../img/lottery_ticket_mnist_fc.png) The above figure shows the result of the fully connected network. `round0-sparsity-0.0` is the performance without pruning. Consistent with the paper, pruning around 80% also obtain similar performance compared to non-pruning, and converges a little faster. If pruning too much, e.g., larger than 94%, the accuracy becomes lower and convergence becomes a little slower. A little different from the paper, the trend of the data in the paper is relatively more clear. + + +## Sensitivity Pruner +For each round, SensitivityPruner prunes the model based on the sensitivity to the accuracy of each layer until meeting the final configured sparsity of the whole model: + 1. Analyze the sensitivity of each layer in the current state of the model. + 2. Prune each layer according to the sensitivity. + +For more details, please refer to [Learning both Weights and Connections for Efficient Neural Networks ](https://arxiv.org/abs/1506.02626). + +#### Usage + +PyTorch code + +```python +from nni.compression.torch import SensitivityPruner +config_list = [{ + 'sparsity': 0.5, + 'op_types': ['Conv2d'] + }] +pruner = SensitivityPruner(model, config_list, finetuner=fine_tuner, evaluator=evaluator) +# eval_args and finetune_args are the parameters passed to the evaluator and finetuner respectively +pruner.compress(eval_args=[model], finetune_args=[model]) +``` + + +#### User configuration for Sensitivity Pruner + +##### PyTorch + +```eval_rst +.. autoclass:: nni.compression.torch.SensitivityPruner +``` \ No newline at end of file diff --git a/src/sdk/pynni/nni/compression/torch/pruning/__init__.py b/src/sdk/pynni/nni/compression/torch/pruning/__init__.py index a6977d634e..919ffb3fd3 100644 --- a/src/sdk/pynni/nni/compression/torch/pruning/__init__.py +++ b/src/sdk/pynni/nni/compression/torch/pruning/__init__.py @@ -11,3 +11,4 @@ from .net_adapt_pruner import NetAdaptPruner from .admm_pruner import ADMMPruner from .auto_compress_pruner import AutoCompressPruner +from .sensitivity_pruner import SensitivityPruner diff --git a/src/sdk/pynni/nni/compression/torch/pruning/sensitivity_pruner.py b/src/sdk/pynni/nni/compression/torch/pruning/sensitivity_pruner.py new file mode 100644 index 0000000000..eee975c770 --- /dev/null +++ b/src/sdk/pynni/nni/compression/torch/pruning/sensitivity_pruner.py @@ -0,0 +1,397 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. +import os +import csv +import copy +import json +import logging +import torch + +from schema import And, Optional +from ..compressor import Pruner +from ..utils.config_validation import CompressorSchema +from .constants_pruner import PRUNER_DICT +from ..utils.sensitivity_analysis import SensitivityAnalysis + + +MAX_PRUNE_RATIO_PER_ITER = 0.95 + +_logger = logging.getLogger('Sensitivity_Pruner') + + +class SensitivityPruner(Pruner): + """ + This function prune the model based on the sensitivity + for each layer. + + Parameters + ---------- + model: torch.nn.Module + model to be compressed + evaluator: function + validation function for the model. This function should return the accuracy + of the validation dataset. The input parameters of evaluator can be specified + in the parameter `eval_args` and 'eval_kwargs' of the compress function if needed. + Example: + >>> def evaluator(model): + >>> device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + >>> val_loader = ... + >>> model.eval() + >>> correct = 0 + >>> with torch.no_grad(): + >>> for data, target in val_loader: + >>> data, target = data.to(device), target.to(device) + >>> output = model(data) + >>> # get the index of the max log-probability + >>> pred = output.argmax(dim=1, keepdim=True) + >>> correct += pred.eq(target.view_as(pred)).sum().item() + >>> accuracy = correct / len(val_loader.dataset) + >>> return accuracy + finetuner: function + finetune function for the model. This parameter is not essential, if is not None, + the sensitivity pruner will finetune the model after pruning in each iteration. + The input parameters of finetuner can be specified in the parameter of compress + called `finetune_args` and `finetune_kwargs` if needed. + Example: + >>> def finetuner(model, epoch=3): + >>> device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + >>> train_loader = ... + >>> criterion = torch.nn.CrossEntropyLoss() + >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.01) + >>> model.train() + >>> for _ in range(epoch): + >>> for _, (data, target) in enumerate(train_loader): + >>> data, target = data.to(device), target.to(device) + >>> optimizer.zero_grad() + >>> output = model(data) + >>> loss = criterion(output, target) + >>> loss.backward() + >>> optimizer.step() + base_algo: str + base pruning algorithm. `level`, `l1` or `l2`, by default `l1`. + sparsity_proportion_calc: function + This function generate the sparsity proportion between the conv layers according to the + sensitivity analysis results. We provide a default function to quantify the sparsity + proportion according to the sensitivity analysis results. Users can also customize + this function according to their needs. The input of this function is a dict, + for example : {'conv1' : {0.1: 0.9, 0.2 : 0.8}, 'conv2' : {0.1: 0.9, 0.2 : 0.8}}, + in which, 'conv1' and is the name of the conv layer, and 0.1:0.9 means when the + sparsity of conv1 is 0.1 (10%), the model's val accuracy equals to 0.9. + sparsity_per_iter: float + The sparsity of the model that the pruner try to prune in each iteration. + acc_drop_threshold : float + The hyperparameter used to quantifiy the sensitivity for each layer. + checkpoint_dir: str + The dir path to save the checkpoints during the pruning. + """ + + def __init__(self, model, config_list, evaluator, + finetuner=None, base_algo='l1', sparsity_proportion_calc=None, + sparsity_per_iter=0.1, acc_drop_threshold=0.05, checkpoint_dir=None): + + self.base_algo = base_algo + self.model = model + super(SensitivityPruner, self).__init__(model, config_list) + # unwrap the model + self._unwrap_model() + _logger.debug(str(self.model)) + self.evaluator = evaluator + self.finetuner = finetuner + self.analyzer = SensitivityAnalysis( + self.model, self.evaluator, prune_type=base_algo, \ + early_stop_mode='dropped', early_stop_value=acc_drop_threshold) + # Get the original accuracy of the pretrained model + self.ori_acc = None + # Copy the original weights before pruning + self.ori_state_dict = copy.deepcopy(self.model.state_dict()) + self.sensitivities = {} + # Save the weight count for each layer + self.weight_count = {} + self.weight_sum = 0 + # Map the layer name to the layer module + self.named_module = {} + + self.Pruner = PRUNER_DICT[self.base_algo] + # Count the total weight count of the model + for name, submodule in self.model.named_modules(): + self.named_module[name] = submodule + if name in self.analyzer.target_layer: + # Currently, only count the weights in the conv layers + # else the fully connected layer (which contains + # the most weights) may make the pruner prune the + # model too hard + # if hasattr(submodule, 'weight'): # Count all the weights of the model + self.weight_count[name] = submodule.weight.data.numel() + self.weight_sum += self.weight_count[name] + # function to generate the sparsity proportion between the conv layers + if sparsity_proportion_calc is None: + self.sparsity_proportion_calc = self._max_prune_ratio + else: + self.sparsity_proportion_calc = sparsity_proportion_calc + # The ratio of remained weights is 1.0 at the begining + self.remained_ratio = 1.0 + self.sparsity_per_iter = sparsity_per_iter + self.acc_drop_threshold = acc_drop_threshold + self.checkpoint_dir = checkpoint_dir + + def validate_config(self, model, config_list): + """ + Parameters + ---------- + model : torch.nn.module + Model to be pruned + config_list : list + List on pruning configs + """ + + if self.base_algo == 'level': + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + Optional('op_types'): [str], + Optional('op_names'): [str], + }], model, _logger) + elif self.base_algo in ['l1', 'l2']: + schema = CompressorSchema([{ + 'sparsity': And(float, lambda n: 0 < n < 1), + 'op_types': ['Conv2d'], + Optional('op_names'): [str] + }], model, _logger) + + schema.validate(config_list) + + def load_sensitivity(self, filepath): + """ + load the sensitivity results exported by the sensitivity analyzer + """ + assert os.path.exists(filepath) + with open(filepath, 'r') as csvf: + csv_r = csv.reader(csvf) + header = next(csv_r) + sparsities = [float(x) for x in header[1:]] + sensitivities = {} + for row in csv_r: + layername = row[0] + accuracies = [float(x) for x in row[1:]] + sensitivities[layername] = {} + for i, accuracy in enumerate(accuracies): + sensitivities[layername][sparsities[i]] = accuracy + return sensitivities + + def _max_prune_ratio(self, ori_acc, threshold, sensitivities): + """ + Find the maximum prune ratio for a single layer whose accuracy + drop is lower than the threshold. + + Parameters + ---------- + ori_acc: float + Original accuracy + threshold: float + Accuracy drop threshold + sensitivities: dict + The dict object that stores the sensitivity results for each layer. + For example: {'conv1' : {0.1: 0.9, 0.2 : 0.8}} + Returns + ------- + max_ratios: dict + return the maximum prune ratio for each layer. For example: + {'conv1':0.1, 'conv2':0.2} + """ + max_ratio = {} + for layer in sensitivities: + prune_ratios = sorted(sensitivities[layer].keys()) + last_ratio = 0 + for ratio in prune_ratios: + cur_acc = sensitivities[layer][ratio] + if cur_acc + threshold < ori_acc: + break + last_ratio = ratio + max_ratio[layer] = last_ratio + return max_ratio + + def normalize(self, ratios, target_pruned): + """ + Normalize the prune ratio of each layer according to the + total already pruned ratio and the final target total pruning + ratio + + Parameters + ---------- + ratios: + Dict object that save the prune ratio for each layer + target_pruned: + The amount of the weights expected to be pruned in this + iteration + + Returns + ------- + new_ratios: + return the normalized prune ratios for each layer. + + """ + w_sum = 0 + _Max = 0 + for layername, ratio in ratios.items(): + wcount = self.weight_count[layername] + w_sum += ratio * wcount * \ + (1-self.analyzer.already_pruned[layername]) + target_count = self.weight_sum * target_pruned + for layername in ratios: + ratios[layername] = ratios[layername] * target_count / w_sum + _Max = max(_Max, ratios[layername]) + # Cannot Prune too much in a single iteration + # If a layer's prune ratio is larger than the + # MAX_PRUNE_RATIO_PER_ITER we rescal all prune + # ratios under this threshold + if _Max > MAX_PRUNE_RATIO_PER_ITER: + for layername in ratios: + ratios[layername] = ratios[layername] * \ + MAX_PRUNE_RATIO_PER_ITER / _Max + return ratios + + def create_cfg(self, ratios): + """ + Generate the cfg_list for the pruner according to the prune ratios. + + Parameters + --------- + ratios: + For example: {'conv1' : 0.2} + + Returns + ------- + cfg_list: + For example: [{'sparsity':0.2, 'op_names':['conv1'], 'op_types':['Conv2d']}] + """ + cfg_list = [] + for layername in ratios: + prune_ratio = ratios[layername] + remain = 1 - self.analyzer.already_pruned[layername] + sparsity = remain * prune_ratio + \ + self.analyzer.already_pruned[layername] + if sparsity > 0: + # Pruner does not allow the prune ratio to be zero + cfg = {'sparsity': sparsity, 'op_names': [ + layername], 'op_types': ['Conv2d']} + cfg_list.append(cfg) + return cfg_list + + def current_sparsity(self): + """ + The sparsity of the weight. + """ + pruned_weight = 0 + for layer_name in self.analyzer.already_pruned: + w_count = self.weight_count[layer_name] + prune_ratio = self.analyzer.already_pruned[layer_name] + pruned_weight += w_count * prune_ratio + return pruned_weight / self.weight_sum + + def compress(self, eval_args=None, eval_kwargs=None, + finetune_args=None, finetune_kwargs=None, resume_sensitivity=None): + """ + This function iteratively prune the model according to the results of + the sensitivity analysis. + + Parameters + ---------- + eval_args: list + eval_kwargs: list& dict + Parameters for the val_funtion, the val_function will be called like + evaluator(*eval_args, **eval_kwargs) + finetune_args: list + finetune_kwargs: dict + Parameters for the finetuner function if needed. + resume_sensitivity: + resume the sensitivity results from this file. + """ + # pylint suggest not use the empty list and dict + # as the default input parameter + if not eval_args: + eval_args = [] + if not eval_kwargs: + eval_kwargs = {} + if not finetune_args: + finetune_args = [] + if not finetune_kwargs: + finetune_kwargs = {} + if self.ori_acc is None: + self.ori_acc = self.evaluator(*eval_args, **eval_kwargs) + if not resume_sensitivity: + self.sensitivities = self.analyzer.analysis( + val_args=eval_args, val_kwargs=eval_kwargs) + else: + self.sensitivities = self.load_sensitivity(resume_sensitivity) + self.analyzer.sensitivities = self.sensitivities + # the final target sparsity of the model + target_ratio = 1 - self.config_list[0]['sparsity'] + cur_ratio = self.remained_ratio + ori_acc = self.ori_acc + iteration_count = 0 + if self.checkpoint_dir is not None: + os.makedirs(self.checkpoint_dir, exist_ok=True) + while cur_ratio > target_ratio: + iteration_count += 1 + # Each round have three steps: + # 1) Get the current sensitivity for each layer(the sensitivity + # of each layer may change during the pruning) + # 2) Prune each layer according the sensitivies + # 3) finetune the model + _logger.info('Current base accuracy %f', ori_acc) + _logger.info('Remained %f weights', cur_ratio) + # determine the sparsity proportion between different + # layers according to the sensitivity result + proportion = self.sparsity_proportion_calc( + ori_acc, self.acc_drop_threshold, self.sensitivities) + new_pruneratio = self.normalize(proportion, self.sparsity_per_iter) + cfg_list = self.create_cfg(new_pruneratio) + _logger.debug('Pruner Config: %s', str(cfg_list)) + pruner = self.Pruner(self.model, cfg_list) + pruner.compress() + pruned_acc = self.evaluator(*eval_args, **eval_kwargs) + _logger.info('Accuracy after pruning: %f', pruned_acc) + finetune_acc = pruned_acc + if self.finetuner is not None: + # if the finetune function is None, then skip the finetune + self.finetuner(*finetune_args, **finetune_kwargs) + finetune_acc = self.evaluator(*eval_args, **eval_kwargs) + _logger.info('Accuracy after finetune: %f', finetune_acc) + ori_acc = finetune_acc + # unwrap the pruner + pruner._unwrap_model() + # update the already prune ratio of each layer befor the new + # sensitivity analysis + for layer_cfg in cfg_list: + name = layer_cfg['op_names'][0] + sparsity = layer_cfg['sparsity'] + self.analyzer.already_pruned[name] = sparsity + # update the cur_ratio + cur_ratio = 1 - self.current_sparsity() + del pruner + _logger.info('Currently remained weights: %f', cur_ratio) + + if self.checkpoint_dir is not None: + checkpoint_name = 'Iter_%d_finetune_acc_%.5f_sparsity_%.4f' % ( + iteration_count, finetune_acc, cur_ratio) + checkpoint_path = os.path.join( + self.checkpoint_dir, '%s.pth' % checkpoint_name) + cfg_path = os.path.join( + self.checkpoint_dir, '%s_pruner.json' % checkpoint_name) + sensitivity_path = os.path.join( + self.checkpoint_dir, '%s_sensitivity.csv' % checkpoint_name) + torch.save(self.model.state_dict(), checkpoint_path) + with open(cfg_path, 'w') as jf: + json.dump(cfg_list, jf) + self.analyzer.export(sensitivity_path) + if cur_ratio > target_ratio: + # If this is the last prune iteration, skip the time-consuming + # sensitivity analysis + self.analyzer.load_state_dict(self.model.state_dict()) + self.sensitivities = self.analyzer.analysis( + val_args=eval_args, val_kwargs=eval_kwargs) + + _logger.info('After Pruning: %.2f weights remains', cur_ratio) + return self.model + + def calc_mask(self, wrapper, **kwargs): + return None diff --git a/src/sdk/pynni/nni/compression/torch/utils/sensitivity_analysis.py b/src/sdk/pynni/nni/compression/torch/utils/sensitivity_analysis.py index fc259833b6..341c5ab67e 100644 --- a/src/sdk/pynni/nni/compression/torch/utils/sensitivity_analysis.py +++ b/src/sdk/pynni/nni/compression/torch/utils/sensitivity_analysis.py @@ -9,10 +9,7 @@ import numpy as np import torch.nn as nn -from nni.compression.torch import LevelPruner -from nni.compression.torch import L1FilterPruner -from nni.compression.torch import L2FilterPruner - +from ..pruning.constants_pruner import PRUNER_DICT SUPPORTED_OP_NAME = ['Conv2d', 'Conv1d'] SUPPORTED_OP_TYPE = [getattr(nn, name) for name in SUPPORTED_OP_NAME] @@ -77,11 +74,7 @@ def __init__(self, model, val_func, sparsities=None, prune_type='l1', early_stop else: self.sparsities = np.arange(0.1, 1.0, 0.1) self.sparsities = [np.round(x, 2) for x in self.sparsities] - self.Pruner = L1FilterPruner - if prune_type == 'l2': - self.Pruner = L2FilterPruner - elif prune_type == 'fine-grained': - self.Pruner = LevelPruner + self.Pruner = PRUNER_DICT[prune_type] self.early_stop_mode = early_stop_mode self.early_stop_value = early_stop_value self.ori_metric = None # original validation metric for the model From d5072a29f541b5d5d208eb4e6143de6ff27cc764 Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Wed, 12 Aug 2020 09:31:42 +0800 Subject: [PATCH 12/28] Support save and open experiments (#2750) --- docs/en_US/Tutorial/Nnictl.md | 59 +++++++++++- tools/nni_cmd/common_utils.py | 12 +++ tools/nni_cmd/config_utils.py | 8 +- tools/nni_cmd/constants.py | 2 + tools/nni_cmd/nnictl.py | 35 +++++-- tools/nni_cmd/nnictl_utils.py | 166 +++++++++++++++++++++++++++++++++- 6 files changed, 263 insertions(+), 19 deletions(-) diff --git a/docs/en_US/Tutorial/Nnictl.md b/docs/en_US/Tutorial/Nnictl.md index ed5c9761e1..81caf6f047 100644 --- a/docs/en_US/Tutorial/Nnictl.md +++ b/docs/en_US/Tutorial/Nnictl.md @@ -444,9 +444,6 @@ Debug mode will disable version check function in Trialkeeper. |--all| False| |delete all of experiments| - - - * __nnictl experiment export__ * Description @@ -531,6 +528,62 @@ Debug mode will disable version check function in Trialkeeper. nnictl experiment import [experiment_id] -f experiment_data.json ``` +* __nnictl experiment save__ + * Description + + Save nni experiment metadata and code data. + + * Usage + + ```bash + nnictl experiment save [OPTIONS] + ``` + + * Options + + |Name, shorthand|Required|Default|Description| + |------|------|------ |------| + |id| True| |The id of the experiment you want to save| + |--path, -p| False| |the folder path to store nni experiment data, default current working directory| + |--saveCodeDir, -s| False| |save codeDir data of the experiment, default False| + + * Examples + + > save an expeirment + + ```bash + nnictl experiment save [experiment_id] --saveCodeDir + ``` + +* __nnictl experiment load__ + * Description + + Load an nni experiment. + + * Usage + + ```bash + nnictl experiment load [OPTIONS] + ``` + + * Options + + |Name, shorthand|Required|Default|Description| + |------|------|------ |------| + |--path, -p| True| |the file path of nni package| + |--codeDir, -c| True| |the path of codeDir for loaded experiment, this path will also put the code in the loaded experiment package| + |--logDir, -l| False| |the path of logDir for loaded experiment| + + * Examples + + > load an expeirment + + ```bash + nnictl experiment load --path [path] --codeDir [codeDir] + ``` + + + ### Manage platform information diff --git a/tools/nni_cmd/common_utils.py b/tools/nni_cmd/common_utils.py index 4166bf034c..2edbf667df 100644 --- a/tools/nni_cmd/common_utils.py +++ b/tools/nni_cmd/common_utils.py @@ -4,7 +4,10 @@ import os import sys import json +import tempfile import socket +import string +import random import ruamel.yaml as yaml import psutil from colorama import Fore @@ -83,3 +86,12 @@ def check_tensorboard_version(): print_error('import tensorboard error!') exit(1) +def generate_temp_dir(): + '''generate a temp folder''' + def generate_folder_name(): + return os.path.join(tempfile.gettempdir(), 'nni', ''.join(random.sample(string.ascii_letters + string.digits, 8))) + temp_dir = generate_folder_name() + while os.path.exists(temp_dir): + temp_dir = generate_folder_name() + os.makedirs(temp_dir) + return temp_dir diff --git a/tools/nni_cmd/config_utils.py b/tools/nni_cmd/config_utils.py index 8cc1dc8ada..e6472ee3de 100644 --- a/tools/nni_cmd/config_utils.py +++ b/tools/nni_cmd/config_utils.py @@ -54,13 +54,13 @@ def __init__(self): self.experiment_file = os.path.join(NNICTL_HOME_DIR, '.experiment') self.experiments = self.read_file() - def add_experiment(self, expId, port, time, file_name, platform, experiment_name): + def add_experiment(self, expId, port, startTime, file_name, platform, experiment_name, endTime='N/A', status='INITIALIZED'): '''set {key:value} paris to self.experiment''' self.experiments[expId] = {} self.experiments[expId]['port'] = port - self.experiments[expId]['startTime'] = time - self.experiments[expId]['endTime'] = 'N/A' - self.experiments[expId]['status'] = 'INITIALIZED' + self.experiments[expId]['startTime'] = startTime + self.experiments[expId]['endTime'] = endTime + self.experiments[expId]['status'] = status self.experiments[expId]['fileName'] = file_name self.experiments[expId]['platform'] = platform self.experiments[expId]['experimentName'] = experiment_name diff --git a/tools/nni_cmd/constants.py b/tools/nni_cmd/constants.py index 5a37c3a1f1..0654473ed4 100644 --- a/tools/nni_cmd/constants.py +++ b/tools/nni_cmd/constants.py @@ -6,6 +6,8 @@ NNICTL_HOME_DIR = os.path.join(os.path.expanduser('~'), '.local', 'nnictl') +NNI_HOME_DIR = os.path.join(os.path.expanduser('~'), 'nni-experiments') + ERROR_INFO = 'ERROR: ' NORMAL_INFO = 'INFO: ' WARNING_INFO = 'WARNING: ' diff --git a/tools/nni_cmd/nnictl.py b/tools/nni_cmd/nnictl.py index 6a2991fe50..07afd85fab 100644 --- a/tools/nni_cmd/nnictl.py +++ b/tools/nni_cmd/nnictl.py @@ -11,7 +11,8 @@ from .nnictl_utils import stop_experiment, trial_ls, trial_kill, list_experiment, experiment_status,\ log_trial, experiment_clean, platform_clean, experiment_list, \ monitor_experiment, export_trials_data, trial_codegen, webui_url, \ - get_config, log_stdout, log_stderr, search_space_auto_gen, webui_nas + get_config, log_stdout, log_stderr, search_space_auto_gen, webui_nas, \ + save_experiment, load_experiment from .package_management import package_install, package_uninstall, package_show, package_list from .constants import DEFAULT_REST_PORT from .tensorboard_utils import start_tensorboard, stop_tensorboard @@ -129,15 +130,6 @@ def parse_args(): parser_experiment_clean.add_argument('id', nargs='?', help='the id of experiment') parser_experiment_clean.add_argument('--all', action='store_true', default=False, help='delete all of experiments') parser_experiment_clean.set_defaults(func=experiment_clean) - - #parse experiment command - parser_platform = subparsers.add_parser('platform', help='get platform information') - #add subparsers for parser_experiment - parser_platform_subparsers = parser_platform.add_subparsers() - parser_platform_clean = parser_platform_subparsers.add_parser('clean', help='clean up the platform data') - parser_platform_clean.add_argument('--config', '-c', required=True, dest='config', help='the path of yaml config file') - parser_platform_clean.set_defaults(func=platform_clean) - #import tuning data parser_import_data = parser_experiment_subparsers.add_parser('import', help='import additional data') parser_import_data.add_argument('id', nargs='?', help='the id of experiment') @@ -149,6 +141,29 @@ def parse_args(): parser_trial_export.add_argument('--type', '-t', choices=['json', 'csv'], required=True, dest='type', help='target file type') parser_trial_export.add_argument('--filename', '-f', required=True, dest='path', help='target file path') parser_trial_export.set_defaults(func=export_trials_data) + #save an NNI experiment + parser_save_experiment = parser_experiment_subparsers.add_parser('save', help='save an experiment') + parser_save_experiment.add_argument('id', nargs='?', help='the id of experiment') + parser_save_experiment.add_argument('--path', '-p', required=False, help='the folder path to store nni experiment data, \ + default current working directory') + parser_save_experiment.add_argument('--saveCodeDir', '-s', action='store_true', default=False, help='save codeDir data \ + of the experiment') + parser_save_experiment.set_defaults(func=save_experiment) + #load an NNI experiment + parser_load_experiment = parser_experiment_subparsers.add_parser('load', help='load an experiment') + parser_load_experiment.add_argument('--path', '-p', required=True, help='the path of nni package file') + parser_load_experiment.add_argument('--codeDir', '-c', required=True, help='the path of codeDir for loaded experiment, \ + this path will also put the code in the loaded experiment package') + parser_load_experiment.add_argument('--logDir', '-l', required=False, help='the path of logDir for loaded experiment') + parser_load_experiment.set_defaults(func=load_experiment) + + #parse platform command + parser_platform = subparsers.add_parser('platform', help='get platform information') + #add subparsers for parser_platform + parser_platform_subparsers = parser_platform.add_subparsers() + parser_platform_clean = parser_platform_subparsers.add_parser('clean', help='clean up the platform data') + parser_platform_clean.add_argument('--config', '-c', required=True, dest='config', help='the path of yaml config file') + parser_platform_clean.set_defaults(func=platform_clean) #TODO:finish webui function #parse board command diff --git a/tools/nni_cmd/nnictl_utils.py b/tools/nni_cmd/nnictl_utils.py index bbbf54fcc6..b411cfda77 100644 --- a/tools/nni_cmd/nnictl_utils.py +++ b/tools/nni_cmd/nnictl_utils.py @@ -18,9 +18,9 @@ from .rest_utils import rest_get, rest_delete, check_rest_server_quick, check_response from .url_utils import trial_jobs_url, experiment_url, trial_job_id_url, export_data_url from .config_utils import Config, Experiments -from .constants import NNICTL_HOME_DIR, EXPERIMENT_INFORMATION_FORMAT, EXPERIMENT_DETAIL_FORMAT, \ +from .constants import NNICTL_HOME_DIR, NNI_HOME_DIR, EXPERIMENT_INFORMATION_FORMAT, EXPERIMENT_DETAIL_FORMAT, \ EXPERIMENT_MONITOR_INFO, TRIAL_MONITOR_HEAD, TRIAL_MONITOR_CONTENT, TRIAL_MONITOR_TAIL, REST_TIME_OUT -from .common_utils import print_normal, print_error, print_warning, detect_process, get_yml_content +from .common_utils import print_normal, print_error, print_warning, detect_process, get_yml_content, generate_temp_dir from .command_utils import check_output_command, kill_command from .ssh_utils import create_ssh_sftp_client, remove_remote_directory @@ -736,3 +736,165 @@ def search_space_auto_gen(args): print_warning('Expected search space file \'{}\' generated, but not found.'.format(file_path)) else: print_normal('Generate search space done: \'{}\'.'.format(file_path)) + +def save_experiment(args): + '''save experiment data to a zip file''' + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + if args.id is None: + print_error('Please set experiment id.') + exit(1) + if args.id not in experiment_dict: + print_error('Cannot find experiment {0}.'.format(args.id)) + exit(1) + if experiment_dict[args.id].get('status') != 'STOPPED': + print_error('Can only save stopped experiment!') + exit(1) + print_normal('Saving...') + nni_config = Config(experiment_dict[args.id]['fileName']) + logDir = os.path.join(NNI_HOME_DIR, args.id) + if nni_config.get_config('logDir'): + logDir = os.path.join(nni_config.get_config('logDir'), args.id) + temp_root_dir = generate_temp_dir() + + # Step1. Copy logDir to temp folder + if not os.path.exists(logDir): + print_error('logDir: %s does not exist!' % logDir) + exit(1) + temp_experiment_dir = os.path.join(temp_root_dir, 'experiment') + shutil.copytree(logDir, temp_experiment_dir) + + # Step2. Copy nnictl metadata to temp folder + temp_nnictl_dir = os.path.join(temp_root_dir, 'nnictl') + os.makedirs(temp_nnictl_dir, exist_ok=True) + try: + with open(os.path.join(temp_nnictl_dir, '.experiment'), 'w') as file: + experiment_dict[args.id]['id'] = args.id + json.dump(experiment_dict[args.id], file) + except IOError: + print_error('Write file to %s failed!' % os.path.join(temp_nnictl_dir, '.experiment')) + exit(1) + nnictl_config_dir = os.path.join(NNICTL_HOME_DIR, experiment_dict[args.id]['fileName']) + shutil.copytree(nnictl_config_dir, os.path.join(temp_nnictl_dir, experiment_dict[args.id]['fileName'])) + + # Step3. Copy code dir + if args.saveCodeDir: + temp_code_dir = os.path.join(temp_root_dir, 'code') + shutil.copytree(nni_config.get_config('experimentConfig')['trial']['codeDir'], temp_code_dir) + + # Step4. Archive folder + zip_package_name = 'nni_experiment_%s' % args.id + if args.path: + os.makedirs(args.path, exist_ok=True) + zip_package_name = os.path.join(args.path, zip_package_name) + shutil.make_archive(zip_package_name, 'zip', temp_root_dir) + print_normal('Save to %s.zip success!' % zip_package_name) + + # Step5. Cleanup temp data + shutil.rmtree(temp_root_dir) + +def load_experiment(args): + '''load experiment data''' + package_path = os.path.expanduser(args.path) + if not os.path.exists(args.path): + print_error('file path %s does not exist!' % args.path) + exit(1) + temp_root_dir = generate_temp_dir() + shutil.unpack_archive(package_path, temp_root_dir) + print_normal('Loading...') + # Step1. Validation + if not os.path.exists(args.codeDir): + print_error('Invalid: codeDir path does not exist!') + exit(1) + if args.logDir: + if not os.path.exists(args.logDir): + print_error('Invalid: logDir path does not exist!') + exit(1) + experiment_temp_dir = os.path.join(temp_root_dir, 'experiment') + if not os.path.exists(os.path.join(experiment_temp_dir, 'db')): + print_error('Invalid archive file: db file does not exist!') + shutil.rmtree(temp_root_dir) + exit(1) + nnictl_temp_dir = os.path.join(temp_root_dir, 'nnictl') + if not os.path.exists(os.path.join(nnictl_temp_dir, '.experiment')): + print_error('Invalid archive file: nnictl metadata file does not exist!') + shutil.rmtree(temp_root_dir) + exit(1) + try: + with open(os.path.join(nnictl_temp_dir, '.experiment'), 'r') as file: + experiment_metadata = json.load(file) + except ValueError as err: + print_error('Invalid nnictl metadata file: %s' % err) + shutil.rmtree(temp_root_dir) + exit(1) + experiment_config = Experiments() + experiment_dict = experiment_config.get_all_experiments() + experiment_id = experiment_metadata.get('id') + if experiment_id in experiment_dict: + print_error('Invalid: experiment id already exist!') + shutil.rmtree(temp_root_dir) + exit(1) + if not os.path.exists(os.path.join(nnictl_temp_dir, experiment_metadata.get('fileName'))): + print_error('Invalid: experiment metadata does not exist!') + shutil.rmtree(temp_root_dir) + exit(1) + + # Step2. Copy nnictl metadata + src_path = os.path.join(nnictl_temp_dir, experiment_metadata.get('fileName')) + dest_path = os.path.join(NNICTL_HOME_DIR, experiment_metadata.get('fileName')) + if os.path.exists(dest_path): + shutil.rmtree(dest_path) + shutil.copytree(src_path, dest_path) + + # Step3. Copy experiment data + nni_config = Config(experiment_metadata.get('fileName')) + nnictl_exp_config = nni_config.get_config('experimentConfig') + if args.logDir: + logDir = args.logDir + nnictl_exp_config['logDir'] = logDir + else: + if nnictl_exp_config.get('logDir'): + logDir = nnictl_exp_config['logDir'] + else: + logDir = NNI_HOME_DIR + os.rename(os.path.join(temp_root_dir, 'experiment'), os.path.join(temp_root_dir, experiment_id)) + src_path = os.path.join(os.path.join(temp_root_dir, experiment_id)) + dest_path = os.path.join(os.path.join(logDir, experiment_id)) + if os.path.exists(dest_path): + shutil.rmtree(dest_path) + shutil.copytree(src_path, dest_path) + + # Step4. Copy code dir + codeDir = os.path.expanduser(args.codeDir) + if not os.path.isabs(codeDir): + codeDir = os.path.join(os.getcwd(), codeDir) + print_normal('Expand codeDir to %s' % codeDir) + nnictl_exp_config['trial']['codeDir'] = codeDir + archive_code_dir = os.path.join(temp_root_dir, 'code') + if os.path.exists(archive_code_dir): + file_list = os.listdir(archive_code_dir) + for file_name in file_list: + src_path = os.path.join(archive_code_dir, file_name) + target_path = os.path.join(codeDir, file_name) + if os.path.exists(target_path): + print_error('Copy %s failed, %s exist!' % (file_name, target_path)) + continue + if os.path.isdir(src_path): + shutil.copytree(src_path, target_path) + else: + shutil.copy(src_path, target_path) + + # Step5. Create experiment metadata + nni_config.set_config('experimentConfig', nnictl_exp_config) + experiment_config.add_experiment(experiment_id, + experiment_metadata.get('port'), + experiment_metadata.get('startTime'), + experiment_metadata.get('fileName'), + experiment_metadata.get('platform'), + experiment_metadata.get('experimentName'), + experiment_metadata.get('endTime'), + experiment_metadata.get('status')) + print_normal('Load experiment %s succsss!' % experiment_id) + + # Step6. Cleanup temp data + shutil.rmtree(temp_root_dir) From 118256110a74aea4234bf9d9c37167ef1748f561 Mon Sep 17 00:00:00 2001 From: chicm-ms <38930155+chicm-ms@users.noreply.github.com> Date: Wed, 12 Aug 2020 10:51:37 +0800 Subject: [PATCH 13/28] upgrade pipeline python35 to python36 (#2778) --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 839aa2c4d6..2e68ac8add 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -49,9 +49,9 @@ jobs: sphinx-build -M html . _build -W displayName: 'Sphinx Documentation Build check' -- job: 'ubuntu_1604_python35_legacy_torch_tf' +- job: 'ubuntu_1804_python36_legacy_torch_tf' pool: - vmImage: 'Ubuntu 16.04' + vmImage: 'Ubuntu 18.04' steps: - script: | From bcefce6a814c2c1228e25fcd98c88fea9e8111a9 Mon Sep 17 00:00:00 2001 From: lin bin <756691769@qq.com> Date: Wed, 12 Aug 2020 10:53:01 +0800 Subject: [PATCH 14/28] Add supporting sklearn=0.23.2 for nni (#2777) --- azure-pipelines.yml | 2 +- deployment/docker/Dockerfile | 4 ++-- deployment/docker/README.md | 2 +- deployment/docker/README_zh_CN.md | 2 +- deployment/pypi/setup.py | 2 +- docs/requirements.txt | 2 +- setup.py | 2 +- .../gradient_selector/gradient_selector.py | 2 +- .../feature_engineering/gradient_selector/requirements.txt | 2 +- src/sdk/pynni/nni/metis_tuner/requirments.txt | 2 +- src/sdk/pynni/requirements.txt | 2 +- test/pipelines/pipelines-it-local-tf2.yml | 2 +- test/pipelines/pipelines-it-local-windows.yml | 2 +- test/pipelines/pipelines-it-local.yml | 2 +- test/pipelines/pipelines-it-pai-windows.yml | 4 ++-- test/pipelines/pipelines-it-remote-windows-to-linux.yml | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 2e68ac8add..78917d879f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -141,7 +141,7 @@ jobs: powershell.exe -file install.ps1 displayName: 'Install nni toolkit via source code' - script: | - python -m pip install scikit-learn==0.20.0 --user + python -m pip install scikit-learn==0.23.2 --user python -m pip install keras==2.1.6 --user python -m pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html --user python -m pip install tensorflow==1.15.2 --user diff --git a/deployment/docker/Dockerfile b/deployment/docker/Dockerfile index 5e33cc6047..493cdba17b 100644 --- a/deployment/docker/Dockerfile +++ b/deployment/docker/Dockerfile @@ -60,9 +60,9 @@ RUN python3 -m pip --no-cache-dir install torch==1.4.0 RUN python3 -m pip install torchvision==0.5.0 # -# sklearn 0.20.0 +# sklearn 0.23.2 # -RUN python3 -m pip --no-cache-dir install scikit-learn==0.20.0 +RUN python3 -m pip --no-cache-dir install scikit-learn==0.23.2 # # pandas==0.23.4 lightgbm==2.2.2 diff --git a/deployment/docker/README.md b/deployment/docker/README.md index aa8dae38a4..1ee8d0b3bd 100644 --- a/deployment/docker/README.md +++ b/deployment/docker/README.md @@ -11,7 +11,7 @@ scipy 1.1.0 tensorflow-gpu 1.15.0 keras 2.1.6 torch 1.4.0 -scikit-learn 0.20.0 +scikit-learn 0.23.2 pandas 0.23.4 lightgbm 2.2.2 nni diff --git a/deployment/docker/README_zh_CN.md b/deployment/docker/README_zh_CN.md index a2b243e472..9766f92138 100644 --- a/deployment/docker/README_zh_CN.md +++ b/deployment/docker/README_zh_CN.md @@ -47,4 +47,4 @@ 使用下列命令从 docker Hub 中拉取 NNI docker 映像。 - docker pull msranni/nni:latest \ No newline at end of file + docker pull msranni/nni:latest diff --git a/deployment/pypi/setup.py b/deployment/pypi/setup.py index 61f7ff0178..8968686cd4 100644 --- a/deployment/pypi/setup.py +++ b/deployment/pypi/setup.py @@ -63,7 +63,7 @@ 'scipy', 'coverage', 'colorama', - 'scikit-learn>=0.20,<0.22', + 'scikit-learn>=0.23.2', 'pkginfo', 'websockets' ], diff --git a/docs/requirements.txt b/docs/requirements.txt index 721a587a20..59706a60ea 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -13,6 +13,6 @@ peewee nbsphinx schema tensorboard -scikit-learn==0.20 +scikit-learn>=0.23.2 thop https://download.pytorch.org/whl/cpu/torch-1.3.1%2Bcpu-cp37-cp37m-linux_x86_64.whl diff --git a/setup.py b/setup.py index 30d4f448c6..fc86bbc954 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ def read(fname): 'schema', 'PythonWebHDFS', 'colorama', - 'scikit-learn>=0.20,<0.22', + 'scikit-learn>=0.23.2', 'pkginfo', 'websockets' ], diff --git a/src/sdk/pynni/nni/feature_engineering/gradient_selector/gradient_selector.py b/src/sdk/pynni/nni/feature_engineering/gradient_selector/gradient_selector.py index 9dea3e8a0e..f7cb69f627 100644 --- a/src/sdk/pynni/nni/feature_engineering/gradient_selector/gradient_selector.py +++ b/src/sdk/pynni/nni/feature_engineering/gradient_selector/gradient_selector.py @@ -24,7 +24,7 @@ import pandas as pd from sklearn.base import BaseEstimator -from sklearn.feature_selection.base import SelectorMixin +from sklearn.feature_selection import SelectorMixin from sklearn.utils.validation import check_is_fitted import torch diff --git a/src/sdk/pynni/nni/feature_engineering/gradient_selector/requirements.txt b/src/sdk/pynni/nni/feature_engineering/gradient_selector/requirements.txt index 06d2241d5f..2aafc0c86f 100644 --- a/src/sdk/pynni/nni/feature_engineering/gradient_selector/requirements.txt +++ b/src/sdk/pynni/nni/feature_engineering/gradient_selector/requirements.txt @@ -1,4 +1,4 @@ numpy==1.14.3 -scikit-learn==0.20.0 +scikit-learn>=0.23.2 scipy==1.1.0 torch==1.1.0 diff --git a/src/sdk/pynni/nni/metis_tuner/requirments.txt b/src/sdk/pynni/nni/metis_tuner/requirments.txt index 05c74c49ca..3dfc2232a1 100644 --- a/src/sdk/pynni/nni/metis_tuner/requirments.txt +++ b/src/sdk/pynni/nni/metis_tuner/requirments.txt @@ -1 +1 @@ -scikit-learn==0.20 \ No newline at end of file +scikit-learn>=0.23.2 diff --git a/src/sdk/pynni/requirements.txt b/src/sdk/pynni/requirements.txt index 282f572631..ec1b8705fb 100644 --- a/src/sdk/pynni/requirements.txt +++ b/src/sdk/pynni/requirements.txt @@ -8,4 +8,4 @@ scipy hyperopt==0.1.2 # metis tuner -scikit-learn==0.20 +scikit-learn>=0.23.2 diff --git a/test/pipelines/pipelines-it-local-tf2.yml b/test/pipelines/pipelines-it-local-tf2.yml index 26f3d4c87d..2b9526c142 100644 --- a/test/pipelines/pipelines-it-local-tf2.yml +++ b/test/pipelines/pipelines-it-local-tf2.yml @@ -10,7 +10,7 @@ jobs: displayName: 'Install nni toolkit via source code' - script: | set -e - python3 -m pip install scikit-learn==0.20.0 --user + python3 -m pip install scikit-learn==0.23.2 --user python3 -m pip install torch==1.3.1 torchvision==0.4.2 -f https://download.pytorch.org/whl/torch_stable.html --user python3 -m pip install tensorflow-gpu==2.2.0 tensorflow-estimator==2.2.0 --force --user python3 -m pip install keras==2.4.2 --user diff --git a/test/pipelines/pipelines-it-local-windows.yml b/test/pipelines/pipelines-it-local-windows.yml index bfdf4eed21..4b791c762c 100644 --- a/test/pipelines/pipelines-it-local-windows.yml +++ b/test/pipelines/pipelines-it-local-windows.yml @@ -7,7 +7,7 @@ jobs: powershell.exe -file install.ps1 displayName: 'Install nni toolkit via source code' - script: | - python -m pip install scikit-learn==0.20.0 --user + python -m pip install scikit-learn==0.23.2 --user python -m pip install keras==2.1.6 --user python -m pip install torchvision===0.4.1 torch===1.3.1 -f https://download.pytorch.org/whl/torch_stable.html --user python -m pip install tensorflow-gpu==1.15.2 tensorflow-estimator==1.15.1 --force --user diff --git a/test/pipelines/pipelines-it-local.yml b/test/pipelines/pipelines-it-local.yml index ec6735a473..eb72e4099d 100644 --- a/test/pipelines/pipelines-it-local.yml +++ b/test/pipelines/pipelines-it-local.yml @@ -10,7 +10,7 @@ jobs: displayName: 'Install nni toolkit via source code' - script: | set -e - python3 -m pip install scikit-learn==0.20.0 --user + python3 -m pip install scikit-learn==0.23.2 --user python3 -m pip install torchvision==0.4.2 --user python3 -m pip install torch==1.3.1 --user python3 -m pip install keras==2.1.6 --user diff --git a/test/pipelines/pipelines-it-pai-windows.yml b/test/pipelines/pipelines-it-pai-windows.yml index 3bdb4ee69b..1765868827 100644 --- a/test/pipelines/pipelines-it-pai-windows.yml +++ b/test/pipelines/pipelines-it-pai-windows.yml @@ -62,7 +62,7 @@ jobs: displayName: 'Install nni toolkit via source code' - script: | set PATH=$(ENV_PATH) - python -m pip install scikit-learn==0.21.0 --user + python -m pip install scikit-learn==0.23.2 --user displayName: 'Install dependencies for integration tests' - script: | cd test @@ -71,4 +71,4 @@ jobs: mount -o anon $(pai_nfs_uri) $(local_nfs_uri) python nni_test/nnitest/generate_ts_config.py --ts pai --pai_token $(pai_token) --pai_host $(pai_host) --pai_user $(pai_user) --nni_docker_image $(docker_image) --pai_storage_config_name $(pai_storage_config_name) --nni_manager_nfs_mount_path $(nni_manager_nfs_mount_path) --container_nfs_mount_path $(container_nfs_mount_path) --nni_manager_ip $(nni_manager_ip) --vc $(virtual_cluster) python nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts pai - displayName: 'Examples and advanced features tests on pai' \ No newline at end of file + displayName: 'Examples and advanced features tests on pai' diff --git a/test/pipelines/pipelines-it-remote-windows-to-linux.yml b/test/pipelines/pipelines-it-remote-windows-to-linux.yml index 36a98a9819..d87230201a 100644 --- a/test/pipelines/pipelines-it-remote-windows-to-linux.yml +++ b/test/pipelines/pipelines-it-remote-windows-to-linux.yml @@ -16,7 +16,7 @@ jobs: powershell.exe -file install.ps1 displayName: 'Install nni toolkit via source code' - script: | - python -m pip install scikit-learn==0.20.1 --user + python -m pip install scikit-learn==0.23.2 --user displayName: 'Install dependencies for integration tests' - task: SSH@0 inputs: From d654eff441aa7faafef279b798e64d5dd4f7de57 Mon Sep 17 00:00:00 2001 From: Tab Zhang Date: Wed, 12 Aug 2020 13:30:13 +0800 Subject: [PATCH 15/28] feature: export experiment results (#2706) --- docs/en_US/Tutorial/Nnictl.md | 4 +- tools/nni_cmd/nnictl.py | 2 + tools/nni_cmd/nnictl_utils.py | 80 ++++++++++++++++++++++------------- tools/nni_cmd/url_utils.py | 6 +++ 4 files changed, 60 insertions(+), 32 deletions(-) diff --git a/docs/en_US/Tutorial/Nnictl.md b/docs/en_US/Tutorial/Nnictl.md index 81caf6f047..2c92514b4b 100644 --- a/docs/en_US/Tutorial/Nnictl.md +++ b/docs/en_US/Tutorial/Nnictl.md @@ -462,13 +462,14 @@ Debug mode will disable version check function in Trialkeeper. |id| False| |ID of the experiment | |--filename, -f| True| |File path of the output file | |--type| True| |Type of output file, only support "csv" and "json"| + |--intermediate, -i|False||Are intermediate results included| * Examples > export all trial data in an experiment as json format ```bash - nnictl experiment export [experiment_id] --filename [file_path] --type json + nnictl experiment export [experiment_id] --filename [file_path] --type json --intermediate ``` * __nnictl experiment import__ @@ -903,4 +904,3 @@ Debug mode will disable version check function in Trialkeeper. ```bash nnictl --version ``` - diff --git a/tools/nni_cmd/nnictl.py b/tools/nni_cmd/nnictl.py index 07afd85fab..21fdf13f6d 100644 --- a/tools/nni_cmd/nnictl.py +++ b/tools/nni_cmd/nnictl.py @@ -140,6 +140,8 @@ def parse_args(): parser_trial_export.add_argument('id', nargs='?', help='the id of experiment') parser_trial_export.add_argument('--type', '-t', choices=['json', 'csv'], required=True, dest='type', help='target file type') parser_trial_export.add_argument('--filename', '-f', required=True, dest='path', help='target file path') + parser_trial_export.add_argument('--intermediate', '-i', action='store_true', + default=False, help='are intermediate results included') parser_trial_export.set_defaults(func=export_trials_data) #save an NNI experiment parser_save_experiment = parser_experiment_subparsers.add_parser('save', help='save an experiment') diff --git a/tools/nni_cmd/nnictl_utils.py b/tools/nni_cmd/nnictl_utils.py index b411cfda77..3aa5a6629a 100644 --- a/tools/nni_cmd/nnictl_utils.py +++ b/tools/nni_cmd/nnictl_utils.py @@ -16,7 +16,7 @@ from nni.package_utils import get_nni_installation_path from nni_annotation import expand_annotations from .rest_utils import rest_get, rest_delete, check_rest_server_quick, check_response -from .url_utils import trial_jobs_url, experiment_url, trial_job_id_url, export_data_url +from .url_utils import trial_jobs_url, experiment_url, trial_job_id_url, export_data_url, metric_data_url from .config_utils import Config, Experiments from .constants import NNICTL_HOME_DIR, NNI_HOME_DIR, EXPERIMENT_INFORMATION_FORMAT, EXPERIMENT_DETAIL_FORMAT, \ EXPERIMENT_MONITOR_INFO, TRIAL_MONITOR_HEAD, TRIAL_MONITOR_CONTENT, TRIAL_MONITOR_TAIL, REST_TIME_OUT @@ -681,45 +681,64 @@ def monitor_experiment(args): set_monitor(False, args.time) def export_trials_data(args): - '''export experiment metadata to csv + '''export experiment metadata and intermediate results to json or csv ''' + def groupby_trial_id(intermediate_results): + sorted(intermediate_results, key=lambda x: x['timestamp']) + groupby = dict() + for content in intermediate_results: + groupby.setdefault(content['trialJobId'], []).append(json.loads(content['data'])) + return groupby + nni_config = Config(get_config_filename(args)) rest_port = nni_config.get_config('restServerPort') rest_pid = nni_config.get_config('restServerPid') + if not detect_process(rest_pid): print_error('Experiment is not running...') return running, response = check_rest_server_quick(rest_port) - if running: - response = rest_get(export_data_url(rest_port), 20) - if response is not None and check_response(response): - if args.type == 'json': - with open(args.path, 'w') as file: - file.write(response.text) - elif args.type == 'csv': - content = json.loads(response.text) - trial_records = [] - for record in content: - record_value = json.loads(record['value']) - if not isinstance(record_value, (float, int)): - formated_record = {**record['parameter'], **record_value, **{'id': record['id']}} - else: - formated_record = {**record['parameter'], **{'reward': record_value, 'id': record['id']}} - trial_records.append(formated_record) - if not trial_records: - print_error('No trial results collected! Please check your trial log...') - exit(0) - with open(args.path, 'w', newline='') as file: - writer = csv.DictWriter(file, set.union(*[set(r.keys()) for r in trial_records])) - writer.writeheader() - writer.writerows(trial_records) - else: - print_error('Unknown type: %s' % args.type) - exit(1) + if not running: + print_error('Restful server is not running') + return + response = rest_get(export_data_url(rest_port), 20) + if response is not None and check_response(response): + content = json.loads(response.text) + if args.intermediate: + intermediate_results_response = rest_get(metric_data_url(rest_port), REST_TIME_OUT) + if not intermediate_results_response or not check_response(intermediate_results_response): + print_error('Error getting intermediate results.') + return + intermediate_results = groupby_trial_id(json.loads(intermediate_results_response.text)) + for record in content: + record['intermediate'] = intermediate_results[record['id']] + if args.type == 'json': + with open(args.path, 'w') as file: + file.write(json.dumps(content)) + elif args.type == 'csv': + trial_records = [] + for record in content: + formated_record = dict() + if args.intermediate: + formated_record['intermediate'] = '[' + ','.join(record['intermediate']) + ']' + record_value = json.loads(record['value']) + if not isinstance(record_value, (float, int)): + formated_record.update({**record['parameter'], **record_value, **{'id': record['id']}}) + else: + formated_record.update({**record['parameter'], **{'reward': record_value, 'id': record['id']}}) + trial_records.append(formated_record) + if not trial_records: + print_error('No trial results collected! Please check your trial log...') + exit(0) + with open(args.path, 'w', newline='') as file: + writer = csv.DictWriter(file, set.union(*[set(r.keys()) for r in trial_records])) + writer.writeheader() + writer.writerows(trial_records) else: - print_error('Export failed...') + print_error('Unknown type: %s' % args.type) + return else: - print_error('Restful server is not Running') + print_error('Export failed...') def search_space_auto_gen(args): '''dry run trial code to generate search space file''' @@ -898,3 +917,4 @@ def load_experiment(args): # Step6. Cleanup temp data shutil.rmtree(temp_root_dir) + diff --git a/tools/nni_cmd/url_utils.py b/tools/nni_cmd/url_utils.py index 083a865d65..6d1f7694e1 100644 --- a/tools/nni_cmd/url_utils.py +++ b/tools/nni_cmd/url_utils.py @@ -22,6 +22,12 @@ TENSORBOARD_API = '/tensorboard' +METRIC_DATA_API = '/metric-data' + +def metric_data_url(port): + '''get metric_data url''' + return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, METRIC_DATA_API) + def check_status_url(port): '''get check_status url''' From 5623dbf32952bd7842d3d9c5cf12a8e97e2b1fab Mon Sep 17 00:00:00 2001 From: liuzhe-lz <40699903+liuzhe-lz@users.noreply.github.com> Date: Wed, 12 Aug 2020 14:02:58 +0800 Subject: [PATCH 16/28] Compression for Tensorflow (#2755) --- docs/en_US/Compressor/Pruner.md | 26 +- docs/zh_CN/Compressor/Pruner.md | 12 +- examples/model_compress/model_prune_tf.py | 82 ++++ .../nni/compression/tensorflow/__init__.py | 5 +- .../compression/tensorflow/builtin_pruners.py | 195 --------- .../tensorflow/builtin_quantizers.py | 74 ---- .../nni/compression/tensorflow/compressor.py | 408 +++++++++++------- .../compression/tensorflow/default_layers.py | 34 +- .../tensorflow/pruning/__init__.py | 1 + .../tensorflow/pruning/one_shot.py | 67 +++ ...compressor.py => test_compressor_torch.py} | 55 +-- 11 files changed, 414 insertions(+), 545 deletions(-) create mode 100644 examples/model_compress/model_prune_tf.py delete mode 100644 src/sdk/pynni/nni/compression/tensorflow/builtin_pruners.py delete mode 100644 src/sdk/pynni/nni/compression/tensorflow/builtin_quantizers.py create mode 100644 src/sdk/pynni/nni/compression/tensorflow/pruning/__init__.py create mode 100644 src/sdk/pynni/nni/compression/tensorflow/pruning/one_shot.py rename src/sdk/pynni/tests/{test_compressor.py => test_compressor_torch.py} (87%) diff --git a/docs/en_US/Compressor/Pruner.md b/docs/en_US/Compressor/Pruner.md index e059834eca..9efcce8e7b 100644 --- a/docs/en_US/Compressor/Pruner.md +++ b/docs/en_US/Compressor/Pruner.md @@ -38,7 +38,7 @@ Tensorflow code ```python from nni.compression.tensorflow import LevelPruner config_list = [{ 'sparsity': 0.8, 'op_types': ['default'] }] -pruner = LevelPruner(model_graph, config_list) +pruner = LevelPruner(model, config_list) pruner.compress() ``` @@ -117,17 +117,6 @@ FPGMPruner prune filters with the smallest geometric median. ### Usage -Tensorflow code -```python -from nni.compression.tensorflow import FPGMPruner -config_list = [{ - 'sparsity': 0.5, - 'op_types': ['Conv2D'] -}] -pruner = FPGMPruner(model, config_list) -pruner.compress() -``` - PyTorch code ```python from nni.compression.torch import FPGMPruner @@ -146,11 +135,6 @@ pruner.compress() .. autoclass:: nni.compression.torch.FPGMPruner ``` -##### Tensorflow -```eval_rst -.. autoclass:: nni.compression.tensorflow.FPGMPruner -``` - ## L1Filter Pruner This is an one-shot pruner, In ['PRUNING FILTERS FOR EFFICIENT CONVNETS'](https://arxiv.org/abs/1608.08710), authors Hao Li, Asim Kadav, Igor Durdanovic, Hanan Samet and Hans Peter Graf. @@ -383,12 +367,6 @@ You can view [example](https://github.com/microsoft/nni/blob/master/examples/mod .. autoclass:: nni.compression.torch.AGPPruner ``` -##### Tensorflow - -```eval_rst -.. autoclass:: nni.compression.tensorflow.AGPPruner -``` - *** ## NetAdapt Pruner @@ -620,4 +598,4 @@ pruner.compress(eval_args=[model], finetune_args=[model]) ```eval_rst .. autoclass:: nni.compression.torch.SensitivityPruner -``` \ No newline at end of file +``` diff --git a/docs/zh_CN/Compressor/Pruner.md b/docs/zh_CN/Compressor/Pruner.md index f78b8e0f1c..d11829e4b5 100644 --- a/docs/zh_CN/Compressor/Pruner.md +++ b/docs/zh_CN/Compressor/Pruner.md @@ -37,7 +37,7 @@ TensorFlow 代码 ```python from nni.compression.tensorflow import LevelPruner config_list = [{ 'sparsity': 0.8, 'op_types': ['default'] }] -pruner = LevelPruner(model_graph, config_list) +pruner = LevelPruner(model, config_list) pruner.compress() ``` @@ -102,16 +102,6 @@ pruner.compress() ### 用法 -TensorFlow 代码 -```python -from nni.compression.tensorflow import FPGMPruner -config_list = [{ - 'sparsity': 0.5, - 'op_types': ['Conv2D'] -}] -pruner = FPGMPruner(model, config_list) -pruner.compress() -``` PyTorch 代码 ```python from nni.compression.torch import FPGMPruner diff --git a/examples/model_compress/model_prune_tf.py b/examples/model_compress/model_prune_tf.py new file mode 100644 index 0000000000..99e8278df4 --- /dev/null +++ b/examples/model_compress/model_prune_tf.py @@ -0,0 +1,82 @@ +import argparse + +import tensorflow as tf + +import nni.compression.tensorflow + +prune_config = { + 'level': { + 'dataset_name': 'mnist', + 'model_name': 'naive', + 'pruner_class': nni.compression.tensorflow.LevelPruner, + 'config_list': [{ + 'sparsity': 0.9, + 'op_types': ['default'], + }] + }, +} + + +def get_dataset(dataset_name='mnist'): + assert dataset_name == 'mnist' + + (x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data() + x_train = x_train[..., tf.newaxis] / 255.0 + x_test = x_test[..., tf.newaxis] / 255.0 + return (x_train, y_train), (x_test, y_test) + + +def create_model(model_name='naive'): + assert model_name == 'naive' + return tf.keras.Sequential([ + tf.keras.layers.Conv2D(filters=20, kernel_size=5), + tf.keras.layers.BatchNormalization(), + tf.keras.layers.ReLU(), + tf.keras.layers.MaxPool2D(pool_size=2), + tf.keras.layers.Conv2D(filters=20, kernel_size=5), + tf.keras.layers.BatchNormalization(), + tf.keras.layers.ReLU(), + tf.keras.layers.MaxPool2D(pool_size=2), + tf.keras.layers.Flatten(), + tf.keras.layers.Dense(units=500), + tf.keras.layers.ReLU(), + tf.keras.layers.Dense(units=10), + tf.keras.layers.Softmax() + ]) + + +def create_pruner(model, pruner_name): + pruner_class = prune_config[pruner_name]['pruner_class'] + config_list = prune_config[pruner_name]['config_list'] + return pruner_class(model, config_list) + + +def main(args): + model_name = prune_config[args.pruner_name]['model_name'] + dataset_name = prune_config[args.pruner_name]['dataset_name'] + train_set, test_set = get_dataset(dataset_name) + model = create_model(model_name) + + optimizer = tf.keras.optimizers.SGD(learning_rate=0.1, momentum=0.9, decay=1e-4) + model.compile(optimizer=optimizer, loss='sparse_categorical_crossentropy', metrics=['accuracy']) + + print('start training') + model.fit(train_set[0], train_set[1], batch_size=args.batch_size, epochs=args.pretrain_epochs, validation_data=test_set) + + print('start model pruning') + optimizer_finetune = tf.keras.optimizers.SGD(learning_rate=0.001, momentum=0.9, decay=1e-4) + pruner = create_pruner(model, args.pruner_name) + model = pruner.compress() + model.compile(optimizer=optimizer_finetune, loss='sparse_categorical_crossentropy', metrics=['accuracy']) + model.fit(train_set[0], train_set[1], batch_size=args.batch_size, epochs=args.prune_epochs, validation_data=test_set) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--pruner_name', type=str, default='level') + parser.add_argument('--batch_size', type=int, default=256) + parser.add_argument('--pretrain_epochs', type=int, default=10) + parser.add_argument('--prune_epochs', type=int, default=10) + + args = parser.parse_args() + main(args) diff --git a/src/sdk/pynni/nni/compression/tensorflow/__init__.py b/src/sdk/pynni/nni/compression/tensorflow/__init__.py index 45b6c4e7b8..00d41ee55b 100644 --- a/src/sdk/pynni/nni/compression/tensorflow/__init__.py +++ b/src/sdk/pynni/nni/compression/tensorflow/__init__.py @@ -1,6 +1,5 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -from .compressor import LayerInfo, Compressor, Pruner, Quantizer -from .builtin_pruners import * -from .builtin_quantizers import * +from .compressor import Compressor, Pruner +from .pruning import * diff --git a/src/sdk/pynni/nni/compression/tensorflow/builtin_pruners.py b/src/sdk/pynni/nni/compression/tensorflow/builtin_pruners.py deleted file mode 100644 index 89ea1a722d..0000000000 --- a/src/sdk/pynni/nni/compression/tensorflow/builtin_pruners.py +++ /dev/null @@ -1,195 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -import logging -import numpy as np -import tensorflow as tf -from .compressor import Pruner - -__all__ = ['LevelPruner', 'AGPPruner', 'FPGMPruner'] - -_logger = logging.getLogger(__name__) - - -class LevelPruner(Pruner): - """ - Parameters - ---------- - model : tensorflow model - Model to be pruned - config_list : list - Supported keys: - - sparsity : This is to specify the sparsity operations to be compressed to. - - op_types : Operation types to prune. - """ - def __init__(self, model, config_list): - super().__init__(model, config_list) - self.mask_list = {} - self.if_init_list = {} - - def calc_mask(self, layer, config): - weight = layer.weight - op_name = layer.name - if self.if_init_list.get(op_name, True): - threshold = tf.contrib.distributions.percentile(tf.abs(weight), config['sparsity'] * 100) - mask = tf.cast(tf.math.greater(tf.abs(weight), threshold), weight.dtype) - self.mask_list.update({op_name: mask}) - self.if_init_list.update({op_name: False}) - else: - mask = self.mask_list[op_name] - return mask - - -class AGPPruner(Pruner): - """ - Parameters - ---------- - model : torch.nn.Module - Model to be pruned. - config_list : listlist - Supported keys: - - initial_sparsity: This is to specify the sparsity when compressor starts to compress. - - final_sparsity: This is to specify the sparsity when compressor finishes to compress. - - start_epoch: This is to specify the epoch number when compressor starts to compress, default start from epoch 0. - - end_epoch: This is to specify the epoch number when compressor finishes to compress. - - frequency: This is to specify every *frequency* number epochs compressor compress once, default frequency=1. - """ - - def __init__(self, model, config_list): - super().__init__(model, config_list) - self.mask_list = {} - self.if_init_list = {} - self.now_epoch = tf.Variable(0) - self.assign_handler = [] - - def calc_mask(self, layer, config): - weight = layer.weight - op_name = layer.name - start_epoch = config.get('start_epoch', 0) - freq = config.get('frequency', 1) - if self.now_epoch >= start_epoch and self.if_init_list.get(op_name, True) and ( - self.now_epoch - start_epoch) % freq == 0: - target_sparsity = self.compute_target_sparsity(config) - threshold = tf.contrib.distributions.percentile(weight, target_sparsity * 100) - # stop gradient in case gradient change the mask - mask = tf.stop_gradient(tf.cast(tf.math.greater(weight, threshold), weight.dtype)) - self.assign_handler.append(tf.assign(weight, weight * mask)) - self.mask_list.update({op_name: tf.constant(mask)}) - self.if_init_list.update({op_name: False}) - else: - mask = self.mask_list[op_name] - return mask - - def compute_target_sparsity(self, config): - end_epoch = config.get('end_epoch', 1) - start_epoch = config.get('start_epoch', 0) - freq = config.get('frequency', 1) - final_sparsity = config.get('final_sparsity', 0) - initial_sparsity = config.get('initial_sparsity', 0) - - if end_epoch <= start_epoch or initial_sparsity >= final_sparsity: - _logger.warning('your end epoch <= start epoch or initial_sparsity >= final_sparsity') - return final_sparsity - - now_epoch = tf.minimum(self.now_epoch, tf.constant(end_epoch)) - span = int(((end_epoch - start_epoch - 1) // freq) * freq) - assert span > 0 - base = tf.cast(now_epoch - start_epoch, tf.float32) / span - target_sparsity = (final_sparsity + - (initial_sparsity - final_sparsity) * - (tf.pow(1.0 - base, 3))) - return target_sparsity - - def update_epoch(self, epoch, sess): - sess.run(self.assign_handler) - sess.run(tf.assign(self.now_epoch, int(epoch))) - for k in self.if_init_list: - self.if_init_list[k] = True - - -class FPGMPruner(Pruner): - """ - Parameters - ---------- - model : tensorflow model - Model to be pruned - config_list : list - Supported keys: - - sparsity : percentage of convolutional filters to be pruned. - - op_types : Only Conv2d is supported in FPGM Pruner. - """ - def __init__(self, model, config_list): - super().__init__(model, config_list) - self.mask_dict = {} - self.assign_handler = [] - self.epoch_pruned_layers = set() - - def calc_mask(self, layer, config): - """ - Supports Conv1D, Conv2D - filter dimensions for Conv1D: - LEN: filter length - IN: number of input channel - OUT: number of output channel - - filter dimensions for Conv2D: - H: filter height - W: filter width - IN: number of input channel - OUT: number of output channel - - Parameters - ---------- - layer : LayerInfo - calculate mask for `layer`'s weight - config : dict - the configuration for generating the mask - """ - - weight = layer.weight - op_type = layer.type - op_name = layer.name - assert 0 <= config.get('sparsity') < 1 - assert op_type in ['Conv1D', 'Conv2D'] - assert op_type in config['op_types'] - - if layer.name in self.epoch_pruned_layers: - assert layer.name in self.mask_dict - return self.mask_dict.get(layer.name) - - try: - w = tf.stop_gradient(tf.transpose(tf.reshape(weight, (-1, weight.shape[-1])), [1, 0])) - masks = np.ones(w.shape) - num_filters = w.shape[0] - num_prune = int(num_filters * config.get('sparsity')) - if num_filters < 2 or num_prune < 1: - return masks - min_gm_idx = self._get_min_gm_kernel_idx(w, num_prune) - - for idx in min_gm_idx: - masks[idx] = 0. - finally: - masks = tf.reshape(tf.transpose(masks, [1, 0]), weight.shape) - masks = tf.Variable(masks) - self.mask_dict.update({op_name: masks}) - self.epoch_pruned_layers.add(layer.name) - - return masks - - def _get_min_gm_kernel_idx(self, weight, n): - dist_list = [] - for out_i in range(weight.shape[0]): - dist_sum = self._get_distance_sum(weight, out_i) - dist_list.append((dist_sum, out_i)) - min_gm_kernels = sorted(dist_list, key=lambda x: x[0])[:n] - return [x[1] for x in min_gm_kernels] - - def _get_distance_sum(self, weight, out_idx): - anchor_w = tf.tile(tf.expand_dims(weight[out_idx], 0), [weight.shape[0], 1]) - x = weight - anchor_w - x = tf.math.reduce_sum((x*x), -1) - x = tf.math.sqrt(x) - return tf.math.reduce_sum(x) - - def update_epoch(self, epoch): - self.epoch_pruned_layers = set() diff --git a/src/sdk/pynni/nni/compression/tensorflow/builtin_quantizers.py b/src/sdk/pynni/nni/compression/tensorflow/builtin_quantizers.py deleted file mode 100644 index 3f54cbfb12..0000000000 --- a/src/sdk/pynni/nni/compression/tensorflow/builtin_quantizers.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -import logging -import tensorflow as tf -from .compressor import Quantizer - -__all__ = ['NaiveQuantizer', 'QAT_Quantizer', 'DoReFaQuantizer'] - -_logger = logging.getLogger(__name__) - - -class NaiveQuantizer(Quantizer): - """quantize weight to 8 bits - """ - def __init__(self, model, config_list): - super().__init__(model, config_list) - self.layer_scale = {} - - def quantize_weight(self, weight, config, op_name, **kwargs): - new_scale = tf.reduce_max(tf.abs(weight)) / 127 - scale = tf.maximum(self.layer_scale.get(op_name, tf.constant(0.0)), new_scale) - self.layer_scale[op_name] = scale - orig_type = weight.dtype - return tf.cast(tf.cast(weight / scale, tf.int8), orig_type) * scale - - -class QAT_Quantizer(Quantizer): - """Quantizer using the Quantization and Training scheme, as defined in: - Quantization and Training of Neural Networks for Efficient Integer-Arithmetic-Only Inference - http://openaccess.thecvf.com/content_cvpr_2018/papers/Jacob_Quantization_and_Training_CVPR_2018_paper.pdf - """ - def __init__(self, model, config_list): - """ - config_list: supported keys: - - q_bits - """ - super().__init__(model, config_list) - - def quantize_weight(self, weight, config, **kwargs): - a = tf.stop_gradient(tf.reduce_min(weight)) - b = tf.stop_gradient(tf.reduce_max(weight)) - n = tf.cast(2 ** config['q_bits'], tf.float32) - scale = b-a/(n-1) - - # use gradient_override_map to change round to idetity for gradient - with tf.get_default_graph().gradient_override_map({'Round': 'Identity'}): - qw = tf.round((weight-a)/scale)*scale +a - - return qw - - -class DoReFaQuantizer(Quantizer): - """Quantizer using the DoReFa scheme, as defined in: - Zhou et al., DoReFa-Net: Training Low Bitwidth Convolutional Neural Networks with Low Bitwidth Gradients - (https://arxiv.org/abs/1606.06160) - """ - def __init__(self, model, config_list): - """ - config_list: supported keys: - - q_bits - """ - super().__init__(model, config_list) - - def quantize_weight(self, weight, config, **kwargs): - a = tf.math.tanh(weight) - b = a/(2*tf.reduce_max(tf.abs(weight))) + 0.5 - - scale = pow(2, config['q_bits'] - 1) - # use gradient_override_map to change round to idetity for gradient - with tf.get_default_graph().gradient_override_map({'Round': 'Identity'}): - qw = tf.round(b*scale)/scale - r_qw = 2 * qw - 1 - return r_qw diff --git a/src/sdk/pynni/nni/compression/tensorflow/compressor.py b/src/sdk/pynni/nni/compression/tensorflow/compressor.py index 62580738a3..bbe4a21a52 100644 --- a/src/sdk/pynni/nni/compression/tensorflow/compressor.py +++ b/src/sdk/pynni/nni/compression/tensorflow/compressor.py @@ -1,204 +1,300 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +""" +Abstract base classes for TensorFlow model compression. +""" + import logging import tensorflow as tf from . import default_layers -tf.config.experimental_run_functions_eagerly(True) _logger = logging.getLogger(__name__) class LayerInfo: - def __init__(self, keras_layer): - self.keras_layer = keras_layer - self.name = keras_layer.name - self.type = default_layers.get_op_type(type(keras_layer)) - self.weight_index = default_layers.get_weight_index(self.type) - if self.weight_index is not None: - self.weight = keras_layer.weights[self.weight_index] - self._call = None + """ + This structure contains all infomation needed to compress a TensorFlow ``Layer``. + + + Attributes + ---------- + layer : tf.keras.layers.Layer + The layer. + name : str + The layer's name. Note that it's local to sub-model and may differ from its attribute name. + type : str + Name of the layer's class. + path : list of str/int + The layer object's and its parents' attribute name / list index. + For example, if the path is `['cells', 2, 'conv']`, then the layer can be accessed as `model.cells[2].conv`. + config : JSON object + Selected configuration for this layer. The format is detailed in tutorial. + + Parameters + ---------- + layer : tf.keras.layers.Layer + See attributes section. + path : list of str/int + See attributes section. + """ + + def __init__(self, layer, path=None): + self.layer = layer + self.name = layer.name + self.type = type(layer).__name__ + self.path = path + self.config = None + class Compressor: """ - Abstract base TensorFlow compressor + Common base class for all compressors. + + This class is designed for other base classes. + Algorithms should inherit ``Pruner`` or ``Quantizer`` instead. + + + Attributes + ---------- + bound_model : tf.keras.Model + Compressed user model. + wrappers : list of tf.keras.Model + A wrapper is an instrumented TF ``Layer``, in ``Model`` format. + The list is ordered by preorder traversal. + + Parameters + ---------- + LayerWrapperClass : a class derive from Model + The class used to instrument layers. + model : tf.keras.Model + The user model to be compressed. + config_list : list of JSON object + User configuration. The format is detailed in tutorial. """ - def __init__(self, model, config_list): - """ - Record necessary info in class members + def __init__(self, LayerWrapperClass, model, config_list): + assert isinstance(model, tf.keras.Model) + self.validate_config(model, config_list) - Parameters - ---------- - model : keras model - the model user wants to compress - config_list : list - the configurations that users specify for compression - """ self.bound_model = model - self.config_list = config_list - self.modules_to_compress = [] + self.wrappers = [] - def detect_modules_to_compress(self): - """ - detect all modules should be compressed, and save the result in `self.modules_to_compress`. + for layer_info in _detect_layers_to_compress(model, config_list): + self.wrappers.append(LayerWrapperClass(layer_info, self)) + if not self.wrappers: + _logger.warning('Nothing is configured to compress, please check your model and config list') - The model will be instrumented and user should never edit it after calling this method. - """ - if self.modules_to_compress is None: - self.modules_to_compress = [] - for keras_layer in self.bound_model.layers: - layer = LayerInfo(keras_layer) - config = self.select_config(layer) - if config is not None: - self.modules_to_compress.append((layer, config)) - return self.modules_to_compress + _instrument_model(model, self.wrappers) - def compress(self): + def set_wrappers_attribute(self, name, value): """ - Compress the model with algorithm implemented by subclass. - - The model will be instrumented and user should never edit it after calling this method. - `self.modules_to_compress` records all the to-be-compressed layers + Call ``setattr`` on all wrappers. """ - modules_to_compress = self.detect_modules_to_compress() - for layer, config in modules_to_compress: - self._instrument_layer(layer, config) - return self.bound_model + for wrapper in self.wrappers: + setattr(wrapper, name, value) - def get_modules_to_compress(self): - """ - To obtain all the to-be-compressed layers. - Returns - ------- - self.modules_to_compress : list - a list of the layers, each of which is a tuple (`layer`, `config`), - `layer` is `LayerInfo`, `config` is a `dict` - """ - return self.modules_to_compress +class Pruner(Compressor): + """ + Base class for pruning algorithms. - def select_config(self, layer): - """ - Find the configuration for `layer` by parsing `self.config_list` + End users should use ``compress`` and callback APIs (WIP) to prune their models. - Parameters - ---------- - layer: LayerInfo - one layer + The underlying model is instrumented upon initialization of pruner object. + So if you want to pre-train the model, train it before creating pruner object. - Returns - ------- - ret : config or None - the retrieved configuration for this layer, if None, this layer should - not be compressed - """ - ret = None - if layer.type is None: - return None - for config in self.config_list: - config = config.copy() - config['op_types'] = self._expand_config_op_types(config) - if layer.type not in config['op_types']: - continue - if config.get('op_names') and layer.name not in config['op_names']: - continue - ret = config - if ret is None or ret.get('exclude'): - return None - return ret + The compressed model can only execute in eager mode. - def update_epoch(self, epoch): - """ - If user want to update model every epoch, user can override this method. - This method should be called at the beginning of each epoch + Algorithm developers should override ``calc_masks`` method to specify pruning strategy. - Parameters - ---------- - epoch : num - the current epoch number - """ + Parameters + ---------- + model : tf.keras.Model + The user model to prune. + config_list : list of JSON object + User configuration. The format is detailed in tutorial. + """ + def __init__(self, model, config_list): + super().__init__(PrunerLayerWrapper, model, config_list) + #self.callback = PrunerCallback(self) - def step(self): - """ - If user want to update mask every step, user can override this method + def compress(self): """ + Apply compression on a pre-trained model. + If you want to prune the model during training, use callback API (WIP) instead. - def _instrument_layer(self, layer, config): + Returns + ------- + tf.keras.Model + The compressed model, for convenience. This is exactly the same object to constructor argument. """ - This method is implemented in the subclasses, i.e., `Pruner` and `Quantizer` + self._update_mask() + return self.bound_model - Parameters - ---------- - layer : LayerInfo - the layer to instrument the compression operation - config : dict - the configuration for compressing this layer + def calc_masks(self, wrapper, **kwargs): """ - raise NotImplementedError() - - def _expand_config_op_types(self, config): - if config is None: - return [] - op_types = [] - - for op_type in config.get('op_types', []): - if op_type == 'default': - op_types.extend(default_layers.default_layers) - else: - op_types.append(op_type) - return op_types + Abstract method to be overridden by algorithm. End users should ignore it. - -class Pruner(Compressor): - """ - Abstract base TensorFlow pruner - """ - - def calc_mask(self, layer, config): - """ - Pruners should overload this method to provide mask for weight tensors. - The mask must have the same shape and type comparing to the weight. - It will be applied with `mul()` operation on the weight. - This method is effectively hooked to `forward()` method of the model. + If the callback is set up, this method will be invoked at end of each training minibatch. + If not, it will only be called when end user invokes ``compress``. Parameters ---------- - layer : LayerInfo - calculate mask for `layer`'s weight - config : dict - the configuration for generating the mask - """ - raise NotImplementedError("Pruners must overload calc_mask()") + wrapper : PrunerLayerWrapper + The instrumented layer. + **kwargs + Reserved for forward compatibility. - def _instrument_layer(self, layer, config): - """ - Create a wrapper forward function to replace the original one. - - Parameters - ---------- - layer : LayerInfo - the layer to instrument the mask - config : dict - the configuration for generating the mask + Returns + ------- + dict of (str, tf.Tensor), or None + The key is weight ``Variable``'s name. The value is a mask ``Tensor`` of weight's shape and dtype. + If a weight's key does not appear in the return value, that weight will not be pruned. + Returning ``None`` means the mask is not changed since last time. + Weight names are globally unique, e.g. `model/conv_1/kernel:0`. """ - layer._call = layer.keras_layer.call + # TODO: maybe it should be able to calc on weight-granularity, beside from layer-granularity + raise NotImplementedError("Pruners must overload calc_masks()") - def new_call(*inputs): - weights = [x.numpy() for x in layer.keras_layer.weights] - mask = self.calc_mask(layer, config) - weights[layer.weight_index] = weights[layer.weight_index] * mask - layer.keras_layer.set_weights(weights) - ret = layer._call(*inputs) - return ret + def _update_mask(self): + for wrapper_idx, wrapper in enumerate(self.wrappers): + masks = self.calc_masks(wrapper, wrapper_idx=wrapper_idx) + if masks is not None: + wrapper.masks = masks - layer.keras_layer.call = new_call -class Quantizer(Compressor): +class PrunerLayerWrapper(tf.keras.Model): """ - Abstract base TensorFlow quantizer + Instrumented TF layer. + + Wrappers will be passed to pruner's ``calc_masks`` API, + and the pruning algorithm should use wrapper's attributes to calculate masks. + + Once instrumented, underlying layer's weights will get **modified** by masks before forward pass. + + Attributes + ---------- + layer_info : LayerInfo + All static information of the original layer. + layer : tf.keras.layers.Layer + The original layer. + config : JSON object + Selected configuration. The format is detailed in tutorial. + pruner : Pruner + Bound pruner object. + masks : dict of (str, tf.Tensor) + Current masks. The key is weight's name and the value is mask tensor. + On initialization, `masks` is an empty dict, which means no weight is pruned. + Afterwards, `masks` is the last return value of ``Pruner.calc_masks``. + See ``Pruner.calc_masks`` for details. """ - - def quantize_weight(self, weight, config, op, op_type, op_name): - raise NotImplementedError("Quantizer must overload quantize_weight()") + def __init__(self, layer_info, pruner): + super().__init__() + self.layer_info = layer_info + self.layer = layer_info.layer + self.config = layer_info.config + self.pruner = pruner + self.masks = {} + _logger.info('Layer detected to compress: %s', self.layer.name) + + def call(self, *inputs): + new_weights = [] + for weight in self.layer.weights: + mask = self.masks.get(weight.name) + if mask is not None: + new_weights.append(tf.math.multiply(weight, mask).numpy()) + else: + new_weights.append(weight.numpy()) + self.layer.set_weights(new_weights) + return self.layer(*inputs) + + +# TODO: designed to replace `patch_optimizer` +#class PrunerCallback(tf.keras.callbacks.Callback): +# def __init__(self, pruner): +# super().__init__() +# self._pruner = pruner +# +# def on_train_batch_end(self, batch, logs=None): +# self._pruner.update_mask() + + +def _detect_layers_to_compress(model, config_list): + # Returns list of LayerInfo. + located_layers = _locate_layers(model) + ret = [] + for layer in model.layers: + config = _select_config(LayerInfo(layer), config_list) + if config is not None: + if id(layer) not in located_layers: + _logger.error('Failed to locate layer %s in model. The layer will not be compressed. ' + 'This is a bug in NNI, feel free to fire an issue.', layer.name) + continue + layer_info = located_layers[id(layer)] + layer_info.config = config + ret.append(layer_info) + return ret + +def _locate_layers(model, cur_path=[]): + # Find out how to access layers from model object. + # Returns dict of (layer's object ID, LayerInfo). + # This function is required because TF framework does not track layer's attribute name, + # and to my knowledge `Layer.name` is only useful for read-only access. + # `cur_path`s format is documented in `LayerInfo.path`. + # TODO: it can only find layers in `Model` and `list` for now. + ret = {} + + if isinstance(model, tf.keras.Model): + for key, value in model.__dict__.items(): + if isinstance(value, tf.keras.Model): + ret.update(_locate_layers(value, cur_path + [key])) + elif isinstance(value, list): + ret.update(_locate_layers(value, cur_path + [key])) + elif isinstance(value, tf.keras.layers.Layer): + ret[id(value)] = LayerInfo(value, cur_path + [key]) + + elif isinstance(model, list): + for i, item in enumerate(model): + if isinstance(item, tf.keras.Model): + ret.update(_locate_layers(item, cur_path + [i])) + elif isinstance(item, tf.keras.layers.Layer): + ret[id(item)] = LayerInfo(item, cur_path + [i]) + + else: + raise ValueError('Unexpected model type: {}'.format(type(model))) + return ret + +def _select_config(layer_info, config_list): + # Find the last matching config block for given layer. + # Returns None if the layer should not be compressed. + ret = None + for config in config_list: + if 'op_types' in config: + match = layer_info.type in config['op_types'] + match_default = 'default' in config['op_types'] and layer_info.type in default_layers.weighted_modules + if not match and not match_default: + continue + if 'op_names' in config and layer_info.name not in config['op_names']: + continue + ret = config + if ret is None or 'exclude' in ret: + return None + return ret + + +def _instrument_model(model, wrappers): + # Replace layers to wrappers + for wrapper in reversed(wrappers): + cur = model + for key in wrapper.layer_info.path[:-1]: + if isinstance(key, int): + cur = cur[key] + else: + cur = getattr(cur, key) + key = wrapper.layer_info.path[-1] + if isinstance(key, int): + cur[key] = wrapper + else: + setattr(cur, key, wrapper) diff --git a/src/sdk/pynni/nni/compression/tensorflow/default_layers.py b/src/sdk/pynni/nni/compression/tensorflow/default_layers.py index 2ecc46e3e3..0c729bd883 100644 --- a/src/sdk/pynni/nni/compression/tensorflow/default_layers.py +++ b/src/sdk/pynni/nni/compression/tensorflow/default_layers.py @@ -1,31 +1,9 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -from tensorflow import keras - -supported_layers = { - keras.layers.Conv1D: ('Conv1D', 0), - keras.layers.Conv2D: ('Conv2D', 0), - keras.layers.Conv2DTranspose: ('Conv2DTranspose', 0), - keras.layers.Conv3D: ('Conv3D', 0), - keras.layers.Conv3DTranspose: ('Conv3DTranspose', 0), - keras.layers.ConvLSTM2D: ('ConvLSTM2D', 0), - keras.layers.Dense: ('Dense', 0), - keras.layers.Embedding: ('Embedding', 0), - keras.layers.GRU: ('GRU', 0), - keras.layers.LSTM: ('LSTM', 0), -} - -default_layers = [x[0] for x in supported_layers.values()] - -def get_op_type(layer_type): - if layer_type in supported_layers: - return supported_layers[layer_type][0] - else: - return None - -def get_weight_index(op_type): - for k in supported_layers: - if supported_layers[k][0] == op_type: - return supported_layers[k][1] - return None +weighted_modules = [ + 'Conv1D', 'Conv2D', 'Conv3D', 'Conv1DTranspose', 'Conv2DTranspose', 'Conv3DTranspose', + 'Dense', + 'PReLU', + 'Embedding', +] diff --git a/src/sdk/pynni/nni/compression/tensorflow/pruning/__init__.py b/src/sdk/pynni/nni/compression/tensorflow/pruning/__init__.py new file mode 100644 index 0000000000..f8ac8ea9b9 --- /dev/null +++ b/src/sdk/pynni/nni/compression/tensorflow/pruning/__init__.py @@ -0,0 +1 @@ +from .one_shot import * diff --git a/src/sdk/pynni/nni/compression/tensorflow/pruning/one_shot.py b/src/sdk/pynni/nni/compression/tensorflow/pruning/one_shot.py new file mode 100644 index 0000000000..ace3d39e4e --- /dev/null +++ b/src/sdk/pynni/nni/compression/tensorflow/pruning/one_shot.py @@ -0,0 +1,67 @@ +import tensorflow as tf + +from ..compressor import Pruner + +__all__ = [ + 'OneshotPruner', + 'LevelPruner', +] + +class OneshotPruner(Pruner): + def __init__(self, model, config_list, pruning_algorithm='level', **algo_kwargs): + super().__init__(model, config_list) + self.set_wrappers_attribute('calculated', False) + self.masker = MASKER_DICT[pruning_algorithm](model, self, **algo_kwargs) + + def validate_config(self, model, config_list): + pass # TODO + + def calc_masks(self, wrapper, wrapper_idx=None): + if wrapper.calculated: + return None + sparsity = wrapper.config['sparsity'] + masks = self.masker.calc_masks(sparsity, wrapper, wrapper_idx) + if masks is not None: + wrapper.calculated = True + return masks + + +class LevelPruner(OneshotPruner): + def __init__(self, model, config_list): + super().__init__(model, config_list, pruning_algorithm='level') + + +class WeightMasker: + def __init__(self, model, pruner, **kwargs): + self.model = model + self.pruner = pruner + + def calc_masks(self, sparsity, wrapper, wrapper_idx=None): + raise NotImplementedError() + + +class LevelPrunerMasker(WeightMasker): + def calc_masks(self, sparsity, wrapper, wrapper_idx=None): + masks = {} + for weight_variable in wrapper.layer.weights: + if weight_variable.name == 'bias': + continue + + k = int(tf.size(weight_variable).numpy() * sparsity) + if k == 0: + continue + + weight = weight_variable.read_value() + if wrapper.masks.get(weight_variable.name) is not None: + weight = tf.math.multiply(weight, wrapper.masks[weight_variable.name]) + + w_abs = tf.math.abs(tf.reshape(weight, [-1])) + threshold = tf.math.top_k(w_abs, k)[0][0] + mask = tf.math.greater(w_abs, threshold) + masks[weight_variable.name] = tf.cast(mask, weight.dtype) + return masks + + +MASKER_DICT = { + 'level': LevelPrunerMasker, +} diff --git a/src/sdk/pynni/tests/test_compressor.py b/src/sdk/pynni/tests/test_compressor_torch.py similarity index 87% rename from src/sdk/pynni/tests/test_compressor.py rename to src/sdk/pynni/tests/test_compressor_torch.py index 87afb5f23c..8d631da25a 100644 --- a/src/sdk/pynni/tests/test_compressor.py +++ b/src/sdk/pynni/tests/test_compressor_torch.py @@ -3,33 +3,12 @@ from unittest import TestCase, main import numpy as np -import tensorflow as tf import torch import torch.nn.functional as F import schema import nni.compression.torch as torch_compressor import math -if tf.__version__ >= '2.0': - import nni.compression.tensorflow as tf_compressor - - -def get_tf_model(): - model = tf.keras.models.Sequential([ - tf.keras.layers.Conv2D(filters=5, kernel_size=7, input_shape=[28, 28, 1], activation='relu', padding="SAME"), - tf.keras.layers.MaxPooling2D(pool_size=2), - tf.keras.layers.Conv2D(filters=10, kernel_size=3, activation='relu', padding="SAME"), - tf.keras.layers.MaxPooling2D(pool_size=2), - tf.keras.layers.Flatten(), - tf.keras.layers.Dense(units=128, activation='relu'), - tf.keras.layers.Dropout(0.5), - tf.keras.layers.Dense(units=10, activation='softmax'), - ]) - model.compile(loss="sparse_categorical_crossentropy", - optimizer=tf.keras.optimizers.SGD(lr=1e-3), - metrics=["accuracy"]) - return model - class TorchModel(torch.nn.Module): def __init__(self): @@ -52,13 +31,6 @@ def forward(self, x): return F.log_softmax(x, dim=1) -def tf2(func): - def test_tf2_func(*args): - if tf.__version__ >= '2.0': - func(*args) - - return test_tf2_func - class CompressorTestCase(TestCase): def test_torch_quantizer_modules_detection(self): # test if modules can be detected @@ -92,11 +64,6 @@ def test_torch_level_pruner(self): configure_list = [{'sparsity': 0.8, 'op_types': ['default']}] torch_compressor.LevelPruner(model, configure_list, optimizer).compress() - @tf2 - def test_tf_level_pruner(self): - configure_list = [{'sparsity': 0.8, 'op_types': ['default']}] - tf_compressor.LevelPruner(get_tf_model(), configure_list).compress() - def test_torch_naive_quantizer(self): model = TorchModel() configure_list = [{ @@ -108,10 +75,6 @@ def test_torch_naive_quantizer(self): }] torch_compressor.NaiveQuantizer(model, configure_list).compress() - @tf2 - def test_tf_naive_quantizer(self): - tf_compressor.NaiveQuantizer(get_tf_model(), [{'op_types': ['default']}]).compress() - def test_torch_fpgm_pruner(self): """ With filters(kernels) weights defined as above (w), it is obvious that w[4] and w[5] is the Geometric Median @@ -141,23 +104,7 @@ def test_torch_fpgm_pruner(self): masks = pruner.calc_mask(model.conv2) assert all(torch.sum(masks['weight_mask'], (1, 2, 3)).numpy() == np.array([125., 125., 0., 0., 0., 0., 0., 0., 125., 125.])) - @tf2 - def test_tf_fpgm_pruner(self): - w = np.array([np.ones((5, 3, 3)) * (i+1) for i in range(10)]).astype(np.float32) - model = get_tf_model() - config_list = [{'sparsity': 0.2, 'op_types': ['Conv2D']}] - - pruner = tf_compressor.FPGMPruner(model, config_list) - weights = model.layers[2].weights - weights[0] = np.array(w).astype(np.float32).transpose([2, 3, 0, 1]).transpose([0, 1, 3, 2]) - model.layers[2].set_weights([weights[0], weights[1].numpy()]) - - layer = tf_compressor.compressor.LayerInfo(model.layers[2]) - masks = pruner.calc_mask(layer, config_list[0]).numpy() - masks = masks.reshape((-1, masks.shape[-1])).transpose([1, 0]) - - assert all(masks.sum((1)) == np.array([45., 45., 45., 45., 0., 0., 45., 45., 45., 45.])) - + def test_torch_l1filter_pruner(self): """ Filters with the minimum sum of the weights' L1 norm are pruned in this paper: From e7fccfb4a01285795c7a68ea233af119e0fd5a3c Mon Sep 17 00:00:00 2001 From: liuzhe-lz <40699903+liuzhe-lz@users.noreply.github.com> Date: Wed, 12 Aug 2020 14:46:23 +0800 Subject: [PATCH 17/28] TF NAS fix: avoid checking member during forward (#2781) Co-authored-by: liuzhe --- .../pynni/nni/nas/tensorflow/enas/trainer.py | 8 +++--- src/sdk/pynni/nni/nas/tensorflow/mutables.py | 27 ++++++++----------- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/src/sdk/pynni/nni/nas/tensorflow/enas/trainer.py b/src/sdk/pynni/nni/nas/tensorflow/enas/trainer.py index 2d03a3fbb8..a9645e9203 100644 --- a/src/sdk/pynni/nni/nas/tensorflow/enas/trainer.py +++ b/src/sdk/pynni/nni/nas/tensorflow/enas/trainer.py @@ -136,10 +136,10 @@ def validate_one_epoch(self, epoch): meters = AverageMeterGroup() for x, y in test_loader: self.mutator.reset() - logits = self.model(x) + logits = self.model(x, training=False) if isinstance(logits, tuple): logits, _ = logits - metrics = self.metrics(logits, y) + metrics = self.metrics(y, logits) loss = self.loss(y, logits) metrics['loss'] = tf.reduce_mean(loss).numpy() meters.update(metrics) @@ -151,8 +151,8 @@ def validate_one_epoch(self, epoch): def _create_train_loader(self): train_set = self.train_set.shuffle(1000000).repeat().batch(self.batch_size) - test_set = self.test_set.shuffle(1000000).repeat().batch(self.batch_size) + test_set = self.valid_set.shuffle(1000000).repeat().batch(self.batch_size) return iter(train_set), iter(test_set) def _create_validate_loader(self): - return iter(self.test_set.shuffle(1000000).repeat().batch(self.batch_size)) + return iter(self.test_set.shuffle(1000000).batch(self.batch_size)) diff --git a/src/sdk/pynni/nni/nas/tensorflow/mutables.py b/src/sdk/pynni/nni/nas/tensorflow/mutables.py index b83b6f6325..06183a34c1 100644 --- a/src/sdk/pynni/nni/nas/tensorflow/mutables.py +++ b/src/sdk/pynni/nni/nas/tensorflow/mutables.py @@ -28,20 +28,19 @@ def __init__(self, key=None): def __deepcopy__(self, memodict=None): raise NotImplementedError("Deep copy doesn't work for mutables.") - def __call__(self, *args, **kwargs): - self._check_built() - return super().__call__(*args, **kwargs) - def set_mutator(self, mutator): - if 'mutator' in self.__dict__: + if hasattr(self, 'mutator'): raise RuntimeError('`set_mutator is called more than once. ' 'Did you parse the search space multiple times? ' 'Or did you apply multiple fixed architectures?') - self.__dict__['mutator'] = mutator + self.mutator = mutator def call(self, *inputs): raise NotImplementedError('Method `call` of Mutable must be overridden') + def build(self, input_shape): + self._check_built() + @property def key(self): return self._key @@ -68,7 +67,6 @@ def __repr__(self): class MutableScope(Mutable): def __call__(self, *args, **kwargs): try: - self._check_built() self.mutator.enter_mutable_scope(self) return super().__call__(*args, **kwargs) finally: @@ -80,7 +78,7 @@ def __init__(self, op_candidates, reduction='sum', return_mask=False, key=None): super().__init__(key=key) self.names = [] if isinstance(op_candidates, OrderedDict): - for name, _ in op_candidates.items(): + for name in op_candidates: assert name not in ["length", "reduction", "return_mask", "_key", "key", "names"], \ "Please don't use a reserved name '{}' for your module.".format(name) self.names.append(name) @@ -94,21 +92,18 @@ def __init__(self, op_candidates, reduction='sum', return_mask=False, key=None): self.choices = op_candidates self.reduction = reduction self.return_mask = return_mask - self._built = False def call(self, *inputs): - if not self._built: - for op in self.choices: - if len(inputs) > 1: # FIXME: not tested - op.build([inp.shape for inp in inputs]) - elif len(inputs) == 1: - op.build(inputs[0].shape) - self._built = True out, mask = self.mutator.on_forward_layer_choice(self, *inputs) if self.return_mask: return out, mask return out + def build(self, input_shape): + self._check_built() + for op in self.choices: + op.build(input_shape) + def __len__(self): return len(self.choices) From 5d2a59fd4cf708d285d0db8ff3522c9156d2c4a9 Mon Sep 17 00:00:00 2001 From: Ningxin Zheng <49771382+zheng-ningxin@users.noreply.github.com> Date: Wed, 12 Aug 2020 15:24:37 +0800 Subject: [PATCH 18/28] Successive unpack (#2768) --- src/sdk/pynni/nni/_graph_utils.py | 30 ++++++-- src/sdk/pynni/tests/test_graph_utils.py | 98 ++++++++++++++++++++++++- 2 files changed, 119 insertions(+), 9 deletions(-) diff --git a/src/sdk/pynni/nni/_graph_utils.py b/src/sdk/pynni/nni/_graph_utils.py index 25e513f42e..3fa6cd0eab 100644 --- a/src/sdk/pynni/nni/_graph_utils.py +++ b/src/sdk/pynni/nni/_graph_utils.py @@ -530,8 +530,15 @@ def _is_key_func(self, node_cpp): return True if node_cpp.kind() in [LIST_UNPACK_KIND, TUPLE_UNPACK_KIND]: # We cannot merge the List/Tuple - # Construct/Unpack func into other nodes, else it + # Unpack func into other nodes, else it # may lead to a graph construction error. + # The reason why we donnot take the construct node + # also as a key node is that `cat` operation node need + # the last(previous) visited node to infer the mask. If + # we take the Construct node as the important node, the + # predecessor of the `cat` node will always be a construct + # node, which means we cannot infer the mask for the cat + # operation. return True return False @@ -556,9 +563,13 @@ def unpack_manually(self): _logger.debug('List/Tuple Construct Node(cpp) %s', str(last_cpp)) _logger.debug('List/Tuple Unpack Node(cpp) %s', str(unpack_cpp)) assert len(list(unpack_cpp.outputs())) == len(list(last_cpp.inputs())) - for _input, _output in zip(last_cpp.inputs(), unpack_cpp.outputs()): - _debug_input = _input.debugName() - _debug_output = _output.debugName() + errmsg = '%s Input number: %d if inconsistent with the output number %d' % (unpack_cpp, \ + len(node.inputs), len(list(last_cpp.inputs()))) + + assert len(node.inputs) == len(list(last_cpp.inputs())), errmsg + for _debug_input, _debug_output in zip(node.inputs, node.outputs): + # _debug_input = _input.debugName() + # _debug_output = _output.debugName() if _debug_input in self.input_to_node and _debug_output in self.input_to_node: # input_to_node[_debug_input] is a list of NodePyGroup, because # one tensor can be used as input for multiple nodes at the same time. @@ -570,10 +581,13 @@ def unpack_manually(self): self.input_to_node[_debug_input].remove(node) # add the following nodes of _output into the input_to_node[_debug_input] self.input_to_node[_debug_input].extend(self.input_to_node[_debug_output]) - if _debug_input in self.output_to_node and _debug_output in self.output_to_node: - # output_to_node[_debug_output] is a NodePyGroup, because one output - # tensor only can be generated by one node. - self.output_to_node[_debug_output] = self.output_to_node[_debug_input] + # just remove the _debug_output from the grapgh index. So that we can also skip + # the construct and tuple + if _debug_output in self.input_to_node: + for following_node in self.input_to_node[_debug_output]: + _tmp_index = following_node.inputs.index(_debug_output) + following_node.inputs[_tmp_index] = _debug_input + self.unpacked = True diff --git a/src/sdk/pynni/tests/test_graph_utils.py b/src/sdk/pynni/tests/test_graph_utils.py index 92851bc91c..f6181d5482 100644 --- a/src/sdk/pynni/tests/test_graph_utils.py +++ b/src/sdk/pynni/tests/test_graph_utils.py @@ -15,7 +15,7 @@ import unittest from unittest import TestCase, main -from nni._graph_utils import build_module_graph, build_graph, TorchModuleGraph +from nni._graph_utils import build_module_graph, build_graph, TorchModuleGraph, TUPLE_UNPACK_KIND class BackboneModel1(nn.Module): def __init__(self): @@ -194,5 +194,101 @@ def forward(self, x): assert len(nodes) == 1 node = nodes[0] + @unittest.skipIf(torch.__version__ < "1.4.0", "not supported") + def test_module_unpack(self): + """ + test the tuple/list unpack function of TorchModuleGraph. + Following models are from the issue 2756 + https://github.com/microsoft/nni/issues/2756. + MyModule will have two successive tuple unpack operations + between the B and C. + """ + class CBR(nn.Module): + def __init__(self, i, o): + super(CBR, self).__init__() + self.conv1 = nn.Conv2d(i, o, kernel_size=1) + self.bn1 = nn.BatchNorm2d(o) + self.act1 = nn.ReLU() + + def forward(self, x): + return self.act1(self.bn1(self.conv1(x))) + + + class A(nn.Module): + def __init__(self): + super(A, self).__init__() + self.conv1 = CBR(3, 6, ) + self.conv2 = CBR(6, 8, ) + self.conv3 = CBR(6, 12) + + def forward(self, x): + x1 = self.conv1(x) + x2 = self.conv2(x1) + x3 = self.conv3(x1) + return (x2, x3) + + + class B1(nn.Module): + def __init__(self): + super(B1, self).__init__() + self.conv1 = CBR(12, 32) + self.conv2 = CBR(32, 32) + self.conv3 = CBR(32, 32) + + def forward(self, x): + x1 = self.conv1(x) + x2 = self.conv2(x1) + x3 = self.conv3(x2) + return (x1, x2, x3) + + class B(nn.Module): + def __init__(self): + super(B, self).__init__() + self.b = B1() + + def forward(self, x): + return self.b(x[-1]) + + class C(nn.Module): + def __init__(self): + super(C, self).__init__() + self.conv1 = CBR(8, 32) + self.conv2 = CBR(12, 32) + self.conv3 = CBR(32, 32) + self.conv4 = CBR(32, 32) + self.conv5 = CBR(32, 32) + + def forward(self, x): + return(self.conv1(x[0]), self.conv2(x[1]), self.conv3(x[2]),self.conv4(x[3]),self.conv5(x[4])) + + class MyModule(nn.Module): + def __init__(self): + super(MyModule, self).__init__() + self.a = A() + self.b = B() + # self.dummy = Dummy() + self.c = C() + + def forward(self, x): + x_a = self.a(x) + x_b = self.b(x_a) + xc = self.c(x_a + x_b) + return xc + + dummy_input = torch.rand(1, 3, 28, 28) + model = MyModule() + graph = TorchModuleGraph(model, dummy_input) + graph.unpack_manually() + for node in graph.nodes_py.nodes_op: + # The input of the function nodes should + # not come from the TupleUnpack node, because + # all the TupleUnpack nodes have been removed(unpacked) + # manually + for _input in node.inputs: + if _input in graph.output_to_node: + preprocessor = graph.output_to_node[_input] + assert preprocessor.op_type != TUPLE_UNPACK_KIND + + if __name__ == '__main__': main() From e2a8689969d4cb8338e2f814c23b9dc006c5f0f7 Mon Sep 17 00:00:00 2001 From: Lijiaoa <61399850+Lijiaoa@users.noreply.github.com> Date: Wed, 12 Aug 2020 16:29:21 +0800 Subject: [PATCH 19/28] deal with all type metrics (#2782) * update * update * Fix metrics for type other than number * update * change master version TableList.tsx * revert unuseful change Co-authored-by: Lijiao Co-authored-by: Lijiao <1425861283@qq.com> Co-authored-by: Yuge Zhang --- src/webui/mock/all-types-metric.json | 2527 +++++++++++++++++ src/webui/src/App.tsx | 32 +- .../src/components/trial-detail/Para.tsx | 2 +- src/webui/src/static/function.ts | 27 +- src/webui/src/static/interface.ts | 2 +- src/webui/src/static/model/experiment.ts | 4 +- src/webui/src/static/model/searchspace.ts | 7 +- src/webui/src/static/model/trial.ts | 71 +- 8 files changed, 2621 insertions(+), 51 deletions(-) create mode 100644 src/webui/mock/all-types-metric.json diff --git a/src/webui/mock/all-types-metric.json b/src/webui/mock/all-types-metric.json new file mode 100644 index 0000000000..ec6bc19457 --- /dev/null +++ b/src/webui/mock/all-types-metric.json @@ -0,0 +1,2527 @@ + +{ + "checkStatus": { + "status": "DONE", + "errors": [] + }, + "experiment": { + "id": "Tkaxm2mb", + "revision": 118, + "execDuration": 150, + "logDir": "/***/nni/experiments/Tkaxm2mb", + "nextSequenceId": 110, + "params": { + "authorName": "default", + "experimentName": "default", + "trialConcurrency": 10, + "maxExecDuration": 3600, + "maxTrialNum": 100, + "searchSpace": "{\"intermediate1\": {\"_type\": \"choice\", \"_value\": [\"normal\", \"inf\", \"neginf\", \"nan\", \"string\", \"dict-empty\", \"dict-normal\", \"dict-nodefault\", \"dict-defaultdict\"]}, \"intermediate2\": {\"_type\": \"choice\", \"_value\": [\"normal\", \"inf\", \"neginf\", \"nan\", \"string\", \"dict-empty\", \"dict-normal\", \"dict-nodefault\", \"dict-defaultdict\"]}, \"intermediate3\": {\"_type\": \"choice\", \"_value\": [\"normal\", \"inf\", \"neginf\", \"nan\", \"string\", \"dict-empty\", \"dict-normal\", \"dict-nodefault\", \"dict-defaultdict\"]}, \"intermediate_count\": {\"_type\": \"choice\", \"_value\": [0, 1, 2, 3]}, \"final1\": {\"_type\": \"choice\", \"_value\": [\"normal\", \"inf\", \"neginf\", \"nan\", \"string\", \"dict-empty\", \"dict-normal\", \"dict-nodefault\", \"dict-defaultdict\"]}, \"final2\": {\"_type\": \"choice\", \"_value\": [\"normal\", \"inf\", \"neginf\", \"nan\", \"string\", \"dict-empty\", \"dict-normal\", \"dict-nodefault\", \"dict-defaultdict\"]}, \"final_count\": {\"_type\": \"choice\", \"_value\": [0, 1, 2]}}", + "trainingServicePlatform": "local", + "tuner": { + "codeDir": "/***/nnidev/src/webui/tests/metrics-test/.", + "classFileName": "naive_random.py", + "className": "NaiveRandomTuner", + "checkpointDir": "/***/nni/experiments/Tkaxm2mb/checkpoint" + }, + "versionCheck": true, + "clusterMetaData": [ + { + "key": "codeDir", + "value": "/***/nnidev/src/webui/tests/metrics-test/." + }, + { + "key": "command", + "value": "python trial.py" + } + ] + }, + "startTime": 1595901129833, + "endTime": 1595901290657 + }, + "metricData": [ + { + "timestamp": 1595901141232, + "trialJobId": "sXvMz", + "parameterId": "0", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{\\\"default\\\": -7.823851251971656, \\\"other\\\": -9.844189628757352}\"" + }, + { + "timestamp": 1595901141321, + "trialJobId": "y3owq", + "parameterId": "1", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"\\\"-5.8373125018382055\\\"\"" + }, + { + "timestamp": 1595901141347, + "trialJobId": "etEUl", + "parameterId": "2", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"-Infinity\"" + }, + { + "timestamp": 1595901141374, + "trialJobId": "r5pwY", + "parameterId": "3", + "type": "FINAL", + "sequence": 0, + "data": "\"{}\"" + }, + { + "timestamp": 1595901141455, + "trialJobId": "JxX0I", + "parameterId": "4", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"NaN\"" + }, + { + "timestamp": 1595901141543, + "trialJobId": "ywQvm", + "parameterId": "5", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{\\\"default\\\": {\\\"tensor\\\": 0, \\\"data\\\": -9.945796251990785}}\"" + }, + { + "timestamp": 1595901141643, + "trialJobId": "tkxcP", + "parameterId": "6", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"NaN\"" + }, + { + "timestamp": 1595901141708, + "trialJobId": "MjX3O", + "parameterId": "7", + "type": "FINAL", + "sequence": 0, + "data": "\"{}\"" + }, + { + "timestamp": 1595901141754, + "trialJobId": "MQlPp", + "parameterId": "9", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{\\\"default\\\": {\\\"tensor\\\": 0, \\\"data\\\": -6.887609164015767}}\"" + }, + { + "timestamp": 1595901141756, + "trialJobId": "LKVCX", + "parameterId": "8", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901142236, + "trialJobId": "sXvMz", + "parameterId": "0", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"{\\\"default\\\": -8.08656113718457, \\\"other\\\": 7.483152033140179}\"" + }, + { + "timestamp": 1595901142326, + "trialJobId": "y3owq", + "parameterId": "1", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"default\\\": {\\\"tensor\\\": 0, \\\"data\\\": 6.896445698700774}}\"" + }, + { + "timestamp": 1595901142355, + "trialJobId": "etEUl", + "parameterId": "2", + "type": "FINAL", + "sequence": 0, + "data": "\"\\\"3.727416078457388\\\"\"" + }, + { + "timestamp": 1595901142458, + "trialJobId": "JxX0I", + "parameterId": "4", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"{\\\"default\\\": {\\\"tensor\\\": 0, \\\"data\\\": 8.963738323502998}}\"" + }, + { + "timestamp": 1595901142548, + "trialJobId": "ywQvm", + "parameterId": "5", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"4.024454725511186\"" + }, + { + "timestamp": 1595901142758, + "trialJobId": "MQlPp", + "parameterId": "9", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"NaN\"" + }, + { + "timestamp": 1595901142760, + "trialJobId": "LKVCX", + "parameterId": "8", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"\\\"0.982665154375141\\\"\"" + }, + { + "timestamp": 1595901143239, + "trialJobId": "sXvMz", + "parameterId": "0", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"default\\\": {\\\"tensor\\\": 0, \\\"data\\\": -8.10531137074138}}\"" + }, + { + "timestamp": 1595901143362, + "trialJobId": "etEUl", + "parameterId": "2", + "type": "FINAL", + "sequence": 0, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901143462, + "trialJobId": "JxX0I", + "parameterId": "4", + "type": "PERIODICAL", + "sequence": 2, + "data": "\"{\\\"other\\\": -5.433157293214572}\"" + }, + { + "timestamp": 1595901143552, + "trialJobId": "ywQvm", + "parameterId": "5", + "type": "PERIODICAL", + "sequence": 2, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901143761, + "trialJobId": "MQlPp", + "parameterId": "9", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"other\\\": 8.927687040316364}\"" + }, + { + "timestamp": 1595901143764, + "trialJobId": "LKVCX", + "parameterId": "8", + "type": "FINAL", + "sequence": 0, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901144556, + "trialJobId": "ywQvm", + "parameterId": "5", + "type": "FINAL", + "sequence": 0, + "data": "\"-4.804921436452929\"" + }, + { + "timestamp": 1595901144765, + "trialJobId": "MQlPp", + "parameterId": "9", + "type": "FINAL", + "sequence": 0, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901156846, + "trialJobId": "fJHIW", + "parameterId": "10", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{\\\"default\\\": -6.3254623036120545, \\\"other\\\": 6.661583778582873}\"" + }, + { + "timestamp": 1595901156921, + "trialJobId": "z7WgL", + "parameterId": "13", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"NaN\"" + }, + { + "timestamp": 1595901156954, + "trialJobId": "Ofyt2", + "parameterId": "12", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"1.7787198770217199\"" + }, + { + "timestamp": 1595901157264, + "trialJobId": "aKV3K", + "parameterId": "17", + "type": "FINAL", + "sequence": 0, + "data": "\"NaN\"" + }, + { + "timestamp": 1595901157336, + "trialJobId": "EFsFo", + "parameterId": "19", + "type": "FINAL", + "sequence": 0, + "data": "\"-0.9452602480917385\"" + }, + { + "timestamp": 1595901157852, + "trialJobId": "fJHIW", + "parameterId": "10", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"{}\"" + }, + { + "timestamp": 1595901157925, + "trialJobId": "z7WgL", + "parameterId": "13", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"NaN\"" + }, + { + "timestamp": 1595901157959, + "trialJobId": "Ofyt2", + "parameterId": "12", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901158930, + "trialJobId": "z7WgL", + "parameterId": "13", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"other\\\": 1.8045794393579122}\"" + }, + { + "timestamp": 1595901158961, + "trialJobId": "Ofyt2", + "parameterId": "12", + "type": "PERIODICAL", + "sequence": 2, + "data": "\"{\\\"default\\\": {\\\"tensor\\\": 0, \\\"data\\\": 8.685460178518326}}\"" + }, + { + "timestamp": 1595901159931, + "trialJobId": "z7WgL", + "parameterId": "13", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"other\\\": -7.794922103589295}\"" + }, + { + "timestamp": 1595901159966, + "trialJobId": "Ofyt2", + "parameterId": "12", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"default\\\": {\\\"tensor\\\": 0, \\\"data\\\": 7.483634247448858}}\"" + }, + { + "timestamp": 1595901160970, + "trialJobId": "Ofyt2", + "parameterId": "12", + "type": "FINAL", + "sequence": 0, + "data": "\"\\\"-3.905177892216985\\\"\"" + }, + { + "timestamp": 1595901172384, + "trialJobId": "dUJTL", + "parameterId": "20", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901172404, + "trialJobId": "xAoeQ", + "parameterId": "21", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{\\\"default\\\": 4.1133250278375915, \\\"other\\\": -2.4983824090454387}\"" + }, + { + "timestamp": 1595901172422, + "trialJobId": "de6XT", + "parameterId": "22", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{\\\"default\\\": -4.1861178094861495, \\\"other\\\": -1.8025564533646659}\"" + }, + { + "timestamp": 1595901172467, + "trialJobId": "Rofrb", + "parameterId": "24", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"other\\\": 8.461683188282755}\"" + }, + { + "timestamp": 1595901172471, + "trialJobId": "MOOrR", + "parameterId": "25", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{\\\"default\\\": {\\\"tensor\\\": 0, \\\"data\\\": -6.970771731370702}}\"" + }, + { + "timestamp": 1595901172521, + "trialJobId": "A7C0a", + "parameterId": "23", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"NaN\"" + }, + { + "timestamp": 1595901172629, + "trialJobId": "p2m5y", + "parameterId": "26", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"-4.874888030632755\"" + }, + { + "timestamp": 1595901172661, + "trialJobId": "mSPRF", + "parameterId": "27", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"-Infinity\"" + }, + { + "timestamp": 1595901172822, + "trialJobId": "G5nv9", + "parameterId": "28", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"-Infinity\"" + }, + { + "timestamp": 1595901172910, + "trialJobId": "ciSWN", + "parameterId": "29", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"NaN\"" + }, + { + "timestamp": 1595901173388, + "trialJobId": "dUJTL", + "parameterId": "20", + "type": "FINAL", + "sequence": 0, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901173408, + "trialJobId": "xAoeQ", + "parameterId": "21", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"-Infinity\"" + }, + { + "timestamp": 1595901173428, + "trialJobId": "de6XT", + "parameterId": "22", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"NaN\"" + }, + { + "timestamp": 1595901173472, + "trialJobId": "Rofrb", + "parameterId": "24", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"default\\\": -8.764260228545442, \\\"other\\\": 1.191253727619479}\"" + }, + { + "timestamp": 1595901173476, + "trialJobId": "MOOrR", + "parameterId": "25", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"\\\"-9.609870583781277\\\"\"" + }, + { + "timestamp": 1595901173524, + "trialJobId": "A7C0a", + "parameterId": "23", + "type": "FINAL", + "sequence": 0, + "data": "\"{}\"" + }, + { + "timestamp": 1595901173633, + "trialJobId": "p2m5y", + "parameterId": "26", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"{\\\"other\\\": 6.950769342806488}\"" + }, + { + "timestamp": 1595901173664, + "trialJobId": "mSPRF", + "parameterId": "27", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"NaN\"" + }, + { + "timestamp": 1595901173827, + "trialJobId": "G5nv9", + "parameterId": "28", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"\\\"8.83440457248663\\\"\"" + }, + { + "timestamp": 1595901173915, + "trialJobId": "ciSWN", + "parameterId": "29", + "type": "FINAL", + "sequence": 0, + "data": "\"NaN\"" + }, + { + "timestamp": 1595901174413, + "trialJobId": "xAoeQ", + "parameterId": "21", + "type": "PERIODICAL", + "sequence": 2, + "data": "\"1.8151933189474878\"" + }, + { + "timestamp": 1595901174434, + "trialJobId": "de6XT", + "parameterId": "22", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"other\\\": 0.8472658215331563}\"" + }, + { + "timestamp": 1595901174481, + "trialJobId": "MOOrR", + "parameterId": "25", + "type": "PERIODICAL", + "sequence": 2, + "data": "\"{}\"" + }, + { + "timestamp": 1595901174636, + "trialJobId": "p2m5y", + "parameterId": "26", + "type": "FINAL", + "sequence": 0, + "data": "\"9.902729745066438\"" + }, + { + "timestamp": 1595901174667, + "trialJobId": "mSPRF", + "parameterId": "27", + "type": "FINAL", + "sequence": 0, + "data": "\"\\\"-2.5183912965656763\\\"\"" + }, + { + "timestamp": 1595901174831, + "trialJobId": "G5nv9", + "parameterId": "28", + "type": "PERIODICAL", + "sequence": 2, + "data": "\"{\\\"default\\\": 5.406128202621579, \\\"other\\\": -6.350852877668696}\"" + }, + { + "timestamp": 1595901175418, + "trialJobId": "xAoeQ", + "parameterId": "21", + "type": "FINAL", + "sequence": 0, + "data": "\"-8.43771544998285\"" + }, + { + "timestamp": 1595901175438, + "trialJobId": "de6XT", + "parameterId": "22", + "type": "FINAL", + "sequence": 0, + "data": "\"-Infinity\"" + }, + { + "timestamp": 1595901175485, + "trialJobId": "MOOrR", + "parameterId": "25", + "type": "FINAL", + "sequence": 0, + "data": "\"\\\"2.8954800063491586\\\"\"" + }, + { + "timestamp": 1595901175671, + "trialJobId": "mSPRF", + "parameterId": "27", + "type": "FINAL", + "sequence": 0, + "data": "\"{}\"" + }, + { + "timestamp": 1595901175834, + "trialJobId": "G5nv9", + "parameterId": "28", + "type": "FINAL", + "sequence": 0, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901176422, + "trialJobId": "xAoeQ", + "parameterId": "21", + "type": "FINAL", + "sequence": 0, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901176489, + "trialJobId": "MOOrR", + "parameterId": "25", + "type": "FINAL", + "sequence": 0, + "data": "\"-Infinity\"" + }, + { + "timestamp": 1595901176838, + "trialJobId": "G5nv9", + "parameterId": "28", + "type": "FINAL", + "sequence": 0, + "data": "\"3.5218235306581356\"" + }, + { + "timestamp": 1595901187944, + "trialJobId": "zaTFd", + "parameterId": "33", + "type": "FINAL", + "sequence": 0, + "data": "\"-Infinity\"" + }, + { + "timestamp": 1595901187975, + "trialJobId": "WrtVY", + "parameterId": "30", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{\\\"default\\\": 6.446947454739629, \\\"other\\\": 4.2394889873504695}\"" + }, + { + "timestamp": 1595901188002, + "trialJobId": "RZ45L", + "parameterId": "32", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"8.53321110060542\"" + }, + { + "timestamp": 1595901188047, + "trialJobId": "Ss6eU", + "parameterId": "34", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{\\\"default\\\": -2.6862091423857564, \\\"other\\\": 8.839298350682931}\"" + }, + { + "timestamp": 1595901188087, + "trialJobId": "J5lYo", + "parameterId": "31", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901188183, + "trialJobId": "tb6Tr", + "parameterId": "35", + "type": "FINAL", + "sequence": 0, + "data": "\"NaN\"" + }, + { + "timestamp": 1595901188267, + "trialJobId": "ZMzvY", + "parameterId": "36", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"-Infinity\"" + }, + { + "timestamp": 1595901188275, + "trialJobId": "PNJDQ", + "parameterId": "39", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"\\\"6.952238868136657\\\"\"" + }, + { + "timestamp": 1595901188309, + "trialJobId": "VFEj6", + "parameterId": "37", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{\\\"other\\\": 5.022546354803907}\"" + }, + { + "timestamp": 1595901188338, + "trialJobId": "mcAWe", + "parameterId": "38", + "type": "FINAL", + "sequence": 0, + "data": "\"5.528136238632005\"" + }, + { + "timestamp": 1595901188979, + "trialJobId": "WrtVY", + "parameterId": "30", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"-4.361541180657595\"" + }, + { + "timestamp": 1595901189006, + "trialJobId": "RZ45L", + "parameterId": "32", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"2.781596441148668\"" + }, + { + "timestamp": 1595901189053, + "trialJobId": "Ss6eU", + "parameterId": "34", + "type": "FINAL", + "sequence": 0, + "data": "\"-3.3592681835773286\"" + }, + { + "timestamp": 1595901189093, + "trialJobId": "J5lYo", + "parameterId": "31", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"\\\"-8.400621787401052\\\"\"" + }, + { + "timestamp": 1595901189269, + "trialJobId": "ZMzvY", + "parameterId": "36", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901189279, + "trialJobId": "PNJDQ", + "parameterId": "39", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"{\\\"default\\\": {\\\"tensor\\\": 0, \\\"data\\\": 2.1053341113677}}\"" + }, + { + "timestamp": 1595901189343, + "trialJobId": "mcAWe", + "parameterId": "38", + "type": "FINAL", + "sequence": 0, + "data": "\"NaN\"" + }, + { + "timestamp": 1595901189984, + "trialJobId": "WrtVY", + "parameterId": "30", + "type": "PERIODICAL", + "sequence": 2, + "data": "\"{\\\"default\\\": -9.29846727931565, \\\"other\\\": -3.5575764805061}\"" + }, + { + "timestamp": 1595901190056, + "trialJobId": "Ss6eU", + "parameterId": "34", + "type": "FINAL", + "sequence": 0, + "data": "\"6.581757373301858\"" + }, + { + "timestamp": 1595901190098, + "trialJobId": "J5lYo", + "parameterId": "31", + "type": "PERIODICAL", + "sequence": 2, + "data": "\"-Infinity\"" + }, + { + "timestamp": 1595901190273, + "trialJobId": "ZMzvY", + "parameterId": "36", + "type": "PERIODICAL", + "sequence": 2, + "data": "\"{\\\"other\\\": 3.39802649436532}\"" + }, + { + "timestamp": 1595901190283, + "trialJobId": "PNJDQ", + "parameterId": "39", + "type": "FINAL", + "sequence": 0, + "data": "\"-1.8105252216174517\"" + }, + { + "timestamp": 1595901190988, + "trialJobId": "WrtVY", + "parameterId": "30", + "type": "FINAL", + "sequence": 0, + "data": "\"9.357586503792628\"" + }, + { + "timestamp": 1595901191101, + "trialJobId": "J5lYo", + "parameterId": "31", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"default\\\": {\\\"tensor\\\": 0, \\\"data\\\": -4.784440856817207}}\"" + }, + { + "timestamp": 1595901191277, + "trialJobId": "ZMzvY", + "parameterId": "36", + "type": "FINAL", + "sequence": 0, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901191287, + "trialJobId": "PNJDQ", + "parameterId": "39", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"other\\\": 3.1762929944716927}\"" + }, + { + "timestamp": 1595901192106, + "trialJobId": "J5lYo", + "parameterId": "31", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"other\\\": 9.239821847210145}\"" + }, + { + "timestamp": 1595901203447, + "trialJobId": "B0prO", + "parameterId": "41", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"\\\"-2.479154993253598\\\"\"" + }, + { + "timestamp": 1595901203492, + "trialJobId": "ggpj9", + "parameterId": "43", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"-Infinity\"" + }, + { + "timestamp": 1595901203506, + "trialJobId": "ta3sm", + "parameterId": "40", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{\\\"other\\\": 9.24069239452307}\"" + }, + { + "timestamp": 1595901203549, + "trialJobId": "IZ5SL", + "parameterId": "44", + "type": "FINAL", + "sequence": 0, + "data": "\"\\\"-0.9142325374848674\\\"\"" + }, + { + "timestamp": 1595901203646, + "trialJobId": "MInUq", + "parameterId": "45", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901203705, + "trialJobId": "YWceT", + "parameterId": "46", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901203869, + "trialJobId": "idTj5", + "parameterId": "47", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{\\\"other\\\": 6.469325465919496}\"" + }, + { + "timestamp": 1595901203924, + "trialJobId": "LLkId", + "parameterId": "49", + "type": "FINAL", + "sequence": 0, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901204452, + "trialJobId": "B0prO", + "parameterId": "41", + "type": "FINAL", + "sequence": 0, + "data": "\"\\\"5.017413317607618\\\"\"" + }, + { + "timestamp": 1595901204496, + "trialJobId": "ggpj9", + "parameterId": "43", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"\\\"4.245668260906047\\\"\"" + }, + { + "timestamp": 1595901204511, + "trialJobId": "ta3sm", + "parameterId": "40", + "type": "FINAL", + "sequence": 0, + "data": "\"NaN\"" + }, + { + "timestamp": 1595901204553, + "trialJobId": "IZ5SL", + "parameterId": "44", + "type": "FINAL", + "sequence": 0, + "data": "\"-Infinity\"" + }, + { + "timestamp": 1595901204651, + "trialJobId": "MInUq", + "parameterId": "45", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"\\\"0.25272592242281533\\\"\"" + }, + { + "timestamp": 1595901204710, + "trialJobId": "YWceT", + "parameterId": "46", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"{\\\"default\\\": {\\\"tensor\\\": 0, \\\"data\\\": -6.908361500557971}}\"" + }, + { + "timestamp": 1595901204874, + "trialJobId": "idTj5", + "parameterId": "47", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901204929, + "trialJobId": "LLkId", + "parameterId": "49", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"default\\\": -1.7045174390916955, \\\"other\\\": 9.14883282895672}\"" + }, + { + "timestamp": 1595901205501, + "trialJobId": "ggpj9", + "parameterId": "43", + "type": "PERIODICAL", + "sequence": 2, + "data": "\"{}\"" + }, + { + "timestamp": 1595901205516, + "trialJobId": "ta3sm", + "parameterId": "40", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"default\\\": 1.7906001963110256, \\\"other\\\": 0.7111312975095512}\"" + }, + { + "timestamp": 1595901205714, + "trialJobId": "YWceT", + "parameterId": "46", + "type": "PERIODICAL", + "sequence": 2, + "data": "\"{\\\"other\\\": -5.9019641918541055}\"" + }, + { + "timestamp": 1595901205878, + "trialJobId": "idTj5", + "parameterId": "47", + "type": "PERIODICAL", + "sequence": 2, + "data": "\"{\\\"other\\\": 2.823040228107409}\"" + }, + { + "timestamp": 1595901206505, + "trialJobId": "ggpj9", + "parameterId": "43", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"default\\\": {\\\"tensor\\\": 0, \\\"data\\\": 4.182220602389556}}\"" + }, + { + "timestamp": 1595901206882, + "trialJobId": "idTj5", + "parameterId": "47", + "type": "FINAL", + "sequence": 0, + "data": "\"-4.4221564350515274\"" + }, + { + "timestamp": 1595901219027, + "trialJobId": "ZbXHn", + "parameterId": "52", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901219044, + "trialJobId": "En80l", + "parameterId": "51", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{}\"" + }, + { + "timestamp": 1595901219069, + "trialJobId": "l99Rx", + "parameterId": "50", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{\\\"default\\\": {\\\"tensor\\\": 0, \\\"data\\\": -9.014544365514672}}\"" + }, + { + "timestamp": 1595901219082, + "trialJobId": "ZnEue", + "parameterId": "54", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"NaN\"" + }, + { + "timestamp": 1595901219271, + "trialJobId": "elkq7", + "parameterId": "55", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{\\\"other\\\": 5.0465681238608}\"" + }, + { + "timestamp": 1595901219313, + "trialJobId": "eE79m", + "parameterId": "56", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{}\"" + }, + { + "timestamp": 1595901219316, + "trialJobId": "glY0F", + "parameterId": "57", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"default\\\": {\\\"tensor\\\": 0, \\\"data\\\": -7.127630310607653}}\"" + }, + { + "timestamp": 1595901219433, + "trialJobId": "RQQYv", + "parameterId": "58", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"-Infinity\"" + }, + { + "timestamp": 1595901219441, + "trialJobId": "mYziy", + "parameterId": "59", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"-5.80358242411701\"" + }, + { + "timestamp": 1595901220032, + "trialJobId": "ZbXHn", + "parameterId": "52", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901220048, + "trialJobId": "En80l", + "parameterId": "51", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"default\\\": 2.6298452518160538, \\\"other\\\": -0.7910217651464624}\"" + }, + { + "timestamp": 1595901220075, + "trialJobId": "l99Rx", + "parameterId": "50", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"\\\"5.408334907304216\\\"\"" + }, + { + "timestamp": 1595901220086, + "trialJobId": "ZnEue", + "parameterId": "54", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"{\\\"other\\\": -1.0699150731178424}\"" + }, + { + "timestamp": 1595901220276, + "trialJobId": "elkq7", + "parameterId": "55", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"-Infinity\"" + }, + { + "timestamp": 1595901220437, + "trialJobId": "RQQYv", + "parameterId": "58", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"{\\\"default\\\": 5.991596839229384, \\\"other\\\": -5.791984484999113}\"" + }, + { + "timestamp": 1595901221036, + "trialJobId": "ZbXHn", + "parameterId": "52", + "type": "PERIODICAL", + "sequence": 2, + "data": "\"-5.745821975821488\"" + }, + { + "timestamp": 1595901221079, + "trialJobId": "l99Rx", + "parameterId": "50", + "type": "PERIODICAL", + "sequence": 2, + "data": "\"{\\\"other\\\": -2.8010065229085024}\"" + }, + { + "timestamp": 1595901221280, + "trialJobId": "elkq7", + "parameterId": "55", + "type": "PERIODICAL", + "sequence": 2, + "data": "\"{\\\"default\\\": -7.3137237874911705, \\\"other\\\": -7.995517504106601}\"" + }, + { + "timestamp": 1595901221441, + "trialJobId": "RQQYv", + "parameterId": "58", + "type": "PERIODICAL", + "sequence": 2, + "data": "\"\\\"-1.3969094674689302\\\"\"" + }, + { + "timestamp": 1595901222041, + "trialJobId": "ZbXHn", + "parameterId": "52", + "type": "FINAL", + "sequence": 0, + "data": "\"NaN\"" + }, + { + "timestamp": 1595901222284, + "trialJobId": "elkq7", + "parameterId": "55", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"default\\\": {\\\"tensor\\\": 0, \\\"data\\\": 8.087980079012624}}\"" + }, + { + "timestamp": 1595901222446, + "trialJobId": "RQQYv", + "parameterId": "58", + "type": "FINAL", + "sequence": 0, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901223290, + "trialJobId": "elkq7", + "parameterId": "55", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"other\\\": 4.0721479933987474}\"" + }, + { + "timestamp": 1595901223449, + "trialJobId": "RQQYv", + "parameterId": "58", + "type": "FINAL", + "sequence": 0, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901234574, + "trialJobId": "VmI7f", + "parameterId": "60", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{\\\"other\\\": -1.820722180261674}\"" + }, + { + "timestamp": 1595901234697, + "trialJobId": "VSWkZ", + "parameterId": "63", + "type": "FINAL", + "sequence": 0, + "data": "\"-Infinity\"" + }, + { + "timestamp": 1595901234732, + "trialJobId": "EZUe0", + "parameterId": "62", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901234745, + "trialJobId": "zHVA2", + "parameterId": "64", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"\\\"5.633256430913928\\\"\"" + }, + { + "timestamp": 1595901234746, + "trialJobId": "a1MOX", + "parameterId": "61", + "type": "FINAL", + "sequence": 0, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901234806, + "trialJobId": "u8t3k", + "parameterId": "66", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{}\"" + }, + { + "timestamp": 1595901234898, + "trialJobId": "OuLsc", + "parameterId": "65", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{\\\"default\\\": 8.765460410421554, \\\"other\\\": 6.246732298977708}\"" + }, + { + "timestamp": 1595901234949, + "trialJobId": "eGrff", + "parameterId": "67", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901234962, + "trialJobId": "Ujc39", + "parameterId": "68", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{\\\"other\\\": -1.1268667476429037}\"" + }, + { + "timestamp": 1595901235066, + "trialJobId": "wg7hy", + "parameterId": "69", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"-Infinity\"" + }, + { + "timestamp": 1595901235579, + "trialJobId": "VmI7f", + "parameterId": "60", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"other\\\": 5.584444622345025}\"" + }, + { + "timestamp": 1595901235703, + "trialJobId": "VSWkZ", + "parameterId": "63", + "type": "FINAL", + "sequence": 0, + "data": "\"\\\"-3.8928731668227456\\\"\"" + }, + { + "timestamp": 1595901235735, + "trialJobId": "EZUe0", + "parameterId": "62", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"{\\\"default\\\": -9.739986972816562, \\\"other\\\": -0.357173900732942}\"" + }, + { + "timestamp": 1595901235743, + "trialJobId": "zHVA2", + "parameterId": "64", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"{}\"" + }, + { + "timestamp": 1595901235747, + "trialJobId": "a1MOX", + "parameterId": "61", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"default\\\": {\\\"tensor\\\": 0, \\\"data\\\": -2.2757225430354033}}\"" + }, + { + "timestamp": 1595901235809, + "trialJobId": "u8t3k", + "parameterId": "66", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"2.6146452798999604\"" + }, + { + "timestamp": 1595901235903, + "trialJobId": "OuLsc", + "parameterId": "65", + "type": "FINAL", + "sequence": 0, + "data": "\"-Infinity\"" + }, + { + "timestamp": 1595901235953, + "trialJobId": "eGrff", + "parameterId": "67", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"-Infinity\"" + }, + { + "timestamp": 1595901235967, + "trialJobId": "Ujc39", + "parameterId": "68", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"5.879185588587639\"" + }, + { + "timestamp": 1595901236070, + "trialJobId": "wg7hy", + "parameterId": "69", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"{}\"" + }, + { + "timestamp": 1595901236584, + "trialJobId": "VmI7f", + "parameterId": "60", + "type": "FINAL", + "sequence": 0, + "data": "\"{}\"" + }, + { + "timestamp": 1595901236740, + "trialJobId": "EZUe0", + "parameterId": "62", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"default\\\": {\\\"tensor\\\": 0, \\\"data\\\": 1.091872290620957}}\"" + }, + { + "timestamp": 1595901236749, + "trialJobId": "zHVA2", + "parameterId": "64", + "type": "PERIODICAL", + "sequence": 2, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901236813, + "trialJobId": "u8t3k", + "parameterId": "66", + "type": "PERIODICAL", + "sequence": 2, + "data": "\"8.789904140828813\"" + }, + { + "timestamp": 1595901236956, + "trialJobId": "eGrff", + "parameterId": "67", + "type": "PERIODICAL", + "sequence": 2, + "data": "\"{}\"" + }, + { + "timestamp": 1595901236971, + "trialJobId": "Ujc39", + "parameterId": "68", + "type": "PERIODICAL", + "sequence": 2, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901237078, + "trialJobId": "wg7hy", + "parameterId": "69", + "type": "FINAL", + "sequence": 0, + "data": "\"NaN\"" + }, + { + "timestamp": 1595901237754, + "trialJobId": "zHVA2", + "parameterId": "64", + "type": "FINAL", + "sequence": 0, + "data": "\"\\\"6.067140454523734\\\"\"" + }, + { + "timestamp": 1595901237817, + "trialJobId": "u8t3k", + "parameterId": "66", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"default\\\": 6.738254874676596, \\\"other\\\": 3.407365737620623}\"" + }, + { + "timestamp": 1595901237960, + "trialJobId": "eGrff", + "parameterId": "67", + "type": "FINAL", + "sequence": 0, + "data": "\"-8.799684391921716\"" + }, + { + "timestamp": 1595901250174, + "trialJobId": "rCq1z", + "parameterId": "70", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901250225, + "trialJobId": "lCV9F", + "parameterId": "71", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{\\\"default\\\": -8.842244100829086, \\\"other\\\": -2.386236945799789}\"" + }, + { + "timestamp": 1595901250260, + "trialJobId": "IiCNj", + "parameterId": "72", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{}\"" + }, + { + "timestamp": 1595901250333, + "trialJobId": "e9bF7", + "parameterId": "74", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{\\\"default\\\": {\\\"tensor\\\": 0, \\\"data\\\": 1.0496251402087449}}\"" + }, + { + "timestamp": 1595901250335, + "trialJobId": "QPHAP", + "parameterId": "73", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{\\\"other\\\": 2.32920853708144}\"" + }, + { + "timestamp": 1595901250425, + "trialJobId": "XURUT", + "parameterId": "75", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{\\\"default\\\": -1.0644703377033373, \\\"other\\\": -9.313141516349681}\"" + }, + { + "timestamp": 1595901250515, + "trialJobId": "QafjF", + "parameterId": "76", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"-Infinity\"" + }, + { + "timestamp": 1595901250567, + "trialJobId": "ognsb", + "parameterId": "77", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{}\"" + }, + { + "timestamp": 1595901250642, + "trialJobId": "bsqIF", + "parameterId": "78", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"-8.862311093464992\"" + }, + { + "timestamp": 1595901250807, + "trialJobId": "OiDp3", + "parameterId": "79", + "type": "FINAL", + "sequence": 0, + "data": "\"NaN\"" + }, + { + "timestamp": 1595901251178, + "trialJobId": "rCq1z", + "parameterId": "70", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"6.248246468866515\"" + }, + { + "timestamp": 1595901251229, + "trialJobId": "lCV9F", + "parameterId": "71", + "type": "FINAL", + "sequence": 0, + "data": "\"-4.715082675645508\"" + }, + { + "timestamp": 1595901251264, + "trialJobId": "IiCNj", + "parameterId": "72", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"NaN\"" + }, + { + "timestamp": 1595901251338, + "trialJobId": "e9bF7", + "parameterId": "74", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"{\\\"default\\\": {\\\"tensor\\\": 0, \\\"data\\\": -5.9184589897185536}}\"" + }, + { + "timestamp": 1595901251339, + "trialJobId": "QPHAP", + "parameterId": "73", + "type": "FINAL", + "sequence": 0, + "data": "\"NaN\"" + }, + { + "timestamp": 1595901251429, + "trialJobId": "XURUT", + "parameterId": "75", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901251519, + "trialJobId": "QafjF", + "parameterId": "76", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"2.8245498927483315\"" + }, + { + "timestamp": 1595901251572, + "trialJobId": "ognsb", + "parameterId": "77", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"\\\"-8.995058023766827\\\"\"" + }, + { + "timestamp": 1595901251646, + "trialJobId": "bsqIF", + "parameterId": "78", + "type": "FINAL", + "sequence": 0, + "data": "\"0.8160856187719805\"" + }, + { + "timestamp": 1595901251812, + "trialJobId": "OiDp3", + "parameterId": "79", + "type": "FINAL", + "sequence": 0, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901252345, + "trialJobId": "QPHAP", + "parameterId": "73", + "type": "FINAL", + "sequence": 0, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901252345, + "trialJobId": "e9bF7", + "parameterId": "74", + "type": "FINAL", + "sequence": 0, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901252434, + "trialJobId": "XURUT", + "parameterId": "75", + "type": "PERIODICAL", + "sequence": 2, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901252523, + "trialJobId": "QafjF", + "parameterId": "76", + "type": "PERIODICAL", + "sequence": 2, + "data": "\"{\\\"other\\\": 1.0945194250479169}\"" + }, + { + "timestamp": 1595901252575, + "trialJobId": "ognsb", + "parameterId": "77", + "type": "FINAL", + "sequence": 0, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901253438, + "trialJobId": "XURUT", + "parameterId": "75", + "type": "FINAL", + "sequence": 0, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901253527, + "trialJobId": "QafjF", + "parameterId": "76", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"default\\\": 2.541955892720406, \\\"other\\\": -4.377781317201417}\"" + }, + { + "timestamp": 1595901265903, + "trialJobId": "wkVrB", + "parameterId": "82", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"NaN\"" + }, + { + "timestamp": 1595901265939, + "trialJobId": "bQhQx", + "parameterId": "81", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{\\\"other\\\": -7.7245400592833535}\"" + }, + { + "timestamp": 1595901266079, + "trialJobId": "VstNm", + "parameterId": "84", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{\\\"default\\\": -6.8832078296276, \\\"other\\\": -7.67458445595935}\"" + }, + { + "timestamp": 1595901266081, + "trialJobId": "GRUrH", + "parameterId": "85", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"NaN\"" + }, + { + "timestamp": 1595901266190, + "trialJobId": "RzOte", + "parameterId": "87", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{\\\"default\\\": {\\\"tensor\\\": 0, \\\"data\\\": -2.6660304258057117}}\"" + }, + { + "timestamp": 1595901266208, + "trialJobId": "Sb2tj", + "parameterId": "86", + "type": "FINAL", + "sequence": 0, + "data": "\"-Infinity\"" + }, + { + "timestamp": 1595901266317, + "trialJobId": "NB1ou", + "parameterId": "89", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{\\\"default\\\": -6.409147608132686, \\\"other\\\": 3.281989187926694}\"" + }, + { + "timestamp": 1595901266894, + "trialJobId": "wkVrB", + "parameterId": "82", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"{}\"" + }, + { + "timestamp": 1595901266944, + "trialJobId": "bQhQx", + "parameterId": "81", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901267086, + "trialJobId": "GRUrH", + "parameterId": "85", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"-Infinity\"" + }, + { + "timestamp": 1595901267194, + "trialJobId": "RzOte", + "parameterId": "87", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"{\\\"default\\\": -5.66892041315209, \\\"other\\\": 2.3283168851019287}\"" + }, + { + "timestamp": 1595901267322, + "trialJobId": "NB1ou", + "parameterId": "89", + "type": "FINAL", + "sequence": 0, + "data": "\"{}\"" + }, + { + "timestamp": 1595901267898, + "trialJobId": "wkVrB", + "parameterId": "82", + "type": "PERIODICAL", + "sequence": 2, + "data": "\"{}\"" + }, + { + "timestamp": 1595901268197, + "trialJobId": "RzOte", + "parameterId": "87", + "type": "PERIODICAL", + "sequence": 2, + "data": "\"-Infinity\"" + }, + { + "timestamp": 1595901268902, + "trialJobId": "wkVrB", + "parameterId": "82", + "type": "FINAL", + "sequence": 0, + "data": "\"\\\"-8.643383856668647\\\"\"" + }, + { + "timestamp": 1595901269906, + "trialJobId": "wkVrB", + "parameterId": "82", + "type": "FINAL", + "sequence": 0, + "data": "\"\\\"5.674545659125172\\\"\"" + }, + { + "timestamp": 1595901281386, + "trialJobId": "utKiW", + "parameterId": "90", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901281478, + "trialJobId": "uPdSU", + "parameterId": "92", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"NaN\"" + }, + { + "timestamp": 1595901281487, + "trialJobId": "dPeSr", + "parameterId": "91", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"-Infinity\"" + }, + { + "timestamp": 1595901281537, + "trialJobId": "pCXHB", + "parameterId": "93", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901281579, + "trialJobId": "okxUn", + "parameterId": "95", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"other\\\": -2.023400446813328}\"" + }, + { + "timestamp": 1595901281605, + "trialJobId": "QO9EO", + "parameterId": "94", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"{\\\"other\\\": -7.37571843162065}\"" + }, + { + "timestamp": 1595901281742, + "trialJobId": "MrNMC", + "parameterId": "97", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"\\\"-0.03936926816899877\\\"\"" + }, + { + "timestamp": 1595901281745, + "trialJobId": "zZquy", + "parameterId": "99", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"\\\"-5.576159193327383\\\"\"" + }, + { + "timestamp": 1595901281765, + "trialJobId": "tXlrm", + "parameterId": "98", + "type": "PERIODICAL", + "sequence": 0, + "data": "\"\\\"2.8878423268479807\\\"\"" + }, + { + "timestamp": 1595901282391, + "trialJobId": "utKiW", + "parameterId": "90", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"-Infinity\"" + }, + { + "timestamp": 1595901282484, + "trialJobId": "uPdSU", + "parameterId": "92", + "type": "FINAL", + "sequence": 0, + "data": "\"{}\"" + }, + { + "timestamp": 1595901282540, + "trialJobId": "pCXHB", + "parameterId": "93", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"{\\\"other\\\": 1.9549026751543792}\"" + }, + { + "timestamp": 1595901282583, + "trialJobId": "okxUn", + "parameterId": "95", + "type": "FINAL", + "sequence": 0, + "data": "\"NaN\"" + }, + { + "timestamp": 1595901282611, + "trialJobId": "QO9EO", + "parameterId": "94", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"-3.3552487681830145\"" + }, + { + "timestamp": 1595901282746, + "trialJobId": "MrNMC", + "parameterId": "97", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"{\\\"default\\\": 3.641608082036285, \\\"other\\\": -8.348670460176342}\"" + }, + { + "timestamp": 1595901282749, + "trialJobId": "zZquy", + "parameterId": "99", + "type": "PERIODICAL", + "sequence": 1, + "data": "\"\\\"7.462277129845603\\\"\"" + }, + { + "timestamp": 1595901283396, + "trialJobId": "utKiW", + "parameterId": "90", + "type": "FINAL", + "sequence": 0, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901283488, + "trialJobId": "uPdSU", + "parameterId": "92", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"other\\\": 5.491624461329462}\"" + }, + { + "timestamp": 1595901283544, + "trialJobId": "pCXHB", + "parameterId": "93", + "type": "FINAL", + "sequence": 0, + "data": "\"-6.732500732566957\"" + }, + { + "timestamp": 1595901283618, + "trialJobId": "QO9EO", + "parameterId": "94", + "type": "FINAL", + "sequence": 0, + "data": "\"Infinity\"" + }, + { + "timestamp": 1595901283751, + "trialJobId": "MrNMC", + "parameterId": "97", + "type": "PERIODICAL", + "sequence": 2, + "data": "\"{\\\"default\\\": {\\\"tensor\\\": 0, \\\"data\\\": -3.38303756126507}}\"" + }, + { + "timestamp": 1595901284400, + "trialJobId": "utKiW", + "parameterId": "90", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"default\\\": {\\\"tensor\\\": 0, \\\"data\\\": -5.853483580522072}}\"" + }, + { + "timestamp": 1595901284547, + "trialJobId": "pCXHB", + "parameterId": "93", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"other\\\": -8.024581781976966}\"" + }, + { + "timestamp": 1595901284621, + "trialJobId": "QO9EO", + "parameterId": "94", + "type": "FINAL", + "sequence": 0, + "data": "\"\\\"-2.538557600204168\\\"\"" + }, + { + "timestamp": 1595901284756, + "trialJobId": "MrNMC", + "parameterId": "97", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"other\\\": -0.5479590285769511}\"" + }, + { + "timestamp": 1595901285760, + "trialJobId": "MrNMC", + "parameterId": "97", + "type": "FINAL", + "sequence": 0, + "data": "\"Infinity\"" + } + ], + "trialJobs": [ + { + "id": "sXvMz", + "status": "SUCCEEDED", + "hyperParameters": [ + "{\"parameter_id\":0,\"parameter_source\":\"algorithm\",\"parameters\":{\"intermediate1\":\"dict-normal\",\"intermediate2\":\"dict-normal\",\"intermediate3\":\"normal\",\"intermediate_count\":2,\"final1\":\"dict-defaultdict\",\"final2\":\"dict-nodefault\",\"final_count\":1},\"parameter_index\":0}" + ], + "logPath": "file://localhost:/***/nni/experiments/Tkaxm2mb/trials/sXvMz", + "startTime": 1595901139862, + "sequenceId": 0, + "endTime": 1595901143276, + "finalMetricData": [ + { + "timestamp": 1595901143239, + "trialJobId": "sXvMz", + "parameterId": "0", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"default\\\": {\\\"tensor\\\": 0, \\\"data\\\": -8.10531137074138}}\"" + } + ] + }, + { + "id": "y3owq", + "status": "SUCCEEDED", + "hyperParameters": [ + "{\"parameter_id\":1,\"parameter_source\":\"algorithm\",\"parameters\":{\"intermediate1\":\"string\",\"intermediate2\":\"dict-nodefault\",\"intermediate3\":\"dict-empty\",\"intermediate_count\":1,\"final1\":\"dict-defaultdict\",\"final2\":\"neginf\",\"final_count\":1},\"parameter_index\":0}" + ], + "logPath": "file://localhost:/***/nni/experiments/Tkaxm2mb/trials/y3owq", + "startTime": 1595901139885, + "sequenceId": 1, + "endTime": 1595901142374, + "finalMetricData": [ + { + "timestamp": 1595901142326, + "trialJobId": "y3owq", + "parameterId": "1", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"default\\\": {\\\"tensor\\\": 0, \\\"data\\\": 6.896445698700774}}\"" + } + ] + }, + { + "id": "etEUl", + "status": "SUCCEEDED", + "hyperParameters": [ + "{\"parameter_id\":2,\"parameter_source\":\"algorithm\",\"parameters\":{\"intermediate1\":\"neginf\",\"intermediate2\":\"inf\",\"intermediate3\":\"string\",\"intermediate_count\":1,\"final1\":\"string\",\"final2\":\"inf\",\"final_count\":2},\"parameter_index\":0}" + ], + "logPath": "file://localhost:/***/nni/experiments/Tkaxm2mb/trials/etEUl", + "startTime": 1595901139903, + "sequenceId": 2, + "endTime": 1595901143415, + "finalMetricData": [ + { + "timestamp": 1595901142355, + "trialJobId": "etEUl", + "parameterId": "2", + "type": "FINAL", + "sequence": 0, + "data": "\"\\\"3.727416078457388\\\"\"" + } + ] + }, + { + "id": "r5pwY", + "status": "SUCCEEDED", + "hyperParameters": [ + "{\"parameter_id\":3,\"parameter_source\":\"algorithm\",\"parameters\":{\"intermediate1\":\"inf\",\"intermediate2\":\"dict-empty\",\"intermediate3\":\"dict-nodefault\",\"intermediate_count\":0,\"final1\":\"dict-empty\",\"final2\":\"dict-normal\",\"final_count\":1},\"parameter_index\":0}" + ], + "logPath": "file://localhost:/***/nni/experiments/Tkaxm2mb/trials/r5pwY", + "startTime": 1595901139918, + "sequenceId": 3, + "endTime": 1595901141410, + "finalMetricData": [ + { + "timestamp": 1595901141374, + "trialJobId": "r5pwY", + "parameterId": "3", + "type": "FINAL", + "sequence": 0, + "data": "\"{}\"" + } + ] + }, + { + "id": "JxX0I", + "status": "SUCCEEDED", + "hyperParameters": [ + "{\"parameter_id\":4,\"parameter_source\":\"algorithm\",\"parameters\":{\"intermediate1\":\"nan\",\"intermediate2\":\"dict-defaultdict\",\"intermediate3\":\"dict-nodefault\",\"intermediate_count\":3,\"final1\":\"dict-defaultdict\",\"final2\":\"string\",\"final_count\":0},\"parameter_index\":0}" + ], + "logPath": "file://localhost:/***/nni/experiments/Tkaxm2mb/trials/JxX0I", + "startTime": 1595901139981, + "sequenceId": 4, + "endTime": 1595901143497 + }, + { + "id": "ywQvm", + "status": "SUCCEEDED", + "hyperParameters": [ + "{\"parameter_id\":5,\"parameter_source\":\"algorithm\",\"parameters\":{\"intermediate1\":\"dict-defaultdict\",\"intermediate2\":\"normal\",\"intermediate3\":\"inf\",\"intermediate_count\":3,\"final1\":\"normal\",\"final2\":\"dict-nodefault\",\"final_count\":1},\"parameter_index\":0}" + ], + "logPath": "file://localhost:/***/nni/experiments/Tkaxm2mb/trials/ywQvm", + "startTime": 1595901140071, + "sequenceId": 5, + "endTime": 1595901144603, + "finalMetricData": [ + { + "timestamp": 1595901144556, + "trialJobId": "ywQvm", + "parameterId": "5", + "type": "FINAL", + "sequence": 0, + "data": "\"-4.804921436452929\"" + } + ] + }, + { + "id": "tkxcP", + "status": "SUCCEEDED", + "hyperParameters": [ + "{\"parameter_id\":6,\"parameter_source\":\"algorithm\",\"parameters\":{\"intermediate1\":\"nan\",\"intermediate2\":\"dict-empty\",\"intermediate3\":\"inf\",\"intermediate_count\":1,\"final1\":\"nan\",\"final2\":\"nan\",\"final_count\":0},\"parameter_index\":0}" + ], + "logPath": "file://localhost:/***/nni/experiments/Tkaxm2mb/trials/tkxcP", + "startTime": 1595901140239, + "sequenceId": 6, + "endTime": 1595901141679 + }, + { + "id": "MjX3O", + "status": "SUCCEEDED", + "hyperParameters": [ + "{\"parameter_id\":7,\"parameter_source\":\"algorithm\",\"parameters\":{\"intermediate1\":\"dict-defaultdict\",\"intermediate2\":\"dict-nodefault\",\"intermediate3\":\"inf\",\"intermediate_count\":0,\"final1\":\"dict-empty\",\"final2\":\"dict-defaultdict\",\"final_count\":1},\"parameter_index\":0}" + ], + "logPath": "file://localhost:/***/nni/experiments/Tkaxm2mb/trials/MjX3O", + "startTime": 1595901134932, + "sequenceId": 7, + "endTime": 1595901141756, + "finalMetricData": [ + { + "timestamp": 1595901141708, + "trialJobId": "MjX3O", + "parameterId": "7", + "type": "FINAL", + "sequence": 0, + "data": "\"{}\"" + } + ] + }, + { + "id": "LKVCX", + "status": "SUCCEEDED", + "hyperParameters": [ + "{\"parameter_id\":8,\"parameter_source\":\"algorithm\",\"parameters\":{\"intermediate1\":\"inf\",\"intermediate2\":\"string\",\"intermediate3\":\"dict-defaultdict\",\"intermediate_count\":2,\"final1\":\"inf\",\"final2\":\"dict-defaultdict\",\"final_count\":1},\"parameter_index\":0}" + ], + "logPath": "file://localhost:/***/nni/experiments/Tkaxm2mb/trials/LKVCX", + "startTime": 1595901134943, + "sequenceId": 8, + "endTime": 1595901143811, + "finalMetricData": [ + { + "timestamp": 1595901143764, + "trialJobId": "LKVCX", + "parameterId": "8", + "type": "FINAL", + "sequence": 0, + "data": "\"Infinity\"" + } + ] + }, + { + "id": "MQlPp", + "status": "SUCCEEDED", + "hyperParameters": [ + "{\"parameter_id\":9,\"parameter_source\":\"algorithm\",\"parameters\":{\"intermediate1\":\"dict-defaultdict\",\"intermediate2\":\"nan\",\"intermediate3\":\"dict-defaultdict\",\"intermediate_count\":2,\"final1\":\"dict-nodefault\",\"final2\":\"inf\",\"final_count\":2},\"parameter_index\":0}" + ], + "logPath": "file://localhost:/***/nni/experiments/Tkaxm2mb/trials/MQlPp", + "startTime": 1595901134954, + "sequenceId": 9, + "endTime": 1595901144801, + "finalMetricData": [ + { + "timestamp": 1595901143761, + "trialJobId": "MQlPp", + "parameterId": "9", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"other\\\": 8.927687040316364}\"" + } + ] + }, + { + "id": "fJHIW", + "status": "SUCCEEDED", + "hyperParameters": [ + "{\"parameter_id\":10,\"parameter_source\":\"algorithm\",\"parameters\":{\"intermediate1\":\"dict-normal\",\"intermediate2\":\"dict-empty\",\"intermediate3\":\"nan\",\"intermediate_count\":2,\"final1\":\"neginf\",\"final2\":\"nan\",\"final_count\":0},\"parameter_index\":0}" + ], + "logPath": "file://localhost:/***/nni/experiments/Tkaxm2mb/trials/fJHIW", + "startTime": 1595901155426, + "sequenceId": 10, + "endTime": 1595901157888 + }, + { + "id": "RDBG2", + "status": "SUCCEEDED", + "hyperParameters": [ + "{\"parameter_id\":11,\"parameter_source\":\"algorithm\",\"parameters\":{\"intermediate1\":\"normal\",\"intermediate2\":\"string\",\"intermediate3\":\"dict-nodefault\",\"intermediate_count\":0,\"final1\":\"inf\",\"final2\":\"neginf\",\"final_count\":0},\"parameter_index\":0}" + ], + "logPath": "file://localhost:/***/nni/experiments/Tkaxm2mb/trials/RDBG2", + "startTime": 1595901155437, + "sequenceId": 11, + "endTime": 1595901155983 + }, + { + "id": "Ofyt2", + "status": "SUCCEEDED", + "hyperParameters": [ + "{\"parameter_id\":12,\"parameter_source\":\"algorithm\",\"parameters\":{\"intermediate1\":\"normal\",\"intermediate2\":\"inf\",\"intermediate3\":\"dict-defaultdict\",\"intermediate_count\":3,\"final1\":\"dict-defaultdict\",\"final2\":\"string\",\"final_count\":2},\"parameter_index\":0}" + ], + "logPath": "file://localhost:/***/nni/experiments/Tkaxm2mb/trials/Ofyt2", + "startTime": 1595901155450, + "sequenceId": 12, + "endTime": 1595901161007, + "finalMetricData": [ + { + "timestamp": 1595901159966, + "trialJobId": "Ofyt2", + "parameterId": "12", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"default\\\": {\\\"tensor\\\": 0, \\\"data\\\": 7.483634247448858}}\"" + } + ] + }, + { + "id": "z7WgL", + "status": "SUCCEEDED", + "hyperParameters": [ + "{\"parameter_id\":13,\"parameter_source\":\"algorithm\",\"parameters\":{\"intermediate1\":\"nan\",\"intermediate2\":\"nan\",\"intermediate3\":\"dict-normal\",\"intermediate_count\":2,\"final1\":\"dict-nodefault\",\"final2\":\"dict-nodefault\",\"final_count\":2},\"parameter_index\":0}" + ], + "logPath": "file://localhost:/***/nni/experiments/Tkaxm2mb/trials/z7WgL", + "startTime": 1595901155464, + "sequenceId": 13, + "endTime": 1595901159968, + "finalMetricData": [ + { + "timestamp": 1595901158930, + "trialJobId": "z7WgL", + "parameterId": "13", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"other\\\": 1.8045794393579122}\"" + } + ] + }, + { + "id": "OotJc", + "status": "SUCCEEDED", + "hyperParameters": [ + "{\"parameter_id\":14,\"parameter_source\":\"algorithm\",\"parameters\":{\"intermediate1\":\"dict-empty\",\"intermediate2\":\"inf\",\"intermediate3\":\"dict-empty\",\"intermediate_count\":0,\"final1\":\"dict-nodefault\",\"final2\":\"dict-empty\",\"final_count\":0},\"parameter_index\":0}" + ], + "logPath": "file://localhost:/***/nni/experiments/Tkaxm2mb/trials/OotJc", + "startTime": 1595901155490, + "sequenceId": 14, + "endTime": 1595901156062 + }, + { + "id": "WxWLk", + "status": "SUCCEEDED", + "hyperParameters": [ + "{\"parameter_id\":15,\"parameter_source\":\"algorithm\",\"parameters\":{\"intermediate1\":\"nan\",\"intermediate2\":\"normal\",\"intermediate3\":\"string\",\"intermediate_count\":0,\"final1\":\"nan\",\"final2\":\"dict-empty\",\"final_count\":0},\"parameter_index\":0}" + ], + "logPath": "file://localhost:/***/nni/experiments/Tkaxm2mb/trials/WxWLk", + "startTime": 1595901155536, + "sequenceId": 15, + "endTime": 1595901156147 + }, + { + "id": "Zzazj", + "status": "SUCCEEDED", + "hyperParameters": [ + "{\"parameter_id\":16,\"parameter_source\":\"algorithm\",\"parameters\":{\"intermediate1\":\"dict-empty\",\"intermediate2\":\"dict-normal\",\"intermediate3\":\"normal\",\"intermediate_count\":0,\"final1\":\"neginf\",\"final2\":\"nan\",\"final_count\":0},\"parameter_index\":0}" + ], + "logPath": "file://localhost:/***/nni/experiments/Tkaxm2mb/trials/Zzazj", + "startTime": 1595901155644, + "sequenceId": 16, + "endTime": 1595901156212 + }, + { + "id": "aKV3K", + "status": "SUCCEEDED", + "hyperParameters": [ + "{\"parameter_id\":17,\"parameter_source\":\"algorithm\",\"parameters\":{\"intermediate1\":\"dict-defaultdict\",\"intermediate2\":\"inf\",\"intermediate3\":\"normal\",\"intermediate_count\":0,\"final1\":\"nan\",\"final2\":\"inf\",\"final_count\":1},\"parameter_index\":0}" + ], + "logPath": "file://localhost:/***/nni/experiments/Tkaxm2mb/trials/aKV3K", + "startTime": 1595901155795, + "sequenceId": 17, + "endTime": 1595901157314, + "finalMetricData": [ + { + "timestamp": 1595901157264, + "trialJobId": "aKV3K", + "parameterId": "17", + "type": "FINAL", + "sequence": 0, + "data": "\"NaN\"" + } + ] + }, + { + "id": "WR5fG", + "status": "SUCCEEDED", + "hyperParameters": [ + "{\"parameter_id\":18,\"parameter_source\":\"algorithm\",\"parameters\":{\"intermediate1\":\"inf\",\"intermediate2\":\"dict-empty\",\"intermediate3\":\"inf\",\"intermediate_count\":0,\"final1\":\"normal\",\"final2\":\"nan\",\"final_count\":0},\"parameter_index\":0}" + ], + "logPath": "file://localhost:/***/nni/experiments/Tkaxm2mb/trials/WR5fG", + "startTime": 1595901155825, + "sequenceId": 18, + "endTime": 1595901156321 + }, + { + "id": "EFsFo", + "status": "SUCCEEDED", + "hyperParameters": [ + "{\"parameter_id\":19,\"parameter_source\":\"algorithm\",\"parameters\":{\"intermediate1\":\"inf\",\"intermediate2\":\"dict-nodefault\",\"intermediate3\":\"nan\",\"intermediate_count\":0,\"final1\":\"normal\",\"final2\":\"dict-defaultdict\",\"final_count\":1},\"parameter_index\":0}" + ], + "logPath": "file://localhost:/***/nni/experiments/Tkaxm2mb/trials/EFsFo", + "startTime": 1595901150521, + "sequenceId": 19, + "endTime": 1595901157386, + "finalMetricData": [ + { + "timestamp": 1595901157336, + "trialJobId": "EFsFo", + "parameterId": "19", + "type": "FINAL", + "sequence": 0, + "data": "\"-0.9452602480917385\"" + } + ] + }, + { + "id": "dUJTL", + "status": "SUCCEEDED", + "hyperParameters": [ + "{\"parameter_id\":20,\"parameter_source\":\"algorithm\",\"parameters\":{\"intermediate1\":\"inf\",\"intermediate2\":\"string\",\"intermediate3\":\"inf\",\"intermediate_count\":1,\"final1\":\"inf\",\"final2\":\"string\",\"final_count\":1},\"parameter_index\":0}" + ], + "logPath": "file://localhost:/***/nni/experiments/Tkaxm2mb/trials/dUJTL", + "startTime": 1595901170933, + "sequenceId": 20, + "endTime": 1595901173438, + "finalMetricData": [ + { + "timestamp": 1595901173388, + "trialJobId": "dUJTL", + "parameterId": "20", + "type": "FINAL", + "sequence": 0, + "data": "\"Infinity\"" + } + ] + }, + { + "id": "xAoeQ", + "status": "SUCCEEDED", + "hyperParameters": [ + "{\"parameter_id\":21,\"parameter_source\":\"algorithm\",\"parameters\":{\"intermediate1\":\"dict-normal\",\"intermediate2\":\"neginf\",\"intermediate3\":\"normal\",\"intermediate_count\":3,\"final1\":\"normal\",\"final2\":\"inf\",\"final_count\":2},\"parameter_index\":0}" + ], + "logPath": "file://localhost:/***/nni/experiments/Tkaxm2mb/trials/xAoeQ", + "startTime": 1595901170945, + "sequenceId": 21, + "endTime": 1595901176469, + "finalMetricData": [ + { + "timestamp": 1595901175418, + "trialJobId": "xAoeQ", + "parameterId": "21", + "type": "FINAL", + "sequence": 0, + "data": "\"-8.43771544998285\"" + } + ] + }, + { + "id": "de6XT", + "status": "SUCCEEDED", + "hyperParameters": [ + "{\"parameter_id\":22,\"parameter_source\":\"algorithm\",\"parameters\":{\"intermediate1\":\"dict-normal\",\"intermediate2\":\"nan\",\"intermediate3\":\"string\",\"intermediate_count\":2,\"final1\":\"dict-nodefault\",\"final2\":\"neginf\",\"final_count\":2},\"parameter_index\":0}" + ], + "logPath": "file://localhost:/***/nni/experiments/Tkaxm2mb/trials/de6XT", + "startTime": 1595901170962, + "sequenceId": 22, + "endTime": 1595901175475, + "finalMetricData": [ + { + "timestamp": 1595901174434, + "trialJobId": "de6XT", + "parameterId": "22", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"other\\\": 0.8472658215331563}\"" + } + ] + }, + { + "id": "A7C0a", + "status": "SUCCEEDED", + "hyperParameters": [ + "{\"parameter_id\":23,\"parameter_source\":\"algorithm\",\"parameters\":{\"intermediate1\":\"nan\",\"intermediate2\":\"normal\",\"intermediate3\":\"neginf\",\"intermediate_count\":1,\"final1\":\"dict-empty\",\"final2\":\"dict-defaultdict\",\"final_count\":1},\"parameter_index\":0}" + ], + "logPath": "file://localhost:/***/nni/experiments/Tkaxm2mb/trials/A7C0a", + "startTime": 1595901170977, + "sequenceId": 23, + "endTime": 1595901173571, + "finalMetricData": [ + { + "timestamp": 1595901173524, + "trialJobId": "A7C0a", + "parameterId": "23", + "type": "FINAL", + "sequence": 0, + "data": "\"{}\"" + } + ] + }, + { + "id": "Rofrb", + "status": "SUCCEEDED", + "hyperParameters": [ + "{\"parameter_id\":24,\"parameter_source\":\"algorithm\",\"parameters\":{\"intermediate1\":\"inf\",\"intermediate2\":\"dict-nodefault\",\"intermediate3\":\"neginf\",\"intermediate_count\":0,\"final1\":\"dict-nodefault\",\"final2\":\"dict-normal\",\"final_count\":2},\"parameter_index\":0}" + ], + "logPath": "file://localhost:/***/nni/experiments/Tkaxm2mb/trials/Rofrb", + "startTime": 1595901170990, + "sequenceId": 24, + "endTime": 1595901173521, + "finalMetricData": [ + { + "timestamp": 1595901172467, + "trialJobId": "Rofrb", + "parameterId": "24", + "type": "FINAL", + "sequence": 0, + "data": "\"{\\\"other\\\": 8.461683188282755}\"" + } + ] + }, + { + "id": "MOOrR", + "status": "SUCCEEDED", + "hyperParameters": [ + "{\"parameter_id\":25,\"parameter_source\":\"algorithm\",\"parameters\":{\"intermediate1\":\"dict-defaultdict\",\"intermediate2\":\"string\",\"intermediate3\":\"dict-empty\",\"intermediate_count\":3,\"final1\":\"string\",\"final2\":\"neginf\",\"final_count\":2},\"parameter_index\":0}" + ], + "logPath": "file://localhost:/***/nni/experiments/Tkaxm2mb/trials/MOOrR", + "startTime": 1595901171032, + "sequenceId": 25, + "endTime": 1595901176529, + "finalMetricData": [ + { + "timestamp": 1595901175485, + "trialJobId": "MOOrR", + "parameterId": "25", + "type": "FINAL", + "sequence": 0, + "data": "\"\\\"2.8954800063491586\\\"\"" + } + ] + }, + { + "id": "p2m5y", + "status": "SUCCEEDED", + "hyperParameters": [ + "{\"parameter_id\":26,\"parameter_source\":\"algorithm\",\"parameters\":{\"intermediate1\":\"normal\",\"intermediate2\":\"dict-nodefault\",\"intermediate3\":\"inf\",\"intermediate_count\":2,\"final1\":\"normal\",\"final2\":\"dict-defaultdict\",\"final_count\":1},\"parameter_index\":0}" + ], + "logPath": "file://localhost:/***/nni/experiments/Tkaxm2mb/trials/p2m5y", + "startTime": 1595901171129, + "sequenceId": 26, + "endTime": 1595901174686, + "finalMetricData": [ + { + "timestamp": 1595901174636, + "trialJobId": "p2m5y", + "parameterId": "26", + "type": "FINAL", + "sequence": 0, + "data": "\"9.902729745066438\"" + } + ] + }, + { + "id": "mSPRF", + "status": "SUCCEEDED", + "hyperParameters": [ + "{\"parameter_id\":27,\"parameter_source\":\"algorithm\",\"parameters\":{\"intermediate1\":\"neginf\",\"intermediate2\":\"nan\",\"intermediate3\":\"dict-nodefault\",\"intermediate_count\":2,\"final1\":\"string\",\"final2\":\"dict-empty\",\"final_count\":2},\"parameter_index\":0}" + ], + "logPath": "file://localhost:/***/nni/experiments/Tkaxm2mb/trials/mSPRF", + "startTime": 1595901171272, + "sequenceId": 27, + "endTime": 1595901175714, + "finalMetricData": [ + { + "timestamp": 1595901174667, + "trialJobId": "mSPRF", + "parameterId": "27", + "type": "FINAL", + "sequence": 0, + "data": "\"\\\"-2.5183912965656763\\\"\"" + } + ] + }, + { + "id": "G5nv9", + "status": "SUCCEEDED", + "hyperParameters": [ + "{\"parameter_id\":28,\"parameter_source\":\"algorithm\",\"parameters\":{\"intermediate1\":\"neginf\",\"intermediate2\":\"string\",\"intermediate3\":\"dict-normal\",\"intermediate_count\":3,\"final1\":\"inf\",\"final2\":\"normal\",\"final_count\":2},\"parameter_index\":0}" + ], + "logPath": "file://localhost:/***/nni/experiments/Tkaxm2mb/trials/G5nv9", + "startTime": 1595901171428, + "sequenceId": 28, + "endTime": 1595901176884, + "finalMetricData": [ + { + "timestamp": 1595901175834, + "trialJobId": "G5nv9", + "parameterId": "28", + "type": "FINAL", + "sequence": 0, + "data": "\"Infinity\"" + } + ] + } + ] +} diff --git a/src/webui/src/App.tsx b/src/webui/src/App.tsx index 2405b02de0..690efc23c4 100644 --- a/src/webui/src/App.tsx +++ b/src/webui/src/App.tsx @@ -15,13 +15,13 @@ interface AppState { isillegalFinal: boolean; expWarningMessage: string; bestTrialEntries: string; // for overview page: best trial entreis + isUpdate: boolean; } class App extends React.Component<{}, AppState> { private timerId!: number | undefined; private dataFormatimer!: number; private firstLoad: boolean = false; // when click refresh selector options - constructor(props: {}) { super(props); this.state = { @@ -32,16 +32,19 @@ class App extends React.Component<{}, AppState> { metricGraphMode: 'max', isillegalFinal: false, expWarningMessage: '', - bestTrialEntries: '10' + bestTrialEntries: '10', + isUpdate: true }; } async componentDidMount(): Promise { await Promise.all([EXPERIMENT.init(), TRIALS.init()]); - this.setState(state => ({ experimentUpdateBroadcast: state.experimentUpdateBroadcast + 1 })); - this.setState(state => ({ trialsUpdateBroadcast: state.trialsUpdateBroadcast + 1 })); - this.timerId = window.setTimeout(this.refresh, this.state.interval * 1000); - this.setState({ metricGraphMode: (EXPERIMENT.optimizeMode === 'minimize' ? 'min' : 'max') }); + this.setState(state => ({ + experimentUpdateBroadcast: state.experimentUpdateBroadcast + 1, + trialsUpdateBroadcast: state.trialsUpdateBroadcast + 1, + metricGraphMode: (EXPERIMENT.optimizeMode === 'minimize' ? 'min' : 'max') + })); + this.timerId = window.setTimeout(this.refresh, this.state.interval * 100); // final result is legal // get a succeed trial,see final result data's format // eslint-disable-next-line @typescript-eslint/no-non-null-assertion @@ -99,6 +102,15 @@ class App extends React.Component<{}, AppState> { this.setState({ bestTrialEntries: entries }); } + shouldComponentUpdate(nextProps: any, nextState: AppState): boolean { + + if(!(nextState.isUpdate || nextState.isUpdate === undefined)){ + nextState.isUpdate = true; + return false; + } + return true; + } + render(): React.ReactNode { const { interval, columnList, experimentUpdateBroadcast, trialsUpdateBroadcast, metricGraphMode, isillegalFinal, expWarningMessage, bestTrialEntries @@ -106,7 +118,6 @@ class App extends React.Component<{}, AppState> { if (experimentUpdateBroadcast === 0 || trialsUpdateBroadcast === 0) { return null; // TODO: render a loading page } - const errorList = [ { errorWhere: TRIALS.jobListError(), errorMessage: TRIALS.getJobErrorMessage() }, { errorWhere: EXPERIMENT.experimentError(), errorMessage: EXPERIMENT.getExperimentMessage() }, @@ -158,7 +169,6 @@ class App extends React.Component<{}, AppState> { } private refresh = async (): Promise => { - // resolve this question: 10s -> 20s, page refresh twice. // only refresh this page after clicking the refresh options if (this.firstLoad !== true) { @@ -177,8 +187,7 @@ class App extends React.Component<{}, AppState> { // experiment status and /trial-jobs api's status could decide website update if (['DONE', 'ERROR', 'STOPPED'].includes(EXPERIMENT.status) || TRIALS.jobListError()) { // experiment finished, refresh once more to ensure consistency - this.setState({ interval: 0 }); - this.lastRefresh(); + this.setState(() => ({ interval: 0, isUpdate: false })); return; } @@ -189,8 +198,7 @@ class App extends React.Component<{}, AppState> { public async lastRefresh(): Promise { await EXPERIMENT.update(); await TRIALS.update(true); - this.setState(state => ({ experimentUpdateBroadcast: state.experimentUpdateBroadcast + 1 })); - this.setState(state => ({ trialsUpdateBroadcast: state.trialsUpdateBroadcast + 1 })); + this.setState(state => ({ experimentUpdateBroadcast: state.experimentUpdateBroadcast + 1, trialsUpdateBroadcast: state.trialsUpdateBroadcast + 1 })); } } diff --git a/src/webui/src/components/trial-detail/Para.tsx b/src/webui/src/components/trial-detail/Para.tsx index 6a018b71e3..f6fd0a7884 100644 --- a/src/webui/src/components/trial-detail/Para.tsx +++ b/src/webui/src/components/trial-detail/Para.tsx @@ -195,7 +195,7 @@ class Para extends React.Component { }]); } - if (convertedTrials.length === 0) { + if (convertedTrials.length === 0 || dimensions.length <= 1) { return; } diff --git a/src/webui/src/static/function.ts b/src/webui/src/static/function.ts index 221d4da0e2..fda4cd89cf 100644 --- a/src/webui/src/static/function.ts +++ b/src/webui/src/static/function.ts @@ -51,7 +51,7 @@ const convertDuration = (num: number): string => { }; function parseMetrics(metricData: string): any { - if (metricData.includes('NaN')) { + if (metricData.includes('NaN') || metricData.includes('Infinity')) { return JSON5.parse(JSON5.parse(metricData)); } else { return JSON.parse(JSON.parse(metricData)); @@ -84,15 +84,18 @@ const getFinalResult = (final?: MetricDataRecord[]): number => { } }; +function isNaNorInfinity(val: number): boolean { + return Object.is(val, NaN) || Object.is(val, Infinity); +} + // get final result value // acc obj const getFinal = (final?: MetricDataRecord[]): FinalType | undefined => { let showDefault: FinalType; if (final) { showDefault = parseMetrics(final[final.length - 1].data); if (typeof showDefault === 'number') { - if(!isNaN(showDefault)){ - showDefault = { default: showDefault }; - return showDefault; + if(!isNaNorInfinity(showDefault)){ + return { default: showDefault }; } } else if (isArrayType(showDefault)) { // not support final type @@ -165,11 +168,9 @@ const killJob = (key: number, id: string, status: string, updateList?: Function) .catch(error => { if (error.response.status === 500) { if (error.response.data.error) { - alert(123); - // message.error(error.response.data.error); + alert(error.response.data.error); } else { - alert(234); - // message.error('500 error, fail to cancel the job'); + alert('500 error, fail to cancel the job'); } } }); @@ -229,9 +230,17 @@ function formatAccuracy(accuracy: number): string { return accuracy.toFixed(6).replace(/0+$/, '').replace(/\.$/, ''); } +function formatComplexTypeValue(value: any): string | number { + if (['number', 'string'].includes(typeof value)) { + return value; + } else { + return value.toString(); + } +} + export { convertTime, convertDuration, getFinalResult, getFinal, downFile, intermediateGraphOption, killJob, filterByStatus, filterDuration, formatAccuracy, formatTimestamp, metricAccuracy, parseMetrics, - isArrayType, requestAxios + isArrayType, requestAxios, isNaNorInfinity, formatComplexTypeValue }; diff --git a/src/webui/src/static/interface.ts b/src/webui/src/static/interface.ts index 493ffc4b41..734921ead3 100644 --- a/src/webui/src/static/interface.ts +++ b/src/webui/src/static/interface.ts @@ -46,7 +46,7 @@ interface TableRecord { duration: number; status: string; intermediateCount: number; - accuracy?: number; + accuracy?: number | any; latestAccuracy: number | undefined; formattedLatestAccuracy: string; // format (LATEST/FINAL), accDictionary: FinalType | undefined; diff --git a/src/webui/src/static/model/experiment.ts b/src/webui/src/static/model/experiment.ts index 2f899eccdf..3d8d088789 100644 --- a/src/webui/src/static/model/experiment.ts +++ b/src/webui/src/static/model/experiment.ts @@ -58,7 +58,7 @@ class Experiment { await requestAxios(`${MANAGER_IP}/experiment`) .then(data => { - updated = updated || compareProfiles(this.profileField, data); + updated = updated || !compareProfiles(this.profileField, data); this.profileField = data; }) .catch(error => { @@ -69,7 +69,7 @@ class Experiment { await requestAxios(`${MANAGER_IP}/check-status`) .then(data => { - updated = JSON.stringify(this.statusField) === JSON.stringify(data); + updated = JSON.stringify(this.statusField) !== JSON.stringify(data); this.statusField = data; }) .catch(error => { diff --git a/src/webui/src/static/model/searchspace.ts b/src/webui/src/static/model/searchspace.ts index 75006bc1cd..cd1cfede23 100644 --- a/src/webui/src/static/model/searchspace.ts +++ b/src/webui/src/static/model/searchspace.ts @@ -1,5 +1,6 @@ import { SingleAxis, MultipleAxes, TableObj } from '../interface'; import { SUPPORTED_SEARCH_SPACE_TYPE } from '../const'; +import { formatComplexTypeValue } from '../function'; function fullNameJoin(prefix: string, name: string): string { return prefix ? (prefix + '/' + name) : name; @@ -52,7 +53,7 @@ class SimpleOrdinalAxis implements SingleAxis { this.baseName = baseName; this.fullName = fullName; this.type = type; - this.domain = value; + this.domain = Array.from(value).map(formatComplexTypeValue); } } @@ -115,7 +116,7 @@ export class SearchSpace implements MultipleAxes { trial.parameters(searchSpace); } catch (unexpectedEntries) { // eslint-disable-next-line no-console - console.log(unexpectedEntries); + console.warn(unexpectedEntries); for (const [k, v] of unexpectedEntries as Map) { const column = addingColumns.get(k); if (column === undefined) { @@ -164,7 +165,7 @@ export class MetricSpace implements MultipleAxes { if (value.every(v => typeof v === 'number')) { this.axes.set(key, new NumericAxis(key, key, 'uniform', [Math.min(...value), Math.max(...value)])); } else { - // TODO: skip for now + this.axes.set(key, new SimpleOrdinalAxis(key, key, 'choice', value)); } }); } diff --git a/src/webui/src/static/model/trial.ts b/src/webui/src/static/model/trial.ts index b1431a0a5f..c5366fe8c9 100644 --- a/src/webui/src/static/model/trial.ts +++ b/src/webui/src/static/model/trial.ts @@ -1,5 +1,6 @@ +import * as JSON5 from 'json5'; import { MetricDataRecord, TrialJobInfo, TableObj, TableRecord, Parameters, FinalType, MultipleAxes, SingleAxis } from '../interface'; -import { getFinal, formatAccuracy, metricAccuracy, parseMetrics, isArrayType } from '../function'; +import { getFinal, formatAccuracy, metricAccuracy, parseMetrics, isArrayType, isNaNorInfinity, formatComplexTypeValue } from '../function'; /** * Get a structured representation of parameters @@ -27,10 +28,10 @@ function inferTrialParameters(paramObj: object, space: MultipleAxes, prefix: str subUnexpected.forEach((v, k) => unexpectedEntries.set(k, v)); } } else { - parameters.set(axisKey, v); + parameters.set(axisKey, formatComplexTypeValue(v)); } } else { - unexpectedEntries.set(prefix + k, v); + unexpectedEntries.set(prefix + k, formatComplexTypeValue(v)); } } return [parameters, unexpectedEntries]; @@ -110,7 +111,15 @@ class Trial implements TableObj { const endTime = this.info.endTime || new Date().getTime(); // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const duration = (endTime - this.info.startTime!) / 1000; - + let accuracy; + if(this.acc !== undefined && this.acc.default !== undefined){ + if(typeof this.acc.default === 'number'){ + accuracy = JSON5.parse(this.acc.default); + }else { + accuracy = this.acc.default; + } + } + return { key: this.info.id, sequenceId: this.info.sequenceId, @@ -121,8 +130,7 @@ class Trial implements TableObj { duration, status: this.info.status, intermediateCount: this.intermediates.length, - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - accuracy: this.acc !== undefined ? JSON.parse(this.acc!.default) : undefined, + accuracy: accuracy, latestAccuracy: this.latestAccuracy, formattedLatestAccuracy: this.formatLatestAccuracy(), accDictionary: this.acc @@ -152,6 +160,9 @@ class Trial implements TableObj { } get acc(): FinalType | undefined { + if (this.info === undefined) { + return undefined; + } return getFinal(this.info.finalMetricData); } @@ -190,10 +201,10 @@ class Trial implements TableObj { } public parameters(axes: MultipleAxes): Map { - const tempHyper = this.info.hyperParameters; - if (tempHyper === undefined) { - throw new Map([['error', 'This trial\'s parameters are not available.']]); + if (this.info === undefined || this.info.hyperParameters === undefined) { + throw new Map(); } else { + const tempHyper = this.info.hyperParameters; let params = JSON.parse(tempHyper[tempHyper.length - 1]).parameters; if (typeof params === 'string') { params = JSON.parse(params); @@ -216,6 +227,7 @@ class Trial implements TableObj { Object.entries(acc).forEach(item => { const [k, v] = item; const column = space.axes.get(k); + if (column !== undefined) { ret.set(column, v); } else { @@ -233,8 +245,11 @@ class Trial implements TableObj { } public finalKeys(): string[] { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - return Object.keys(this.acc!); + if(this.acc !== undefined){ + return Object.keys(this.acc); + } else { + return []; + } } /* table obj end */ @@ -288,24 +303,34 @@ class Trial implements TableObj { return !same; } - public formatLatestAccuracy(): string { // TODO: this should be private - if (this.accuracy !== undefined) { - if (isNaN(this.accuracy)) { - return this.accuracy.toString(); + private renderNumber(val: any): string { + if(typeof val === 'number'){ + if (isNaNorInfinity(val)) { + return `${val}`; // show 'NaN' or 'Infinity' } else { - return `${formatAccuracy(this.accuracy)} (FINAL)`; + return `${formatAccuracy(val)} (FINAL)`; } - } else if (this.intermediates.length === 0) { - return '--'; } else { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const latest = this.intermediates[this.intermediates.length - 1]!; - if (isNaN(metricAccuracy(latest))) { - return 'NaN'; + // show other types, such as {tensor: {data: }} + return JSON.stringify(val); + } + } + + public formatLatestAccuracy(): string { // TODO: this should be private + if(this.status === 'SUCCEEDED'){ + return (this.accuracy === undefined ? '--': this.renderNumber(this.accuracy)); + } else { + if (this.accuracy !== undefined) { + return this.renderNumber(this.accuracy); + } else if (this.intermediates.length === 0) { + return '--'; } else { - return `${formatAccuracy(metricAccuracy(latest))} (LATEST)`; + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const latest = this.intermediates[this.intermediates.length - 1]!; + return this.renderNumber(metricAccuracy(latest)); } } + } } From 10c177c2a9c426f7fac6d5229e6e36d5ba7c40c8 Mon Sep 17 00:00:00 2001 From: Junwei Sun <30487595+JunweiSUN@users.noreply.github.com> Date: Wed, 12 Aug 2020 16:56:13 +0800 Subject: [PATCH 20/28] support display trial log on local mode (#2718) --- src/nni_manager/common/manager.ts | 4 ++- src/nni_manager/common/trainingService.ts | 5 ++- src/nni_manager/core/nnimanager.ts | 6 +++- .../core/test/mockedTrainingService.ts | 6 +++- src/nni_manager/rest_server/restHandler.ts | 14 ++++++++ .../rest_server/test/mockedNNIManager.ts | 5 ++- .../dlts/dltsTrainingService.ts | 7 +++- .../kubernetes/kubernetesTrainingService.ts | 7 +++- .../local/localTrainingService.ts | 18 ++++++++-- .../pai/paiTrainingService.ts | 7 +++- .../remoteMachineTrainingService.ts | 13 +++++-- .../reusable/routerTrainingService.ts | 7 +++- .../reusable/trialDispatcher.ts | 8 +++-- .../test/localTrainingService.test.ts | 36 +++++++++++++++++-- .../src/components/public-child/OpenRow.tsx | 24 ++++++++++++- 15 files changed, 147 insertions(+), 20 deletions(-) diff --git a/src/nni_manager/common/manager.ts b/src/nni_manager/common/manager.ts index f37745de16..c003598abc 100644 --- a/src/nni_manager/common/manager.ts +++ b/src/nni_manager/common/manager.ts @@ -4,7 +4,7 @@ 'use strict'; import { MetricDataRecord, MetricType, TrialJobInfo } from './datastore'; -import { TrialJobStatus } from './trainingService'; +import { TrialJobStatus, LogType } from './trainingService'; type ProfileUpdateType = 'TRIAL_CONCURRENCY' | 'MAX_EXEC_DURATION' | 'SEARCH_SPACE' | 'MAX_TRIAL_NUM'; type ExperimentStatus = 'INITIALIZED' | 'RUNNING' | 'ERROR' | 'STOPPING' | 'STOPPED' | 'DONE' | 'NO_MORE_TRIAL' | 'TUNER_NO_MORE_TRIAL'; @@ -101,6 +101,8 @@ abstract class Manager { public abstract getMetricDataByRange(minSeqId: number, maxSeqId: number): Promise; public abstract getLatestMetricData(): Promise; + public abstract getTrialLog(trialJobId: string, logType: LogType): Promise; + public abstract getTrialJobStatistics(): Promise; public abstract getStatus(): NNIManagerStatus; } diff --git a/src/nni_manager/common/trainingService.ts b/src/nni_manager/common/trainingService.ts index 83bd51e884..4edcf16ab6 100644 --- a/src/nni_manager/common/trainingService.ts +++ b/src/nni_manager/common/trainingService.ts @@ -8,6 +8,8 @@ */ type TrialJobStatus = 'UNKNOWN' | 'WAITING' | 'RUNNING' | 'SUCCEEDED' | 'FAILED' | 'USER_CANCELED' | 'SYS_CANCELED' | 'EARLY_STOPPED'; +type LogType = 'TRIAL_LOG' | 'TRIAL_ERROR'; + interface TrainingServiceMetadata { readonly key: string; readonly value: string; @@ -79,6 +81,7 @@ abstract class TrainingService { public abstract updateTrialJob(trialJobId: string, form: TrialJobApplicationForm): Promise; public abstract get isMultiPhaseJobSupported(): boolean; public abstract cancelTrialJob(trialJobId: string, isEarlyStopped?: boolean): Promise; + public abstract getTrialLog(trialJobId: string, logType: LogType): Promise; public abstract setClusterMetadata(key: string, value: string): Promise; public abstract getClusterMetadata(key: string): Promise; public abstract cleanUp(): Promise; @@ -98,5 +101,5 @@ class NNIManagerIpConfig { export { TrainingService, TrainingServiceError, TrialJobStatus, TrialJobApplicationForm, TrainingServiceMetadata, TrialJobDetail, TrialJobMetric, HyperParameters, - NNIManagerIpConfig + NNIManagerIpConfig, LogType }; diff --git a/src/nni_manager/core/nnimanager.ts b/src/nni_manager/core/nnimanager.ts index 038fe9ef9a..ad243f4835 100644 --- a/src/nni_manager/core/nnimanager.ts +++ b/src/nni_manager/core/nnimanager.ts @@ -16,7 +16,7 @@ import { NNIManagerStatus, ProfileUpdateType, TrialJobStatistics } from '../common/manager'; import { - TrainingService, TrialJobApplicationForm, TrialJobDetail, TrialJobMetric, TrialJobStatus + TrainingService, TrialJobApplicationForm, TrialJobDetail, TrialJobMetric, TrialJobStatus, LogType } from '../common/trainingService'; import { delay, getCheckpointDir, getExperimentRootDir, getLogDir, getMsgDispatcherCommand, mkDirP, getTunerProc, getLogLevel, isAlive, killPid } from '../common/utils'; import { @@ -325,6 +325,10 @@ class NNIManager implements Manager { // FIXME: unit test } + public async getTrialLog(trialJobId: string, logType: LogType): Promise { + return this.trainingService.getTrialLog(trialJobId, logType); + } + public getExperimentProfile(): Promise { // TO DO: using Promise.resolve() const deferred: Deferred = new Deferred(); diff --git a/src/nni_manager/core/test/mockedTrainingService.ts b/src/nni_manager/core/test/mockedTrainingService.ts index 546a36e494..5dfec86427 100644 --- a/src/nni_manager/core/test/mockedTrainingService.ts +++ b/src/nni_manager/core/test/mockedTrainingService.ts @@ -7,7 +7,7 @@ import { Deferred } from 'ts-deferred'; import { Provider } from 'typescript-ioc'; import { MethodNotImplementedError } from '../../common/errors'; -import { TrainingService, TrialJobApplicationForm, TrialJobDetail, TrialJobMetric } from '../../common/trainingService'; +import { TrainingService, TrialJobApplicationForm, TrialJobDetail, TrialJobMetric, LogType } from '../../common/trainingService'; const testTrainingServiceProvider: Provider = { get: () => { return new MockedTrainingService(); } @@ -63,6 +63,10 @@ class MockedTrainingService extends TrainingService { return deferred.promise; } + public getTrialLog(trialJobId: string, logType: LogType): Promise { + throw new MethodNotImplementedError(); + } + async run(): Promise { } diff --git a/src/nni_manager/rest_server/restHandler.ts b/src/nni_manager/rest_server/restHandler.ts index 457f154b69..af44d71a01 100644 --- a/src/nni_manager/rest_server/restHandler.ts +++ b/src/nni_manager/rest_server/restHandler.ts @@ -57,6 +57,7 @@ class NNIRestHandler { this.getMetricData(router); this.getMetricDataByRange(router); this.getLatestMetricData(router); + this.getTrialLog(router); this.exportData(router); // Express-joi-validator configuration @@ -268,6 +269,19 @@ class NNIRestHandler { }); } + private getTrialLog(router: Router): void { + router.get('/trial-log/:id/:type', async(req: Request, res: Response) => { + this.nniManager.getTrialLog(req.params.id, req.params.type).then((log: string) => { + if (log === '') { + log = 'No logs available.' + } + res.send(log); + }).catch((err: Error) => { + this.handleError(err, res); + }); + }); + } + private exportData(router: Router): void { router.get('/export-data', (req: Request, res: Response) => { this.nniManager.exportData().then((exportedData: string) => { diff --git a/src/nni_manager/rest_server/test/mockedNNIManager.ts b/src/nni_manager/rest_server/test/mockedNNIManager.ts index 5c8bc267b7..e45819d6cb 100644 --- a/src/nni_manager/rest_server/test/mockedNNIManager.ts +++ b/src/nni_manager/rest_server/test/mockedNNIManager.ts @@ -13,7 +13,7 @@ import { TrialJobStatistics, NNIManagerStatus } from '../../common/manager'; import { - TrialJobApplicationForm, TrialJobDetail, TrialJobStatus + TrialJobApplicationForm, TrialJobDetail, TrialJobStatus, LogType } from '../../common/trainingService'; export const testManagerProvider: Provider = { @@ -118,6 +118,9 @@ export class MockedNNIManager extends Manager { public getLatestMetricData(): Promise { throw new MethodNotImplementedError(); } + public getTrialLog(trialJobId: string, logType: LogType): Promise { + throw new MethodNotImplementedError(); + } public getExperimentProfile(): Promise { const profile: ExperimentProfile = { params: { diff --git a/src/nni_manager/training_service/dlts/dltsTrainingService.ts b/src/nni_manager/training_service/dlts/dltsTrainingService.ts index ba707fbb13..30d8fbcf8d 100644 --- a/src/nni_manager/training_service/dlts/dltsTrainingService.ts +++ b/src/nni_manager/training_service/dlts/dltsTrainingService.ts @@ -12,9 +12,10 @@ import { EventEmitter } from 'events'; import { String } from 'typescript-string-operations'; import { getExperimentId } from '../../common/experimentStartupInfo'; import { getLogger, Logger } from '../../common/log'; +import { MethodNotImplementedError } from '../../common/errors'; import { NNIManagerIpConfig, TrainingService, - TrialJobApplicationForm, TrialJobDetail, TrialJobMetric + TrialJobApplicationForm, TrialJobDetail, TrialJobMetric, LogType } from '../../common/trainingService'; import { DLTS_TRIAL_COMMAND_FORMAT } from './dltsData'; import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../common/containerJobData'; @@ -246,6 +247,10 @@ class DLTSTrainingService implements TrainingService { return trialJob } + public async getTrialLog(_trialJobId: string, _logType: LogType): Promise { + throw new MethodNotImplementedError(); + } + public addTrialJobMetricListener(listener: (metric: TrialJobMetric) => void): void { this.metricsEmitter.on('metric', listener); } diff --git a/src/nni_manager/training_service/kubernetes/kubernetesTrainingService.ts b/src/nni_manager/training_service/kubernetes/kubernetesTrainingService.ts index f21ac9ad69..11a54c453c 100644 --- a/src/nni_manager/training_service/kubernetes/kubernetesTrainingService.ts +++ b/src/nni_manager/training_service/kubernetes/kubernetesTrainingService.ts @@ -12,8 +12,9 @@ import { Base64 } from 'js-base64'; import { String } from 'typescript-string-operations'; import { getExperimentId } from '../../common/experimentStartupInfo'; import { getLogger, Logger } from '../../common/log'; +import { MethodNotImplementedError } from '../../common/errors'; import { - NNIManagerIpConfig, TrialJobDetail, TrialJobMetric + NNIManagerIpConfig, TrialJobDetail, TrialJobMetric, LogType } from '../../common/trainingService'; import { delay, getExperimentRootDir, getIPV4Address, getJobCancelStatus, getVersion, uniqueString } from '../../common/utils'; import { AzureStorageClientUtility } from './azureStorageClientUtils'; @@ -98,6 +99,10 @@ abstract class KubernetesTrainingService { return Promise.resolve(kubernetesTrialJob); } + public async getTrialLog(_trialJobId: string, _logType: LogType): Promise { + throw new MethodNotImplementedError(); + } + public addTrialJobMetricListener(listener: (metric: TrialJobMetric) => void): void { this.metricsEmitter.on('metric', listener); } diff --git a/src/nni_manager/training_service/local/localTrainingService.ts b/src/nni_manager/training_service/local/localTrainingService.ts index 71a1c5719c..a69bff8df8 100644 --- a/src/nni_manager/training_service/local/localTrainingService.ts +++ b/src/nni_manager/training_service/local/localTrainingService.ts @@ -14,7 +14,7 @@ import { getExperimentId } from '../../common/experimentStartupInfo'; import { getLogger, Logger } from '../../common/log'; import { HyperParameters, TrainingService, TrialJobApplicationForm, - TrialJobDetail, TrialJobMetric, TrialJobStatus + TrialJobDetail, TrialJobMetric, TrialJobStatus, LogType } from '../../common/trainingService'; import { delay, generateParamFileName, getExperimentRootDir, getJobCancelStatus, getNewLine, isAlive, uniqueString @@ -184,6 +184,18 @@ class LocalTrainingService implements TrainingService { return trialJob; } + public async getTrialLog(trialJobId: string, logType: LogType): Promise { + let logPath: string; + if (logType === 'TRIAL_LOG') { + logPath = path.join(this.rootDir, 'trials', trialJobId, 'trial.log'); + } else if (logType === 'TRIAL_ERROR') { + logPath = path.join(this.rootDir, 'trials', trialJobId, 'stderr'); + } else { + throw new Error('unexpected log type'); + } + return fs.promises.readFile(logPath, 'utf8'); + } + public addTrialJobMetricListener(listener: (metric: TrialJobMetric) => void): void { this.eventEmitter.on('metric', listener); } @@ -450,8 +462,8 @@ class LocalTrainingService implements TrainingService { while (!this.stopping) { while (!this.stopping && this.jobQueue.length !== 0) { const trialJobId: string = this.jobQueue[0]; - const trialJobDeatil: LocalTrialJobDetail | undefined = this.jobMap.get(trialJobId); - if (trialJobDeatil !== undefined && trialJobDeatil.status === 'WAITING') { + const trialJobDetail: LocalTrialJobDetail | undefined = this.jobMap.get(trialJobId); + if (trialJobDetail !== undefined && trialJobDetail.status === 'WAITING') { const [success, resource] = this.tryGetAvailableResource(); if (!success) { break; diff --git a/src/nni_manager/training_service/pai/paiTrainingService.ts b/src/nni_manager/training_service/pai/paiTrainingService.ts index aff583de54..56756b708d 100644 --- a/src/nni_manager/training_service/pai/paiTrainingService.ts +++ b/src/nni_manager/training_service/pai/paiTrainingService.ts @@ -11,9 +11,10 @@ import { EventEmitter } from 'events'; import { Deferred } from 'ts-deferred'; import { getExperimentId } from '../../common/experimentStartupInfo'; import { getLogger, Logger } from '../../common/log'; +import { MethodNotImplementedError } from '../../common/errors'; import { NNIManagerIpConfig, TrainingService, - TrialJobApplicationForm, TrialJobDetail, TrialJobMetric + TrialJobApplicationForm, TrialJobDetail, TrialJobMetric, LogType } from '../../common/trainingService'; import { delay } from '../../common/utils'; import { PAIJobInfoCollector } from './paiJobInfoCollector'; @@ -117,6 +118,10 @@ abstract class PAITrainingService implements TrainingService { return jobs; } + public async getTrialLog(_trialJobId: string, _logType: LogType): Promise { + throw new MethodNotImplementedError(); + } + public async getTrialJob(trialJobId: string): Promise { if (this.paiClusterConfig === undefined) { throw new Error('PAI Cluster config is not initialized'); diff --git a/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts b/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts index b291690a0b..8736bc09b7 100644 --- a/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts +++ b/src/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts @@ -10,13 +10,13 @@ import * as path from 'path'; import { ShellExecutor } from 'training_service/remote_machine/shellExecutor'; import { Deferred } from 'ts-deferred'; import * as component from '../../common/component'; -import { NNIError, NNIErrorNames } from '../../common/errors'; +import { NNIError, NNIErrorNames, MethodNotImplementedError } from '../../common/errors'; import { getExperimentId } from '../../common/experimentStartupInfo'; import { getLogger, Logger } from '../../common/log'; import { ObservableTimer } from '../../common/observableTimer'; import { HyperParameters, NNIManagerIpConfig, TrainingService, TrialJobApplicationForm, - TrialJobDetail, TrialJobMetric + TrialJobDetail, TrialJobMetric, LogType } from '../../common/trainingService'; import { delay, generateParamFileName, getExperimentRootDir, getIPV4Address, getJobCancelStatus, @@ -180,6 +180,15 @@ class RemoteMachineTrainingService implements TrainingService { } } + /** + * Get trial job log + * @param _trialJobId ID of trial job + * @param _logType 'TRIAL_LOG' | 'TRIAL_STDERR' + */ + public async getTrialLog(_trialJobId: string, _logType: LogType): Promise { + throw new MethodNotImplementedError(); + } + /** * Add job metrics listener * @param listener callback listener diff --git a/src/nni_manager/training_service/reusable/routerTrainingService.ts b/src/nni_manager/training_service/reusable/routerTrainingService.ts index 1fd28604be..1e3b75cc86 100644 --- a/src/nni_manager/training_service/reusable/routerTrainingService.ts +++ b/src/nni_manager/training_service/reusable/routerTrainingService.ts @@ -6,7 +6,8 @@ import { Container, Scope } from 'typescript-ioc'; import * as component from '../../common/component'; import { getLogger, Logger } from '../../common/log'; -import { TrainingService, TrialJobApplicationForm, TrialJobDetail, TrialJobMetric } from '../../common/trainingService'; +import { MethodNotImplementedError } from '../../common/errors' +import { TrainingService, TrialJobApplicationForm, TrialJobDetail, TrialJobMetric, LogType } from '../../common/trainingService'; import { delay } from '../../common/utils'; import { TrialConfigMetadataKey } from '../common/trialConfigMetadataKey'; import { PAIClusterConfig } from '../pai/paiConfig'; @@ -47,6 +48,10 @@ class RouterTrainingService implements TrainingService { return await this.internalTrainingService.getTrialJob(trialJobId); } + public async getTrialLog(_trialJobId: string, _logType: LogType): Promise { + throw new MethodNotImplementedError(); + } + public addTrialJobMetricListener(listener: (metric: TrialJobMetric) => void): void { if (this.internalTrainingService === undefined) { throw new Error("TrainingService is not assigned!"); diff --git a/src/nni_manager/training_service/reusable/trialDispatcher.ts b/src/nni_manager/training_service/reusable/trialDispatcher.ts index 1b310ef9e0..046f389ca2 100644 --- a/src/nni_manager/training_service/reusable/trialDispatcher.ts +++ b/src/nni_manager/training_service/reusable/trialDispatcher.ts @@ -9,10 +9,10 @@ import * as path from 'path'; import { Writable } from 'stream'; import { String } from 'typescript-string-operations'; import * as component from '../../common/component'; -import { NNIError, NNIErrorNames } from '../../common/errors'; +import { NNIError, NNIErrorNames, MethodNotImplementedError } from '../../common/errors'; import { getBasePort, getExperimentId, getPlatform } from '../../common/experimentStartupInfo'; import { getLogger, Logger } from '../../common/log'; -import { NNIManagerIpConfig, TrainingService, TrialJobApplicationForm, TrialJobMetric, TrialJobStatus } from '../../common/trainingService'; +import { NNIManagerIpConfig, TrainingService, TrialJobApplicationForm, TrialJobMetric, TrialJobStatus, LogType } from '../../common/trainingService'; import { delay, getExperimentRootDir, getIPV4Address, getLogLevel, getVersion, mkDirPSync, uniqueString } from '../../common/utils'; import { GPU_INFO, INITIALIZED, KILL_TRIAL_JOB, NEW_TRIAL_JOB, REPORT_METRIC_DATA, SEND_TRIAL_JOB_PARAMETER, STDOUT, TRIAL_END, VERSION_CHECK } from '../../core/commands'; import { ScheduleResultType } from '../../training_service/common/gpuData'; @@ -111,6 +111,10 @@ class TrialDispatcher implements TrainingService { return trial; } + public async getTrialLog(_trialJobId: string, _logType: LogType): Promise { + throw new MethodNotImplementedError(); + } + public async submitTrialJob(form: TrialJobApplicationForm): Promise { if (this.trialConfig === undefined) { throw new Error(`trialConfig not initialized!`); diff --git a/src/nni_manager/training_service/test/localTrainingService.test.ts b/src/nni_manager/training_service/test/localTrainingService.test.ts index bc47e747ba..fbaaedcd41 100644 --- a/src/nni_manager/training_service/test/localTrainingService.test.ts +++ b/src/nni_manager/training_service/test/localTrainingService.test.ts @@ -3,14 +3,14 @@ 'use strict'; -import * as assert from 'assert'; import * as chai from 'chai'; import * as chaiAsPromised from 'chai-as-promised'; import * as fs from 'fs'; +import * as path from 'path'; import * as tmp from 'tmp'; import * as component from '../../common/component'; -import { TrialJobApplicationForm, TrialJobDetail, TrainingService } from '../../common/trainingService'; -import { cleanupUnitTest, delay, prepareUnitTest } from '../../common/utils'; +import { TrialJobApplicationForm, TrialJobDetail} from '../../common/trainingService'; +import { cleanupUnitTest, delay, prepareUnitTest, getExperimentRootDir } from '../../common/utils'; import { TrialConfigMetadataKey } from '../common/trialConfigMetadataKey'; import { LocalTrainingService } from '../local/localTrainingService'; @@ -72,6 +72,36 @@ describe('Unit Test for LocalTrainingService', () => { chai.expect(jobDetail.status).to.be.equals('USER_CANCELED'); }).timeout(20000); + it('Get trial log', async () => { + await localTrainingService.setClusterMetadata(TrialConfigMetadataKey.TRIAL_CONFIG, trialConfig); + + // submit job + const form: TrialJobApplicationForm = { + sequenceId: 0, + hyperParameters: { + value: 'mock hyperparameters', + index: 0 + } + }; + + const jobDetail: TrialJobDetail = await localTrainingService.submitTrialJob(form); + + // get trial log + const rootDir: string = getExperimentRootDir() + fs.mkdirSync(path.join(rootDir, 'trials')) + fs.mkdirSync(jobDetail.workingDirectory) + fs.writeFileSync(path.join(jobDetail.workingDirectory, 'trial.log'), 'trial log') + fs.writeFileSync(path.join(jobDetail.workingDirectory, 'stderr'), 'trial stderr') + chai.expect(await localTrainingService.getTrialLog(jobDetail.id, 'TRIAL_LOG')).to.be.equals('trial log'); + chai.expect(await localTrainingService.getTrialLog(jobDetail.id, 'TRIAL_ERROR')).to.be.equals('trial stderr'); + fs.unlinkSync(path.join(jobDetail.workingDirectory, 'trial.log')) + fs.unlinkSync(path.join(jobDetail.workingDirectory, 'stderr')) + fs.rmdirSync(jobDetail.workingDirectory) + fs.rmdirSync(path.join(rootDir, 'trials')) + + await localTrainingService.cancelTrialJob(jobDetail.id); + }).timeout(20000); + it('Read metrics, Add listener, and remove listener', async () => { // set meta data const trialConfig: string = `{\"command\":\"python3 mockedTrial.py\", \"codeDir\":\"${localCodeDir}\",\"gpuNum\":0}` diff --git a/src/webui/src/components/public-child/OpenRow.tsx b/src/webui/src/components/public-child/OpenRow.tsx index a0c6c274c1..a20cf5313f 100644 --- a/src/webui/src/components/public-child/OpenRow.tsx +++ b/src/webui/src/components/public-child/OpenRow.tsx @@ -2,6 +2,7 @@ import * as React from 'react'; import * as copy from 'copy-to-clipboard'; import { Stack, PrimaryButton, Pivot, PivotItem } from 'office-ui-fabric-react'; import { Trial } from '../../static/model/trial'; +import { MANAGER_IP } from '../../static/const'; import { EXPERIMENT, TRIALS } from '../../static/datamodel'; import JSONTree from 'react-json-tree'; import PaiTrialLog from '../public-child/PaiTrialLog'; @@ -9,6 +10,7 @@ import TrialLog from '../public-child/TrialLog'; import MessageInfo from '../Modals/MessageInfo'; import '../../static/style/overview.scss'; import '../../static/style/copyParameter.scss'; +import '../../static/style/openRow.scss'; interface OpenRowProps { trialId: string; @@ -55,6 +57,10 @@ class OpenRow extends React.Component { } } + openTrialLog = (type: string): void => { + window.open(`${MANAGER_IP}/trial-log/${this.props.trialId}/${type}`); + } + render(): React.ReactNode { const { isHidenInfo, typeInfo, info } = this.state; const trialId = this.props.trialId; @@ -105,7 +111,23 @@ class OpenRow extends React.Component { logCollection={EXPERIMENT.logCollectionEnabled} /> : - +
+ + {/* view each trial log in drawer*/} +
+
+ + +
+
+
} From 44954e0caa172b198d6b144fc25026f0acc3ecab Mon Sep 17 00:00:00 2001 From: Tab Zhang Date: Wed, 12 Aug 2020 18:10:29 +0800 Subject: [PATCH 21/28] add nnictl command to list trial results with highest/lowest metric (#2747) --- docs/en_US/Tutorial/Nnictl.md | 6 +++++- tools/nni_cmd/nnictl.py | 2 ++ tools/nni_cmd/nnictl_utils.py | 23 +++++++++++++++++++++++ tools/nni_cmd/url_utils.py | 1 - 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/docs/en_US/Tutorial/Nnictl.md b/docs/en_US/Tutorial/Nnictl.md index 2c92514b4b..3c71ccf8a5 100644 --- a/docs/en_US/Tutorial/Nnictl.md +++ b/docs/en_US/Tutorial/Nnictl.md @@ -305,12 +305,14 @@ Debug mode will disable version check function in Trialkeeper. * Description - You can use this command to show trial's information. + You can use this command to show trial's information. Note that if `head` or `tail` is set, only complete trials will be listed. * Usage ```bash nnictl trial ls + nnictl trial ls --head 10 + nnictl trial ls --tail 10 ``` * Options @@ -318,6 +320,8 @@ Debug mode will disable version check function in Trialkeeper. |Name, shorthand|Required|Default|Description| |------|------|------ |------| |id| False| |ID of the experiment you want to set| + |--head|False||the number of items to be listed with the highest default metric| + |--tail|False||the number of items to be listed with the lowest default metric| * __nnictl trial kill__ diff --git a/tools/nni_cmd/nnictl.py b/tools/nni_cmd/nnictl.py index 21fdf13f6d..213554d5e8 100644 --- a/tools/nni_cmd/nnictl.py +++ b/tools/nni_cmd/nnictl.py @@ -103,6 +103,8 @@ def parse_args(): parser_trial_subparsers = parser_trial.add_subparsers() parser_trial_ls = parser_trial_subparsers.add_parser('ls', help='list trial jobs') parser_trial_ls.add_argument('id', nargs='?', help='the id of experiment') + parser_trial_ls.add_argument('--head', type=int, help='list the highest experiments on the default metric') + parser_trial_ls.add_argument('--tail', type=int, help='list the lowest experiments on the default metric') parser_trial_ls.set_defaults(func=trial_ls) parser_trial_kill = parser_trial_subparsers.add_parser('kill', help='kill trial jobs') parser_trial_kill.add_argument('id', nargs='?', help='the id of experiment') diff --git a/tools/nni_cmd/nnictl_utils.py b/tools/nni_cmd/nnictl_utils.py index 3aa5a6629a..3fad64b1fc 100644 --- a/tools/nni_cmd/nnictl_utils.py +++ b/tools/nni_cmd/nnictl_utils.py @@ -9,6 +9,7 @@ import re import shutil import subprocess +from functools import cmp_to_key from datetime import datetime, timezone from pathlib import Path from subprocess import Popen @@ -248,6 +249,20 @@ def stop_experiment(args): def trial_ls(args): '''List trial''' + def final_metric_data_cmp(lhs, rhs): + metric_l = json.loads(json.loads(lhs['finalMetricData'][0]['data'])) + metric_r = json.loads(json.loads(rhs['finalMetricData'][0]['data'])) + if isinstance(metric_l, float): + return metric_l - metric_r + elif isinstance(metric_l, dict): + return metric_l['default'] - metric_r['default'] + else: + print_error('Unexpected data format. Please check your data.') + raise ValueError + + if args.head and args.tail: + print_error('Head and tail cannot be set at the same time.') + return nni_config = Config(get_config_filename(args)) rest_port = nni_config.get_config('restServerPort') rest_pid = nni_config.get_config('restServerPid') @@ -259,6 +274,14 @@ def trial_ls(args): response = rest_get(trial_jobs_url(rest_port), REST_TIME_OUT) if response and check_response(response): content = json.loads(response.text) + if args.head: + assert args.head > 0, 'The number of requested data must be greater than 0.' + content = sorted(filter(lambda x: 'finalMetricData' in x, content), + key=cmp_to_key(final_metric_data_cmp), reverse=True)[:args.head] + elif args.tail: + assert args.tail > 0, 'The number of requested data must be greater than 0.' + content = sorted(filter(lambda x: 'finalMetricData' in x, content), + key=cmp_to_key(final_metric_data_cmp))[:args.tail] for index, value in enumerate(content): content[index] = convert_time_stamp_to_date(value) print(json.dumps(content, indent=4, sort_keys=True, separators=(',', ':'))) diff --git a/tools/nni_cmd/url_utils.py b/tools/nni_cmd/url_utils.py index 6d1f7694e1..59a28837a6 100644 --- a/tools/nni_cmd/url_utils.py +++ b/tools/nni_cmd/url_utils.py @@ -28,7 +28,6 @@ def metric_data_url(port): '''get metric_data url''' return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, METRIC_DATA_API) - def check_status_url(port): '''get check_status url''' return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, CHECK_STATUS_API) From f82ef623c1b813e7676849f885c12bcdc98d2a8e Mon Sep 17 00:00:00 2001 From: Junwei Sun <30487595+JunweiSUN@users.noreply.github.com> Date: Wed, 12 Aug 2020 19:58:51 +0800 Subject: [PATCH 22/28] update nnicli (#2713) --- docs/en_US/Tutorial/Nnictl.md | 2 +- docs/en_US/conf.py | 1 + docs/en_US/nnicli_ref.md | 41 ++ docs/en_US/sdk_reference.rst | 3 +- src/sdk/pycli/nnicli/nni_client.py | 544 ++++++++++++++++++++++---- test/config/integration_tests.yml | 4 +- test/config/integration_tests_tf2.yml | 4 +- test/config/pr_tests.yml | 8 +- test/nni_test/nnitest/validators.py | 12 +- tools/nni_cmd/updater.py | 2 +- 10 files changed, 520 insertions(+), 101 deletions(-) create mode 100644 docs/en_US/nnicli_ref.md diff --git a/docs/en_US/Tutorial/Nnictl.md b/docs/en_US/Tutorial/Nnictl.md index 3c71ccf8a5..adcc4b91e3 100644 --- a/docs/en_US/Tutorial/Nnictl.md +++ b/docs/en_US/Tutorial/Nnictl.md @@ -262,7 +262,7 @@ Debug mode will disable version check function in Trialkeeper. |Name, shorthand|Required|Default|Description| |------|------|------ |------| |id| False| |ID of the experiment you want to set| - |--value, -v| True| |the experiment duration will be NUMBER seconds. SUFFIX may be 's' for seconds (the default), 'm' for minutes, 'h' for hours or 'd' for days.| + |--value, -v| True| | Strings like '1m' for one minute or '2h' for two hours. SUFFIX may be 's' for seconds, 'm' for minutes, 'h' for hours or 'd' for days.| * Example diff --git a/docs/en_US/conf.py b/docs/en_US/conf.py index 6962037bfb..2d6ce2fb75 100644 --- a/docs/en_US/conf.py +++ b/docs/en_US/conf.py @@ -17,6 +17,7 @@ import os import sys sys.path.insert(0, os.path.abspath('../../src/sdk/pynni')) +sys.path.insert(1, os.path.abspath('../../src/sdk/pycli')) # -- Project information --------------------------------------------------- diff --git a/docs/en_US/nnicli_ref.md b/docs/en_US/nnicli_ref.md new file mode 100644 index 0000000000..02c8cbbb30 --- /dev/null +++ b/docs/en_US/nnicli_ref.md @@ -0,0 +1,41 @@ +# NNI Client + +NNI client is a python API of `nnictl`, which implements the most commonly used commands. Users can use this API to control their experiments, collect experiment results and conduct advanced analyses based on experiment results in python code directly instead of using command line. Here is an example: + +``` +from nnicli import Experiment + +# create an experiment instance +exp = Experiment() + +# start an experiment, then connect the instance to this experiment +# you can also use `resume_experiment`, `view_experiment` or `connect_experiment` +# only one of them should be called in one instance +exp.start_experiment('nni/examples/trials/mnist-pytorch/config.yml', port=9090) + +# update the experiment's concurrency +exp.update_concurrency(3) + +# get some information about the experiment +print(exp.get_experiment_status()) +print(exp.get_job_statistics()) +print(exp.list_trial_jobs()) + +# stop the experiment, then disconnect the instance from the experiment. +exp.stop_experiment() +``` + +## References + +```eval_rst +.. autoclass:: nnicli.Experiment + :members: +.. autoclass:: nnicli.TrialJob + :members: +.. autoclass:: nnicli.TrialHyperParameters + :members: +.. autoclass:: nnicli.TrialMetricData + :members: +.. autoclass:: nnicli.TrialResult + :members: +``` diff --git a/docs/en_US/sdk_reference.rst b/docs/en_US/sdk_reference.rst index 2602e257b9..ca87bf7500 100644 --- a/docs/en_US/sdk_reference.rst +++ b/docs/en_US/sdk_reference.rst @@ -8,4 +8,5 @@ Python API Reference Auto Tune NAS - Compression Utilities \ No newline at end of file + Compression Utilities + NNI Client \ No newline at end of file diff --git a/src/sdk/pycli/nnicli/nni_client.py b/src/sdk/pycli/nnicli/nni_client.py index ca083c2c7f..571e2bd036 100644 --- a/src/sdk/pycli/nnicli/nni_client.py +++ b/src/sdk/pycli/nnicli/nni_client.py @@ -5,67 +5,47 @@ Example: -import nnicli as nc +from nnicli import Experiment -nc.start_nni('../../../../examples/trials/mnist/config.yml') +exp = Experiment() +exp.start_experiment('../../../../examples/trials/mnist-pytorch/config.yml') -nc.set_endpoint('http://localhost:8080') +exp.update_concurrency(3) -print(nc.version()) -print(nc.get_experiment_status()) +print(exp.get_experiment_status()) +print(exp.get_job_statistics()) +print(exp.list_trial_jobs()) -print(nc.get_job_statistics()) -print(nc.list_trial_jobs()) - -nc.stop_nni() +exp.stop_experiment() """ import sys import os import subprocess +import re +import json import requests __all__ = [ - 'start_nni', - 'stop_nni', - 'set_endpoint', - 'version', - 'get_experiment_status', - 'get_experiment_profile', - 'get_trial_job', - 'list_trial_jobs', - 'get_job_statistics', - 'get_job_metrics', - 'export_data' + 'Experiment', + 'TrialResult', + 'TrialMetricData', + 'TrialHyperParameters', + 'TrialJob' ] EXPERIMENT_PATH = 'experiment' -VERSION_PATH = 'version' STATUS_PATH = 'check-status' JOB_STATISTICS_PATH = 'job-statistics' TRIAL_JOBS_PATH = 'trial-jobs' METRICS_PATH = 'metric-data' EXPORT_DATA_PATH = 'export-data' - API_ROOT_PATH = 'api/v1/nni' -_api_endpoint = None - -def set_endpoint(endpoint): - """set endpoint of nni rest server for nnicli, for example: - http://localhost:8080 - """ - global _api_endpoint - _api_endpoint = endpoint - -def _check_endpoint(): - if _api_endpoint is None: - raise AssertionError("Please call set_endpoint to specify nni endpoint") - -def _nni_rest_get(api_path, response_type='json'): - _check_endpoint() - uri = '{}/{}/{}'.format(_api_endpoint, API_ROOT_PATH, api_path) +def _nni_rest_get(endpoint, api_path, response_type='json'): + _check_endpoint(endpoint) + uri = '{}/{}/{}'.format(endpoint.strip('/'), API_ROOT_PATH, api_path) res = requests.get(uri) if _http_succeed(res.status_code): if response_type == 'json': @@ -73,7 +53,7 @@ def _nni_rest_get(api_path, response_type='json'): elif response_type == 'text': return res.text else: - raise AssertionError('Incorrect response_type') + raise RuntimeError('Incorrect response_type') else: return None @@ -92,48 +72,444 @@ def _create_process(cmd): print(output.decode('utf-8').strip()) return process.returncode -def start_nni(config_file): - """start nni experiment with specified configuration file""" - cmd = 'nnictl create --config {}'.format(config_file).split(' ') - if _create_process(cmd) != 0: - raise RuntimeError('Failed to start nni.') - -def stop_nni(): - """stop nni experiment""" - cmd = 'nnictl stop'.split(' ') - if _create_process(cmd) != 0: - raise RuntimeError('Failed to stop nni.') - -def version(): - """return version of nni""" - return _nni_rest_get(VERSION_PATH, 'text') - -def get_experiment_status(): - """return experiment status as a dict""" - return _nni_rest_get(STATUS_PATH) - -def get_experiment_profile(): - """return experiment profile as a dict""" - return _nni_rest_get(EXPERIMENT_PATH) - -def get_trial_job(trial_job_id): - """return trial job information as a dict""" - assert trial_job_id is not None - return _nni_rest_get(os.path.join(TRIAL_JOBS_PATH, trial_job_id)) - -def list_trial_jobs(): - """return information for all trial jobs as a list""" - return _nni_rest_get(TRIAL_JOBS_PATH) - -def get_job_statistics(): - """return trial job statistics information as a dict""" - return _nni_rest_get(JOB_STATISTICS_PATH) - -def get_job_metrics(trial_job_id=None): - """return trial job metrics""" - api_path = METRICS_PATH if trial_job_id is None else os.path.join(METRICS_PATH, trial_job_id) - return _nni_rest_get(api_path) - -def export_data(): - """return exported information for all trial jobs""" - return _nni_rest_get(EXPORT_DATA_PATH) +def _check_endpoint(endpoint): + if endpoint is None: + raise RuntimeError("This instance hasn't been connect to an experiment.") + +class TrialResult: + """ + TrialResult stores the result information of a trial job. + + Parameters + ---------- + json_obj: dict + Json object that stores the result information. + + Attributes + ---------- + parameter: dict + Hyper parameters for this trial. + value: serializable object, usually a number, or a dict with key "default" and other extra keys + Final result. + trialJobId: str + Trial job id. + """ + def __init__(self, json_obj): + self.parameter = None + self.value = None + self.trialJobId = None + for key in json_obj.keys(): + if key == 'id': + setattr(self, 'trialJobId', json_obj[key]) + elif hasattr(self, key): + setattr(self, key, json_obj[key]) + self.value = json.loads(self.value) + + def __repr__(self): + return "TrialResult(parameter: {} value: {} trialJobId: {})".format(self.parameter, self.value, self.trialJobId) + +class TrialMetricData: + """ + TrialMetricData stores the metric data of a trial job. + A trial job may have both intermediate metric and final metric. + + Parameters + ---------- + json_obj: dict + Json object that stores the metric data. + + Attributes + ---------- + timestamp: int + Time stamp. + trialJobId: str + Trial job id. + parameterId: int + Parameter id. + type: str + Metric type, `PERIODICAL` for intermediate result and `FINAL` for final result. + sequence: int + Sequence number in this trial. + data: serializable object, usually a number, or a dict with key "default" and other extra keys + Metric data. + """ + def __init__(self, json_obj): + self.timestamp = None + self.trialJobId = None + self.parameterId = None + self.type = None + self.sequence = None + self.data = None + for key in json_obj.keys(): + setattr(self, key, json_obj[key]) + self.data = json.loads(json.loads(self.data)) + + def __repr__(self): + return "TrialMetricData(timestamp: {} trialJobId: {} parameterId: {} type: {} sequence: {} data: {})" \ + .format(self.timestamp, self.trialJobId, self.parameterId, self.type, self.sequence, self.data) + +class TrialHyperParameters: + """ + TrialHyperParameters stores the hyper parameters of a trial job. + + Parameters + ---------- + json_obj: dict + Json object that stores the hyper parameters. + + Attributes + ---------- + parameter_id: int + Parameter id. + parameter_source: str + Parameter source. + parameters: dict + Hyper parameters. + parameter_index: int + Parameter index. + """ + def __init__(self, json_obj): + self.parameter_id = None + self.parameter_source = None + self.parameters = None + self.parameter_index = None + for key in json_obj.keys(): + if hasattr(self, key): + setattr(self, key, json_obj[key]) + + def __repr__(self): + return "TrialHyperParameters(parameter_id: {} parameter_source: {} parameters: {} parameter_index: {})" \ + .format(self.parameter_id, self.parameter_source, self.parameters, self.parameter_index) + +class TrialJob: + """ + TrialJob stores the information of a trial job. + + Parameters + ---------- + json_obj: dict + json object that stores the hyper parameters + + Attributes + ---------- + trialJobId: str + Trial job id. + status: str + Job status. + hyperParameters: list of `nnicli.TrialHyperParameters` + See `nnicli.TrialHyperParameters`. + logPath: str + Log path. + startTime: int + Job start time (timestamp). + endTime: int + Job end time (timestamp). + finalMetricData: list of `nnicli.TrialMetricData` + See `nnicli.TrialMetricData`. + parameter_index: int + Parameter index. + """ + def __init__(self, json_obj): + self.trialJobId = None + self.status = None + self.hyperParameters = None + self.logPath = None + self.startTime = None + self.endTime = None + self.finalMetricData = None + self.stderrPath = None + for key in json_obj.keys(): + if key == 'id': + setattr(self, 'trialJobId', json_obj[key]) + elif hasattr(self, key): + setattr(self, key, json_obj[key]) + if self.hyperParameters: + self.hyperParameters = [TrialHyperParameters(json.loads(e)) for e in self.hyperParameters] + if self.finalMetricData: + self.finalMetricData = [TrialMetricData(e) for e in self.finalMetricData] + + def __repr__(self): + return ("TrialJob(trialJobId: {} status: {} hyperParameters: {} logPath: {} startTime: {} " + "endTime: {} finalMetricData: {} stderrPath: {})") \ + .format(self.trialJobId, self.status, self.hyperParameters, self.logPath, + self.startTime, self.endTime, self.finalMetricData, self.stderrPath) + +class Experiment: + def __init__(self): + self._endpoint = None + self._exp_id = None + self._port = None + + @property + def endpoint(self): + return self._endpoint + + @property + def exp_id(self): + return self._exp_id + + @property + def port(self): + return self._port + + def _exec_command(self, cmd, port=None): + if self._endpoint is not None: + raise RuntimeError('This instance has been connected to an experiment.') + if _create_process(cmd) != 0: + raise RuntimeError('Failed to establish experiment, please check your config.') + else: + if port: + self._port = port + else: + self._port = 8080 + self._endpoint = 'http://localhost:{}'.format(self._port) + self._exp_id = self.get_experiment_profile()['id'] + + def start_experiment(self, config_file, port=None, debug=False): + """ + Start an experiment with specified configuration file and connect to it. + + Parameters + ---------- + config_file: str + Path to the config file. + port: int + The port of restful server, bigger than 1024. + debug: boolean + Set debug mode. + """ + cmd = 'nnictl create --config {}'.format(config_file).split(' ') + if port: + cmd += '--port {}'.format(port).split(' ') + if debug: + cmd += ['--debug'] + self._exec_command(cmd, port) + + def resume_experiment(self, exp_id, port=None, debug=False): + """ + Resume a stopped experiment with specified experiment id + + Parameters + ---------- + exp_id: str + Experiment id. + port: int + The port of restful server, bigger than 1024. + debug: boolean + Set debug mode. + """ + cmd = 'nnictl resume {}'.format(exp_id).split(' ') + if port: + cmd += '--port {}'.format(port).split(' ') + if debug: + cmd += ['--debug'] + self._exec_command(cmd, port) + + def view_experiment(self, exp_id, port=None): + """ + View a stopped experiment with specified experiment id. + + Parameters + ---------- + exp_id: str + Experiment id. + port: int + The port of restful server, bigger than 1024. + """ + cmd = 'nnictl view {}'.format(exp_id).split(' ') + if port: + cmd += '--port {}'.format(port).split(' ') + self._exec_command(cmd, port) + + def connect_experiment(self, endpoint): + """ + Connect to an existing experiment. + + Parameters + ---------- + endpoint: str + The endpoint of nni rest server, i.e, the url of Web UI. Should be a format like `http://ip:port`. + """ + if self._endpoint is not None: + raise RuntimeError('This instance has been connected to an experiment.') + self._endpoint = endpoint + try: + self._exp_id = self.get_experiment_profile()['id'] + except TypeError: + raise RuntimeError('Invalid experiment endpoint.') + self._port = int(re.search(r':[0-9]+', self._endpoint).group().replace(':', '')) + + def stop_experiment(self): + """Stop the experiment. + """ + _check_endpoint(self._endpoint) + cmd = 'nnictl stop {}'.format(self._exp_id).split(' ') + if _create_process(cmd) != 0: + raise RuntimeError('Failed to stop experiment.') + self._endpoint = None + self._exp_id = None + self._port = None + + def update_searchspace(self, filename): + """ + Update the experiment's search space. + + Parameters + ---------- + filename: str + Path to the searchspace file. + """ + _check_endpoint(self._endpoint) + cmd = 'nnictl update searchspace {} --filename {}'.format(self._exp_id, filename).split(' ') + if _create_process(cmd) != 0: + raise RuntimeError('Failed to update searchspace.') + + def update_concurrency(self, value): + """ + Update an experiment's concurrency + + Parameters + ---------- + value: int + New concurrency value. + """ + _check_endpoint(self._endpoint) + cmd = 'nnictl update concurrency {} --value {}'.format(self._exp_id, value).split(' ') + if _create_process(cmd) != 0: + raise RuntimeError('Failed to update concurrency.') + + def update_duration(self, value): + """ + Update an experiment's duration + + Parameters + ---------- + value: str + Strings like '1m' for one minute or '2h' for two hours. + SUFFIX may be 's' for seconds, 'm' for minutes, 'h' for hours or 'd' for days. + """ + _check_endpoint(self._endpoint) + cmd = 'nnictl update duration {} --value {}'.format(self._exp_id, value).split(' ') + if _create_process(cmd) != 0: + raise RuntimeError('Failed to update duration.') + + def update_trailnum(self, value): + """ + Update an experiment's maxtrialnum + + Parameters + ---------- + value: int + New trailnum value. + """ + _check_endpoint(self._endpoint) + cmd = 'nnictl update trialnum {} --value {}'.format(self._exp_id, value).split(' ') + if _create_process(cmd) != 0: + raise RuntimeError('Failed to update trailnum.') + + def get_experiment_status(self): + """ + Return experiment status as a dict. + + Returns + ---------- + dict + Experiment status. + """ + _check_endpoint(self._endpoint) + return _nni_rest_get(self._endpoint, STATUS_PATH) + + def get_trial_job(self, trial_job_id): + """ + Return a trial job. + + Parameters + ---------- + trial_job_id: str + Trial job id. + + Returns + ---------- + nnicli.TrialJob + A `nnicli.TrialJob` instance corresponding to `trial_job_id`. + """ + _check_endpoint(self._endpoint) + assert trial_job_id is not None + trial_job = _nni_rest_get(self._endpoint, os.path.join(TRIAL_JOBS_PATH, trial_job_id)) + return TrialJob(trial_job) + + def list_trial_jobs(self): + """ + Return information for all trial jobs as a list. + + Returns + ---------- + list + List of `nnicli.TrialJob`. + """ + _check_endpoint(self._endpoint) + trial_jobs = _nni_rest_get(self._endpoint, TRIAL_JOBS_PATH) + return [TrialJob(e) for e in trial_jobs] + + def get_job_statistics(self): + """ + Return trial job statistics information as a dict. + + Returns + ---------- + list + Job statistics information. + """ + _check_endpoint(self._endpoint) + return _nni_rest_get(self._endpoint, JOB_STATISTICS_PATH) + + def get_job_metrics(self, trial_job_id=None): + """ + Return trial job metrics. + + Parameters + ---------- + trial_job_id: str + trial job id. if this parameter is None, all trail jobs' metrics will be returned. + + Returns + ---------- + dict + Each key is a trialJobId, the corresponding value is a list of `nnicli.TrialMetricData`. + """ + _check_endpoint(self._endpoint) + api_path = METRICS_PATH if trial_job_id is None else os.path.join(METRICS_PATH, trial_job_id) + output = {} + trail_metrics = _nni_rest_get(self._endpoint, api_path) + for metric in trail_metrics: + trial_id = metric["trialJobId"] + if trial_id not in output: + output[trial_id] = [TrialMetricData(metric)] + else: + output[trial_id].append(TrialMetricData(metric)) + return output + + def export_data(self): + """ + Return exported information for all trial jobs. + + Returns + ---------- + list + List of `nnicli.TrialResult`. + """ + _check_endpoint(self._endpoint) + trial_results = _nni_rest_get(self._endpoint, EXPORT_DATA_PATH) + return [TrialResult(e) for e in trial_results] + + def get_experiment_profile(self): + """ + Return experiment profile as a dict. + + Returns + ---------- + dict + The profile of the experiment. + """ + _check_endpoint(self._endpoint) + return _nni_rest_get(self._endpoint, EXPERIMENT_PATH) diff --git a/test/config/integration_tests.yml b/test/config/integration_tests.yml index b3802239da..4078321479 100644 --- a/test/config/integration_tests.yml +++ b/test/config/integration_tests.yml @@ -140,8 +140,8 @@ testCases: config: maxTrialNum: 4 trialConcurrency: 4 - launchCommand: python3 -c 'import nnicli as nc; nc.start_nni("$configFile")' - stopCommand: python3 -c 'import nnicli as nc; nc.stop_nni()' + launchCommand: python3 -c 'from nnicli import Experiment; exp = Experiment(); exp.start_experiment("$configFile")' + stopCommand: python3 -c 'from nnicli import Experiment; exp = Experiment(); exp.connect_experiment("http://localhost:8080/"); exp.stop_experiment()' validator: class: NnicliValidator platform: linux darwin diff --git a/test/config/integration_tests_tf2.yml b/test/config/integration_tests_tf2.yml index e060511289..2002f36367 100644 --- a/test/config/integration_tests_tf2.yml +++ b/test/config/integration_tests_tf2.yml @@ -110,8 +110,8 @@ testCases: config: maxTrialNum: 4 trialConcurrency: 4 - launchCommand: python3 -c 'import nnicli as nc; nc.start_nni("$configFile")' - stopCommand: python3 -c 'import nnicli as nc; nc.stop_nni()' + launchCommand: python3 -c 'from nnicli import Experiment; exp = Experiment(); exp.start_experiment("$configFile")' + stopCommand: python3 -c 'from nnicli import Experiment; exp = Experiment(); exp.connect_experiment("http://localhost:8080/"); exp.stop_experiment()' validator: class: NnicliValidator platform: linux darwin diff --git a/test/config/pr_tests.yml b/test/config/pr_tests.yml index d49bf9d7ec..f82143b836 100644 --- a/test/config/pr_tests.yml +++ b/test/config/pr_tests.yml @@ -45,10 +45,10 @@ testCases: - name: nnicli configFile: test/config/examples/sklearn-regression.yml config: - maxTrialNum: 2 - trialConcurrency: 2 - launchCommand: python3 -c 'import nnicli as nc; nc.start_nni("$configFile")' - stopCommand: python3 -c 'import nnicli as nc; nc.stop_nni()' + maxTrialNum: 4 + trialConcurrency: 4 + launchCommand: python3 -c 'from nnicli import Experiment; exp = Experiment(); exp.start_experiment("$configFile")' + stopCommand: python3 -c 'from nnicli import Experiment; exp = Experiment(); exp.connect_experiment("http://localhost:8080/"); exp.stop_experiment()' validator: class: NnicliValidator platform: linux darwin diff --git a/test/nni_test/nnitest/validators.py b/test/nni_test/nnitest/validators.py index 1cdadb8669..5ad9090c18 100644 --- a/test/nni_test/nnitest/validators.py +++ b/test/nni_test/nnitest/validators.py @@ -6,7 +6,7 @@ import subprocess import json import requests -import nnicli as nc +from nnicli import Experiment from utils import METRICS_URL @@ -80,8 +80,8 @@ def get_metric_results(self, metrics): class NnicliValidator(ITValidator): def __call__(self, rest_endpoint, experiment_dir, nni_source_dir, **kwargs): print(rest_endpoint) - nc.set_endpoint(rest_endpoint) - #print(nc.version()) - print(nc.get_job_statistics()) - print(nc.get_experiment_status()) - print(nc.list_trial_jobs()) + exp = Experiment() + exp.connect_experiment(rest_endpoint) + print(exp.get_job_statistics()) + print(exp.get_experiment_status()) + print(exp.list_trial_jobs()) diff --git a/tools/nni_cmd/updater.py b/tools/nni_cmd/updater.py index 13ee679c49..c9991b8bab 100644 --- a/tools/nni_cmd/updater.py +++ b/tools/nni_cmd/updater.py @@ -14,7 +14,7 @@ def validate_digit(value, start, end): '''validate if a digit is valid''' if not str(value).isdigit() or int(value) < start or int(value) > end: - raise ValueError('%s must be a digit from %s to %s' % (value, start, end)) + raise ValueError('value (%s) must be a digit from %s to %s' % (value, start, end)) def validate_file(path): '''validate if a file exist''' From 3757cf271d10dee5b284151cda7f82b132e7a808 Mon Sep 17 00:00:00 2001 From: colorjam Date: Wed, 12 Aug 2020 19:59:25 +0800 Subject: [PATCH 23/28] Change local sort per page to global search of all pages (#2773) --- .../src/components/trial-detail/TableList.tsx | 111 +++++++++--------- 1 file changed, 54 insertions(+), 57 deletions(-) diff --git a/src/webui/src/components/trial-detail/TableList.tsx b/src/webui/src/components/trial-detail/TableList.tsx index 5b45cc8283..ee11f0fdec 100644 --- a/src/webui/src/components/trial-detail/TableList.tsx +++ b/src/webui/src/components/trial-detail/TableList.tsx @@ -75,7 +75,7 @@ interface TableListState { tableSourceForSort: Array; sortMessage: SortInfo; offset: number; - data: Array; + tablePerPage: Array; perPage: number; currentPage: number; pageCount: number; @@ -111,7 +111,7 @@ class TableList extends React.Component { allColumnList: this.getAllColumnKeys(), sortMessage: { field: '', isDescend: false }, offset: 0, - data: [], + tablePerPage: [], perPage: 20, currentPage: 0, pageCount: 0, @@ -121,7 +121,7 @@ class TableList extends React.Component { // sort for table column onColumnClick = (ev: React.MouseEvent, getColumn: IColumn): void => { - const { tableColumns, tableSourceForSort } = this.state; + const { tableColumns } = this.state; const newColumns: IColumn[] = tableColumns.slice(); const currColumn: IColumn = newColumns.filter(item => getColumn.key === item.key)[0]; newColumns.forEach((newCol: IColumn) => { @@ -133,26 +133,12 @@ class TableList extends React.Component { newCol.isSortedDescending = true; } }); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const newItems = this.copyAndSort(tableSourceForSort, currColumn.fieldName!, currColumn.isSortedDescending); + this.setState({ tableColumns: newColumns, - tableSourceForSort: newItems, sortMessage: { field: getColumn.key, isDescend: currColumn.isSortedDescending } - }); - - }; - - private copyAndSort(items: T[], columnKey: string, isSortedDescending?: boolean): any { - const key = columnKey as keyof T; - return items.slice(0).sort(function (a: T, b: T): any { - if (a[key] === undefined) { - return 1; - } - if (b[key] === undefined) { - return -1; - } - return (isSortedDescending ? a[key] < b[key] : a[key] > b[key]) ? 1 : -1; + }, () => { + this.updateData(); }); } @@ -567,61 +553,86 @@ class TableList extends React.Component { componentDidMount(): void { window.addEventListener('resize', this.onWindowResize); - this.updateData() + this.updateData(); } componentDidUpdate(prevProps: TableListProps): void { - if (this.props.columnList !== prevProps.columnList || this.props.tableSource !== prevProps.tableSource) { + if (this.props.columnList !== prevProps.columnList || this.props.tableSource !== prevProps.tableSource || prevProps.trialsUpdateBroadcast !== this.props.trialsUpdateBroadcast) { const { columnList } = this.props; this.setState({ tableColumns: this.initTableColumnList(columnList), allColumnList: this.getAllColumnKeys() - }, () => {this.updateData(); - }); + }, () => { + this.updateData(); + }); } } + // slice all table data into current page data updateData(): void { - const tableSource: Array = JSON.parse(JSON.stringify(this.props.tableSource)); + const tableSource: Array = this.props.tableSource; + const { offset, perPage, sortMessage } = this.state; + + if (sortMessage.field !== '') { + tableSource.sort(function (a, b): any { + if (a[sortMessage.field] === undefined || Object.is(a[sortMessage.field], NaN) || Object.is(a[sortMessage.field], Infinity) || Object.is(a[sortMessage.field], -Infinity) || typeof a[sortMessage.field] === 'object' ) { + return 1; + } + if (b[sortMessage.field] === undefined || Object.is(b[sortMessage.field], NaN) || Object.is(b[sortMessage.field], Infinity) || Object.is(b[sortMessage.field], -Infinity) || typeof b[sortMessage.field] === 'object' ) { + return -1; + } + return (sortMessage.isDescend ? a[sortMessage.field] < b[sortMessage.field] : a[sortMessage.field] > b[sortMessage.field]) ? 1 : -1; + }); + } - const tableSlice = tableSource.slice(this.state.offset, this.state.offset + this.state.perPage) - + const tableSlice = tableSource.slice(offset, offset + perPage) + const curPageCount = Math.ceil(tableSource.length / perPage) this.setState({ - tableSourceForSort: tableSlice, - pageCount: Math.ceil(tableSource.length / this.state.perPage), + tablePerPage: tableSlice, + pageCount: curPageCount, }); } + // update data when click the page index of pagination handlePageClick = (evt: any): void => { const selectedPage = evt.selected; const offset = selectedPage * this.state.perPage; this.setState({ currentPage: selectedPage, - offset: offset }, - () => { this.updateData(); + offset: offset + }, () => { + this.updateData(); }); } - updateperPage = (event: React.FormEvent, item: IDropdownOption | undefined): void => { - // clear input value and re-render table + // update per page items when click the dropdown of pagination + updatePerPage = (event: React.FormEvent, item: IDropdownOption | undefined): void => { + const { pageCount } = this.state; + if (item !== undefined) { + const currentPerPage = item.key === 'all' ? this.props.tableSource.length: Number(item.key) + const currentPageCount = this.props.tableSource.length <= currentPerPage ? 1 : pageCount + this.setState({ - perPage: item.key === 'all' ? this.props.tableSource.length: Number(item.key) }, - () => {this.updateData(); + perPage: currentPerPage, + offset: 0, + currentPage: 0, + pageCount: currentPageCount + }, () => { + this.updateData(); }); } } - render(): React.ReactNode { const { intermediateKey, modalIntermediateWidth, modalIntermediateHeight, tableColumns, allColumnList, isShowColumn, modalVisible, selectRows, isShowCompareModal, intermediateOtherKeys, - isShowCustomizedModal, copyTrialId, intermediateOption, sortMessage + isShowCustomizedModal, copyTrialId, intermediateOption, + tablePerPage } = this.state; const { columnList } = this.props; - const tableSource = this.state.tableSourceForSort const perPageOptions = [ { key: '10', text: '10 items per page'}, { key: '20', text: '20 items per page'}, @@ -629,25 +640,12 @@ class TableList extends React.Component { { key: 'all', text: 'All items'}, ]; - - if (sortMessage.field !== '') { - tableSource.sort(function (a, b): any { - if (a[sortMessage.field] === undefined) { - return 1; - } - if (b[sortMessage.field] === undefined) { - return -1; - } - return (sortMessage.isDescend ? a[sortMessage.field] < b[sortMessage.field] : a[sortMessage.field] > b[sortMessage.field]) ? 1 : -1; - }); - } - return (
{ - {/* this.props.tableSource.length > this.state.perPage && */} "} @@ -676,11 +673,11 @@ class TableList extends React.Component { containerClassName={(this.props.tableSource.length == 0 ? "pagination hidden" : "pagination" )} subContainerClassName={"pages pagination"} disableInitialCallback={false} - activeClassName={"active"}/> - + activeClassName={"active"} + forcePage={this.state.currentPage} + /> - {/* /> */}
{/* Intermediate Result Modal */} Date: Wed, 12 Aug 2020 20:00:43 +0800 Subject: [PATCH 24/28] AutoML for model compression (#2573) --- azure-pipelines.yml | 4 + docs/en_US/Compressor/Pruner.md | 34 + docs/img/amc_pruner.jpg | Bin 0 -> 59888 bytes examples/model_compress/amc/amc_search.py | 136 ++++ examples/model_compress/amc/amc_train.py | 234 +++++++ examples/model_compress/amc/data.py | 156 +++++ examples/model_compress/amc/utils.py | 138 ++++ examples/model_compress/models/mobilenet.py | 83 +++ .../model_compress/models/mobilenet_v2.py | 128 ++++ .../pynni/nni/compression/torch/compressor.py | 28 +- .../nni/compression/torch/pruning/__init__.py | 2 + .../compression/torch/pruning/amc/__init__.py | 4 + .../torch/pruning/amc/amc_pruner.py | 329 ++++++++++ .../torch/pruning/amc/channel_pruning_env.py | 602 ++++++++++++++++++ .../torch/pruning/amc/lib/__init__.py | 0 .../torch/pruning/amc/lib/agent.py | 232 +++++++ .../torch/pruning/amc/lib/memory.py | 227 +++++++ .../torch/pruning/amc/lib/net_measure.py | 123 ++++ .../torch/pruning/amc/lib/utils.py | 124 ++++ .../torch/pruning/structured_pruning.py | 168 ++++- .../tests/models/pytorch_models/mobilenet.py | 83 +++ src/sdk/pynni/tests/test_pruners.py | 28 +- 22 files changed, 2851 insertions(+), 12 deletions(-) create mode 100644 docs/img/amc_pruner.jpg create mode 100644 examples/model_compress/amc/amc_search.py create mode 100644 examples/model_compress/amc/amc_train.py create mode 100644 examples/model_compress/amc/data.py create mode 100644 examples/model_compress/amc/utils.py create mode 100644 examples/model_compress/models/mobilenet.py create mode 100644 examples/model_compress/models/mobilenet_v2.py create mode 100644 src/sdk/pynni/nni/compression/torch/pruning/amc/__init__.py create mode 100644 src/sdk/pynni/nni/compression/torch/pruning/amc/amc_pruner.py create mode 100644 src/sdk/pynni/nni/compression/torch/pruning/amc/channel_pruning_env.py create mode 100644 src/sdk/pynni/nni/compression/torch/pruning/amc/lib/__init__.py create mode 100644 src/sdk/pynni/nni/compression/torch/pruning/amc/lib/agent.py create mode 100644 src/sdk/pynni/nni/compression/torch/pruning/amc/lib/memory.py create mode 100644 src/sdk/pynni/nni/compression/torch/pruning/amc/lib/net_measure.py create mode 100644 src/sdk/pynni/nni/compression/torch/pruning/amc/lib/utils.py create mode 100644 src/sdk/pynni/tests/models/pytorch_models/mobilenet.py diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 78917d879f..2889a8df67 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -28,6 +28,7 @@ jobs: set -e sudo apt-get install -y pandoc python3 -m pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html --user + python3 -m pip install tensorboardX==1.9 python3 -m pip install tensorflow==2.2.0 --user python3 -m pip install keras==2.4.2 --user python3 -m pip install gym onnx peewee thop --user @@ -68,6 +69,7 @@ jobs: - script: | set -e python3 -m pip install torch==1.3.1+cpu torchvision==0.4.2+cpu -f https://download.pytorch.org/whl/torch_stable.html --user + python3 -m pip install tensorboardX==1.9 python3 -m pip install tensorflow==1.15.2 --user python3 -m pip install keras==2.1.6 --user python3 -m pip install gym onnx peewee --user @@ -117,6 +119,7 @@ jobs: set -e # pytorch Mac binary does not support CUDA, default is cpu version python3 -m pip install torchvision==0.6.0 torch==1.5.0 --user + python3 -m pip install tensorboardX==1.9 python3 -m pip install tensorflow==1.15.2 --user brew install swig@3 rm -f /usr/local/bin/swig @@ -144,6 +147,7 @@ jobs: python -m pip install scikit-learn==0.23.2 --user python -m pip install keras==2.1.6 --user python -m pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html --user + python -m pip install tensorboardX==1.9 python -m pip install tensorflow==1.15.2 --user displayName: 'Install dependencies' - script: | diff --git a/docs/en_US/Compressor/Pruner.md b/docs/en_US/Compressor/Pruner.md index 9efcce8e7b..0901c2a46d 100644 --- a/docs/en_US/Compressor/Pruner.md +++ b/docs/en_US/Compressor/Pruner.md @@ -20,6 +20,7 @@ We provide several pruning algorithms that support fine-grained weight pruning a * [NetAdapt Pruner](#netadapt-pruner) * [SimulatedAnnealing Pruner](#simulatedannealing-pruner) * [AutoCompress Pruner](#autocompress-pruner) +* [AutoML for Model Compression Pruner](#automl-for-model-compression-pruner) * [Sensitivity Pruner](#sensitivity-pruner) **Others** @@ -476,6 +477,39 @@ You can view [example](https://github.com/microsoft/nni/blob/master/examples/mod .. autoclass:: nni.compression.torch.AutoCompressPruner ``` +## AutoML for Model Compression Pruner + +AutoML for Model Compression Pruner (AMCPruner) leverages reinforcement learning to provide the model compression policy. +This learning-based compression policy outperforms conventional rule-based compression policy by having higher compression ratio, +better preserving the accuracy and freeing human labor. + +![](../../img/amc_pruner.jpg) + +For more details, please refer to [AMC: AutoML for Model Compression and Acceleration on Mobile Devices](https://arxiv.org/pdf/1802.03494.pdf). + + +#### Usage + +PyTorch code + +```python +from nni.compression.torch import AMCPruner +config_list = [{ + 'op_types': ['Conv2d', 'Linear'] + }] +pruner = AMCPruner(model, config_list, evaluator, val_loader, flops_ratio=0.5) +pruner.compress() +``` + +You can view [example](https://github.com/microsoft/nni/blob/master/examples/model_compress/amc/) for more information. + +#### User configuration for AutoCompress Pruner + +##### PyTorch + +```eval_rst +.. autoclass:: nni.compression.torch.AMCPruner +``` ## ADMM Pruner Alternating Direction Method of Multipliers (ADMM) is a mathematical optimization technique, diff --git a/docs/img/amc_pruner.jpg b/docs/img/amc_pruner.jpg new file mode 100644 index 0000000000000000000000000000000000000000..456dcbc318403bd85bc370ac007737fce1f894cb GIT binary patch literal 59888 zcmeFY1z26nwk|la;2{v)g9mq)5P}B{t_iLScUgf12o8$?2|fL@AcPv@)fhHR?Vs!f5{j%#wge{Y#Df>AR{jWARr(BZ{dFcY~{(R zw5PQN04ON|%m4tO0muj<03tkw4NoY9SOduLI08JblbrJV4UhuBli^)*02Q8x1y75^ z13>qI?egOc0K>j2CAF>5n(r^6!mM#AOIE((K`ELaNM&NG*{zl+$1pWg7 zZcc7aVNO9|PCjZbUSVEAVQyaFzm*37Qh*g;4uAklz#g!Mzp4Mo`7MyZRxqioTwNW7 zIXFNr?C;DROw8F$9qc(g-#Kz{v2$_&q7t5t?@VpYU8zmXEv-Rfj7JSEjMUa{ky_Z4#>2&C#=~PKz{e%XZNX(i^KZ?Ynf|kNM>l7?KiW1kJm5y1>!k;-KO9 zN2mW+wF!slpN;-E;fK=*FC#4FZ2r#G{1rTi{~lmAPC+(KK8=4Lbas9L9#M{eDJRPD z8;n17^nY#7|EJIr7d0~#HvQdq2j_n@|Hj78!V6SA z0T2<8kPwlPP*9NJ_Dcjm`0oHRE(+c=F3Cqv)!w1fIN@^##HORszAS4cP#-y_<1uj# zM8_Z`A|@fFXJBM{&dkflFCZu+EcHrSMpjN<;f;o-mbQ+ro~fC+g{76Xjf<ppgJ~25p zJu|zqy0*TtxwXBsdvbbqesOtqeRKPpE(8GSA7uTLvVWlq7fu%TkLb5Ix`l z2^SgV8P_8`Nj21WPETpL1JLka#-^9GqSNxI9}}23k6;ke@vhLH{3h)W%KqmF3;e%C z**^*U54vUnEF=W@;344x62PrY*b0=^GxN=H$7BF|(k@$8gL{hIZu9D=@w6asa_8BK3c}qj4{2xG zF7VD8l%9uz<6uomIIuU%?|XGW@?D3({kJ9`)1N^RCor-sUQ%>yL zrqn_p|2Jpss6syd#g7jHg^RwW58 z=rA}><6$qRRE$EdR$ckI=6l2whTH2u>{O4=8+PCsybh|N==XnxfIjd4{;7(k!`|(< zeqqW4!Iu4rck6*Y8eCh;KSiQsG3XNIQxsu)MFhj!VJj7<*mcTv1U`llG=*iImv4d*WJU z^_dP2!h3`@F_sse7cJG1Wg{TM4nCFroF*Oy(X}f%(-*QKy;j|X4Gt=$-}R)V^L z#FugzgmxPpEUwGVf@|$AZkwcls)jHN0q^3CrZUJ=OlflFBBV@UW~+7p)36Z+puY76 z8)=J#vAnOn&MBMeakKuuV9@4Z;&7;IA$_`v*F(~&(!bO|_ZXT{+;f6{aNq!4$mGjj zNR9~(^O=TlLh{W#KBp9jtD|itn+j!S!+?^u>S{QCA1klFKIA)CK7?i{g2nm;xIZiA zwOdig$iRTc9xQZ;=MG^RL$tyANt{!I&9`O zx~b;ecxQBr*7l9Hp5q_Jq9h9!X`1S4PC|m%mQ&?c>g#KD*R+1dV0yeoI|@W78lv#r zJ^D4pP5+U?8ASi*(q%Ue-{$(CA?-|AQILVx(qi~cF6hdL>Di>KL>?@{wk zy8KY~sUr4^Vg=9mn?ea}c*#Lh^Y9tkFu5!wmNs&;m zu1jsj(>hb_!0c_dBo;~FN%k9ku~q!I$3=W81KQZ?GuU^#bbBiNK-T8&O)=4b37MCh z!j*b@bLy~4&zwtV2En0=i_;aL!a1-cN+{c%jUbgiAK;rzd@8%%-S522^>Vk!S)mYc z_BI=}fl^PpE(J~9Ui9;q*>KG?wLAUPU5ln=fxH+*=vjWuoU+kqgjrED#dyM>E|7>7 z?c(6&Y(W};%d+>=z>E*Qyk)m;Lw$7ki0w>sicm`f@sTrx>1NBrtB=Em5~FyRH7< zxc6A4AO(^V5_9D}+-wOZG9*PlBIJfKMD|d~CTDRH3yL~PA4T~}y}Rras+`lVFK^ai zXFpg?wK54!$1D6YOd_0;Oa3@F>N#I_+Qrz#kI5_JDdTI}%A??7ioXqV5sW5E zWn@b!@u{LG-Ii#_*8164ug09Iwrpi&c2yW!8?Tk0npc@4cC4YlV7Th=Wfdjvtkzr zt+I%o4eC3Kn&U^k&nQ7f47qkQrC-R8doylUUJ+|vQy!b?7COO?UD9oagd`$mDpuCv zQe%wkCh~^&Os9kXAhK!SS(>7)2(O2rUkwp~+)UirS(Y|w%z@N)c+X*Frl@kQNNik9 zl35N0SfU*_W`Du0jiGrL5V7vMnXn@c1GaK6hW+#o9whei?fpr1M)drG=iZE`k8K2^ zaj+3&vUrv;NwX_}kPCwBgPc)ps8DzIi?F5y?KRy@&p2*}CFidEkC398&tU-io_5k^ zq7!?z14`3C)&6Bv^`RkpWbZm2!WK!1Pe);a(rU=*lt%N|+X{oZZSSyFC;R6X4{Kdb zSnuESN5r1i^jd)iB=%ai1K3ER4P@R+ABl7(U_j;fNj?hxm16AVG&8OP&u~K8$B`f6 zu7c4HbT9oebnA%ZX#wiUzvFoUR-=y8QK`ad{z43 z0gkzFxIH)iT55khCs#FcSUFzbVpUSPmMeEeLv(u8eOFO)WTCZwG8UGod5j&ETfm=H zGnG)&Y=@Usc4J3t6+7Zs5|PV7Ye{_;$Q8Cx)M#B~Xp1**5N}JuXN5P&v~Fg#=0hc7 zyX@N*BPZ6gk0LJdNt`uKS+Qlmi9$>1>7CW{RCO)BJ(9HqA!0F9QBwbwsaY7n&vv;7 z1F)fTySyq0GggVg8+@31`TX;F0$3rT3ubYGYH@YW;<SA?9$^x|i!^wU)It zIrnGgmurvZR6X9B=21>`8~DoAsXUo&MX9Q+R$tK9tM7~x{bYf8#v9hG{=~pH!uiEP z3`_d4*v7zB2&HXLsu#+`DymCd7067J>IFa*B*NQLl6JgW?|tdFee1bLEvkF zL0zgSgz%*q{ZH2Eg=EXH>H3C7LwPpVjDVg`-2*J~UDR*iQt1ioNt|_dL1BOd40!s2 zMAcW~_@e1_fxfB7iCKC3s)EfJYXM_X$y!l=sc5MP1`Lr=-QB~0 zdnK|%GOLI_2jJR!mN9z2qMumi{k)OV^GI69?N7R)npocth*8NmbH+u zi;7HN=I0C*u>yX1d0|DMk$F-(`b;d`4&9%W!GM+_7|=e_q6-63`6aGyV8C?}&{=zJ zzv7ceh|X%EgHl`Vnc4W2(5j6P@kNP1On(m)-7r{rF@3lc4S{OKPoB`t>Ch^7S$eoAoOmNKKG$j2>!O1a`j2+9?&f|G0{M?0J$8xq z-oElHUG7WFDivW~P-x8{-NN)2XcR93#s zk<&e0b;S!JN|mGbN>v|?G#GU0S!!$g4-KQ_#dJgOZ7BZxps+s*B1rqol>N&xd_jt@ zWx8m?e?{2mhEkfyo55`wA4-C}g(916J&YiYulmo}V_ZKMq-FE>URiQ2aK_9263s%S?W;aAcXLcdj?DIgd)HacW zSVI_4A*~T*yTlU?|jR))CwKvcqJZy2QDp2c(17mhEkNpR(7esjl zW1*#rsC&~Yz$l|Yp6~XXG5a6`p6LNe zZCBiw;Eg{Bnj*H-!!|Qc7kOZ+uUaSHK4T&3_{M?arf28q1))^|*Sw9)FI~DHK_JaC za+*EnwkSpWwHDLS6uR5M?6Rm{2FtEg>`u{=6mjEkEx8InowSxNXz=?;RguX!yB1i4 zRV8Pj#mGgi%PF5sK2;G-%T1#Glfk;+1zSyY_b|#V^XIr-8cg@7PQuusqnP<+*>e$p zhtaGv5L_Y>Z7J`%G^{(jKl&#AjUQU~9g|nfu0C?1IC*Swy5Jxlk1yDPv(^GL?li(w z=txfkP?N_0m!LY~hQ9CLosbZ?Yny)`uN7xf+I=W5C>9fyBKIBhTvL~!>FHdNdn~WA zgzu&=@z9UCGk=p$n@s|5=n`U!FeKcl)98oN4O=)itq>;&UK!X}s#uf32k6mX2MF?e zfLPXs7<|19g|S-%z{;NyKw-t?HNm$!>s68V@h;@Q>?%?PS!$vlh7>~C;bYW8yK`JT zY)6Oq2d4XLEyM>$zvqmzihqmw>3#;7hstW8Q#AE!SKryy`OcAwJRH`OrXpJ#qUVC& zSi7S5;o~55@b6>Ps}1=@P^Z=I7@k}hCBz1ACtATZZWmF+l;Sh%QM8y+C$h0r&F0b0 z3nHe1U-~TvNF-GedI#NUgEyh)Gc^|jvJ{;e_Lg3(&vZXScez|*sjuu?IM%J;i$p>G z0@nda))Z5r>#7^%NK4t}FmQtrH_!kb#(hYTC_flV+wz_HqY^+jBVShh`@33honPh8 z9(u!*L&b9)`ff+%o$@gSjYQD&R-%t+1OzoG)2q*JFn}g$Eo)Hc_p;(5i}4aY7k*+& z&zYLk`OUB7w*mU4+`aVA(<%a-(gaF)PVbwVyoXVGnZxI8OjV-f&ad@$zTu$P|H|8n zxf43*L^ZaF-)JEH=+tFPrWfBgXwmk3>-{&cgF#tin}4j^OWRWJL3;GGirD|C@a_%i z+8a`Dxq4rQ?P^jaw;-fckNvG0;y*@WRMC^;ncRV?tvddK4R~f%GPmurc$(9%aHFf_AuWelo}r2839EZ-3BCr)GcS}b?M;2>#0Y**-iL=7^? zA`O{}V;M{S9!TXV{Wc$Fd0W!Cf=!`UGnS%`e{2A7V%78`U+*0KV}7yzF~2+Hg4{2U9W!cCT0oyyLd8}WyN_0S*vyxl!DJ|qVBh9nxrW@ zv+T?v@P)gfkNbYRrElc7u7!Df zG%T5qv`DZXOHx(>m1@#}W!epiHna%`8qD&tV(31i5C#~9zyK+N^cN`jTBs3?k`lq^ zRMoRT;}k}VY^X%E;hfcv>v}?lU0qE^)21_0K;4H;>6Rt!*z(i*34idEHkT~^Nw29} z#IYyx6C7%n=$})PuQJvc3GoX$tN7H(#9oFmc%8F?Z$Ra zz?1xr=(z};U`Njki9h=pxEx>R@erriM{oC`wXlI>r^JIP1j*CJvf4}RREc7G7!Xdm zK}l>+d;0Ce=0+%$`X(Xq^D1pILgGPmZTqo&#aT)c{{-GvK@O2dbH=?^Od#*6x_RjbL zrSN6Nh4Do__`$Cf8aVMAQnkMZwUNx4aD`CTP3sku%kBl0ewW*I&E4#*9`bqv73}=V zm-A9akty$bTE0Lg zeMT#8CD{5n3#a|$dZPC~sppuhMjD6+O(rzjEojzj)Tk1!X_x$mekN6D9GPl2j(sg@ z3{0d?5M3J**Z5#*tVWV1UCwTvhMHbzqOS^5~tjy58hM{SXx1UUK{T*%!Z|1Fr}UF6v-UBcS%&n zN&U>O6WcjGZRG@}xEkx6NM%CEz65$-E|WSbD|d&*1qlib%&_%;GZbHJ(@!1#;6SGM z&FkI6+$$ZBCTe{$XKQz_>t;0Gb;;2$cPsYvuuFO6X+wPrMjv5}?l}@s`vgE$_ITob_IIY8fE9%*fMP5qFXn z)05oojdy}H^NexmzUhgF$gZTgiOQyT<*_m#_N%Qi3epZ=5fmF-1VfVC#uND2?Xn`g zVz}AWb5y}8Kl8xoPk z9@k*XpGxR4gl`&_YSQW>RXT1>luN2rSY^stK~CM&F|wQ3hWXaH7WO31I~3S=f}B%n z0#HBgr>=NNGHJe578a1`*VfjZO^vqne>D}cw{fZ?{z&9vn`N-y@w8oaBd8?F`yhQP zC@+7!LCDWmWw64rIxIjNG(8M1=NSvQp0wZSn&U7_W*Fk!0=a}O*7;#f`g}nKX)s}w z_0$=}PBEsFT;v!@VSgNza&VkHzLCy&hUr!EVof5EmUzH7iV-7gi>XKNHtWdJ!~S6F z+PZ9iwAdVfY)e$B-zX;F1e^oW<7CctOYHqC)izE`H%hkhp*NEsd zS)FOIrku2<`cWbbka(NLMMQ}O37z)(uMQ_+6f;X4;3*X@?Axp9P~xAMw}cgM zc~bj$u}!1O2rX|RQ~jKLc%89&z&4Pt8Dz!$d0~v_&wHTmZRzcT1hjbyew4Er&gcCH zC-}_`;Nf{(?+^VR-j6|%id+~v%UI{+Y2QnHew}(Itf`K}x8O*&R!K{8H9IYJZk-Pa zs4<)>sJsZHn)afG;rP&DW)SyWnCq_7tx3@Y8>OJh=&GXa2Nz(Et_lnYieE{)Im!bsXB@$R zNbTp#4mXUc6foe2VGT4)Z=p0f)k(0YE7r=}sa3jgz6l0kvb0H@EvdqQR_P@eK$;09 z)D8x3HK)UX!0;9*pN3UFbT;jjqYV(w`P_1UGmW@>-|YZ~BRy~jiMSK|`L zY9274vmg>5Y_D=dv`F>;Xkm)SL90zv?Qw?ESq6%9)d5%|efF%Z0ax~du^KV%HMFDE zEWyVP_tB{|FrbtS=i%{YO`}?A-^v|}Jeyt6{#9bEZpTxl4eUW>bUX6LVl0bo6K>Wr z_8_l|XFURxoeKp0x}JS~E#r2Lik6ho>}mEC^W}bdis+$zOd`iw{v5*` zBYIOB0aspW&*^pWRR__ipS+B5bZQ*{Sx$B|nJ8MFPufoFSv5`vNbkkb1W5$6-ZNjg zB&9{1f8jN6DjzSVJEIL=<<@_;0(p@hk}tsztj7+Iw{h^G&WzklTS@i{3PI#* zinq$+9j*mQ3l4^)T{4{&?whxjB~S3C1%Wh$4^TY$^9CUYQB!-%41E@6S*GT*4`oAC zPL7^yC05Xu(#FSuVXemGp4JK5ktR!opD5SpqM7gj)F^aT&L(AVRbQu-?25phDjz3; z_8hdcm;vg}Hz01Z*jAFw_X2q(FUPu03>WAay6922y!CiUM#c!l`{ z8`|VyneOR7yE>)Zaz{UYwsd1B3!!@EPVT%da}2H z74h-ZXoU2xXyUukr2-|i%GaU%HESs%N~4tzNqoJp{pXx9lbFY&d3-JUqBv`CJ}2o((>UT3PFyz*{O%8t-t zv-D?M4HOcHmqJ#@J{Yb!dhGObXOLS_#K5=rzDcNUPj=@Tak^D z?6IZW{3q9Dwuq}Bb}XOBy7|lv5oNOE%bgqd_@B|Cvg#-Pk-69Y1D2x>?6ZsZ3U+yk zPrBE7jEcBFY>85>EF?!&7@R!%wlW~NxO+uI+-Dd0g7vg+`%`9u={5?X6@JFW>(Pu_ z7=Ys0rgkDzQ7q+`lau0LL2YXl@okfN<A9r^6`QtHCaMKa?y1Fo`iT0O7R?D-7VLkuK$*3UI1mZft88m%vKi}) zom22DJ{awa=v!U0Z{>^OBlK}?13hAeUygGlOBg~i$!W0*0L zZOtOxaZqz^K^roWvyu~e>|B|dF}p{^+f)@ap*6-IJJKm7?B@PPxsuZO>P&yOng<^! z&L|gh#OU!O8mw7T`5f;l+GN|UXeAEP@I89{02PMv_MM@Pft0!#sGc|eY*JX>I?pcW zq7a{-3w(B(*oT^D;B>2seEvT;>b;rZ{l;d~QFOudYPbwLB1Vz|jW)e7UPW`Fs8XM> zaXjbAuPk;)dAf7paZa)W*u$CG*;&Hc)~$K+Ovoi8k{flTGFq4{IIcg$HU+}qfoivf zmF>R%_F=KJAUBkHgf?8GF4)$`iwp5*rXz%pm`sI=^KJVUIiH^JJZWU1Evdl}-4$;l9-=ixmgL7j%Ja35L&kI!AuzZG4>(#!zMW{9C zV8x(YSrzr&6+TWwz*QaNcQNL5Z&Ocwn>$Yj(d&&-T<`6EDPQCMOBR*lWJmSLPid{M zx6Yj|Z%fP8i-3S4EQSEOb16O%^bo#EbxVo^1sJbK{uCdtax083VqTUO;Afs1I2tYdQj1}Rf*4^iWN+eqJ~l)E{wGY z#S4HOs2LX1ehE5s<%sP~2t6(ci$aD7;xZ%-#$fRrVAkF)yXG??Q7IqiL}uGK+Ys_A z610E4slg)>a}RWsz$o(yPthBk3M{kOlssQSeK>c40YJg6N7%W&_M14BvL;;+?NDFk z`{}*P*bmp5DFRnlM2R7S5MySEZ?q=n;B&UElfp#4hc#01uEZ<;n)qV>2EwgCk#Anq zDIT&k@#YtgDwc!zu*KIDCSsILDGkztMaL!FU;ysT6F(O9@{3R+mD3B8-f)N0c|vN? zr&RD0#}~Iz7tB|~lUfy<)9#Vqt?Q9-2ooJP(8*jN9pRKSjH8u>0y0zQ3;qimqlcm! zk=3=#8uz9&2MeN??6B>G0v}5Zm=L2MV#nT zw*MT&{;o10r#}8!s8gc*PZB}$2b)G*6|wHRdK`p^?of<@&*pURrV*x)as`Kq0o=JB zCs%e30@*lPGV>^zyX+TzQ3#->+5rRjd6qdLnvQ$NNvx`_`cp!?Y~D>2m}h8B+on1d zG!RDdf~mF-gg%zv1=nU({65J4mWw9Nq=|sz6Ygj ziC}&TvMroicnlrV9nLwP%+Fis@#DRGs;^J-tPal+O_sSaQoUodmIF z56PO{yVcx?Y?>DiYKe2f>%;XzOwd{;r}Sc^GGZaVmmGbUFw1hiRSDUB*f-HtaPGUe z#QsHPJX$w#JbpXX5h;Gxp2Wn}mY&H!Jw4aqOg}ihfKDRn&(~KP!tpFbrK$ONW21NO zx?s~fM&~OX|G`N08a?`!7)wqnWw^V;ZHs>ncOf6)D74sRsuBh~(Kl+#Qxdf|C5Y^s zYfNME)VIbSIvX zp&q8YMd}jLvpI|^C>ey^~;Y-{Q zVen+Czjac(-TKtB*VzVF)zuY)Nv_SVbGno(&1)cN ze8eCSZSCL;?Mn})pF)gcIFuLNvsN|Y)Kj9Gxw^JQ^}H3a>L$SwE`4_v!SpcB-4_@Uss~2x)VMKQ|Q$4EQCx3a-H9 z5I9&=YF?5!vLBi`sc|5IUJ*=J!e#4gQ=GkX*7Et>_TTCxr{efx zi<$m9?Z`=JIRSsl{xI-~z_{GfDfVUB1ybf)D1i?YS4;{TqVfd1o(ltrv+dz#;nHCd zFR*d`ko6+^)U?&5aI!b|^bV=rt#>!3GDxa`x#3lJ!K0_2=bhy@t3}zCb?yZUnsyn& zvNO0g+oYpLZm1tFOq{w@jteD@LwBjpg*`vO0D6h*4ywNF75 zzwMN86dz_+PLyu?Sn6y%?#2(n4dmv`kG3gA{Q!z^FOytpdAyC=ny+spIL1zLzd}N{ z?w2Yq>H_DMLR1pIxYO=U{v`<+16|#>bUPUL-QM-jl|I@ve%I`yUFlO% z8fmmBMDfu%Xzru?tU@wUH};39$^n7a+x(M{;&cI6Njhxc*9m|nq(tLyeNNoA_i9?m9%VjU3ay`T_?<`1fq|k z1a9}SlsyY1HjCc)c{k(e)umk3VVjCOxpDkZG~4~7LIr1M52dbxd)_ZQK~CT0`wLAj zwBE?BP4VPQ6~Rrh2=C; zp7c~I*2ZKW#_)vgP+$AxfPd+`6zyrv8cb=EcFnQTc@k{663!v}d_hDr=YDH>XgU9{ zv=YFbCV!n~AAYGZ;yiiUd-b}yU%cm3e!W{>qITzp;L*6Bl%D;AJyebot_9GC0bd_~ zU0HfEXPuP#!N)YaFWW{yJCc6fJ0j3L%!{Hnyt{Lmig+p7+boshL?(Y_&KS#PFNYxw zJ&2tTjh!N^HjEz`0rv&&H!I{LYO^)IQW$`i3fDhK<$jVleFFm`)c!R?UC*B?_~*qv zn4;H+;@T^F=Ssliw)tzWp$)f<2T_^9js6T$uc6SpHb7Q6B)xEvtSY`yc2!9;7J}KA zxB8K@^91W12{fn%VHcltsXD0mfF@`QvXc2C$J(j+R_ukK_rgy82HO>pO?=bCA}E^;iB zuj@P#Cu&~LML~pDwkdO!*k|%P0=G2$-DcK>N2wrQKZ070eh&}Rn_nT&JBfW)(b6!K zl(dhBmDv0QL;}^2xtSUL(*BOm5o_gvUZy_}ZF7n^uv$stG#i*m;l6ifUit~lB`-_- zfNf!nqX>M>*t58H<)XT+vGiQu+r-e4(X;l$V8Lmz=0wb=4hZ`L$zCNAjWVdm0plCZ zb?(ia6>)7`QXVdEE9lM?RANDc=`yS0XO9~YhF-@H+BgvQ(TQVC(gn>55%#K=L`c5* zaf-I1rwVUWsLd$)K_Y&oU(_lTyc%3lEg?y z>U*lPZmWimF+bzAea${ii)Sxr(24pO0TE+yPowd6eNj06VM}Gv4_qpRqv0p=V%V&_ zCfanKHjD5C@pOn5Dfw}cyb!B@)>Zy$wASRgpZHjNf9#BIE56f?YPBgnKK7WveWhK* z!QF?EPYarra=@lUqA#{W5Yf5---oqng+~oR$f?IRq9Vgh6&GgYkpo1 z#6Oc|_HGW;v_xJZYAWgPdoZ!OSIlM%S&q7e0ZR2ey%8NI?}^PMH#t#JxsfV;+r8b2 zsVcg!N{ei#T0rCc6jR*q^AX)xNY5iB-b8egoQU!(X?ZYp>O!c9wO=0-<$9|p^L(=hB5!R`_ZezWo7O)0 zotT({1)~>j^Q5{^G>xrhiaXup+1P)VX(20pnZ$!I9Jpz3z*@aw(-ERGg#M2+LrYN+5uy4F9 z*=WobmGgfq;5Y1YRTkzEwx)YNr|2cKZP`*2r7?u}EXG_VPpr!uZpj!?=ff!AEXV|K z6ohD|I3$@bXgO@bUhze)+jhQFQgk&>t8Wf1{7wg#7TrbD)=>cj!^B@lH;k7s;Nr#) z24uTva~R70HAhP)$SNypzgk(7ZU*jsU%sc^wYp}N>IfYorEb8rquTR*Xs9t26YO-- zt~$b1$}-f~)3HhlF7t3M*(3NKsY&1j8KPy5%0v$;4Jd4* zQson=QJqJQek;m6cK+M|FPktELb@O1m1EZyYAQD;Pf|Wrd!gc<4{iGPZ9=}A-mh+^T0>l1YzN@}S^Ftt2e#1K=#O7s6)2l$atbo972O9+u+xnJ|FtYOb2_K?^0EUKWR7JZlWA8UG45H?IsbiyLqZBbB_DrGQ%-6X2>cEV%m^Ei(+5f-{lf z4l6DANjEkgyf+L3woNLTfTA=!Pj4K&9f@~!v8!#Nc(1l>;e6u$sor=`ZrChg!m36X z-`x-`irNK3vVM3yP1ZB@$4|9YTq3JRLW0yWMV^zM66+th$TH39Nu4r`BCMyr-}=0# zCRw2G3D%OYLOu0e?c8#ee*OAB{tDr8zNQ#k;v+}6TcyK@^y{qkBe<a9INczU4*GlG? zlIR@O{JY|&X;TdEIetqq>c5Z3m6XBnMRR{HovMh6z<{?64iETM4F*RcY@%;6ueBA% zK)yr;yp!&#bKN5gGX$)k@MFJ4eI>}<@{EIiOfiN5h(+KVnr7}CRJ_gE5A#>FpRzKV zRv!qY22U6t(1lx|C^|6URGQKwRAO5m?udTO+y$4}StRmIz_IZcVaEN&8F^ldbnv4S zqc#lqSx5zy%gG-4WHva2vO5r{v3l6P9F++bXSxk$ydm~8NjxLv)x;5LPdlgEg}a*) zz_d6x-h-@diA4y%l?R) zx=zmJwYV^fklRIBxLxN^egRVZaN%Dp^%9;srQ8JRjUWmjSMWtHr(-+iS7IFKW?( zLZKT&(7>vymkS@OqjPur4N;*l7vavaKOit*hpowQ61qh_)BKMfbK_a(56iIMlY>C)w0gEGIWP70EG-` zyBgmB)8MH}@GV+_-qn3jqYw;u#{dIHb+@;f?j1JaefsvhPc_hu8n}~7l}^3GePsR8 zUBc2Wrs~zpsL}HWwJM!vxZWt15q=yhWokLcg%i7cbc5&WaC;gK?*+aa%CUszZAfU# z+MF_+iuf961OfaTcq`HNBG6x2OSha_6BLs7Xe@cxa6V24U4pi7w49gNtd&+_`Uw+2 zHUn6$>ZH*m36Rx+_JYK)XM|iNvQW zHvT)!Fe?is z6{0lOQlS*|(DpMM0mV#P+KyAGn>NPJ_Qyn{q*y{;0;ziZ_tV;p67^ql2ozg|7CK`k(Gl{Nh!-uHJ4do`Nd&Xi=wD4U@L zp5FWLYdP)JxQpNN?2(v5lGD|QX*U4|OfKY$E>XMdvsdG8OE|5BVQjOPlhD~y!9#-# z`3*Vp{I}>NNF~_pDQkibbPT(Lii#cqhtyxQw9ZW51j2v5y$f-al;%+^^w;U$7Lr2#06QPvl(C-gLz%vGBU#^~8FW8tK<1Rh8;NQdu`L z_qWGLmN|HI^kxXR~6i^T>z=!d6aq?et4V95ECrDlG_5gfr(ye-U|DMB&S%x8-6Re9onZ5BzqIlaL5? zr7F`KjOi^eDlit&2O`M*T&}eqER2okJDi*oI`Q>&kcT$&Ma~-DOA_BTdF_psVCNgU za<)&4&>V9#B|U z)l|^IxuZ1ti_y0g&Lj##$hfT=Vjb$WCv4--Y0zk%-c+E~SlWI1_EW;31ZUiP{7%TI zqfwKd5^id%)N)VEPm$3N;<~XwEa>(>trmalA6#ie>f~;5oNObTBm(zlll$?{+bre! z8cBRfyIKKk&meOBX1LF$aj3VDlP!sN7QaBLpL2bj?%OOxaeLhI+~m7Usz3auvoFMl z94^7LRJ%}8T)Q}Bd&Vr;5UCo><{X3uzW%yXiyQ5q8^bcp^29SQ$#ZBv0WG8Ic>+?s7;jS9ouJUcZYVIkvEGqt}8=fuhLi!_e_Xmv^)!dzb8F zUX9s65=&ZKJ5LAia3fb?D8rLfCmI@|`&P9y6XwIXg64GAG9}UuR<2HS$$R#WitD$C zVv&sK^iEy4%IX^`hG2xh__`E5Xg`49w=;}w;aho?l?DaE>m)oBxG6z=*0g2>@3qDRsfZV~rdA^JH zrx9AZ$+AjwBib6sjD$~Ir{L|x?!Q~Q;qOB01NTd&*+5ng68qj1R7Q#+JI$orY#Ev{ zQ%SGOVN8D*Ahd8sc_Sw5kB{IS#=^eUev6VaUtQz9`3IC3W_-@MV|-0U1p^3V@9CkWa49520WO8Wui;!50Mi-pD=sS>$5Rhj!rIVB z@J~A6x+ipSaLe|)(UxO=2`KY>ex@Suwco5-xdcHcBnq^K_72M5Pz;j&@jL z#rj3?(@-Z#*W}hNxkl|=8ly@do7Yfvs1PU{ZCr*Um77bUta9=35mU5Pet)bcFIyKa-zwg z#wf@{c(DzXSV|v1ip#tXywZ1JyLgxwufgqcmee=qglN%kH+eZU`Z;yj6e=d|uO%R+ zRj5Y$8NV64mtL_q)wuR4*{$zMlrIlaNz!4(U-kO*QtCB0z~{y^?*5f?mCfo+(7CsJ ziakSme!qaM^<$ozyF(SUv;%=*vlFAtRcuO$C!P*!Bb)InQP7!##aBQ%syaH!5nKaw zgl(NlQfBMNjz=oh>=UZ*TZ8kk<`3o1>c|h)zSD|bFCXdawVK-!R3&rG zwPJjXVv1qThr44_c3#p4BubkBhYv-5JIioEJlXINN{=7W|KSGV+m?z@iye9!4#neBTtc5un7x$ zK6$obPNzxwv)&NWF{LVqf%**X#d>+=I)WE(nVA98XJ}t5v0_kEd@A`aIY&1s97*D# zG0ug3(r>7+L{a{T5M{-E3R2bfe`(#m|wAYo(eO4cgubq(6O<0ZI8eMXqr+q z-Pul5(dHncoU;r@E~$0vSi@q3C&7E{V>*`!mb5=`Mzhp2?Hn%*mEPL5w5Nz!){92T zZh-Fav(@v{N?c2XnYO^rOkf$suW<%SPF}7*ofSHj1u0{?UCD~6pJ=m-mkzZt7Y`e! z^p}6NnT66Imx+9>AdZE%#$V@5>D5n3^T%p{Kkf!%k)8$Ya;Ckdk_J~LQq=^r@lu%{C~@=b)$N{61bU_3J9Ni!zu zg}3@r$#>NZUZ3rPcC|>dT_(}x5z_8kcc1ZpqtX53#Txi*(cX^rItz9XniS_Mv7`!e za}>(RNW-iD*=b>?~UBv~_6UGm0__~wR zpEhrn{ZjENJyI%!xX+rHR0_|3UU^v}f?Bn*B%}Li7Jo>_cri=HG&Q3P*b_8-Gn5rb zSPJYKT&c$RvR!4O&{lBp=pRxv@Tow5z}AjjN@6yvLPcFgU0GhlnsHkGp5%gimuAV8 z!gt|4XZizS6prL9{{D-UCFN{jT& zqZ~K=qY|)4xznz9#xWAXoEF9DPQA(_R2N|K!&jwUYeh0fPy+=}zhF&B^Z%Y+VFFhu za)ndBM&O50#VE=8zZ0|@Y@vQ{>cC#5+AIKN_!2Cs(uBA3LrQTsaki2lO8>$efK$1K z2EzYCDIrNUU%q-cuTuf`vUG6N{PdxQ$?Py*74-3t{014A$GXk`z#+qR#UHDxSX#Rt zE%0894&5u}n2HHjL4yiFlYb@h@QKg#u%FXn2Y)5vUx#2g#w%|M=6*bN@XlW$5a-b< zdoVHxXJq9BCBO^dR9nw$-yNblN~yiI9L-5!M-uIF^?#rLkNyAo>AvIhbW1(yY0HU{ zpDeztfKNY$*JTuZB)JkReI64yWub->X=lc%fy?1%nS}`m+?D~!sh}(rC`nupiv5Z` zA*F%^Xo~@}+NSj417%W$q3)JE=@gdndz5N8JPNaHeScZ7>i_cj(qBvlZ}X|e#JU@%G8QBZzT>#_y0r%o<&F1$dJp)pI@n5rXsb7$G=ooQFd8O;xa_r}Hq-;tus5MsVRa38-Z@S>@2 z@`(Gil#ke?Tzo+uSJL(Oe~0{U0~lkuYCMkD5Kl4VdfA5OzU_SCuvW!1K(36m!=Ry3 z6C<}1GreZ~F8C9{OYa+z*6h()lfUi#_u-f2Iuo-*AFQ2k9B03f>c+EIC6lAb?#0HK z&#IA1;L4+HgDd`Gn*RjsO6h8mKb^rByeqKibP)sZC zoia#xQk$UK6FBqPtJ1O)RV-yztJ>1Fq-mmbCFdA$S;2w-+`@vPQrB`B8=18?{pN>> zIG;QwaG8+BJw_bcrd{kmN3XL2rGx{qn*<`bMOgog*a4~aA2@$};F|i|2IDDGZVs}^ z;<6L}cq%2N!@E(1+2 zRDw1ZyC7>`>SKee-4}}7C zj%dI-sK*Q1^O$5p)%B0#?CYq&i~n)@F$R&&Nq3TdqVlEJz@0xZxkwrV6(kT)VcT{E zSsrEI@Em8~-?72?WbYb@C(bZRwXQ@sBd*0xpc=5y4i^}#4*)0*U=wp<2eu1h=O-r5 z_OYj5__9~Y)ysFhEpYch|2Uqhe;(`dqJvnoXra#F%`m`yv-Ib5(l59^o$6n!1)|u> zn?C8On^{`-d@PGW?|Uw9sZvOin+wcvx1`?Mq{G7u6jN%vM+i|3m!k(K$j<4NB>iq1YULS zn2VHle5aXF|4rp__iEMddC&oZre7u)wj{b%2rc(Wzbks;UC**iWOX^M18x7M&-<_H z`oAM`+&<65fn!@Y*2K+k`Lk|v<4c96{u7@;hxr)o8#iNKw~Xl41uM0*E>Totbms2$ z+DO3-rEHy~heYcQPDDez=>Vy98Z7?dQ);XQsY-)R`{m2NvA*Iv-)enbC6UsIJMFc* z2KNz}(x_4@IF2(FlFxA~a)$>M*_e>@4;Ot2t2T+GSDDoYy={BV4rh7j@=zpo@x7Lb zmyjxhOQgsK13&%O%%`Sxfws1C zDd#2KoP2>$`Qm(D`#`VQTl+I(tlyAr-WO-D_h$DED~6}ba05W!KE=TBo`wh2!YJQd6(mh+(&+qg*C7s-{s2Sd6mz%6ZlIiW!hzUB!9Dj9J zC+NK&A&_r^4}p7|v=9BbAJI}~TeDO@pNv9!Ru)$=Vz&Q!NPR_;>*J~6eg?LC#tRwx z8XfJHa%l@I{DigTwGkH|v?+t3_z#Np?{llELXcNP`mq8TP4Dz}3fO{vn{XI9Q-HEl9+abtx7FuwQ71 zO%^_D$`)gt+4d6g14?5{Z$E6d|X%RL-$xF#>2_pBnJX8c2+)Xq5 zgby|!Jp0Fil-#hQ%9?Y9Up|68I#&9NBS_G|E#A)5AQXdMoHR`4i!(?NF5y4H_P?e5 z|M48UjwaAENCL{c3va}3@GHw&j5J^h+Q7}o5NI4s8n>4H`%cgar zD}-MMF+VvS8VP0&qrV;;$FSKe3=h#yEGAwTN*n*4OE7ec2Y_ga6nSMf_%6#*U^}b( z4`YF4i`};s_w%=}C)*BcL(~#{$Fm`)F0;Ke1*c5U`E-ziF*5}X9qp%Ss8XxU$1?fT zL&Je*TF}iVP#W%fB0AV_2;AMY-qu6K*pqF87$90650&(oS(#RySok>P8$8Bp_13W{{a$IDV&COS5v4bS>PGy z^fEW_vvuz0@dV+)BR(Z6aMFA+#rpJtvyj1^j_?56iuV#BL29RICY)Vb6hVKfi+h<_c@~u-H?b)-~P`fXz+{bV2X?#+Lh1YGEQzh@kHQj`7 zg`MWxuQ#>!@-2S@t&o#b1l3NT$XV^SM$Z?)M#25QhZ=S41c<)| z$bWDta4JVND>rlJC%L6CVje8J`DNjQoF%W$;jnXly9!Vdavv-5vk>q@KwIH^Z~Mfu-AW9U zXb-M;pU>Tu@##=9oIUqKrUB=kcCztx&;@!}PuhG{`2eSpgayewQE;P25uaZLu7q|J z*6f*K&`g%C=#jv)>?M-gw0#50u@EWm@Efzx;oZ|^%kM)emhTAys^Pez_63S0yjU7$ z3=JSiz43DQD(+8gEO1_W?${%D?O0#=sor1wE){V7>s~g5ON=G!Vy8a(_ z2a<;;j#>tUHQONZU`0FLT)^0h#R3uUElx}3yzG;zA$>Q6-72Toqx!0~yAIyn1zS%Q z_KK{-n!va-vm@~c(-mt z^dA$xlkI=tz$S-)-D4vl{o@rQfF#)Ii=Yd!Fdv!NXHCmTCA`*@Mbtn-I422zzbzu; zqLs2`T=XgH@TN_(ai;#z!3|@BOY~qb4NtClp9NilzgvW{v93s^?D)>(@OL)$3Zn|o z*ihi(JOg#omRt}Mc#$b2S^@zs7RavUz$YI)?K+8$^B9<6y$3T()t=5{^+B#K2`BQx zfF+Ki0m!Ip;_*PI4RqYu6`((}Mb^ffU4oU%GdIe6se)p#Y((ih{nDI?`cp=ZP^Y*e z>Td+CDp4w%KX6Np_`I!tA;aL=%Qce<9SpR}-r@5owp0{eXv?RN^EeyNn>M+VGnSRj z`-mcZ6rxniS@2F09`71~Jkdq%OX27Omod$@DXpE_C%!;mJp^!XtcxR$F5cFzF(o09 znpaD$HlS?&2I*($Vmp3>RVrKyq3QHxFQuW!SJXQ;B<<9&jE1w`hEQvSYLl?9Yg#r7 zREmhwn%F!tOV^**8p`fAoLCDniR+*7ZyCm(Q*n`hin(p_VLT!u5+!egI8}CjH`VnY zrK?rNa;ks98bHu}zMSjHl4bj#(qrc-noEssgxgX_W+oZSeCg588}o{5gZpOBp|G!t z@yyKH8=xo8dDcX6TWS(KTIcdss04dz3FCE~OGa>aWow26#=OZxZi_DQLtIbwHZet3 zQK+Pd9c9ko&$tX5B52b~8=zG^=h!Hr$zBq+D>9na5vlH_ZQZDvB=8H=Zakn5b7%E_ zoM>Xo7f9dUH8dH;%kD4nPTlfJW%nmQ)Xk1IN&ysV;h9Fi3i^7T0&7HPrIP%l(Hb#7kGn=aU9AO3L5TY{uo})WtTES?JdyO%bXsEA7%iJbKYqFB z{7}n#uJS!rKWM;1-DX)OsM4W+se(FPlanWN{?{dYeKVDiJR#x>feGcX2qztDEOp;F zmy?JzAYW3!JquFeBNB~RCG?6pJMr-d!GPX@7S9!ymq|rYd_+XYdVS{Uw;F@1l?Fs!Z)(B}0oYs@x`4xi1#w zAZ6Jh^hS~8^QQ5(Lcu$E; z#j$=Jm4cxZSFniom|ahvrG*3t3$E~++!NKddI-U9GwK{_s66#N_wRxeZ-NY|QrT2{ zD}%-A&P3H&R`XdNf&L}d{r(fnGYS$)%e*SG7UB8gP}ea9JNys#U!TP0yk%@#5L_NAmGZi0}o zw^nV)x7|ntUa@)h!R@mhOT=rP+Z}C4uJcJwt<~L6-55>6O0L*Spsa9nIHTmk>ExN( zubkn^uztAE+FF@iDD|@c;EXESYE`^bvt&hY7V}L%+_7;8S{uvgLJHRq^>y$ye0+NW zV>5Ia`#rk+c4C(NR^2m=J&P}!EkRw(2+e8_NqWEDV)8_}rPDo+XxA6ko6vxY{7&sO z(n&?yJ_?0`1i!CqF^~c8VTUvwA@&<+lBnkWyNF}ej`?XjZfUeAN*{y9mPXgi^xdB! zF`|Y+6Ou7pKZ}iz^euyLbhFjk;-m?%WIo3irMaKv3s@68I@{_uv*G1+^^&q=O5)70 zB4djla!R3#Fv#$^bC;_9Lc8Po+%gYHOqm6z3`i;Ev!rGyRAV{outg__{p|&F_{Dk+ za(F}-jm+fJC1&y%Xm0DPr76POfA}j`4!>3#g!7nqQ4jd``8799QB#f!X(u096Puk~ z%e!1BF2*5|w(3YF6cba86f&6FtUH(~zoxSexhod1Z497FrHt=uO&X%XOhBS5-G|d5 zE`to;tIttI{&beud0#6lTFy)b<)gyuql_Nc}!tOv{UU^ECd(hgwQ|exuP>Dmj&J50Iv0WOqM!<;U7T;Chg- z>Y`>#h-2iXO;eirFz$TibQ(I;Ja0%R6;ho!vEOBIDjA$IV4e1;&2{D!bSwq~{#6n5 zFHmkn>Q|ClOyhq1kQ$Yx zZG+qt*4xdivPTY%+W1zOS(1BP9ExNAow)gnwEEive)uT+O1<9Xagz=fcsr-VR0bMS zm;xOKXFsAWXoA&Qx@5!&O!uYdW$PZ&?x|JN=7R|+hnQz+v3U^~q zj8H}0ufy*+vstPI@ja_vX8Le~MDtisOpdcROj!F5Tw^2THOIqta7j>zrn2qip9ZzTL9I9YpFfn>jq*y z8c=IFW5$!8U)jXsY1jTT`O_T(-tZjlTbg<%#waALui}pjb6Enz$7ypbCL@UI!W_SL z-E)7M3MB}UD~vbH*@KIMp?MB0x*RI(n;o9_JS!X4^C^S=b3|wT2*2$~Av|Noo`yZG zat^syGKz%C)A<6Y8L*^lNINfw=H2&Qx2u`P^V~zVa*2Utg+A-)>GNJZFsix1YVBL{ zRg(BP41bq6(ZB}NOR3ncyO}7p$QP&fiPz2JWdv1*W=N!h5#6>wC}wjK#yQ_LS^Tuu z>+Y8y7Z?3;XoL-+laoudkz-9zWAAwByS36}-pFw}Q|NAiYo)AZ9KB)S4@%wN`+g~} zp;_P9`rvxp62unU@cL$C?@6R;g|eCH#i<<_b0e9`-C4Nt{Lp}@!d5?NZu;PIDYzCZ zOt0s&r3I=u#dAJV>6pMXTA^HDnquau;;0kAN%CodfVjR3HFWKczf)j&l8=|2$OPfa;y3DY z(K>ha6Ss@Btrxo$(Ux=!&nSI`X7Y%4Ape8{eqUVvDtC<{wW*b92xa6xtrRN!$VXWA ze4U%gYa?pwSCP&A^V0@x4HL-&F3As}(}WUcGf3-=?=HXvro3K7oXnHTi&QVSRP&o8 zV{A^Pnf|3ci4RN`RNpnxRTy z5#>(XG2&-Ql^8$&@du6-U~=>;ZF{n>_&nQST|h5(TF$oczdd1vP*@=53T3dySQAr# zyUIoYC`XZtb7?4nZ6m+w_Z;E}Z=gY(XMQb<-MME|PfPB#w-tXh zU|oq7f5qv?q#vOHR=1fp)7&JD1M=|rProM~$c3c={EwIXk#x*M>lcIzCOd%zT8IvV zg}E5b`86Uzm&w8=04X1pkkRCcuWY_bkxh+P@-@1nEcAE81kJBQ+4-c6(&Sdai)^Zj+uqo&6ZSfep!Io-rjh9YbbRHyN|YC z-JIH@)1)oB8x@5Y#xqYn*+8lyH>dT5+fHzJkJuKXwcmye2 zTRBx7KOhc*mvYsHwFb>?JRUe-LS}Rwi1h|=z@kZBWh%v`yfp)uSJUs^IY2xa+@5sy zE25;Nj!al&@u~rUah8NCnwv*Lr=l=^CAT*oL;dq- z(x75hlh<1Fw=535-mZ_v#iRuuM19p>EWarXk!GS&KMwE@@*~Z9;Ldt-^?4V6yS1oq zxtcso4i(X4bZfs>cu5w7RcAIe-)h}1l5fZuUSCeP!t7FMw-AmA7dun-6;9-9pCnK| z!x+fvareDzZS8EaO)pQ8mSl=UlKu@>mgC1*-*vmhb`5e8Tml(wJg~LzRJJQsN;^~; zHRj`3FDp%xp z6%id83tPJPs~AF-li704Ia_Q#LwA{O6C*n2yTi6_6cnDJ**zX~H!z^Ct)<*ax+XMJ zePfs`BB))Fg?I7Xq@BWPmFA;l9+5mvtU1cq<;q3^zlfe$=BFV!-7uIRt2blkFip2X zwEC+S(LpV~&#=KX=G(q7%CD}I>kHiKdlG+LLZ)k=`aBIH*ZtnRMX4L*WL2z#3)zSUJ zjBRpSQLcmUqkEnmWfpy}Pj>|GPq_GIypK+YlUk<^J-lg+DK575jd#w5%+}O-(*lU$ zgv0$uJ?I(n^*5_d5#Y_Lmxt?Q)mI<1?>W=+z1sB)tZ~|2AP~{_exyL=m@xJ#$^rw2 zCiWY;<--Ir0D>3xb}uuJ65FXEgu{DG>*X(XBNk4jieCnpMSW$**@NwcQL~uR64eAN z<10K97+6mmh=sr+uFYsYL8$7o2+Te%q3h<3&gdZI;|4LTXTF6jx_MS^KZzVuD%&4Y zA?RKROh;l0dhPNTN84IK8qLkq)Ll*RpISU664(Vl)x<|-0Lu%+WG`d_9c6Uj8Z=ob z4R<4J*o=&vIQ4m`{Mc$Z%yMp#CXYE1y=lH&&E+5$Sh_OwCAy)3@7v40hE1@-IN_b2zn}?8IQ-rZFG#G1mMi%a5^ydL z8Y&LXT(@B_kL)CovdgXWPi)<4Ps!0}C)fxdpo7OTw;K!aIAs3dqS%_qjua~h=gHM& z_PWpMUw@f0*kVI?W=q{)K7al`GVz3{4>~wbbK?VO?SxbSqWu`D0wr}(23z~S{@o8E zeeR+vz9=BKyy(aPnHdNpi7E{3!Kcs(bzHZR*2T<;wmIBovW?&=`aMC2hpc^Tu7<Zp`=2-#NANrdt0<+(v6xvQ| zdu2lIS{pROe!#E!+%VQ zedPwQo*A$fq&HvtM!yML!dvhgod`(;rKY~wI5DUc(h)Mk4}N#WO-pWumwxuy_l?Ix zYTPqyrFFYWVYzYO6~eZEuoxf}&+rU_wF4OYq|&GPeL&z&M+Ovcs@`|rf>|O%Jrf5bD9A}(iK_7VW(OK|Et}Xj9<8!r+@uCF@s&lF_5tWI7ST;x z58}FO-_8&eFrJdg!(G!w@cTUUwVUX|Os7xkBR#omwq9p`@5c#8CHN@3)|252Sf}6~ zdbej0sSxO`*`&>ev!;#`*~0}2=*e9>zNNWqA@@NL;Tp5sfP_ibb;V4mwg<3sBIlP1 z$sfq~<6rjK(kpwf_&}Hcz@(=5NEXnye$y0C)zIeiKC)y8Bir%o*AJ{_hS_(& z;zYzNr*5IFB6_X^9Z|--9QYzC6qZGPt#c)S7n-#i^6HQ6Wt zwN4o!K3rgO+~be>KsumhNRmCAJY;5o;ubK{OQ{;=T|MyXEA4->*8g)A0feMEv7JJz zuG?QK(3i($l~j6j*yQ2*R_q`8UtV35i8}xOs*{z?=j+O|G6`_U)LHluTB|XIx|W2m zvZaXwqAfNDPg(O*E6F4^-xMjIQ((#b*@*f=BKNdiupGU)9W z`ieQGyu5yQ*`mAxC+NMa^Wx9rt1EWTei1=y?cV=hV_9Y@9O8>+_Z_!(D$2h|MdZ9nl5 z(_(qrg~=n>1=gcu@`y`PbAoRs_J92tKOxUH++{U*0=_^vGRIFhD!f zHGkl|9iSw)(+fNo_b@2~^7|rS2AeQq`L}`2&u3MB2b9Ea*gObI)m(X1(=NMd#MI}c zCtWGNfT_0%Equ5BzU_;7Pr!W&y7RyA=)98|u2OVQeh=9UYi)0Uj%L{0VyWLXtv$Lb z+TlN<(16u+Ek6Z8^LQ6j3J+Q>gQ{46$E+^(IZ-3Cot^s*%+h{}ic>*LtcQ=f42VKL z-Hv`2P8ov7KlQ$hzFSogP6#;{FQ~W&Y-CgZ#a zZ4n8Kia5abx^-ByssQ8ED7-2Pp$UDXdAh9Q$n0SmFfc4Z*?6l&Iink~F0|`zVEf}N ze~F5gT4uH7=Z<&KPh1-U)20y9K<)@`0!?W+`nwm!sU2fl1WeTRvT=|iArX`<8ff3_ zG0`75(Nms8YYGk#uY^+|V zcw*2tn%G0QPNK=4JVi!w0a#ggBATYe;-A&_@l8Vbpe&iV=z$D-vw-F7@d0}P7O(Pu9B59@4k$Ktwix&^NDn_GNQ%tb$CBok?W6O(P*#Y} z&+_VX5@o9klDSMybpbiY3Leyg=Wf2r?ja_N{fLPWpd+m_uEF6jC>o@a`(9&Ol7pI< z;CUubTXSpj+G=^d%dGsUvCAgwC!yW6H(^`TQKd0@#Hyo`8Gvkdq{4|h#j~g6dl7%CO+T#5j~wW zG~SjJT<;tF7qcwC`{R~+C+lWgtxpKc;0^M0Ebn0NMOVNnH(^&jn}uLj?MOqF#fR5F zkK*Ylc^=KPc7S^}Zn|AtT_uN_w!vS}vSp@?PMy`Y&0|qw)%&Z+sXbku=8-IXYnE#= zp%{K1{l_ipnP&^qOWm`gWL~+|=w1@Lfl8*yf~lLOs^mKZ6w6li*Qt3g15R9|`{Rp^ z)(Yq5qT3g+pi9OSx@N_+t=q;7)C)IYm~6NPTRH7=!rEcEektv({lxlVQI@xJ3K_YN zqJiy3qk?(r=6Bui1PKT9$R2KRX4KvY8dhZQIBeLhDVI<&xU0k4e_L2<(5%gnyb|D< zZ^x2nc9tiq>y?+GN}~R_eNmD>^To8&XYrImv6#R*nVqzYtbuGvo`(nO%BM@Hk(NYg zVEa(n1+q9^1uVaG1dcG*YYRhqgpHlQ{rFLH-!FQ;Jsl~7W`V$Ltng#@+aI`Qj+u+X zmh0$(jn5QHHSM))d}gup$s1#MHGVlkyg&D_X50m!RxcC))P~UpKlXFn1H0Fi5`DCe z|5V1?3>Yr{a)Zj_DANCS-$L&`Ruu3`R zCJYm%q}Eli z{!-8{qheqcBB9u)-=Jvw1!)p35^3o7#F|V6jpEDl|8@LuC(SLPe;?ARzYnKAplt;G zeMrgw>rC+E0xtId`>tB^g(Z4RV=la@$DLjt@jl|KNlV{|7jHXayML{U5VUroRVS0Ke+V2Gs`G+fU_HP~@+I`u~k) zW;jal(tg=FaJR88Bb;F=;6~g^-ge)G+@e)AeR?u{c*X2;{+@7v*aPjacnSCOzwNF6 z{wBpf6p9@pO2x~3tGL)^VwTX^Fpx4=bY>?|$*|Y5Eo`%m@F)Bhno$p){R}-6q)a=WhH0Hbaml{~pT27zU9MR7LzN1_a3GIa?VOBFJg zT1`~0;~*U*f*}E>d{ik$&4O)Z!&)EC*wBWcDo_6fRnK__8bm>cL2bm?1;R(7*-90e z8UwF_nrq|bJzBTGEp~Xju3z03tGJ&sE>_KOO?ub)TP}Lz^;h+JTpV{AcjC&v-wsf} zJxqkmsJYQuQQBz75XFjkS%EzOxd&+>u8Q2OhR!rg)s{@OJrz1uYgL0`iOu-vLDctz zgryK*3Nqz-sXi(WXP&rRDqoW8q03VAqwH+;txcAA|Mu;~=7;&wts(t&2HM5VRLon1 zg;^iuUHj3zx$Q_whC&reZi3<8SwW?{5+cq_eFNLOVJXZyD#s%ul^y6OqNJKSUk+b& z*?k5Dg0o}R0`#>Gh-UBcL)YDCX$@t+%Doqn-_Q2GKHO&5n$C|j!At)F0P97XG%~mJ zH_uT_N+MNmFB==z?tK!=EJ3UP zQlDyZw~!eL@+B;sL@9p8K5EQL;aJSu_)w>eV()g~Zx++j_!3Uen2em}(pABIKB^t2oXLt_O&b}26W6IxcF zIh@W;WrkOv^IdyYI7uLWc7hoEA~q`XHU(a#_%L2 zSoiq;`T1g-Skqk{z-j=~`+#^?dkseqNx1F1FN#%~2e3qrr31@|0XXw}B+0${2i(+3 zcq$(fbHFeXE(_hpEAo z50mtOJT-P_5IGz222jhu<9e5F$c1|qGcBhsohcLmEmT$CQh=5m1Olu8V96$6IufV~ z11iEmCeXnLi}41gNQe(mpkax9ngJc`S0?|*v{e!N)-z(GhlryKaB&W`#v}L`j5a+u zfSi*6x8K5#^QJy&Dc8%#u2@ge3*>~`P|s*AU+7pCA(Y^WySKa$)xh@ltZ7o;k4a9L z=Tbd|PZE|IG^zk<_cvQ)Lv?X5P`e~OGV#okaw2wUn#rYJSs@$9zKs#>J<#?itF~1{5C|{yip}H_C5%z#`W1CYK>CLh+{cbOb(1P9WUhx9ULs>Dt4P*f^2M6Y*cVLKzt+&0TL{N@e^)=@ zmEkIKhwIxCM-&2ALPquKfDA5$* zRuh8*7QAVQ^Y}?O{Dk)BPeUTg@)eMJ!UjJH9yYA9UDX18U%MYuim#wRJ1{vqNh7xP zH34GO{Q6J&mC~QkTsn=+L)@8`MO$N`&dXQFCU1oF4hO<{+*PvF4(*GZpxmEg&w_do zFUaZ&Ad=mD>PI@P@qgfKji?nyXOx`Y#buVE>#bh-+|#zKX=rAxACU?vHsa&ozaG!} zw1Oj7@Z!vSo0*>^+dj8dEG%bwttd;xKOLx&>Lzdd)lQ3Fnk_#QpYE#W-eO{^Vvi=1DUg zTczgV7bkAI)#opy`GMYmY6QX9%Qm+u!fq#2^{s*nPs5^{d9p-wSKDb+;6nE&R!5%OX1SJ=b7|Y6FK=9Wxom~>sdKN?}3~fg5jFH_i1GSATX1UE~#f!r+o-B9BPo0^5 z%xtDq3bu2ki7>FDw4T)K<0NnDS1miYYY}&rgWEq#@D5Pvff0vE@$u2hlU+THBwT-q z-Kmd=NKfFDGsbaQfHS|;HKx79-OKmz+k~PC2jHKvHN27WtsNvbSA14!__^w@7JnZj zK9O)Wd2~09lft!RZ)Db#2|r_J zO3lZc_=q5x0OkX46^kHT!!4-<1&Em8XNq4IX!3kyt ze!Os>jy2s#$ZKozfNm<1#OW3`pjHe%@!=JWU>dn``1aU)*Vu$VV#iY;cz>sKI9dqZY{U>4ZPx_DjQmid^p9RY)hl4P*&C-SSQ zsJAMQ&s}3EkIw+!l2tD4B}|SjEbCRgvV2%tylOH8F8Yx&qpyVcw)dhwP>q;$Si%r@ zh=hd9S&%&3*_PMjHIJTM!|Sctk9DtXe{5T<%JAX92fX)jb~={?S}F5GuzdacF7W9i zX$S7uVHRVZoeT!L@j7?wQ8#K4i>PaG%;jPUsN7l|976lGx~Gc8lL{W zYR|dYIaaHunuq~V>suP3^31V`%sYAiu^`_5t9#eA7%_bV5Z_6SUp|@p&porE6PL#s zHZ9{{o5m(IA3mUFuM!+X)R)S@QOF5iWmKxvFvx&Of80f6f(2w&QM&{j+L9ZHT0Kef zE+?P9X=T!u)E7=a5n!&%Z@r4SRu=md`Y6UrNVD61(9Hj{dytxB{~ZSItMq*9PP$pL zy{ko$Rw+^0xH>%Qs5&>6tD56U@LOGCB@x#rp6|qP#(ShYGE*gD>_-xAX5C`z+f>{W z5JkIo5x~rmV>}X+iq=}BaI-ixiM#kd>%sW@w7vf(*LQ}g&UfW{CQY|4E=o1YbLLGo z-wjrmGpn#nR!5s3OdKwD&=t1hJb5IIFHihQn?r#!q?B@sjW$}!nDizA0bzq*7*kNI z<(77qDolHF%RPpF$R{ioep^J7;ioBvNT>wiUKs|;;W|~9?V9{fS0Kmk096Z_HPRlH zJ^R-Sxbs9Q0nIM2>Kf$w5@4BA?!u{#n4;-Dp%r@8oS_KMd8xslKMBdl3yb=QB$1`D zgQIa6%c5Q)d<0-~xusS`mG(1QvEPuoOuocHJs1C&zy?fBcNvIaSymYiUFb2+@jvUp15Vu$=K7%*?W7sjp+A4=q-?}3;+!) zmCMSj)+;?z0XpPKrv|X=!flfziFfg6A(CT>{aJo(ZPtC|ctGwL*2fS^79eklotaYG9<`eipu z*%ovW(|7IxhYWDxSx7urvsx#dc6~$R$2u!@?KrAiGmt}l!K7bbvN`7Y#bt9_PhAmn zCdJ^3g~VouU_WN7q1Z<^>Tn8RnFWDW{sR{+o>(lUO|ZxrkFA5qD?EXZ6mJOjku|T1 z5d?NXj0cQGrG@(!gt}qCApONQ@??vl9a9v|9wiiUrWt zWxhz;AM1Pq%iRbIU5MHZ>eKboiZWg830}TkN6TA~tG<2*|*tm8VUA3f@7%Q)i z?J89Pn>fSSuH$JFDBRrTq!yYFOnOWa)CZ*AZ$jK$z1}(v*WDXj=uW{Dq5C4;rsup7m4*;yG;;)iT+iYVT_4L4le*K zD5Ghb*%~sO{mrs^I&E$>fQ4vX7dK8orb7;^X2kwxD+pD_nFlX+oeu4`GTC|`Pv_GM ztecY4LVDGEua6v1zE;Z;zLa8H!|*<#&q23+QtZ#ZVX*?dii8t^)_3g zCmuZLD0V^Asvvo?0BwM_>Iv#l2-%9BaEK z+5`v^BoN$#B)Gd1+$FdZT$;w+69~a2KyY^p1b6q~E{(gpHLR(8d+&4B`u5D3bIqUm z!PQ+wRhPW}JhyBt#U)&v*Vh&zd+_w7;$XT?Vy9U-XSyGOQBFa&Z-3q)pRR1sK*u#z%J7*vEYZt9P_EnlBEFP2 zA&CYxM1*ck+_2R7Y^iHbv&pw5#g^L3#e&$Gw5Mp8kdIcITSv(Y+s4}nbZ65ifPC!I zNo3t+VuNr&&hncezSKGGc2wX~T0FRR_hIP@J58+ zZF+2x?2d75-=9|jX)Sb-wcw47$0cC0utpkuB+Q(tf6nfKdeF7>@e%RsZ*M}o$T=g2 zEA;0xs{Pp}F!s_0wiS`r0saF;vz~U(hOMkY_c3uj`*2O#F13TQ90UNYZ)JIeKesT7 zTnVHc+Y;KlPTgVs_9Wv4V!L0z0!pPX{GWxSV>hE8kUjhXJD2x>e!+~^241~quAWs_ z-^Iotgo~y=Lr+hnR_@JEY@2ph)q8q=x{|tQ`o<}+-Q+t!<wY(wbzW+^s}IM|7SYOwbFgqAW(Ql!{AD9dj%{yr@A$*9!LqEd&IUr&Pk!|p zz&|~Ml!0EpGUhOg^{$Y|;MDXPf@FN7vydf0I44 zC?G_x@aR`EHTXtP%=?W*Nod6+O5mavHU;U0gJ@@2kubIGn>+RcTSa>BR;9(hjgte# ziNXb>Wcoht_^-xPeDKxRe1wV%`ql;jb`zs=;Jp6{K~Mxxi+KZtL=FU_N&LsfyMaCo z4%kOl=FLK;Uy}CL!xLjA1Ozb{F?{2wk}r#)q2n0Rvf!qFCxwWx=Yy0 zMta|JKiX9doWDllbz>r4Lo;JhRWdvQrhiE|uDbpP=pz4oi$xcLj(}DtCMHme5dH$e zyyCUb^V{Or4oQ8FzeU}NSD7W4%D~2v*=0N=N#`HErV|Z0r*mnJJ&evUT^IR~MumeFgy<)u1tRI9eQ- z%*T}Cdnd!dO@*G!U1UZ57q-$tY?}*ZFjv{(>sQXUG?NnZhN(%8AFn?`<7Vc`!ge8B z9-GfWWaL101qFC<*W#RHBA4Z`bn)&-!R{TP_KpY}0AL_R&-=Qo{wofT+>g_5v?pAc zDA(-@XUIO%tVhV0gi$XqX*4j(s<0!Iq!H;c2+1v}1))3ZMqN`jW17C}mE9h}4vEr` zMCn-W2;iuX?WaKbx2uv#M~Upc1=i^>P$r;os`d7LPxJJhzpE?226@yJ7<{W2n?D5Z zz?Wis{GDO6yvcWXYTzE`dKisq%TLqui{6?CTM=-QvuSoeBg*YFL@mzG zjC0OSate6UA;4`eYwLZ*+N!dL|Hb3CH5=*a{f++k5KhT4@yKV2>-Sr`!tfLc6O?g z&QQ{}w#*(fs_M389yT3=KBi%*5>+x9-*#HUPCK(p8H|vI$}o`&W&mY}3k(h8k#4oU zKB~Y_>;4C1S6=`8#U}Thihi{_DVMKl>a28EfBMa618;r6y`;ZOqqU#0e~V*~(2Ivz zbL%@zg~OCDem=W+XFPFQx3-B%J4nqW#?g{Sx>xkD_rHw+oj_OD?F^E?Krui7tTH?r zz=<6E`;G*b(4&5T6feMI13z433l8AlxV1m1l7cl^{;{bg{c-p|ba&swqTN<+&{F}7 zV85P5_&+w)nX`qGvd8blZuh{COWz!wSw{|o8e(9j&jUxk^yGluhLS4)Bs>F77USai zLVd}}E$lJ{ZcKJ&KW6xbIV7a+f;5?Z_Zg6~-8K48K%l&3s(9{1*JosWjWN zAhKn9F8%Zcz*)(H4<~JZ$3IK8pa8DX zZ3=k8xN6&f+fmiygXE?M^N*eIi0?f2h*q0&g)2GWH`qztxqqY0Df)Z{>i9>~9P9^^6m|ASg^xXvVhdi%hzEcBVZdp9q4C!@MdGf3Z_;k>9#M0o zhEXY7_U=A511Ms~@8fcFSwradmKwG8*oa(|)_M8xtE85yunif7mm14drg3olnLegb zGwg-5rLz8dewl%_jFS8nS1YRkz)|mhS&@I@h;!Im$3DNs!xVY$4dUus_*2p>v?oOM zs&OV@cJ}(9sfS%*jwtJkLPbZ;y_|+ygqefqph&Qds@lrR@3`tJ#7n6Z*I}g)nJ)?k zNSXwEICYA|+Ils#rV%~)uYve~|BJ+}5vF(C9h{xS!|3C6yieBPk%Cz&hZiF%o&f;7 z(O*}b`VVqWL>)k67y8cu@#S%{8is#v0{#KNAw5LyU&}SYN)SdFnMhoDj@Ug!>sh2Q z*n7xEsRy}?SXGDtEJ*A9}^kYd2No2!&!L6H}@?Oh#0fVLm zl}kClBDxWrB6w0+M!e9tL4%{L1eYrNDYJm7rDSEql z>w?seEr`5*F5pnPS*_zI{kg=+UCG+d3KQvhVH$KK)aYkL+sS%Vb9c?LV>X4DVJ^{_ z7DG4IOlvF8QE^oGqJ9@Y;dwus6sPNjn0;aNL~)i*G6>GGqru*sk{_`1IR7t9ZZaa{4dCIqhZi>){l0 z(lIiV@u&<9IT4VF;o@#_=<5vPmUq@Agy5-U*F7VZ^G_}cFz|wn5hM|8iC#E4yhOSv zw&ln-Tui&Y`C~yOE|_`uGILYsvTju8b*^q z5xkuY_}?VDnFeGz$rh^fxK%;t$83caKK4ztVPFsdojX6mDHxTA=UYv>@3tcIp2FGF zHof}h^b0=RQvImUYbNW-J55uNhC1x z=)Kzct?adWl_*pN+nY0 zRcWcJQ2K^`$fuIn7o#Hrh==N2tuHrwHQ%kO`2Sq`+{QbOjX?)}H&+CW;7zhv_N_u~4`0{hELHG&t&ok*>!%nvRR7Ve1?i?U`7DH<>4jbFEH zZXb^EKDe~Y>{Uw*Qiezon482%=YweZ_9az~xVs#_j-)`XSyQPr^fXY$H_4f$^&L}M zFuYXYG-|Gz<#7x?NXD(>&oitnSmG465Ty#6Hp zW1gn2@^b%>5WXAd0!ysXjt2V0Li3-UvpzyJL#PXB+gW~+cRz615};18(y z@8^)8MFQH~FM&bn3|FKp%F)-rolp7rnCdyGPUH^zPsP581qLiVau^B__o;lwYQ=mG zSoD8>QFsRoc*sy-?IH>WX47yVVYIuLr4_q$PpyDnqh>!~*6)xBSr==y$J!GHf~ZjH4PijD40 ze(`S;hpCP^2K!Xm$Xtrh9EAuz;l_TUQok@g+{PYTKe!41b>g7O&Eu&~2+7Sr(SAdC zF}|}ev_ys~ifz=!{m$9knYb>u)JCgbE|#WzxsQc?;uCoP7cx%za$IM<3)j@@Q{|E6 zqX;Ckm94#CrQ7{kAmx`R$&WJ<+bwuc=880< zQ<=nLyae)oUIsYwP=G%8N-+ZOY^goiWrc=Ipp>qR@*CdHcoB6Dcv~CSoQh8q!HjX}J5ly{_p9B)J3>Qcu6$-P%0Mw@KNZlLmboPN$r84vej%~cja7nJJIreaE4i0gQWs#f6kyF8Sof>h_hRE&o7{e(A|#zddSv zM1&h@pKV1>%&604*W^KC9N=Np7GK-My9YaFOa@Hp1vdcUP=? zNQC$4!ktBMmXLz!S~>5XFa(3JqOG5^oJ08f63#lJAIYArVY5(qGfae({4ioZDm1d+?x2J!@tB4YFO!O#svkM1lf|0YEOn*ov;9vE;VjT5w!d?GDWjmHqec5qJ zz&UIW=#&228l(%R*BCX{RmbtW`O`t&G9~^ecVkd4b(}Y95lm*Z#6B(VXau!(EmCP! zb(Txa(w%cc5{IEi`Z0Ye4>9(EZ4-$EJ0(tC9;{ zR8^AMJ_)NX?|$Yypn11Y!Gg)xDJ8I6yh(g$RWok~6}7i$sx$j0rDxR;%_zFpwi^+{ zA8UZCe_sEUGx{ZrVLJb_V{C;T^yFdfG{uQIXFHX&nHGc*fEq4&7E-v8UQaZ%Y)!=B zRG0PRq@w@P%o&yZBs%SOX?(A=T#xU4(r?}_5uWkwoc8*byMSVwXGm<}NF0BVx=1pa zmiq^32Yxe)`ae=%_5UIDHRn7?g{VQ6my*`O}P`{<$HvuXC+BI@MNuKEQvL_xTY+MYYsHgb`Vae&d3_nf5M;A7DxP_g=NzLEbM&8sjvTN{gQCCT|8L7s^4T_G;K1mP?P@4w%o)@f}pZ5 z#B_aWtVXuBAV_9Ilzo!zW3UDZ5Ntmz)m4TC+W8+sMV=6})S)QVXQyATOAnKka!cFN z`G@`412lYoeUJ2#%IJLpk?APJkldD#lz`TPcg zF77I{z1}(vSs>ccP>hYt3YIV&V)u$ua#A)a&au*ke078^30qmr>EuZCh7J&eQWGLd2v{Q}IsoUqgDWGsGZ)! z6GqnO%Hau@V{ASu*d~0tKASk^H%j>NH1hi1mgL*QMnMn7>U$qWa|E z6*$;W+B_=~1Va+}T8na0%U6DaqEba|{1b_9Wq^RHetOPPSCLkE*`3d-aW4Du&Z*vI zV&sme>(#oRcQpH&{+NT*4oVFzVF!7p$k%+AY1NN(e8VYvsj9h3!CXoy)nbx3>M7s3 zPVe!#SRSqbR+uQ4j|;B2ArRwnRs&)@vOtVSt6%INep$$S)l^hP<|aXZV2ApvAy-VM zlCzMMn`h!?lnKNZt9;?HUjboh0S&^ft%tyzAGI)Vpn5;H#jV{;P4D$4kdA{rJaaky z`jn&*)Cb2^Oq`j9+6LGyN{h1V=YxgU@ZHCZR>uLiVhQfY&>!Dl5c@N=nkl2_& z)fi+HuyWJLu+5*Q`o;vm#J;_IDN5_PY0r1O+R7~uK5JXa-m4;=NpatCzv8&GeSA0z z*nz3*9;y;~$u&9E4%%^&TY+eI)8}{V?XMZ*4a&FJ`mM-b62?I|(Z5dF^JTr_)R5V2 zv97?Ar;&&tr-7~ctag(#v98#NENrM2vMjnAcisd>c%AfLWxbl#(0SjY7m6*w<&oxh zqs)uR_{8=bYs_tqBQMb=mf~qT!$H{Os*gA8mAVF&D-iRwOjb(E*>yu8w<4gLa}&uI zmOaIjHzAE{l0iy?I*@6tN2dU{tC5>>TxlL9FB(I?!`Ct{pTWBnCGf0)y<+oT*;7wR zqY*v$^D!ud#JQQo{dh3|Uo(o&W#gBAs`@3BuoZ77HXZyqFL|Uv+2MqvGkT4hb*Z2e zuJ!aN(swZgxBH*c^>P#c9gEU_pN_o*aEh6gcY~p|Hro+O;p{La# z1rfXe_DD%OepTtFSc|y85Q70Y{(v{1PFHK}`*=*#ywoNnK_-W7tiVuOq`hD>Cq;Og zYDXcZ*FZ?^LP%M+f;(MWWG*B|__9Hn+_niLVS@#;v)tSn85VT)(n$X zimlYeq96V;zusDR_FaQY+;_xjAVMa+tJtD;!%>tvz(fqk?6`e?+m2CFSE4zJ%YNS@ zmHT_t9oR!w1IP5g=~>m9rYC2$l;2rLINrzo2oTo292_Lv^B@$|Sj=}XV4AL^Tvmij zWFyHDYUjV~r`RmsGA&mJ`zjnbUp(gcgSG7`^s_-EKLsr*Os6xXa)wJ#tGNn?m(0srHEK<28@@mO7ZQ;i7(D z9T4B=iDcc&lo$vxcVWh}&WmBSYczBqa1KCOAA6tiJ^s#V)J@7FSx4b3eTO|15GUCs zjuPUKJ~=_((6^6k-!rIf54z4gI?|>uwLcO^i_l0Zwk+s06&gu={JKc)O(#O12~jpG zE_F>S_r*x@fweuaOX8?8{#RAYPekogI3S;`TQSdI+D8b)| zefOp|fZDf$`Lzi2$nlx#5+`#B@14inDA(9sI|}iPp3D7PMYZ^@8(`g7mf`lan7vYS zvJ`=L?`I#dw&krpyNHJOOX6ISz6`QMFr%uGieMIyOTxAv#-y|qlt|P1>8n3PXUFwv zXOG%y#qBad(=OVMuJamKC@~*W#0(C{+Ad3rFlICPe1e)5R>>V`%^4(REgtgyL=#@v znBsH4SVQdo{Y0!HN7wO;b-(+TSrdnHZRt&e1pDRUg z{HUZJKk1!ZSgtMWp);svQ61QF2IjtVm6+UyG18QxK{s_&N)4XFP(2Ab zwiP0ig;y@4a;U)&mm*hh)w9WyjSV2NjCFWVBatEtum2_5#JDF~06?2g2^H5Rt=t>I z+Hq(~<=bJHmoUPU8Qhg=JZ@)$m4PZT1Tb|qkE9B_EDX`(T{uG-KKtH7LE74#--)6a z=iyTLH`12Mgx%FHIdUO~<)3@>C$;O30;J?hQPcVYHc*__#7EzrD0{OgS_9H*Pvf3v zBvRgdhF%et=Oth?C-L1$^1mbB@&59r-x)%3#_N$Uy4RgcNBHt*Z8VX!?z^8&1|wd< zy@#UAf~BOwwsBP_3=Q`-^MnY_I&CNr^mK;q_Jw-ZK{uDoD_8jt@PxTf*GmV7JHi7@ z+~ktGuTp2HeRBY(hVg1X|LlPp;>zAf+BV(~|2|%e#`@yn86QX~QJ!Kzl>1h9T;BMI zogL|tww04k&*9DC8<|Ej9n-`?L^4-?1f8OHt_9eDdHR}|i{>1=nR@W?RPHYqp}O)A zcIhWT=1WqOp5>GP?tC_s*Pz3lT-V$Am zc1s}1iv+dej}AR<5vW)=z-Mu)XJd?8K}uNAq56at*v*W(J6U0yPz$tzLKrsXeVp&{ z1JEFZbSF=s&!fc6baCFEUtGebUXZJDZfki6v;G18i{sB$#T4MfFi z_RV1u7Nv~ZnF$(;aV8ZYE*$cA4*q|A2e|kZvl}D2IW>xG_%uyretI#|LhfHk@7|To zpewG2*rJ@cB2-5)#5qz0zlkRnm+O z*>JNn9+Pf2Gqpc#R2N^gLwBRTR_23ra!T4t3;V5-lzr{$E=Y^nl0UHR1~`8nf7YNf zT+H9l6&hi1rdW;c{D(z%Xchj9@8A3_spF~L z9XNnyQK1p@hWrtl841$PDIwKT#`|=%y5#$#Zh|MR8Rcpv%_7hHP-Z6#zbV-KhXpk| zXcE(8^(+fzuL%2jQZ_%BIosNVFaaT;#&p>YX2VvpzcaED{s{T?a?#>4$-sUl+* z>{fr%x1eXM@XjFM0dJIJ0JeWxq$6dFZO~D$OD1Zq?%CMKeL1E&H5QBp%utxMjM8T(-&J9I-9VMqfro3A6^SFVqDJx+efRd$rp zB0)6%X)41X72qd|-z*u``Nki!J{1zTWU@l~E1)$!(?`@HI;O4lR%^ToJtY$UrzGch zf^4{?{Yb#aRBWW2Bh6(leZQ$EJogp+(!v3|NZByJLJ#-Ve^oW7O{Xq1 zxOwLR!@_J>adFXt>`gTD#Y)@nOcA>hpE65<#(OWawvYfuRSjZ|smb5W(B}nkdRe*t98X;lxDm+sm!c0r-Y%c%a(S1Lawf5}X#z-|r63)&8(n9rWoh zkaq?@SxtAz{>F$w>Z6I>?-~UQgD(&Eqa31`?W~d9TgyS?p{XOUrYkV~<@BhqrjCC% zFra+%`RNthPo6L7StN#LLAw)Jkr$bQOeTs}hy+5rK-aFbCYpVhjNpWWIPUT>N)jXs zP&*Ou-P~>`7~NIx!^#NIVm}}D&zAa|{GlB8wx1pXelwPLmmgavT98U@h}28bw_6*n z%*@bmt3kYiWVHj(KTy87xh^&xPDeunLy43QT8v(uNvJyEoebz#z}U+10u0EZreM$^E(*XtsZU zh<{u$Bl8^T(fi}XI-opiKSN#J7R490PK-B3e-!J4ysjjLiGZbb%EASP^d&XDOD++P zwJlie=A05obCTzbOIk$wQI;*LIot{tGlLh3Y5P@wTy#K5rEV=Ok5e-2SvIQC|Cb z{FhX4$~IOGoc8XsE-halM~2lZKk1 z1-Z8Wt@Ne;_B?Z(8N5FZ=5)c4!F{=YbI~Mtql3s%oFQ~1iAP8&xoK&2{4S<28S9+T zeJ{kB<$@u-!CdRy=m9Jl(^wSh?ay@Zgh)X5$O1VW+LUc=FSm33d#V5ZO0(plFlBd0 z)*gSxXZU4*YeT(~&Wn&Z0Hd&6|7z)dJr{0$kGzR1Os7uG5_`?|16pC4RSo$LLUJGFqRz)3*|Sa+(who zf>b#8j7&PaOk-Q@{=T_ezgH6#aVIE{zIu!Ve@6PUI4tIVhA5DEH<}B&(kQIW?MX!A zHtfOZ;q=+k_U{G%&+4XAlj4~hv%ExXp+l%4AF`aa#zV@^*RT3{A^b0j z<&TI$+TzQU1C8Th2DI1aPROuBssHH|`u~|$zhEeJZTXe5C>4uswa9IimJ5<%Cvd`S zC?m!1oY>c`w$V!ID=ae*w+EN6s(6gW5>mu8V>v0!40Fr&>+bF4S~{#~+w_HM=g=qe z*`b*|Ocmt>U}2f66lJTsfQ(Uf|C1t~G9fiuF_52hQbiJ-&;S1}{QqlJ_V-V^N+KCh zEN%xMT!?3(swn!!_y9F+b4Q1r^Zg|9C%Aw}kbi41InQjSBuL8mOW*6BV7LLbT==s> zfZ5V_)jI?+u1sKnY&O{9NcU+1xYFva>2Z_M|q#@G*+1_v~SW8VE;Z5TM1 z4nZ}jM=Glw`W85_%pb=i!H2`rl4YLR6{XUCa`ctAg``DnCsmTUim>5MB#U8NlXX

zuX1R+IgqD0d^hmpuM6&9-}@PTPvuB&mJ*?_aE)YT>^a~L5N+NT_If^clG4j~DMlz9 z57E zSSol^wJ4jR;bjn|oXpmwrI#7onSi2CS zeK8({MHnHGiZW=Bi*%OH+UI7$DIk2J?$fR;7w9U0G_BjoQl4^+@2rRNeIR$b!a3}d zEx&$JxHVb6)f{7fB5t~T41ap#=u6CPQM?{2YIX#bE#o>qkw{K4(JxyS6QA!i z4Z}Ce$&G4w_+5{bf_CIp8#|aN*%06|oNHna`0~#2zky*wq$OOnz1PQAZHoB`MK5Jn z33-=AL2z+B0f-=5f=_S7B9WDw6UbR6lZM6K#!#M8Ay$p;>btCBWliO@D!NOxPqCD3 zMw!~ajfBl-h!MEBe#(1kj66U!*%WS|rQ8~(V^?`xK@||o(97Ns5tSV(=*Q_9XeFg+_h129oM)?S1zjo`|&MQPZf#m0Z0qKNL8)*^x?SQ8Dp7cX=~ z;FQWcgdy7F56Gyc3N3F;Er4RAd_gdt0^;Z*cjN+-wM+HSBsCy z`d5p`*~GCB-NtD;BPIz88B_T+O?Ao0yw;=u-$W=LLH3nmCJba8#;P4UjsMs~d*4=* zAt7v(wEb>m*M1^;C@&{~=u<>>l_X=kOYA@)AW-pVePwyr#MK_kDNs&-y)DBWZ(1Yq z5}t1@Mj9g53oHjbbJ2AlRkWJk7#Zq8`R~JP$?&leRqKN-22GaZ2g);F+hSRfk+#j) zzuc+`%vym?Pka9b%7nrHsy+j^*34?*69r#y|IS~6qF9&_ujH_q+7xN1ly|7`ZSAHFmU!qJ z*}`j1)yl6>K4s%w{I;Sp;WP%2782$I)N29AgU7uNx6jn)eko0PlZTqWm{U!xi+=P5 z7;z4b5HX&ti;91~Z9MGNYdcS@6Ewxq$he^0t<}kB?{KcqQu^q2Cvo(; zg$SJW>m^wTyngSiRotaLUHeR1XGeE>>*@pS zm2i(di>FHpE%u=wl1rDbWw%^TZMDzI8aG(bp{w%#wgpRz2K#5i8AT)mlb+Zdp4rXG ztPN|yry0lxSI@P7f!Kx>0T!vq!2T%=(ZRA|Ex9k9T_bn~2kQQq;R=xD!L2<@xktU9p{~hI}TT z8{Bt*VtB2dHqB?l{VaWHUn-^UQ3Z1}W9QXfkw^vC8ejZX7j9_vwZB`TCuU%T=4A0X z+0QvNDlGl4-_Yp4QrM{@FO2oY)Crg8>rWvkK z$R>BS?W}{m*|_D%Wlp`6z-2-oEIaFX%YsI$A^11s+2F@g-$+1iBO4wUl@AYFA*{GYF z8JB|K;|dD1OL&Q9_4Fb2#|Yym-ZFIO_0yZ+;HboMrVYgse=8gQ$dRGy`r+Cnn<4~$ zzf%whuk-+~4fo6<)1-W_VTtQDtO{Jb1lX$O0@k zw0FRL`v{JkXIWT8&UbYX8+InpM2h&?C-8eRT4e0Pe@iVKlC*tac_}0}yK)>mD_>k8 zFEdI6aNLzXpl3Poqit;#a|wA$6>ps4&yUQ;{J!~C?vgiMkij6M`QG}~I{8R~n?yFN zHumyXY!n!i=574GA!wQz!sY4qNS)eZ3-i+ZpX=aPopITB{J?qjvd~%0H zE9SI>vEUs>Bi7c35}Gw7ZYrmnB+k4rIZ@SFkK-}-xF?ns_)1uM17(}*OD(XuG}8Tz zR||2c)M09PInw`XpuukYrwW5*vt!iv)EfOvv0i8S1hRBVM>6c$+J)<30Y&?VS3T35 zX43&t^O9S_+lW#qL$J+CLGO)*R$HZvgWdkQ2@m@zHE+Gb1z(!;Fk2tp9zRi>r_~l= zllcAFwAQ&w%R2#7;Txz?SmUva)^`JsRqbDi6Ia6vm3HN3Q^)V!Ya*y>x`g_qt>KfA zV_30@;Ujj_#4I;N-e~JC?bqG9Yu)lZj20AZFnB;^aziwUvCLczy)HadL`lrJ%YcM1 z%Vk4+lk&p?5K=yz)+GO2>)mSGRj1XI)0Gs5gw3dsM_Mu1ylnw5Al6|TJ^3YVPY8lW+W2i058mu{)#X1NbR{bd^62dOR6MKDzUV<(x%yf%y>s3(bvUfY;9jH`9fj{S<`n% z<=NzM{li)W2vhd}2Jc^L^Nd|P<2pQ0@vcpbo|@Iu7W-;xdh0yScTczU^Thl=B*Ff^ z2>Cz#zV64MGwhzX7yNcQKMdUh#S+X;N1Z|2)n*`J{IgG*LKCkFy)JhR^NsZby+_l- z&PGolUb(gBrPE8QOBM4dFBU;>tCYa^Y#CgZAa zF#a5<6~zM4iX^Xx7=@_=h4&_x?v&yzSMVfExo96haqb~0wA?&#QAS(V7L3Zyd>&CW zx-A7m zEX^H-5(CCpqKT-_2T7ALm|)xYo$!=G`Hdu|Tt}R$!enHfoH*0Skjt@ z)>j{al6SRv3kNSwA2sK6ZJXm#2=(EpIOOK?j$K@NJE&vk`$=r~Gv0U`p?i3(bH0yO zC8M=IF?*!L4vn}7I&zE&M!oRp#66r=si{wbVO}7W|KUple4$-KxcZ+{ocm?=S16l} z(MC?UOc*38f@!zRza^JtPWob7(gscvYinev-JTz0D>G>?3z z@j_B;*fx3gRbVrHv+bf(A?%MnFF8pIRp&Vy**-zOU z9yRFfGK?0{@`*CXN4GRx(u^5a>IB%Yvz24uQC^vQ^4 zDqu8@*8aJxvbzf&UUXD(3m4D&Uw^By!KhBX+Vu`DM~n9IRn zKlwayUWcNW)bu_%>Sk*g{8~RX&@CmVuAc-KY{m(*pw;NroOV%#^g{>Y4DjPY=pVst z5P@f-Et<27d+m)jQdvu$NFQ_V)O`8;C0UwmZHqeYqy8a2Q$kRM*II!0X3CY^?+G7~ zf;%s$RZS6@h_EuZHBItPoK^>MiZtVh186HFA^#8DN9M(UqM4BG%b6;TFCgHJhH$6) zBzh$MVgwShwE*dCT-=j{!UOH+iJx`N^w9(yU6sL(Uk#-{fDqhET~uBQ`n^8e;+7f( zJU*i42jU2Tkupg<;S<{+Qg&hQI`{=~Y4eX7MWOetn)1nrB^Kt87J|(f@6U6sHWQ*a z+moCNv*xMFwK2*>y|f-U_o0o8<^2?9Yh*7K)swpX$TC%ZKY%fny$4CnyhCcmsn}!0 z?faaNNgh$>a-!-n64}hpAR(6ZGVjiwlc`mQrl=uS!5Z`1KhRv?SC%WfuBW3us>mO% z5Gyw{M}8=~02~_w$IT+nmnLgzvrxU<5^VFk1vQI<8#3g-y#x(6_68T#)|*)wzpfYvODopsgF|@1Hd#I)N&@-jN3!4%yb(!VS5qUSOHN@uKDZr1 z{p&-o^PBX3YRjrV&x%ALe1)jh0R!QsVRq`W0$hPS#b*8UVf50^(_pd$w=ld^_&H~G za&#-Vxm%vP2cZaeEt_FS_9Y5yao@FmlFzGityYp~DtdXZ`(nGou&5;AG(#FM@# zS9Kb4r(1JHirw?XxfA5b)`c!Z^{i&G^jAumdW!7=^U-fVlQ?1e8(tSng0#ZAMp1GT ziDjtbXnjIgs6cr>qyaaEKpH&D`o^Z7zAgr_`O_Rof{l$HCuwts7vwzusZ3Rr;MW~b zHxCDjy`V^eEXB7^XtSX*fRH$B5jPc=-uP{ACFhDUk`GylH~gysdw*PHd^oAtm* zieb=ex8vn(W6qJOF_ n!_%BtTIeKg!HO)#k-=5*0`cN9{lhQs@7{U;x4y&uweUXxX!{S% literal 0 HcmV?d00001 diff --git a/examples/model_compress/amc/amc_search.py b/examples/model_compress/amc/amc_search.py new file mode 100644 index 0000000000..b08060efb9 --- /dev/null +++ b/examples/model_compress/amc/amc_search.py @@ -0,0 +1,136 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import sys +import argparse +import time + +import torch +import torch.nn as nn + +from nni.compression.torch import AMCPruner +from data import get_split_dataset +from utils import AverageMeter, accuracy + +sys.path.append('../models') + +def parse_args(): + parser = argparse.ArgumentParser(description='AMC search script') + parser.add_argument('--model_type', default='mobilenet', type=str, choices=['mobilenet', 'mobilenetv2'], help='model to prune') + parser.add_argument('--dataset', default='cifar10', type=str, choices=['cifar10', 'imagenet'], help='dataset to use (cifar/imagenet)') + parser.add_argument('--batch_size', default=50, type=int, help='number of data batch size') + parser.add_argument('--data_root', default='./cifar10', type=str, help='dataset path') + parser.add_argument('--flops_ratio', default=0.5, type=float, help='target flops ratio to preserve of the model') + parser.add_argument('--lbound', default=0.2, type=float, help='minimum sparsity') + parser.add_argument('--rbound', default=1., type=float, help='maximum sparsity') + parser.add_argument('--ckpt_path', default=None, type=str, help='manual path of checkpoint') + + parser.add_argument('--train_episode', default=800, type=int, help='number of training episode') + parser.add_argument('--n_gpu', default=1, type=int, help='number of gpu to use') + parser.add_argument('--n_worker', default=16, type=int, help='number of data loader worker') + parser.add_argument('--job', default='train_export', type=str, choices=['train_export', 'export_only'], + help='search best pruning policy and export or just export model with searched policy') + parser.add_argument('--export_path', default=None, type=str, help='path for exporting models') + parser.add_argument('--searched_model_path', default=None, type=str, help='path for searched best wrapped model') + + return parser.parse_args() + + +def get_model_and_checkpoint(model, dataset, checkpoint_path, n_gpu=1): + if model == 'mobilenet' and dataset == 'imagenet': + from mobilenet import MobileNet + net = MobileNet(n_class=1000) + elif model == 'mobilenetv2' and dataset == 'imagenet': + from mobilenet_v2 import MobileNetV2 + net = MobileNetV2(n_class=1000) + elif model == 'mobilenet' and dataset == 'cifar10': + from mobilenet import MobileNet + net = MobileNet(n_class=10) + elif model == 'mobilenetv2' and dataset == 'cifar10': + from mobilenet_v2 import MobileNetV2 + net = MobileNetV2(n_class=10) + else: + raise NotImplementedError + if checkpoint_path: + print('loading {}...'.format(checkpoint_path)) + sd = torch.load(checkpoint_path, map_location=torch.device('cpu')) + if 'state_dict' in sd: # a checkpoint but not a state_dict + sd = sd['state_dict'] + sd = {k.replace('module.', ''): v for k, v in sd.items()} + net.load_state_dict(sd) + + if torch.cuda.is_available() and n_gpu > 0: + net = net.cuda() + if n_gpu > 1: + net = torch.nn.DataParallel(net, range(n_gpu)) + + return net + +def init_data(args): + # split the train set into train + val + # for CIFAR, split 5k for val + # for ImageNet, split 3k for val + val_size = 5000 if 'cifar' in args.dataset else 3000 + train_loader, val_loader, _ = get_split_dataset( + args.dataset, args.batch_size, + args.n_worker, val_size, + data_root=args.data_root, + shuffle=False + ) # same sampling + return train_loader, val_loader + +def validate(val_loader, model, verbose=False): + batch_time = AverageMeter() + losses = AverageMeter() + top1 = AverageMeter() + top5 = AverageMeter() + + criterion = nn.CrossEntropyLoss().cuda() + # switch to evaluate mode + model.eval() + end = time.time() + + t1 = time.time() + with torch.no_grad(): + for i, (input, target) in enumerate(val_loader): + target = target.to(device) + input_var = torch.autograd.Variable(input).to(device) + target_var = torch.autograd.Variable(target).to(device) + + # compute output + output = model(input_var) + loss = criterion(output, target_var) + + # measure accuracy and record loss + prec1, prec5 = accuracy(output.data, target, topk=(1, 5)) + losses.update(loss.item(), input.size(0)) + top1.update(prec1.item(), input.size(0)) + top5.update(prec5.item(), input.size(0)) + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + t2 = time.time() + if verbose: + print('* Test loss: %.3f top1: %.3f top5: %.3f time: %.3f' % + (losses.avg, top1.avg, top5.avg, t2 - t1)) + return top5.avg + + +if __name__ == "__main__": + args = parse_args() + + device = torch.device('cuda') if torch.cuda.is_available() and args.n_gpu > 0 else torch.device('cpu') + + model = get_model_and_checkpoint(args.model_type, args.dataset, checkpoint_path=args.ckpt_path, n_gpu=args.n_gpu) + _, val_loader = init_data(args) + + config_list = [{ + 'op_types': ['Conv2d', 'Linear'] + }] + pruner = AMCPruner( + model, config_list, validate, val_loader, model_type=args.model_type, dataset=args.dataset, + train_episode=args.train_episode, job=args.job, export_path=args.export_path, + searched_model_path=args.searched_model_path, + flops_ratio=args.flops_ratio, lbound=args.lbound, rbound=args.rbound) + pruner.compress() diff --git a/examples/model_compress/amc/amc_train.py b/examples/model_compress/amc/amc_train.py new file mode 100644 index 0000000000..bedebc044c --- /dev/null +++ b/examples/model_compress/amc/amc_train.py @@ -0,0 +1,234 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import sys +import os +import time +import argparse +import shutil +import math +import numpy as np + +import torch +import torch.nn as nn +import torch.optim as optim +from tensorboardX import SummaryWriter + +from nni.compression.torch.pruning.amc.lib.net_measure import measure_model +from nni.compression.torch.pruning.amc.lib.utils import get_output_folder + +from data import get_dataset +from utils import AverageMeter, accuracy, progress_bar + +sys.path.append('../models') +from mobilenet import MobileNet +from mobilenet_v2 import MobileNetV2 + +def parse_args(): + parser = argparse.ArgumentParser(description='AMC train / fine-tune script') + parser.add_argument('--model_type', default='mobilenet', type=str, help='name of the model to train') + parser.add_argument('--dataset', default='cifar10', type=str, help='name of the dataset to train') + parser.add_argument('--lr', default=0.1, type=float, help='learning rate') + parser.add_argument('--n_gpu', default=1, type=int, help='number of GPUs to use') + parser.add_argument('--batch_size', default=128, type=int, help='batch size') + parser.add_argument('--n_worker', default=4, type=int, help='number of data loader worker') + parser.add_argument('--lr_type', default='exp', type=str, help='lr scheduler (exp/cos/step3/fixed)') + parser.add_argument('--n_epoch', default=50, type=int, help='number of epochs to train') + parser.add_argument('--wd', default=4e-5, type=float, help='weight decay') + parser.add_argument('--seed', default=None, type=int, help='random seed to set') + parser.add_argument('--data_root', default='./data', type=str, help='dataset path') + # resume + parser.add_argument('--ckpt_path', default=None, type=str, help='checkpoint path to fine tune') + # run eval + parser.add_argument('--eval', action='store_true', help='Simply run eval') + parser.add_argument('--calc_flops', action='store_true', help='Calculate flops') + + return parser.parse_args() + +def get_model(args): + print('=> Building model..') + + if args.dataset == 'imagenet': + n_class = 1000 + elif args.dataset == 'cifar10': + n_class = 10 + else: + raise NotImplementedError + + if args.model_type == 'mobilenet': + net = MobileNet(n_class=n_class).cuda() + elif args.model_type == 'mobilenetv2': + net = MobileNetV2(n_class=n_class).cuda() + else: + raise NotImplementedError + + if args.ckpt_path is not None: + # the checkpoint can be a saved whole model object exported by amc_search.py, or a state_dict + print('=> Loading checkpoint {} ..'.format(args.ckpt_path)) + ckpt = torch.load(args.ckpt_path) + if type(ckpt) == dict: + net.load_state_dict(ckpt['state_dict']) + else: + net = ckpt + + net.to(args.device) + if torch.cuda.is_available() and args.n_gpu > 1: + net = torch.nn.DataParallel(net, list(range(args.n_gpu))) + return net + +def train(epoch, train_loader, device): + print('\nEpoch: %d' % epoch) + net.train() + + batch_time = AverageMeter() + losses = AverageMeter() + top1 = AverageMeter() + top5 = AverageMeter() + end = time.time() + + for batch_idx, (inputs, targets) in enumerate(train_loader): + inputs, targets = inputs.to(device), targets.to(device) + optimizer.zero_grad() + outputs = net(inputs) + loss = criterion(outputs, targets) + + loss.backward() + optimizer.step() + + # measure accuracy and record loss + prec1, prec5 = accuracy(outputs.data, targets.data, topk=(1, 5)) + losses.update(loss.item(), inputs.size(0)) + top1.update(prec1.item(), inputs.size(0)) + top5.update(prec5.item(), inputs.size(0)) + # timing + batch_time.update(time.time() - end) + end = time.time() + + progress_bar(batch_idx, len(train_loader), 'Loss: {:.3f} | Acc1: {:.3f}% | Acc5: {:.3f}%' + .format(losses.avg, top1.avg, top5.avg)) + writer.add_scalar('loss/train', losses.avg, epoch) + writer.add_scalar('acc/train_top1', top1.avg, epoch) + writer.add_scalar('acc/train_top5', top5.avg, epoch) + +def test(epoch, test_loader, device, save=True): + global best_acc + net.eval() + + batch_time = AverageMeter() + losses = AverageMeter() + top1 = AverageMeter() + top5 = AverageMeter() + end = time.time() + + with torch.no_grad(): + for batch_idx, (inputs, targets) in enumerate(test_loader): + inputs, targets = inputs.to(device), targets.to(device) + outputs = net(inputs) + loss = criterion(outputs, targets) + + # measure accuracy and record loss + prec1, prec5 = accuracy(outputs.data, targets.data, topk=(1, 5)) + losses.update(loss.item(), inputs.size(0)) + top1.update(prec1.item(), inputs.size(0)) + top5.update(prec5.item(), inputs.size(0)) + # timing + batch_time.update(time.time() - end) + end = time.time() + + progress_bar(batch_idx, len(test_loader), 'Loss: {:.3f} | Acc1: {:.3f}% | Acc5: {:.3f}%' + .format(losses.avg, top1.avg, top5.avg)) + + if save: + writer.add_scalar('loss/test', losses.avg, epoch) + writer.add_scalar('acc/test_top1', top1.avg, epoch) + writer.add_scalar('acc/test_top5', top5.avg, epoch) + + is_best = False + if top1.avg > best_acc: + best_acc = top1.avg + is_best = True + + print('Current best acc: {}'.format(best_acc)) + save_checkpoint({ + 'epoch': epoch, + 'model': args.model_type, + 'dataset': args.dataset, + 'state_dict': net.module.state_dict() if isinstance(net, nn.DataParallel) else net.state_dict(), + 'acc': top1.avg, + 'optimizer': optimizer.state_dict(), + }, is_best, checkpoint_dir=log_dir) + +def adjust_learning_rate(optimizer, epoch): + if args.lr_type == 'cos': # cos without warm-up + lr = 0.5 * args.lr * (1 + math.cos(math.pi * epoch / args.n_epoch)) + elif args.lr_type == 'exp': + step = 1 + decay = 0.96 + lr = args.lr * (decay ** (epoch // step)) + elif args.lr_type == 'fixed': + lr = args.lr + else: + raise NotImplementedError + print('=> lr: {}'.format(lr)) + for param_group in optimizer.param_groups: + param_group['lr'] = lr + return lr + +def save_checkpoint(state, is_best, checkpoint_dir='.'): + filename = os.path.join(checkpoint_dir, 'ckpt.pth.tar') + print('=> Saving checkpoint to {}'.format(filename)) + torch.save(state, filename) + if is_best: + shutil.copyfile(filename, filename.replace('.pth.tar', '.best.pth.tar')) + +if __name__ == '__main__': + args = parse_args() + + if torch.cuda.is_available(): + torch.backends.cudnn.benchmark = True + args.device = torch.device('cuda') if torch.cuda.is_available() and args.n_gpu > 0 else torch.device('cpu') + + best_acc = 0 # best test accuracy + start_epoch = 0 # start from epoch 0 or last checkpoint epoch + + if args.seed is not None: + np.random.seed(args.seed) + torch.manual_seed(args.seed) + torch.cuda.manual_seed(args.seed) + + print('=> Preparing data..') + train_loader, val_loader, n_class = get_dataset(args.dataset, args.batch_size, args.n_worker, + data_root=args.data_root) + + net = get_model(args) # for measure + + if args.calc_flops: + IMAGE_SIZE = 224 if args.dataset == 'imagenet' else 32 + n_flops, n_params = measure_model(net, IMAGE_SIZE, IMAGE_SIZE) + print('=> Model Parameter: {:.3f} M, FLOPs: {:.3f}M'.format(n_params / 1e6, n_flops / 1e6)) + exit(0) + + criterion = nn.CrossEntropyLoss() + print('Using SGD...') + print('weight decay = {}'.format(args.wd)) + optimizer = optim.SGD(net.parameters(), lr=args.lr, momentum=0.9, weight_decay=args.wd) + + if args.eval: # just run eval + print('=> Start evaluation...') + test(0, val_loader, args.device, save=False) + else: # train + print('=> Start training...') + print('Training {} on {}...'.format(args.model_type, args.dataset)) + train_type = 'train' if args.ckpt_path is None else 'finetune' + log_dir = get_output_folder('./logs', '{}_{}_{}'.format(args.model_type, args.dataset, train_type)) + print('=> Saving logs to {}'.format(log_dir)) + # tf writer + writer = SummaryWriter(logdir=log_dir) + + for epoch in range(start_epoch, start_epoch + args.n_epoch): + lr = adjust_learning_rate(optimizer, epoch) + train(epoch, train_loader, args.device) + test(epoch, val_loader, args.device) + + writer.close() + print('=> Best top-1 acc: {}%'.format(best_acc)) diff --git a/examples/model_compress/amc/data.py b/examples/model_compress/amc/data.py new file mode 100644 index 0000000000..71935b3517 --- /dev/null +++ b/examples/model_compress/amc/data.py @@ -0,0 +1,156 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch +import torch.nn.parallel +import torch.optim +import torch.utils.data +import torchvision +import torchvision.transforms as transforms +import torchvision.datasets as datasets +from torch.utils.data.sampler import SubsetRandomSampler +import numpy as np + +import os + + +def get_dataset(dset_name, batch_size, n_worker, data_root='../../data'): + cifar_tran_train = [ + transforms.RandomCrop(32, padding=4), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), + ] + cifar_tran_test = [ + transforms.ToTensor(), + transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), + ] + print('=> Preparing data..') + if dset_name == 'cifar10': + transform_train = transforms.Compose(cifar_tran_train) + transform_test = transforms.Compose(cifar_tran_test) + trainset = torchvision.datasets.CIFAR10(root=data_root, train=True, download=True, transform=transform_train) + train_loader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True, + num_workers=n_worker, pin_memory=True, sampler=None) + testset = torchvision.datasets.CIFAR10(root=data_root, train=False, download=True, transform=transform_test) + val_loader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, + num_workers=n_worker, pin_memory=True) + n_class = 10 + elif dset_name == 'imagenet': + # get dir + traindir = os.path.join(data_root, 'train') + valdir = os.path.join(data_root, 'val') + + # preprocessing + input_size = 224 + imagenet_tran_train = [ + transforms.RandomResizedCrop(input_size, scale=(0.2, 1.0)), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), + ] + imagenet_tran_test = [ + transforms.Resize(int(input_size / 0.875)), + transforms.CenterCrop(input_size), + transforms.ToTensor(), + transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), + ] + + train_loader = torch.utils.data.DataLoader( + datasets.ImageFolder(traindir, transforms.Compose(imagenet_tran_train)), + batch_size=batch_size, shuffle=True, + num_workers=n_worker, pin_memory=True, sampler=None) + + val_loader = torch.utils.data.DataLoader( + datasets.ImageFolder(valdir, transforms.Compose(imagenet_tran_test)), + batch_size=batch_size, shuffle=False, + num_workers=n_worker, pin_memory=True) + n_class = 1000 + + else: + raise NotImplementedError + + return train_loader, val_loader, n_class + + +def get_split_dataset(dset_name, batch_size, n_worker, val_size, data_root='../data', shuffle=True): + ''' + split the train set into train / val for rl search + ''' + if shuffle: + index_sampler = SubsetRandomSampler + else: # every time we use the same order for the split subset + class SubsetSequentialSampler(SubsetRandomSampler): + def __iter__(self): + return (self.indices[i] for i in torch.arange(len(self.indices)).int()) + index_sampler = SubsetSequentialSampler + + print('=> Preparing data: {}...'.format(dset_name)) + if dset_name == 'cifar10': + transform_train = transforms.Compose([ + transforms.RandomCrop(32, padding=4), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), + ]) + transform_test = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), + ]) + trainset = torchvision.datasets.CIFAR100(root=data_root, train=True, download=True, transform=transform_train) + valset = torchvision.datasets.CIFAR10(root=data_root, train=True, download=True, transform=transform_test) + n_train = len(trainset) + indices = list(range(n_train)) + # now shuffle the indices + #np.random.shuffle(indices) + assert val_size < n_train + train_idx, val_idx = indices[val_size:], indices[:val_size] + + train_sampler = index_sampler(train_idx) + val_sampler = index_sampler(val_idx) + + train_loader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=False, sampler=train_sampler, + num_workers=n_worker, pin_memory=True) + val_loader = torch.utils.data.DataLoader(valset, batch_size=batch_size, shuffle=False, sampler=val_sampler, + num_workers=n_worker, pin_memory=True) + n_class = 10 + elif dset_name == 'imagenet': + train_dir = os.path.join(data_root, 'train') + val_dir = os.path.join(data_root, 'val') + normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]) + input_size = 224 + train_transform = transforms.Compose([ + transforms.RandomResizedCrop(input_size), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + normalize, + ]) + test_transform = transforms.Compose([ + transforms.Resize(int(input_size/0.875)), + transforms.CenterCrop(input_size), + transforms.ToTensor(), + normalize, + ]) + + trainset = datasets.ImageFolder(train_dir, train_transform) + valset = datasets.ImageFolder(train_dir, test_transform) + n_train = len(trainset) + indices = list(range(n_train)) + np.random.shuffle(indices) + assert val_size < n_train + train_idx, val_idx = indices[val_size:], indices[:val_size] + + train_sampler = index_sampler(train_idx) + val_sampler = index_sampler(val_idx) + + train_loader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, sampler=train_sampler, + num_workers=n_worker, pin_memory=True) + val_loader = torch.utils.data.DataLoader(valset, batch_size=batch_size, sampler=val_sampler, + num_workers=n_worker, pin_memory=True) + + n_class = 1000 + else: + raise NotImplementedError + + return train_loader, val_loader, n_class diff --git a/examples/model_compress/amc/utils.py b/examples/model_compress/amc/utils.py new file mode 100644 index 0000000000..d1b17be065 --- /dev/null +++ b/examples/model_compress/amc/utils.py @@ -0,0 +1,138 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import sys +import os +import time + +class AverageMeter(object): + """Computes and stores the average and current value""" + def __init__(self): + self.reset() + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + if self.count > 0: + self.avg = self.sum / self.count + + def accumulate(self, val, n=1): + self.sum += val + self.count += n + if self.count > 0: + self.avg = self.sum / self.count + + +def accuracy(output, target, topk=(1, 5)): + """Computes the precision@k for the specified values of k""" + batch_size = target.size(0) + num = output.size(1) + target_topk = [] + appendices = [] + for k in topk: + if k <= num: + target_topk.append(k) + else: + appendices.append([0.0]) + topk = target_topk + maxk = max(topk) + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].view(-1).float().sum(0) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + appendices + + +# Custom progress bar +_, term_width = os.popen('stty size', 'r').read().split() +term_width = int(term_width) +TOTAL_BAR_LENGTH = 40. +last_time = time.time() +begin_time = last_time + + +def progress_bar(current, total, msg=None): + def format_time(seconds): + days = int(seconds / 3600 / 24) + seconds = seconds - days * 3600 * 24 + hours = int(seconds / 3600) + seconds = seconds - hours * 3600 + minutes = int(seconds / 60) + seconds = seconds - minutes * 60 + secondsf = int(seconds) + seconds = seconds - secondsf + millis = int(seconds * 1000) + + f = '' + i = 1 + if days > 0: + f += str(days) + 'D' + i += 1 + if hours > 0 and i <= 2: + f += str(hours) + 'h' + i += 1 + if minutes > 0 and i <= 2: + f += str(minutes) + 'm' + i += 1 + if secondsf > 0 and i <= 2: + f += str(secondsf) + 's' + i += 1 + if millis > 0 and i <= 2: + f += str(millis) + 'ms' + i += 1 + if f == '': + f = '0ms' + return f + + global last_time, begin_time + if current == 0: + begin_time = time.time() # Reset for new bar. + + cur_len = int(TOTAL_BAR_LENGTH*current/total) + rest_len = int(TOTAL_BAR_LENGTH - cur_len) - 1 + + sys.stdout.write(' [') + for i in range(cur_len): + sys.stdout.write('=') + sys.stdout.write('>') + for i in range(rest_len): + sys.stdout.write('.') + sys.stdout.write(']') + + cur_time = time.time() + step_time = cur_time - last_time + last_time = cur_time + tot_time = cur_time - begin_time + + L = [] + L.append(' Step: %s' % format_time(step_time)) + L.append(' | Tot: %s' % format_time(tot_time)) + if msg: + L.append(' | ' + msg) + + msg = ''.join(L) + sys.stdout.write(msg) + for i in range(term_width-int(TOTAL_BAR_LENGTH)-len(msg)-3): + sys.stdout.write(' ') + + # Go back to the center of the bar. + for i in range(term_width-int(TOTAL_BAR_LENGTH/2)+2): + sys.stdout.write('\b') + sys.stdout.write(' %d/%d ' % (current+1, total)) + + if current < total-1: + sys.stdout.write('\r') + else: + sys.stdout.write('\n') + sys.stdout.flush() diff --git a/examples/model_compress/models/mobilenet.py b/examples/model_compress/models/mobilenet.py new file mode 100644 index 0000000000..8d60c90a4c --- /dev/null +++ b/examples/model_compress/models/mobilenet.py @@ -0,0 +1,83 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch.nn as nn +import math + + +def conv_bn(inp, oup, stride): + return nn.Sequential( + nn.Conv2d(inp, oup, 3, stride, 1, bias=False), + nn.BatchNorm2d(oup), + nn.ReLU(inplace=True) + ) + + +def conv_dw(inp, oup, stride): + return nn.Sequential( + nn.Conv2d(inp, inp, 3, stride, 1, groups=inp, bias=False), + nn.BatchNorm2d(inp), + nn.ReLU(inplace=True), + + nn.Conv2d(inp, oup, 1, 1, 0, bias=False), + nn.BatchNorm2d(oup), + nn.ReLU(inplace=True), + ) + + +class MobileNet(nn.Module): + def __init__(self, n_class, profile='normal'): + super(MobileNet, self).__init__() + + # original + if profile == 'normal': + in_planes = 32 + cfg = [64, (128, 2), 128, (256, 2), 256, (512, 2), 512, 512, 512, 512, 512, (1024, 2), 1024] + # 0.5 AMC + elif profile == '0.5flops': + in_planes = 24 + cfg = [48, (96, 2), 80, (192, 2), 200, (328, 2), 352, 368, 360, 328, 400, (736, 2), 752] + else: + raise NotImplementedError + + self.conv1 = conv_bn(3, in_planes, stride=2) + + self.features = self._make_layers(in_planes, cfg, conv_dw) + + self.classifier = nn.Sequential( + nn.Linear(cfg[-1], n_class), + ) + + self._initialize_weights() + + def forward(self, x): + x = self.conv1(x) + x = self.features(x) + x = x.mean(3).mean(2) # global average pooling + + x = self.classifier(x) + return x + + def _make_layers(self, in_planes, cfg, layer): + layers = [] + for x in cfg: + out_planes = x if isinstance(x, int) else x[0] + stride = 1 if isinstance(x, int) else x[1] + layers.append(layer(in_planes, out_planes, stride)) + in_planes = out_planes + return nn.Sequential(*layers) + + def _initialize_weights(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + if m.bias is not None: + m.bias.data.zero_() + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + n = m.weight.size(1) + m.weight.data.normal_(0, 0.01) + m.bias.data.zero_() diff --git a/examples/model_compress/models/mobilenet_v2.py b/examples/model_compress/models/mobilenet_v2.py new file mode 100644 index 0000000000..b77e85e60b --- /dev/null +++ b/examples/model_compress/models/mobilenet_v2.py @@ -0,0 +1,128 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch.nn as nn +import math + + +def conv_bn(inp, oup, stride): + return nn.Sequential( + nn.Conv2d(inp, oup, 3, stride, 1, bias=False), + nn.BatchNorm2d(oup), + nn.ReLU6(inplace=True) + ) + + +def conv_1x1_bn(inp, oup): + return nn.Sequential( + nn.Conv2d(inp, oup, 1, 1, 0, bias=False), + nn.BatchNorm2d(oup), + nn.ReLU6(inplace=True) + ) + + +class InvertedResidual(nn.Module): + def __init__(self, inp, oup, stride, expand_ratio): + super(InvertedResidual, self).__init__() + self.stride = stride + assert stride in [1, 2] + + hidden_dim = round(inp * expand_ratio) + self.use_res_connect = self.stride == 1 and inp == oup + + if expand_ratio == 1: + self.conv = nn.Sequential( + # dw + nn.Conv2d(hidden_dim, hidden_dim, 3, stride, 1, groups=hidden_dim, bias=False), + nn.BatchNorm2d(hidden_dim), + nn.ReLU6(inplace=True), + # pw-linear + nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False), + nn.BatchNorm2d(oup), + ) + else: + self.conv = nn.Sequential( + # pw + nn.Conv2d(inp, hidden_dim, 1, 1, 0, bias=False), + nn.BatchNorm2d(hidden_dim), + nn.ReLU6(inplace=True), + # dw + nn.Conv2d(hidden_dim, hidden_dim, 3, stride, 1, groups=hidden_dim, bias=False), + nn.BatchNorm2d(hidden_dim), + nn.ReLU6(inplace=True), + # pw-linear + nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False), + nn.BatchNorm2d(oup), + ) + + def forward(self, x): + if self.use_res_connect: + return x + self.conv(x) + else: + return self.conv(x) + + +class MobileNetV2(nn.Module): + def __init__(self, n_class=1000, input_size=224, width_mult=1.): + super(MobileNetV2, self).__init__() + block = InvertedResidual + input_channel = 32 + last_channel = 1280 + interverted_residual_setting = [ + # t, c, n, s + [1, 16, 1, 1], + [6, 24, 2, 2], + [6, 32, 3, 2], + [6, 64, 4, 2], + [6, 96, 3, 1], + [6, 160, 3, 2], + [6, 320, 1, 1], + ] + + # building first layer + assert input_size % 32 == 0 + input_channel = int(input_channel * width_mult) + self.last_channel = int(last_channel * width_mult) if width_mult > 1.0 else last_channel + self.features = [conv_bn(3, input_channel, 2)] + # building inverted residual blocks + for t, c, n, s in interverted_residual_setting: + output_channel = int(c * width_mult) + for i in range(n): + if i == 0: + self.features.append(block(input_channel, output_channel, s, expand_ratio=t)) + else: + self.features.append(block(input_channel, output_channel, 1, expand_ratio=t)) + input_channel = output_channel + # building last several layers + self.features.append(conv_1x1_bn(input_channel, self.last_channel)) + # make it nn.Sequential + self.features = nn.Sequential(*self.features) + + # building classifier + self.classifier = nn.Sequential( + nn.Dropout(0.2), + nn.Linear(self.last_channel, n_class), + ) + + self._initialize_weights() + + def forward(self, x): + x = self.features(x) + x = x.mean(3).mean(2) + x = self.classifier(x) + return x + + def _initialize_weights(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + if m.bias is not None: + m.bias.data.zero_() + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + n = m.weight.size(1) + m.weight.data.normal_(0, 0.01) + m.bias.data.zero_() diff --git a/src/sdk/pynni/nni/compression/torch/compressor.py b/src/sdk/pynni/nni/compression/torch/compressor.py index 51880ece12..e5e3017a07 100644 --- a/src/sdk/pynni/nni/compression/torch/compressor.py +++ b/src/sdk/pynni/nni/compression/torch/compressor.py @@ -54,20 +54,34 @@ def __init__(self, model, config_list, optimizer=None): self._fwd_hook_handles = {} self._fwd_hook_id = 0 - for layer, config in self._detect_modules_to_compress(): - wrapper = self._wrap_modules(layer, config) - self.modules_wrapper.append(wrapper) + self.reset() + if not self.modules_wrapper: _logger.warning('Nothing is configured to compress, please check your model and config_list') - self._wrap_model() - def validate_config(self, model, config_list): """ subclass can optionally implement this method to check if config_list if valid """ pass + def reset(self, checkpoint=None): + """ + reset model state dict and model wrapper + """ + self._unwrap_model() + if checkpoint is not None: + self.bound_model.load_state_dict(checkpoint) + + self.modules_to_compress = None + self.modules_wrapper = [] + + for layer, config in self._detect_modules_to_compress(): + wrapper = self._wrap_modules(layer, config) + self.modules_wrapper.append(wrapper) + + self._wrap_model() + def _detect_modules_to_compress(self): """ detect all modules should be compressed, and save the result in `self.modules_to_compress`. @@ -346,7 +360,7 @@ def _wrap_modules(self, layer, config): config : dict the configuration for generating the mask """ - _logger.info("Module detected to compress : %s.", layer.name) + _logger.debug("Module detected to compress : %s.", layer.name) wrapper = PrunerModuleWrapper(layer.module, layer.name, layer.type, config, self) assert hasattr(layer.module, 'weight'), "module %s does not have 'weight' attribute" % layer.name # move newly registered buffers to the same device of weight @@ -381,7 +395,7 @@ def export_model(self, model_path, mask_path=None, onnx_path=None, input_shape=N if weight_mask is not None: mask_sum = weight_mask.sum().item() mask_num = weight_mask.numel() - _logger.info('Layer: %s Sparsity: %.4f', wrapper.name, 1 - mask_sum / mask_num) + _logger.debug('Layer: %s Sparsity: %.4f', wrapper.name, 1 - mask_sum / mask_num) wrapper.module.weight.data = wrapper.module.weight.data.mul(weight_mask) if bias_mask is not None: wrapper.module.bias.data = wrapper.module.bias.data.mul(bias_mask) diff --git a/src/sdk/pynni/nni/compression/torch/pruning/__init__.py b/src/sdk/pynni/nni/compression/torch/pruning/__init__.py index 919ffb3fd3..9787ba5291 100644 --- a/src/sdk/pynni/nni/compression/torch/pruning/__init__.py +++ b/src/sdk/pynni/nni/compression/torch/pruning/__init__.py @@ -12,3 +12,5 @@ from .admm_pruner import ADMMPruner from .auto_compress_pruner import AutoCompressPruner from .sensitivity_pruner import SensitivityPruner +from .amc import AMCPruner + diff --git a/src/sdk/pynni/nni/compression/torch/pruning/amc/__init__.py b/src/sdk/pynni/nni/compression/torch/pruning/amc/__init__.py new file mode 100644 index 0000000000..3c89a879c6 --- /dev/null +++ b/src/sdk/pynni/nni/compression/torch/pruning/amc/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from .amc_pruner import AMCPruner diff --git a/src/sdk/pynni/nni/compression/torch/pruning/amc/amc_pruner.py b/src/sdk/pynni/nni/compression/torch/pruning/amc/amc_pruner.py new file mode 100644 index 0000000000..5f3e1ce6be --- /dev/null +++ b/src/sdk/pynni/nni/compression/torch/pruning/amc/amc_pruner.py @@ -0,0 +1,329 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +from copy import deepcopy +from argparse import Namespace +import numpy as np +import torch + +from nni.compression.torch.compressor import Pruner +from .channel_pruning_env import ChannelPruningEnv +from .lib.agent import DDPG +from .lib.utils import get_output_folder + +torch.backends.cudnn.deterministic = True + +class AMCPruner(Pruner): + """ + A pytorch implementation of AMC: AutoML for Model Compression and Acceleration on Mobile Devices. + (https://arxiv.org/pdf/1802.03494.pdf) + + Parameters: + model: nn.Module + The model to be pruned. + config_list: list + Configuration list to configure layer pruning. + Supported keys: + - op_types: operation type to be pruned + - op_names: operation name to be pruned + evaluator: function + function to evaluate the pruned model. + The prototype of the function: + >>> def evaluator(val_loader, model): + >>> ... + >>> return acc + val_loader: torch.utils.data.DataLoader + Data loader of validation dataset. + suffix: str + suffix to help you remember what experiment you ran. Default: None. + job: str + train_export: search best pruned model and export after search. + export_only: export a searched model, searched_model_path and export_path must be specified. + searched_model_path: str + when job == export_only, use searched_model_path to specify the path of the searched model. + export_path: str + path for exporting models + + # parameters for pruning environment + model_type: str + model type to prune, currently 'mobilenet' and 'mobilenetv2' are supported. Default: mobilenet + flops_ratio: float + preserve flops ratio. Default: 0.5 + lbound: float + minimum weight preserve ratio for each layer. Default: 0.2 + rbound: float + maximum weight preserve ratio for each layer. Default: 1.0 + reward: function + reward function type: + - acc_reward: accuracy * 0.01 + - acc_flops_reward: - (100 - accuracy) * 0.01 * np.log(flops) + Default: acc_reward + # parameters for channel pruning + n_calibration_batches: int + number of batches to extract layer information. Default: 60 + n_points_per_layer: int + number of feature points per layer. Default: 10 + channel_round: int + round channel to multiple of channel_round. Default: 8 + + # parameters for ddpg agent + hidden1: int + hidden num of first fully connect layer. Default: 300 + hidden2: int + hidden num of second fully connect layer. Default: 300 + lr_c: float + learning rate for critic. Default: 1e-3 + lr_a: float + learning rate for actor. Default: 1e-4 + warmup: int + number of episodes without training but only filling the replay memory. During warmup episodes, + random actions ares used for pruning. Default: 100 + discount: float + next Q value discount for deep Q value target. Default: 0.99 + bsize: int + minibatch size for training DDPG agent. Default: 64 + rmsize: int + memory size for each layer. Default: 100 + window_length: int + replay buffer window length. Default: 1 + tau: float + moving average for target network being used by soft_update. Default: 0.99 + # noise + init_delta: float + initial variance of truncated normal distribution + delta_decay: float + delta decay during exploration + + # parameters for training ddpg agent + max_episode_length: int + maximum episode length + output_dir: str + output directory to save log files and model files. Default: ./logs + debug: boolean + debug mode + train_episode: int + train iters each timestep. Default: 800 + epsilon: int + linear decay of exploration policy. Default: 50000 + seed: int + random seed to set for reproduce experiment. Default: None + """ + + def __init__( + self, + model, + config_list, + evaluator, + val_loader, + suffix=None, + job='train_export', + export_path=None, + searched_model_path=None, + model_type='mobilenet', + dataset='cifar10', + flops_ratio=0.5, + lbound=0.2, + rbound=1., + reward='acc_reward', + n_calibration_batches=60, + n_points_per_layer=10, + channel_round=8, + hidden1=300, + hidden2=300, + lr_c=1e-3, + lr_a=1e-4, + warmup=100, + discount=1., + bsize=64, + rmsize=100, + window_length=1, + tau=0.01, + init_delta=0.5, + delta_decay=0.99, + max_episode_length=1e9, + output_dir='./logs', + debug=False, + train_episode=800, + epsilon=50000, + seed=None): + + from tensorboardX import SummaryWriter + + self.job = job + self.searched_model_path = searched_model_path + self.export_path = export_path + + if seed is not None: + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + + checkpoint = deepcopy(model.state_dict()) + + super().__init__(model, config_list, optimizer=None) + + # build folder and logs + base_folder_name = '{}_{}_r{}_search'.format(model_type, dataset, flops_ratio) + if suffix is not None: + base_folder_name = base_folder_name + '_' + suffix + self.output_dir = get_output_folder(output_dir, base_folder_name) + + if self.export_path is None: + self.export_path = os.path.join(self.output_dir, '{}_r{}_exported.pth'.format(model_type, flops_ratio)) + + self.env_args = Namespace( + model_type=model_type, + preserve_ratio=flops_ratio, + lbound=lbound, + rbound=rbound, + reward=reward, + n_calibration_batches=n_calibration_batches, + n_points_per_layer=n_points_per_layer, + channel_round=channel_round, + output=self.output_dir + ) + + self.env = ChannelPruningEnv( + self, evaluator, val_loader, checkpoint, args=self.env_args) + + if self.job == 'train_export': + print('=> Saving logs to {}'.format(self.output_dir)) + self.tfwriter = SummaryWriter(logdir=self.output_dir) + self.text_writer = open(os.path.join(self.output_dir, 'log.txt'), 'w') + print('=> Output path: {}...'.format(self.output_dir)) + + nb_states = self.env.layer_embedding.shape[1] + nb_actions = 1 # just 1 action here + + rmsize = rmsize * len(self.env.prunable_idx) # for each layer + print('** Actual replay buffer size: {}'.format(rmsize)) + + self.ddpg_args = Namespace( + hidden1=hidden1, + hidden2=hidden2, + lr_c=lr_c, + lr_a=lr_a, + warmup=warmup, + discount=discount, + bsize=bsize, + rmsize=rmsize, + window_length=window_length, + tau=tau, + init_delta=init_delta, + delta_decay=delta_decay, + max_episode_length=max_episode_length, + debug=debug, + train_episode=train_episode, + epsilon=epsilon + ) + self.agent = DDPG(nb_states, nb_actions, self.ddpg_args) + + + def compress(self): + if self.job == 'train_export': + self.train(self.ddpg_args.train_episode, self.agent, self.env, self.output_dir) + self.export_pruned_model() + + def train(self, num_episode, agent, env, output_dir): + agent.is_training = True + step = episode = episode_steps = 0 + episode_reward = 0. + observation = None + T = [] # trajectory + while episode < num_episode: # counting based on episode + # reset if it is the start of episode + if observation is None: + observation = deepcopy(env.reset()) + agent.reset(observation) + + # agent pick action ... + if episode <= self.ddpg_args.warmup: + action = agent.random_action() + # action = sample_from_truncated_normal_distribution(lower=0., upper=1., mu=env.preserve_ratio, sigma=0.5) + else: + action = agent.select_action(observation, episode=episode) + + # env response with next_observation, reward, terminate_info + observation2, reward, done, info = env.step(action) + + T.append([reward, deepcopy(observation), deepcopy(observation2), action, done]) + + # fix-length, never reach here + # if max_episode_length and episode_steps >= max_episode_length - 1: + # done = True + + # [optional] save intermideate model + if num_episode / 3 <= 1 or episode % int(num_episode / 3) == 0: + agent.save_model(output_dir) + + # update + step += 1 + episode_steps += 1 + episode_reward += reward + observation = deepcopy(observation2) + + if done: # end of episode + print( + '#{}: episode_reward:{:.4f} acc: {:.4f}, ratio: {:.4f}'.format( + episode, episode_reward, + info['accuracy'], + info['compress_ratio'] + ) + ) + self.text_writer.write( + '#{}: episode_reward:{:.4f} acc: {:.4f}, ratio: {:.4f}\n'.format( + episode, episode_reward, + info['accuracy'], + info['compress_ratio'] + ) + ) + final_reward = T[-1][0] + # print('final_reward: {}'.format(final_reward)) + # agent observe and update policy + for _, s_t, s_t1, a_t, done in T: + agent.observe(final_reward, s_t, s_t1, a_t, done) + if episode > self.ddpg_args.warmup: + agent.update_policy() + + #agent.memory.append( + # observation, + # agent.select_action(observation, episode=episode), + # 0., False + #) + + # reset + observation = None + episode_steps = 0 + episode_reward = 0. + episode += 1 + T = [] + + self.tfwriter.add_scalar('reward/last', final_reward, episode) + self.tfwriter.add_scalar('reward/best', env.best_reward, episode) + self.tfwriter.add_scalar('info/accuracy', info['accuracy'], episode) + self.tfwriter.add_scalar('info/compress_ratio', info['compress_ratio'], episode) + self.tfwriter.add_text('info/best_policy', str(env.best_strategy), episode) + # record the preserve rate for each layer + for i, preserve_rate in enumerate(env.strategy): + self.tfwriter.add_scalar('preserve_rate/{}'.format(i), preserve_rate, episode) + + self.text_writer.write('best reward: {}\n'.format(env.best_reward)) + self.text_writer.write('best policy: {}\n'.format(env.best_strategy)) + self.text_writer.close() + + def export_pruned_model(self): + if self.searched_model_path is None: + wrapper_model_ckpt = os.path.join(self.output_dir, 'best_wrapped_model.pth') + else: + wrapper_model_ckpt = self.searched_model_path + self.env.reset() + self.bound_model.load_state_dict(torch.load(wrapper_model_ckpt)) + + print('validate searched model:', self.env._validate(self.env._val_loader, self.env.model)) + self.env.export_model() + self._unwrap_model() + print('validate exported model:', self.env._validate(self.env._val_loader, self.env.model)) + + torch.save(self.bound_model, self.export_path) + print('exported model saved to: {}'.format(self.export_path)) diff --git a/src/sdk/pynni/nni/compression/torch/pruning/amc/channel_pruning_env.py b/src/sdk/pynni/nni/compression/torch/pruning/amc/channel_pruning_env.py new file mode 100644 index 0000000000..fdd0694e1b --- /dev/null +++ b/src/sdk/pynni/nni/compression/torch/pruning/amc/channel_pruning_env.py @@ -0,0 +1,602 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import time +import math +import copy +import numpy as np +import torch +import torch.nn as nn + +from nni.compression.torch.compressor import PrunerModuleWrapper +from .lib.utils import prGreen +from .. import AMCWeightMasker + +# for pruning +def acc_reward(net, acc, flops): + return acc * 0.01 + + +def acc_flops_reward(net, acc, flops): + error = (100 - acc) * 0.01 + return -error * np.log(flops) + + +class ChannelPruningEnv: + """ + Env for channel pruning search. + This class is used to prune model using specified pruner. It prunes one layer when + step() is called. When the last layer is pruned, it evaluate the pruned model using + evaluator, and use the returned value of evaluator as reward of the episode. + + Usage: + env = ChannelPruningEnv(pruner, evaluator, val_loader, checkpoint, env_args) + episode = 0 + T = [] + while episode < num_episode: + action = agent.select_action(observation) + observation2, reward, done, info = env.step(action) + T.append([reward, deepcopy(observation), deepcopy(observation2), action, done]) + + if done: # end of episode, last layer pruned + episode += 1 + # train agent with episode data + for _, s_t, s_t1, a_t, done in T: + agent.observe(final_reward, s_t, s_t1, a_t, done) + agent.update_policy() + T = [] + + Attributes: + prunable_idx: layer indices for pruable layers, the index values are the index + of list(self.model.modules()). Pruable layers are pointwise Conv2d layers and Linear + layers. + buffer_idx: layer indices for buffer layers which refers the depthwise layers. + Each depthwise layer is always followd by a pointwise layer for both mobilenet and + mobilenetv2. The depthwise layer's filters are pruned when its next pointwise layer's + corresponding input channels are pruned. + shared_idx: layer indices for layers which share input. + For example: [[1,4], [8, 10, 15]] means layer 1 and 4 share same input, and layer + 8, 10 and 15 share another input. + layer_embedding: embeddings for each prunable layers, the embedding is used as + observation for DDPG agent. + layer_info_dict: flops and number of parameters of each layer. + min_strategy_dict: key is layer index, value is a tuple, the first value is the minimum + action of input channel, the second value is the minimum action value of output channel. + strategy_dict: key is layer index, value is a tuple, the first value is the action of input + channel, the second value is the action of output channel. + + Parameters: + pruner: Pruner + NNI Pruner instance used to prune model. + evaluator: function + function to evaluate the pruned model. + The prototype of the function: + >>> def evaluator(val_loader, model): + >>> ... + >>> return acc + val_loader: torch.utils.data.DataLoader + Data loader of validation dataset. + checkpoint: dict + checkpoint of the model to be pruned. It is used to reset model at beginning of each + episode. + args: + A Namespace object containing following arguments: + model_type: str + model type to prune, currently 'mobilenet' and 'mobilenetv2' are supported. + flops_ratio: float + preserve flops ratio. + lbound: float + minimum weight preserve ratio for each layer. + rbound: float + maximum weight preserve ratio for each layer. + reward: function + reward function type + + # parameters for channel pruning + n_calibration_batches: int + number of batches to extract layer information. + n_points_per_layer: int + number of feature points per layer. + channel_round: int + round channel to multiple of channel_round. + + """ + def __init__(self, pruner, evaluator, val_loader, checkpoint, args): + self.pruner = pruner + self.model = pruner.bound_model + self.checkpoint = checkpoint + self.batch_size = val_loader.batch_size + self.preserve_ratio = args.preserve_ratio + self.channel_prune_masker = AMCWeightMasker(self.model, self.pruner, args.channel_round) + + # options from args + self.args = args + self.lbound = args.lbound + self.rbound = args.rbound + + self.n_calibration_batches = args.n_calibration_batches + self.n_points_per_layer = args.n_points_per_layer + self.channel_round = args.channel_round + + # sanity check + assert self.preserve_ratio > self.lbound, 'Error! You can not achieve preserve_ratio smaller than lbound!' + + # prepare data + self._val_loader = val_loader + self._validate = evaluator + + # build indexs + self._build_index() + self.n_prunable_layer = len(self.prunable_idx) + + # extract information for preparing + self._extract_layer_information() + + # build embedding (static part) + self._build_state_embedding() + + # build reward + self.reset() # restore weight + self.org_acc = self._validate(self._val_loader, self.model) + print('=> original acc: {:.3f}%'.format(self.org_acc)) + self.org_model_size = sum(self.wsize_list) + print('=> original weight size: {:.4f} M param'.format(self.org_model_size * 1. / 1e6)) + self.org_flops = sum(self.flops_list) + print('=> FLOPs:') + print([self.layer_info_dict[idx]['flops']/1e6 for idx in sorted(self.layer_info_dict.keys())]) + print('=> original FLOPs: {:.4f} M'.format(self.org_flops * 1. / 1e6)) + + self.expected_preserve_computation = self.preserve_ratio * self.org_flops + + self.reward = eval(args.reward) + + self.best_reward = -math.inf + self.best_strategy = None + self.best_d_prime_list = None + self.best_masks = None + + self.org_w_size = sum(self.wsize_list) + + def step(self, action): + # Pseudo prune and get the corresponding statistics. The real pruning happens till the end of all pseudo pruning + if self.visited[self.cur_ind]: + action = self.strategy_dict[self.prunable_idx[self.cur_ind]][0] + preserve_idx = self.index_buffer[self.cur_ind] + else: + action = self._action_wall(action) # percentage to preserve + preserve_idx = None + # prune and update action + action, d_prime, preserve_idx = self.prune_kernel(self.prunable_idx[self.cur_ind], action, preserve_idx) + if not self.visited[self.cur_ind]: + for group in self.shared_idx: + if self.cur_ind in group: # set the shared ones + for g_idx in group: + self.strategy_dict[self.prunable_idx[g_idx]][0] = action + self.strategy_dict[self.prunable_idx[g_idx - 1]][1] = action + self.visited[g_idx] = True + self.index_buffer[g_idx] = preserve_idx.copy() + + self.strategy.append(action) # save action to strategy + self.d_prime_list.append(d_prime) + + self.strategy_dict[self.prunable_idx[self.cur_ind]][0] = action + if self.cur_ind > 0: + self.strategy_dict[self.prunable_idx[self.cur_ind - 1]][1] = action + + # all the actions are made + if self._is_final_layer(): + assert len(self.strategy) == len(self.prunable_idx) + current_flops = self._cur_flops() + acc_t1 = time.time() + acc = self._validate(self._val_loader, self.model) + acc_t2 = time.time() + self.val_time = acc_t2 - acc_t1 + compress_ratio = current_flops * 1. / self.org_flops + info_set = {'compress_ratio': compress_ratio, 'accuracy': acc, 'strategy': self.strategy.copy()} + reward = self.reward(self, acc, current_flops) + + if reward > self.best_reward: + self.best_reward = reward + self.best_strategy = self.strategy.copy() + self.best_d_prime_list = self.d_prime_list.copy() + torch.save(self.model.state_dict(), os.path.join(self.args.output, 'best_wrapped_model.pth')) + prGreen('New best reward: {:.4f}, acc: {:.4f}, compress: {:.4f}'.format(self.best_reward, acc, compress_ratio)) + prGreen('New best policy: {}'.format(self.best_strategy)) + prGreen('New best d primes: {}'.format(self.best_d_prime_list)) + obs = self.layer_embedding[self.cur_ind, :].copy() # actually the same as the last state + done = True + return obs, reward, done, info_set + + info_set = None + reward = 0 + done = False + self.visited[self.cur_ind] = True # set to visited + self.cur_ind += 1 # the index of next layer + # build next state (in-place modify) + self.layer_embedding[self.cur_ind][-3] = self._cur_reduced() * 1. / self.org_flops # reduced + self.layer_embedding[self.cur_ind][-2] = sum(self.flops_list[self.cur_ind + 1:]) * 1. / self.org_flops # rest + self.layer_embedding[self.cur_ind][-1] = self.strategy[-1] # last action + obs = self.layer_embedding[self.cur_ind, :].copy() + + return obs, reward, done, info_set + + def reset(self): + # restore env by loading the checkpoint + self.pruner.reset(self.checkpoint) + self.cur_ind = 0 + self.strategy = [] # pruning strategy + self.d_prime_list = [] + self.strategy_dict = copy.deepcopy(self.min_strategy_dict) + # reset layer embeddings + self.layer_embedding[:, -1] = 1. + self.layer_embedding[:, -2] = 0. + self.layer_embedding[:, -3] = 0. + obs = self.layer_embedding[0].copy() + obs[-2] = sum(self.wsize_list[1:]) * 1. / sum(self.wsize_list) + self.extract_time = 0 + self.fit_time = 0 + self.val_time = 0 + # for share index + self.visited = [False] * len(self.prunable_idx) + self.index_buffer = {} + return obs + + def set_export_path(self, path): + self.export_path = path + + def prune_kernel(self, op_idx, preserve_ratio, preserve_idx=None): + m_list = list(self.model.modules()) + op = m_list[op_idx] + assert (0. < preserve_ratio <= 1.) + assert type(op) == PrunerModuleWrapper + if preserve_ratio == 1: # do not prune + if (preserve_idx is None) or (len(preserve_idx) == op.module.weight.size(1)): + return 1., op.module.weight.size(1), None # should be a full index + op.input_feat = self.layer_info_dict[op_idx]['input_feat'] + op.output_feat = self.layer_info_dict[op_idx]['output_feat'] + + masks = self.channel_prune_masker.calc_mask(sparsity=1-preserve_ratio, wrapper=op, preserve_idx=preserve_idx) + m = masks['weight_mask'].cpu().data + if type(op.module) == nn.Conv2d: + d_prime = (m.sum((0, 2, 3)) > 0).sum().item() + preserve_idx = np.nonzero((m.sum((0, 2, 3)) > 0).numpy())[0] + else: + assert type(op.module) == nn.Linear + d_prime = (m.sum(1) > 0).sum().item() + preserve_idx = np.nonzero((m.sum(1) > 0).numpy())[0] + + op.weight_mask = masks['weight_mask'] + if hasattr(op.module, 'bias') and op.module.bias is not None and 'bias_mask' in masks: + op.bias_mask = masks['bias_mask'] + + action = (m == 1).sum().item() / m.numel() + return action, d_prime, preserve_idx + + def export_model(self): + while True: + self.export_layer(self.prunable_idx[self.cur_ind]) + if self._is_final_layer(): + break + self.cur_ind += 1 + + #TODO replace this speedup implementation with nni.compression.torch.ModelSpeedup + def export_layer(self, op_idx): + m_list = list(self.model.modules()) + op = m_list[op_idx] + assert type(op) == PrunerModuleWrapper + w = op.module.weight.cpu().data + m = op.weight_mask.cpu().data + if type(op.module) == nn.Linear: + w = w.unsqueeze(-1).unsqueeze(-1) + m = m.unsqueeze(-1).unsqueeze(-1) + + d_prime = (m.sum((0, 2, 3)) > 0).sum().item() + preserve_idx = np.nonzero((m.sum((0, 2, 3)) > 0).numpy())[0] + assert d_prime <= w.size(1) + + if d_prime == w.size(1): + return + + mask = np.zeros(w.size(1), bool) + mask[preserve_idx] = True + rec_weight = torch.zeros((w.size(0), d_prime, w.size(2), w.size(3))) + rec_weight = w[:, preserve_idx, :, :] + if type(op.module) == nn.Linear: + rec_weight = rec_weight.squeeze() + # no need to provide bias mask for channel pruning + rec_mask = torch.ones_like(rec_weight) + + # assign new weight and mask + device = op.module.weight.device + op.module.weight.data = rec_weight.to(device) + op.weight_mask = rec_mask.to(device) + if type(op.module) == nn.Conv2d: + op.module.in_channels = d_prime + else: + # Linear + op.module.in_features = d_prime + + # export prev layers + prev_idx = self.prunable_idx[self.prunable_idx.index(op_idx) - 1] + for idx in range(prev_idx, op_idx): + m = m_list[idx] + if type(m) == nn.Conv2d: # depthwise + m.weight.data = m.weight.data[mask, :, :, :] + if m.groups == m.in_channels: + m.groups = int(np.sum(mask)) + m.out_channels = d_prime + elif type(m) == nn.BatchNorm2d: + m.weight.data = m.weight.data[mask] + m.bias.data = m.bias.data[mask] + m.running_mean.data = m.running_mean.data[mask] + m.running_var.data = m.running_var.data[mask] + m.num_features = d_prime + + def _is_final_layer(self): + return self.cur_ind == len(self.prunable_idx) - 1 + + def _action_wall(self, action): + """ + Limit the action generated by DDPG for this layer by two constraints: + 1. The total flops must meet the flops reduce target. + For example: the original flops of entire model is 1000, target flops ratio is 0.5, target flops + is 1000*0.5 = 500. The reduced flops of other layers is 400, so the remaining flops quota is 500-400=100, + if the total original flops of this layer is 250, then the maximum ratio is 100/250 = 0.4. So the + action of this layer can not be greater than 0.4. + 2. The action must be greater than lbound which is stored in self.strategy_dict. + """ + assert len(self.strategy) == self.cur_ind + + action = float(action) + action = np.clip(action, 0, 1) + + other_comp = 0 + this_comp = 0 + for i, idx in enumerate(self.prunable_idx): + flop = self.layer_info_dict[idx]['flops'] + buffer_flop = self._get_buffer_flops(idx) + + if i == self.cur_ind - 1: # TODO: add other member in the set + this_comp += flop * self.strategy_dict[idx][0] + # add buffer (but not influenced by ratio) + other_comp += buffer_flop * self.strategy_dict[idx][0] + elif i == self.cur_ind: + this_comp += flop * self.strategy_dict[idx][1] + # also add buffer here (influenced by ratio) + this_comp += buffer_flop + else: + other_comp += flop * self.strategy_dict[idx][0] * self.strategy_dict[idx][1] + # add buffer + other_comp += buffer_flop * self.strategy_dict[idx][0] # only consider input reduction + + self.expected_min_preserve = other_comp + this_comp * action + max_preserve_ratio = (self.expected_preserve_computation - other_comp) * 1. / this_comp + + action = np.minimum(action, max_preserve_ratio) + action = np.maximum(action, self.strategy_dict[self.prunable_idx[self.cur_ind]][0]) # impossible (should be) + + return action + + def _get_buffer_flops(self, idx): + buffer_idx = self.buffer_dict[idx] + buffer_flop = sum([self.layer_info_dict[_]['flops'] for _ in buffer_idx]) + return buffer_flop + + def _cur_flops(self): + flops = 0 + for idx in self.prunable_idx: + c, n = self.strategy_dict[idx] # input, output pruning ratio + flops += self.layer_info_dict[idx]['flops'] * c * n + # add buffer computation + flops += self._get_buffer_flops(idx) * c # only related to input channel reduction + return flops + + def _cur_reduced(self): + # return the reduced weight + reduced = self.org_flops - self._cur_flops() + return reduced + + def _build_index(self): + """ + Build following information/data for later pruning: + self.prunable_idx: layer indices for pruable layers, the index values are the index + of list(self.model.modules()). Pruable layers are pointwise Conv2d layers and Linear + layers. + self.prunable_ops: prunable modules + self.buffer_idx: layer indices for buffer layers which refers the depthwise layers. + Each depthwise layer is always followd by a pointwise layer for both mobilenet and + mobilenetv2. The depthwise layer's filters are pruned when its next pointwise layer's + corresponding input channels are pruned. + self.shared_idx: layer indices for layers which share input. + For example: [[1,4], [8, 10, 15]] means layer 1 and 4 share same input, and layer + 8, 10 and 15 share another input. + self.org_channels: number of input channels for each layer + self.min_strategy_dict: key is layer index, value is a tuple, the first value is the minimum + action of input channel, the second value is the minimum action value of output channel. + self.strategy_dict: same as self.min_strategy_dict, but it will be updated later. + """ + self.prunable_idx = [] + self.prunable_ops = [] + self.layer_type_dict = {} + self.strategy_dict = {} + self.buffer_dict = {} + this_buffer_list = [] + self.org_channels = [] + # build index and the min strategy dict + for i, m in enumerate(self.model.modules()): + if isinstance(m, PrunerModuleWrapper): + m = m.module + if type(m) == nn.Conv2d and m.groups == m.in_channels: # depth-wise conv, buffer + this_buffer_list.append(i) + else: # really prunable + self.prunable_idx.append(i) + self.prunable_ops.append(m) + self.layer_type_dict[i] = type(m) + self.buffer_dict[i] = this_buffer_list + this_buffer_list = [] # empty + self.org_channels.append(m.in_channels if type(m) == nn.Conv2d else m.in_features) + + self.strategy_dict[i] = [self.lbound, self.lbound] + + self.strategy_dict[self.prunable_idx[0]][0] = 1 # modify the input + self.strategy_dict[self.prunable_idx[-1]][1] = 1 # modify the output + + self.shared_idx = [] + if self.args.model_type == 'mobilenetv2': # TODO: to be tested! Share index for residual connection + connected_idx = [4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32] # to be partitioned + last_ch = -1 + share_group = None + for c_idx in connected_idx: + if self.prunable_ops[c_idx].in_channels != last_ch: # new group + last_ch = self.prunable_ops[c_idx].in_channels + if share_group is not None: + self.shared_idx.append(share_group) + share_group = [c_idx] + else: # same group + share_group.append(c_idx) + self.shared_idx.append(share_group) + print('=> Conv layers to share channels: {}'.format(self.shared_idx)) + + self.min_strategy_dict = copy.deepcopy(self.strategy_dict) + + self.buffer_idx = [] + for _, v in self.buffer_dict.items(): + self.buffer_idx += v + + print('=> Prunable layer idx: {}'.format(self.prunable_idx)) + print('=> Buffer layer idx: {}'.format(self.buffer_idx)) + print('=> Shared idx: {}'.format(self.shared_idx)) + print('=> Initial min strategy dict: {}'.format(self.min_strategy_dict)) + + # added for supporting residual connections during pruning + self.visited = [False] * len(self.prunable_idx) + self.index_buffer = {} + + def _extract_layer_information(self): + m_list = list(self.model.modules()) + + self.data_saver = [] + self.layer_info_dict = dict() + self.wsize_list = [] + self.flops_list = [] + + from .lib.utils import measure_layer_for_pruning + + # extend the forward fn to record layer info + def new_forward(m): + def lambda_forward(x): + m.input_feat = x.clone() + #TODO replace this flops counter with nni.compression.torch.utils.counter.count_flops_params + measure_layer_for_pruning(m, x) + y = m.old_forward(x) + m.output_feat = y.clone() + return y + + return lambda_forward + + device = None + for idx in self.prunable_idx + self.buffer_idx: # get all + m = m_list[idx] + m.old_forward = m.forward + m.forward = new_forward(m) + if device is None and type(m) == PrunerModuleWrapper: + device = m.module.weight.device + + # now let the image flow + print('=> Extracting information...') + with torch.no_grad(): + for i_b, (inputs, target) in enumerate(self._val_loader): # use image from train set + if i_b == self.n_calibration_batches: + break + self.data_saver.append((inputs.clone(), target.clone())) + input_var = torch.autograd.Variable(inputs).to(device) + + # inference and collect stats + _ = self.model(input_var) + + if i_b == 0: # first batch + for idx in self.prunable_idx + self.buffer_idx: + self.layer_info_dict[idx] = dict() + self.layer_info_dict[idx]['params'] = m_list[idx].params + self.layer_info_dict[idx]['flops'] = m_list[idx].flops + self.wsize_list.append(m_list[idx].params) + self.flops_list.append(m_list[idx].flops) + print('flops:', self.flops_list) + for idx in self.prunable_idx: + f_in_np = m_list[idx].input_feat.data.cpu().numpy() + f_out_np = m_list[idx].output_feat.data.cpu().numpy() + if len(f_in_np.shape) == 4: # conv + if self.prunable_idx.index(idx) == 0: # first conv + f_in2save, f_out2save = None, None + elif m_list[idx].module.weight.size(3) > 1: # normal conv + f_in2save, f_out2save = f_in_np, f_out_np + else: # 1x1 conv + # assert f_out_np.shape[2] == f_in_np.shape[2] # now support k=3 + randx = np.random.randint(0, f_out_np.shape[2] - 0, self.n_points_per_layer) + randy = np.random.randint(0, f_out_np.shape[3] - 0, self.n_points_per_layer) + # input: [N, C, H, W] + self.layer_info_dict[idx][(i_b, 'randx')] = randx.copy() + self.layer_info_dict[idx][(i_b, 'randy')] = randy.copy() + + f_in2save = f_in_np[:, :, randx, randy].copy().transpose(0, 2, 1)\ + .reshape(self.batch_size * self.n_points_per_layer, -1) + + f_out2save = f_out_np[:, :, randx, randy].copy().transpose(0, 2, 1) \ + .reshape(self.batch_size * self.n_points_per_layer, -1) + else: + assert len(f_in_np.shape) == 2 + f_in2save = f_in_np.copy() + f_out2save = f_out_np.copy() + if 'input_feat' not in self.layer_info_dict[idx]: + self.layer_info_dict[idx]['input_feat'] = f_in2save + self.layer_info_dict[idx]['output_feat'] = f_out2save + else: + self.layer_info_dict[idx]['input_feat'] = np.vstack( + (self.layer_info_dict[idx]['input_feat'], f_in2save)) + self.layer_info_dict[idx]['output_feat'] = np.vstack( + (self.layer_info_dict[idx]['output_feat'], f_out2save)) + + def _build_state_embedding(self): + # build the static part of the state embedding + print('Building state embedding...') + layer_embedding = [] + module_list = list(self.model.modules()) + for i, ind in enumerate(self.prunable_idx): + m = module_list[ind].module + this_state = [] + if type(m) == nn.Conv2d: + this_state.append(i) # index + this_state.append(0) # layer type, 0 for conv + this_state.append(m.in_channels) # in channels + this_state.append(m.out_channels) # out channels + this_state.append(m.stride[0]) # stride + this_state.append(m.kernel_size[0]) # kernel size + this_state.append(np.prod(m.weight.size())) # weight size + elif type(m) == nn.Linear: + this_state.append(i) # index + this_state.append(1) # layer type, 1 for fc + this_state.append(m.in_features) # in channels + this_state.append(m.out_features) # out channels + this_state.append(0) # stride + this_state.append(1) # kernel size + this_state.append(np.prod(m.weight.size())) # weight size + + # this 3 features need to be changed later + this_state.append(0.) # reduced + this_state.append(0.) # rest + this_state.append(1.) # a_{t-1} + layer_embedding.append(np.array(this_state)) + + # normalize the state + layer_embedding = np.array(layer_embedding, 'float') + print('=> shape of embedding (n_layer * n_dim): {}'.format(layer_embedding.shape)) + assert len(layer_embedding.shape) == 2, layer_embedding.shape + for i in range(layer_embedding.shape[1]): + fmin = min(layer_embedding[:, i]) + fmax = max(layer_embedding[:, i]) + if fmax - fmin > 0: + layer_embedding[:, i] = (layer_embedding[:, i] - fmin) / (fmax - fmin) + + self.layer_embedding = layer_embedding + diff --git a/src/sdk/pynni/nni/compression/torch/pruning/amc/lib/__init__.py b/src/sdk/pynni/nni/compression/torch/pruning/amc/lib/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sdk/pynni/nni/compression/torch/pruning/amc/lib/agent.py b/src/sdk/pynni/nni/compression/torch/pruning/amc/lib/agent.py new file mode 100644 index 0000000000..fe066301b8 --- /dev/null +++ b/src/sdk/pynni/nni/compression/torch/pruning/amc/lib/agent.py @@ -0,0 +1,232 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import numpy as np + +import torch +import torch.nn as nn +from torch.optim import Adam + +from .memory import SequentialMemory +from .utils import to_numpy, to_tensor + +criterion = nn.MSELoss() +USE_CUDA = torch.cuda.is_available() + + +class Actor(nn.Module): + def __init__(self, nb_states, nb_actions, hidden1=400, hidden2=300): + super(Actor, self).__init__() + self.fc1 = nn.Linear(nb_states, hidden1) + self.fc2 = nn.Linear(hidden1, hidden2) + self.fc3 = nn.Linear(hidden2, nb_actions) + self.relu = nn.ReLU() + self.sigmoid = nn.Sigmoid() + + def forward(self, x): + out = self.fc1(x) + out = self.relu(out) + out = self.fc2(out) + out = self.relu(out) + out = self.fc3(out) + out = self.sigmoid(out) + return out + + +class Critic(nn.Module): + def __init__(self, nb_states, nb_actions, hidden1=400, hidden2=300): + super(Critic, self).__init__() + self.fc11 = nn.Linear(nb_states, hidden1) + self.fc12 = nn.Linear(nb_actions, hidden1) + self.fc2 = nn.Linear(hidden1, hidden2) + self.fc3 = nn.Linear(hidden2, 1) + self.relu = nn.ReLU() + + def forward(self, xs): + x, a = xs + out = self.fc11(x) + self.fc12(a) + out = self.relu(out) + out = self.fc2(out) + out = self.relu(out) + out = self.fc3(out) + return out + + +class DDPG(object): + def __init__(self, nb_states, nb_actions, args): + + self.nb_states = nb_states + self.nb_actions = nb_actions + + # Create Actor and Critic Network + net_cfg = { + 'hidden1': args.hidden1, + 'hidden2': args.hidden2, + # 'init_w': args.init_w + } + self.actor = Actor(self.nb_states, self.nb_actions, **net_cfg) + self.actor_target = Actor(self.nb_states, self.nb_actions, **net_cfg) + self.actor_optim = Adam(self.actor.parameters(), lr=args.lr_a) + + self.critic = Critic(self.nb_states, self.nb_actions, **net_cfg) + self.critic_target = Critic(self.nb_states, self.nb_actions, **net_cfg) + self.critic_optim = Adam(self.critic.parameters(), lr=args.lr_c) + + self.hard_update(self.actor_target, self.actor) # Make sure target is with the same weight + self.hard_update(self.critic_target, self.critic) + + # Create replay buffer + self.memory = SequentialMemory(limit=args.rmsize, window_length=args.window_length) + # self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions, theta=args.ou_theta, mu=args.ou_mu, + # sigma=args.ou_sigma) + + # Hyper-parameters + self.batch_size = args.bsize + self.tau = args.tau + self.discount = args.discount + self.depsilon = 1.0 / args.epsilon + self.lbound = 0. # args.lbound + self.rbound = 1. # args.rbound + + # noise + self.init_delta = args.init_delta + self.delta_decay = args.delta_decay + self.warmup = args.warmup + + # + self.epsilon = 1.0 + # self.s_t = None # Most recent state + # self.a_t = None # Most recent action + self.is_training = True + + # + if USE_CUDA: self.cuda() + + # moving average baseline + self.moving_average = None + self.moving_alpha = 0.5 # based on batch, so small + + def update_policy(self): + # Sample batch + state_batch, action_batch, reward_batch, \ + next_state_batch, terminal_batch = self.memory.sample_and_split(self.batch_size) + + # normalize the reward + batch_mean_reward = np.mean(reward_batch) + if self.moving_average is None: + self.moving_average = batch_mean_reward + else: + self.moving_average += self.moving_alpha * (batch_mean_reward - self.moving_average) + reward_batch -= self.moving_average + # if reward_batch.std() > 0: + # reward_batch /= reward_batch.std() + + # Prepare for the target q batch + with torch.no_grad(): + next_q_values = self.critic_target([ + to_tensor(next_state_batch), + self.actor_target(to_tensor(next_state_batch)), + ]) + + target_q_batch = to_tensor(reward_batch) + \ + self.discount * to_tensor(terminal_batch.astype(np.float)) * next_q_values + + # Critic update + self.critic.zero_grad() + + q_batch = self.critic([to_tensor(state_batch), to_tensor(action_batch)]) + + value_loss = criterion(q_batch, target_q_batch) + value_loss.backward() + self.critic_optim.step() + + # Actor update + self.actor.zero_grad() + + policy_loss = -self.critic([ # pylint: disable=all + to_tensor(state_batch), + self.actor(to_tensor(state_batch)) + ]) + + policy_loss = policy_loss.mean() + policy_loss.backward() + self.actor_optim.step() + + # Target update + self.soft_update(self.actor_target, self.actor) + self.soft_update(self.critic_target, self.critic) + + def eval(self): + self.actor.eval() + self.actor_target.eval() + self.critic.eval() + self.critic_target.eval() + + def cuda(self): + self.actor.cuda() + self.actor_target.cuda() + self.critic.cuda() + self.critic_target.cuda() + + def observe(self, r_t, s_t, s_t1, a_t, done): + if self.is_training: + self.memory.append(s_t, a_t, r_t, done) # save to memory + # self.s_t = s_t1 + + def random_action(self): + action = np.random.uniform(self.lbound, self.rbound, self.nb_actions) + # self.a_t = action + return action + + def select_action(self, s_t, episode): + # assert episode >= self.warmup, 'Episode: {} warmup: {}'.format(episode, self.warmup) + action = to_numpy(self.actor(to_tensor(np.array(s_t).reshape(1, -1)))).squeeze(0) + delta = self.init_delta * (self.delta_decay ** (episode - self.warmup)) + # action += self.is_training * max(self.epsilon, 0) * self.random_process.sample() + action = self.sample_from_truncated_normal_distribution(lower=self.lbound, upper=self.rbound, mu=action, sigma=delta) + action = np.clip(action, self.lbound, self.rbound) + + # self.a_t = action + return action + + def reset(self, obs): + pass + # self.s_t = obs + # self.random_process.reset_states() + + def load_weights(self, output): + if output is None: return + + self.actor.load_state_dict( + torch.load('{}/actor.pkl'.format(output)) + ) + + self.critic.load_state_dict( + torch.load('{}/critic.pkl'.format(output)) + ) + + def save_model(self, output): + torch.save( + self.actor.state_dict(), + '{}/actor.pkl'.format(output) + ) + torch.save( + self.critic.state_dict(), + '{}/critic.pkl'.format(output) + ) + + def soft_update(self, target, source): + for target_param, param in zip(target.parameters(), source.parameters()): + target_param.data.copy_( + target_param.data * (1.0 - self.tau) + param.data * self.tau + ) + + def hard_update(self, target, source): + for target_param, param in zip(target.parameters(), source.parameters()): + target_param.data.copy_(param.data) + + def sample_from_truncated_normal_distribution(self, lower, upper, mu, sigma, size=1): + from scipy import stats + return stats.truncnorm.rvs((lower-mu)/sigma, (upper-mu)/sigma, loc=mu, scale=sigma, size=size) + + diff --git a/src/sdk/pynni/nni/compression/torch/pruning/amc/lib/memory.py b/src/sdk/pynni/nni/compression/torch/pruning/amc/lib/memory.py new file mode 100644 index 0000000000..57bbcfceb8 --- /dev/null +++ b/src/sdk/pynni/nni/compression/torch/pruning/amc/lib/memory.py @@ -0,0 +1,227 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from __future__ import absolute_import +from collections import deque, namedtuple +import warnings +import random + +import numpy as np + +# [reference] https://github.com/matthiasplappert/keras-rl/blob/master/rl/memory.py + +# This is to be understood as a transition: Given `state0`, performing `action` +# yields `reward` and results in `state1`, which might be `terminal`. +Experience = namedtuple('Experience', 'state0, action, reward, state1, terminal1') + + +def sample_batch_indexes(low, high, size): + if high - low >= size: + # We have enough data. Draw without replacement, that is each index is unique in the + # batch. We cannot use `np.random.choice` here because it is horribly inefficient as + # the memory grows. See https://github.com/numpy/numpy/issues/2764 for a discussion. + # `random.sample` does the same thing (drawing without replacement) and is way faster. + r = range(low, high) + batch_idxs = random.sample(r, size) + else: + # Not enough data. Help ourselves with sampling from the range, but the same index + # can occur multiple times. This is not good and should be avoided by picking a + # large enough warm-up phase. + warnings.warn( + 'Not enough entries to sample without replacement. ' + 'Consider increasing your warm-up phase to avoid oversampling!') + batch_idxs = np.random.random_integers(low, high - 1, size=size) + assert len(batch_idxs) == size + return batch_idxs + + +class RingBuffer(object): + def __init__(self, maxlen): + self.maxlen = maxlen + self.start = 0 + self.length = 0 + self.data = [None for _ in range(maxlen)] + + def __len__(self): + return self.length + + def __getitem__(self, idx): + if idx < 0 or idx >= self.length: + raise KeyError() + return self.data[(self.start + idx) % self.maxlen] + + def append(self, v): + if self.length < self.maxlen: + # We have space, simply increase the length. + self.length += 1 + elif self.length == self.maxlen: + # No space, "remove" the first item. + self.start = (self.start + 1) % self.maxlen + else: + # This should never happen. + raise RuntimeError() + self.data[(self.start + self.length - 1) % self.maxlen] = v + + +def zeroed_observation(observation): + if hasattr(observation, 'shape'): + return np.zeros(observation.shape) + elif hasattr(observation, '__iter__'): + out = [] + for x in observation: + out.append(zeroed_observation(x)) + return out + else: + return 0. + + +class Memory(object): + def __init__(self, window_length, ignore_episode_boundaries=False): + self.window_length = window_length + self.ignore_episode_boundaries = ignore_episode_boundaries + + self.recent_observations = deque(maxlen=window_length) + self.recent_terminals = deque(maxlen=window_length) + + def sample(self, batch_size, batch_idxs=None): + raise NotImplementedError() + + def append(self, observation, action, reward, terminal, training=True): + self.recent_observations.append(observation) + self.recent_terminals.append(terminal) + + def get_recent_state(self, current_observation): + # This code is slightly complicated by the fact that subsequent observations might be + # from different episodes. We ensure that an experience never spans multiple episodes. + # This is probably not that important in practice but it seems cleaner. + state = [current_observation] + idx = len(self.recent_observations) - 1 + for offset in range(0, self.window_length - 1): + current_idx = idx - offset + current_terminal = self.recent_terminals[current_idx - 1] if current_idx - 1 >= 0 else False + if current_idx < 0 or (not self.ignore_episode_boundaries and current_terminal): + # The previously handled observation was terminal, don't add the current one. + # Otherwise we would leak into a different episode. + break + state.insert(0, self.recent_observations[current_idx]) + while len(state) < self.window_length: + state.insert(0, zeroed_observation(state[0])) + return state + + def get_config(self): + config = { + 'window_length': self.window_length, + 'ignore_episode_boundaries': self.ignore_episode_boundaries, + } + return config + + +class SequentialMemory(Memory): + def __init__(self, limit, **kwargs): + super(SequentialMemory, self).__init__(**kwargs) + + self.limit = limit + + # Do not use deque to implement the memory. This data structure may seem convenient but + # it is way too slow on random access. Instead, we use our own ring buffer implementation. + self.actions = RingBuffer(limit) + self.rewards = RingBuffer(limit) + self.terminals = RingBuffer(limit) + self.observations = RingBuffer(limit) + + def sample(self, batch_size, batch_idxs=None): + if batch_idxs is None: + # Draw random indexes such that we have at least a single entry before each + # index. + batch_idxs = sample_batch_indexes(0, self.nb_entries - 1, size=batch_size) + batch_idxs = np.array(batch_idxs) + 1 + assert np.min(batch_idxs) >= 1 + assert np.max(batch_idxs) < self.nb_entries + assert len(batch_idxs) == batch_size + + # Create experiences + experiences = [] + for idx in batch_idxs: + terminal0 = self.terminals[idx - 2] if idx >= 2 else False + while terminal0: + # Skip this transition because the environment was reset here. Select a new, random + # transition and use this instead. This may cause the batch to contain the same + # transition twice. + idx = sample_batch_indexes(1, self.nb_entries, size=1)[0] + terminal0 = self.terminals[idx - 2] if idx >= 2 else False + assert 1 <= idx < self.nb_entries + + # This code is slightly complicated by the fact that subsequent observations might be + # from different episodes. We ensure that an experience never spans multiple episodes. + # This is probably not that important in practice but it seems cleaner. + state0 = [self.observations[idx - 1]] + for offset in range(0, self.window_length - 1): + current_idx = idx - 2 - offset + current_terminal = self.terminals[current_idx - 1] if current_idx - 1 > 0 else False + if current_idx < 0 or (not self.ignore_episode_boundaries and current_terminal): + # The previously handled observation was terminal, don't add the current one. + # Otherwise we would leak into a different episode. + break + state0.insert(0, self.observations[current_idx]) + while len(state0) < self.window_length: + state0.insert(0, zeroed_observation(state0[0])) + action = self.actions[idx - 1] + reward = self.rewards[idx - 1] + terminal1 = self.terminals[idx - 1] + + # Okay, now we need to create the follow-up state. This is state0 shifted on timestep + # to the right. Again, we need to be careful to not include an observation from the next + # episode if the last state is terminal. + state1 = [np.copy(x) for x in state0[1:]] + state1.append(self.observations[idx]) + + assert len(state0) == self.window_length + assert len(state1) == len(state0) + experiences.append(Experience(state0=state0, action=action, reward=reward, + state1=state1, terminal1=terminal1)) + assert len(experiences) == batch_size + return experiences + + def sample_and_split(self, batch_size, batch_idxs=None): + experiences = self.sample(batch_size, batch_idxs) + + state0_batch = [] + reward_batch = [] + action_batch = [] + terminal1_batch = [] + state1_batch = [] + for e in experiences: + state0_batch.append(e.state0) + state1_batch.append(e.state1) + reward_batch.append(e.reward) + action_batch.append(e.action) + terminal1_batch.append(0. if e.terminal1 else 1.) + + # Prepare and validate parameters. + state0_batch = np.array(state0_batch, 'double').reshape(batch_size, -1) + state1_batch = np.array(state1_batch, 'double').reshape(batch_size, -1) + terminal1_batch = np.array(terminal1_batch, 'double').reshape(batch_size, -1) + reward_batch = np.array(reward_batch, 'double').reshape(batch_size, -1) + action_batch = np.array(action_batch, 'double').reshape(batch_size, -1) + + return state0_batch, action_batch, reward_batch, state1_batch, terminal1_batch + + def append(self, observation, action, reward, terminal, training=True): + super(SequentialMemory, self).append(observation, action, reward, terminal, training=training) + + # This needs to be understood as follows: in `observation`, take `action`, obtain `reward` + # and weather the next state is `terminal` or not. + if training: + self.observations.append(observation) + self.actions.append(action) + self.rewards.append(reward) + self.terminals.append(terminal) + + @property + def nb_entries(self): + return len(self.observations) + + def get_config(self): + config = super(SequentialMemory, self).get_config() + config['limit'] = self.limit + return config diff --git a/src/sdk/pynni/nni/compression/torch/pruning/amc/lib/net_measure.py b/src/sdk/pynni/nni/compression/torch/pruning/amc/lib/net_measure.py new file mode 100644 index 0000000000..b9ba133431 --- /dev/null +++ b/src/sdk/pynni/nni/compression/torch/pruning/amc/lib/net_measure.py @@ -0,0 +1,123 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch + +# [reference] https://github.com/ShichenLiu/CondenseNet/blob/master/utils.py + + +def get_num_gen(gen): + return sum(1 for _ in gen) + + +def is_leaf(model): + return get_num_gen(model.children()) == 0 + + +def get_layer_info(layer): + layer_str = str(layer) + type_name = layer_str[:layer_str.find('(')].strip() + return type_name + + +def get_layer_param(model): + import operator + import functools + + return sum([functools.reduce(operator.mul, i.size(), 1) for i in model.parameters()]) + +count_ops = 0 +count_params = 0 + +def measure_layer(layer, x): + global count_ops, count_params + delta_ops = 0 + delta_params = 0 + multi_add = 1 + type_name = get_layer_info(layer) + + # ops_conv + if type_name in ['Conv2d']: + out_h = int((x.size()[2] + 2 * layer.padding[0] - layer.kernel_size[0]) / + layer.stride[0] + 1) + out_w = int((x.size()[3] + 2 * layer.padding[1] - layer.kernel_size[1]) / + layer.stride[1] + 1) + delta_ops = layer.in_channels * layer.out_channels * layer.kernel_size[0] * \ + layer.kernel_size[1] * out_h * out_w / layer.groups * multi_add + delta_params = get_layer_param(layer) + + # ops_nonlinearity + elif type_name in ['ReLU']: + delta_ops = x.numel() / x.size(0) + delta_params = get_layer_param(layer) + + # ops_pooling + elif type_name in ['AvgPool2d']: + in_w = x.size()[2] + kernel_ops = layer.kernel_size * layer.kernel_size + out_w = int((in_w + 2 * layer.padding - layer.kernel_size) / layer.stride + 1) + out_h = int((in_w + 2 * layer.padding - layer.kernel_size) / layer.stride + 1) + delta_ops = x.size()[1] * out_w * out_h * kernel_ops + delta_params = get_layer_param(layer) + + elif type_name in ['AdaptiveAvgPool2d']: + delta_ops = x.size()[1] * x.size()[2] * x.size()[3] + delta_params = get_layer_param(layer) + + # ops_linear + elif type_name in ['Linear']: + weight_ops = layer.weight.numel() * multi_add + bias_ops = layer.bias.numel() + delta_ops = weight_ops + bias_ops + delta_params = get_layer_param(layer) + + # ops_nothing + elif type_name in ['BatchNorm2d', 'Dropout2d', 'DropChannel', 'Dropout']: + delta_params = get_layer_param(layer) + + # unknown layer type + else: + delta_params = get_layer_param(layer) + + count_ops += delta_ops + count_params += delta_params + + return + + +def measure_model(model, H, W): + global count_ops, count_params + count_ops = 0 + count_params = 0 + data = torch.zeros(2, 3, H, W).cuda() + + def should_measure(x): + return is_leaf(x) + + def modify_forward(model): + for child in model.children(): + if should_measure(child): + def new_forward(m): + def lambda_forward(x): + measure_layer(m, x) + return m.old_forward(x) + return lambda_forward + child.old_forward = child.forward + child.forward = new_forward(child) + else: + modify_forward(child) + + def restore_forward(model): + for child in model.children(): + # leaf node + if is_leaf(child) and hasattr(child, 'old_forward'): + child.forward = child.old_forward + child.old_forward = None + else: + restore_forward(child) + + modify_forward(model) + model.forward(data) + restore_forward(model) + + return count_ops, count_params diff --git a/src/sdk/pynni/nni/compression/torch/pruning/amc/lib/utils.py b/src/sdk/pynni/nni/compression/torch/pruning/amc/lib/utils.py new file mode 100644 index 0000000000..477efccb10 --- /dev/null +++ b/src/sdk/pynni/nni/compression/torch/pruning/amc/lib/utils.py @@ -0,0 +1,124 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +import torch + +class TextLogger(object): + """Write log immediately to the disk""" + def __init__(self, filepath): + self.f = open(filepath, 'w') + self.fid = self.f.fileno() + self.filepath = filepath + + def close(self): + self.f.close() + + def write(self, content): + self.f.write(content) + self.f.flush() + os.fsync(self.fid) + + def write_buf(self, content): + self.f.write(content) + + def print_and_write(self, content): + print(content) + self.write(content+'\n') + +def to_numpy(var): + use_cuda = torch.cuda.is_available() + return var.cpu().data.numpy() if use_cuda else var.data.numpy() + + +def to_tensor(ndarray, requires_grad=False): # return a float tensor by default + tensor = torch.from_numpy(ndarray).float() # by default does not require grad + if requires_grad: + tensor.requires_grad_() + return tensor.cuda() if torch.cuda.is_available() else tensor + + +def measure_layer_for_pruning(wrapper, x): + def get_layer_type(layer): + layer_str = str(layer) + return layer_str[:layer_str.find('(')].strip() + + def get_layer_param(model): + import operator + import functools + + return sum([functools.reduce(operator.mul, i.size(), 1) for i in model.parameters()]) + + multi_add = 1 + layer = wrapper.module + type_name = get_layer_type(layer) + + # ops_conv + if type_name in ['Conv2d']: + out_h = int((x.size()[2] + 2 * layer.padding[0] - layer.kernel_size[0]) / + layer.stride[0] + 1) + out_w = int((x.size()[3] + 2 * layer.padding[1] - layer.kernel_size[1]) / + layer.stride[1] + 1) + wrapper.flops = layer.in_channels * layer.out_channels * layer.kernel_size[0] * \ + layer.kernel_size[1] * out_h * out_w / layer.groups * multi_add + wrapper.params = get_layer_param(layer) + # ops_linear + elif type_name in ['Linear']: + weight_ops = layer.weight.numel() * multi_add + bias_ops = layer.bias.numel() + wrapper.flops = weight_ops + bias_ops + wrapper.params = get_layer_param(layer) + return + + +def least_square_sklearn(X, Y): + from sklearn.linear_model import LinearRegression + reg = LinearRegression(fit_intercept=False) + reg.fit(X, Y) + return reg.coef_ + + +def get_output_folder(parent_dir, env_name): + """Return save folder. + Assumes folders in the parent_dir have suffix -run{run + number}. Finds the highest run number and sets the output folder + to that number + 1. This is just convenient so that if you run the + same script multiple times tensorboard can plot all of the results + on the same plots with different names. + Parameters + ---------- + parent_dir: str + Path of the directory containing all experiment runs. + Returns + ------- + parent_dir/run_dir + Path to this run's save directory. + """ + os.makedirs(parent_dir, exist_ok=True) + experiment_id = 0 + for folder_name in os.listdir(parent_dir): + if not os.path.isdir(os.path.join(parent_dir, folder_name)): + continue + try: + folder_name = int(folder_name.split('-run')[-1]) + if folder_name > experiment_id: + experiment_id = folder_name + except: + pass + experiment_id += 1 + + parent_dir = os.path.join(parent_dir, env_name) + parent_dir = parent_dir + '-run{}'.format(experiment_id) + os.makedirs(parent_dir, exist_ok=True) + return parent_dir + + +# logging +def prRed(prt): print("\033[91m {}\033[00m" .format(prt)) +def prGreen(prt): print("\033[92m {}\033[00m" .format(prt)) +def prYellow(prt): print("\033[93m {}\033[00m" .format(prt)) +def prLightPurple(prt): print("\033[94m {}\033[00m" .format(prt)) +def prPurple(prt): print("\033[95m {}\033[00m" .format(prt)) +def prCyan(prt): print("\033[96m {}\033[00m" .format(prt)) +def prLightGray(prt): print("\033[97m {}\033[00m" .format(prt)) +def prBlack(prt): print("\033[98m {}\033[00m" .format(prt)) diff --git a/src/sdk/pynni/nni/compression/torch/pruning/structured_pruning.py b/src/sdk/pynni/nni/compression/torch/pruning/structured_pruning.py index 8fb203452c..e1b3dc12ce 100644 --- a/src/sdk/pynni/nni/compression/torch/pruning/structured_pruning.py +++ b/src/sdk/pynni/nni/compression/torch/pruning/structured_pruning.py @@ -2,19 +2,40 @@ # Licensed under the MIT license. import logging +import math +import numpy as np import torch from .weight_masker import WeightMasker __all__ = ['L1FilterPrunerMasker', 'L2FilterPrunerMasker', 'FPGMPrunerMasker', \ 'TaylorFOWeightFilterPrunerMasker', 'ActivationAPoZRankFilterPrunerMasker', \ - 'ActivationMeanRankFilterPrunerMasker', 'SlimPrunerMasker'] + 'ActivationMeanRankFilterPrunerMasker', 'SlimPrunerMasker', 'AMCWeightMasker'] logger = logging.getLogger('torch filter pruners') class StructuredWeightMasker(WeightMasker): """ A structured pruning masker base class that prunes convolutional layer filters. + + Parameters + ---------- + model: nn.Module + model to be pruned + pruner: Pruner + A Pruner instance used to prune the model + preserve_round: int + after pruning, preserve filters/channels round to `preserve_round`, for example: + for a Conv2d layer, output channel is 32, sparsity is 0.2, if preserve_round is + 1 (no preserve round), then there will be int(32 * 0.2) = 6 filters pruned, and + 32 - 6 = 26 filters are preserved. If preserve_round is 4, preserved filters will + be round up to 28 (which can be divided by 4) and only 4 filters are pruned. + """ + def __init__(self, model, pruner, preserve_round=1): + self.model = model + self.pruner = pruner + self.preserve_round = preserve_round + def calc_mask(self, sparsity, wrapper, wrapper_idx=None): """ Calculate the mask of given layer. @@ -53,9 +74,16 @@ def calc_mask(self, sparsity, wrapper, wrapper_idx=None): mask_bias = None mask = {'weight_mask': mask_weight, 'bias_mask': mask_bias} - filters = weight.size(0) - num_prune = int(filters * sparsity) - if filters < 2 or num_prune < 1: + num_total = weight.size(0) + num_prune = int(num_total * sparsity) + if self.preserve_round > 1: + num_preserve = num_total - num_prune + num_preserve = int(math.ceil(num_preserve * 1. / self.preserve_round) * self.preserve_round) + if num_preserve > num_total: + num_preserve = int(math.floor(num_total * 1. / self.preserve_round) * self.preserve_round) + num_prune = num_total - num_preserve + + if num_total < 2 or num_prune < 1: return mask # weight*mask_weight: apply base mask for iterative pruning return self.get_mask(mask, weight*mask_weight, num_prune, wrapper, wrapper_idx) @@ -365,3 +393,135 @@ def calc_mask(self, sparsity, wrapper, wrapper_idx=None): mask_bias = mask_weight.clone() mask = {'weight_mask': mask_weight.detach(), 'bias_mask': mask_bias.detach()} return mask + +def least_square_sklearn(X, Y): + from sklearn.linear_model import LinearRegression + reg = LinearRegression(fit_intercept=False) + reg.fit(X, Y) + return reg.coef_ + +class AMCWeightMasker(WeightMasker): + """ + Weight maskser class for AMC pruner. Currently, AMCPruner only supports pruning kernel + size 1x1 pointwise Conv2d layer. Before using this class to prune kernels, AMCPruner + collected input and output feature maps for each layer, the features maps are flattened + and save into wrapper.input_feat and wrapper.output_feat. + + Parameters + ---------- + model: nn.Module + model to be pruned + pruner: Pruner + A Pruner instance used to prune the model + preserve_round: int + after pruning, preserve filters/channels round to `preserve_round`, for example: + for a Conv2d layer, output channel is 32, sparsity is 0.2, if preserve_round is + 1 (no preserve round), then there will be int(32 * 0.2) = 6 filters pruned, and + 32 - 6 = 26 filters are preserved. If preserve_round is 4, preserved filters will + be round up to 28 (which can be divided by 4) and only 4 filters are pruned. + """ + def __init__(self, model, pruner, preserve_round=1): + self.model = model + self.pruner = pruner + self.preserve_round = preserve_round + + def calc_mask(self, sparsity, wrapper, wrapper_idx=None, preserve_idx=None): + """ + Calculate the mask of given layer. + Parameters + ---------- + sparsity: float + pruning ratio, preserved weight ratio is `1 - sparsity` + wrapper: PrunerModuleWrapper + layer wrapper of this layer + wrapper_idx: int + index of this wrapper in pruner's all wrappers + Returns + ------- + dict + dictionary for storing masks, keys of the dict: + 'weight_mask': weight mask tensor + 'bias_mask': bias mask tensor (optional) + """ + msg = 'module type {} is not supported!'.format(wrapper.type) + assert wrapper.type in ['Conv2d', 'Linear'], msg + weight = wrapper.module.weight.data + bias = None + if hasattr(wrapper.module, 'bias') and wrapper.module.bias is not None: + bias = wrapper.module.bias.data + + if wrapper.weight_mask is None: + mask_weight = torch.ones(weight.size()).type_as(weight).detach() + else: + mask_weight = wrapper.weight_mask.clone() + if bias is not None: + if wrapper.bias_mask is None: + mask_bias = torch.ones(bias.size()).type_as(bias).detach() + else: + mask_bias = wrapper.bias_mask.clone() + else: + mask_bias = None + mask = {'weight_mask': mask_weight, 'bias_mask': mask_bias} + + num_total = weight.size(1) + num_prune = int(num_total * sparsity) + if self.preserve_round > 1: + num_preserve = num_total - num_prune + num_preserve = int(math.ceil(num_preserve * 1. / self.preserve_round) * self.preserve_round) + if num_preserve > num_total: + num_preserve = num_total + num_prune = num_total - num_preserve + + if (num_total < 2 or num_prune < 1) and preserve_idx is None: + return mask + + return self.get_mask(mask, weight, num_preserve, wrapper, wrapper_idx, preserve_idx) + + def get_mask(self, base_mask, weight, num_preserve, wrapper, wrapper_idx, preserve_idx): + w = weight.data.cpu().numpy() + if wrapper.type == 'Linear': + w = w[:, :, None, None] + + if preserve_idx is None: + importance = np.abs(w).sum((0, 2, 3)) + sorted_idx = np.argsort(-importance) # sum magnitude along C_in, sort descend + d_prime = num_preserve + preserve_idx = sorted_idx[:d_prime] # to preserve index + else: + d_prime = len(preserve_idx) + + assert len(preserve_idx) == d_prime + mask = np.zeros(w.shape[1], bool) + mask[preserve_idx] = True + + # reconstruct, X, Y <= [N, C] + X, Y = wrapper.input_feat, wrapper.output_feat + masked_X = X[:, mask] + if w.shape[2] == 1: # 1x1 conv or fc + rec_weight = least_square_sklearn(X=masked_X, Y=Y) + rec_weight = rec_weight.reshape(-1, 1, 1, d_prime) # (C_out, K_h, K_w, C_in') + rec_weight = np.transpose(rec_weight, (0, 3, 1, 2)) # (C_out, C_in', K_h, K_w) + else: + raise NotImplementedError('Current code only supports 1x1 conv now!') + rec_weight_pad = np.zeros_like(w) + # pylint: disable=all + rec_weight_pad[:, mask, :, :] = rec_weight + rec_weight = rec_weight_pad + + if wrapper.type == 'Linear': + rec_weight = rec_weight.squeeze() + assert len(rec_weight.shape) == 2 + + # now assign + wrapper.module.weight.data = torch.from_numpy(rec_weight).to(weight.device) + + mask_weight = torch.zeros_like(weight) + if wrapper.type == 'Linear': + mask_weight[:, preserve_idx] = 1. + if base_mask['bias_mask'] is not None and wrapper.module.bias is not None: + mask_bias = torch.ones_like(wrapper.module.bias) + else: + mask_weight[:, preserve_idx, :, :] = 1. + mask_bias = None + + return {'weight_mask': mask_weight.detach(), 'bias_mask': mask_bias} diff --git a/src/sdk/pynni/tests/models/pytorch_models/mobilenet.py b/src/sdk/pynni/tests/models/pytorch_models/mobilenet.py new file mode 100644 index 0000000000..8d60c90a4c --- /dev/null +++ b/src/sdk/pynni/tests/models/pytorch_models/mobilenet.py @@ -0,0 +1,83 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import torch.nn as nn +import math + + +def conv_bn(inp, oup, stride): + return nn.Sequential( + nn.Conv2d(inp, oup, 3, stride, 1, bias=False), + nn.BatchNorm2d(oup), + nn.ReLU(inplace=True) + ) + + +def conv_dw(inp, oup, stride): + return nn.Sequential( + nn.Conv2d(inp, inp, 3, stride, 1, groups=inp, bias=False), + nn.BatchNorm2d(inp), + nn.ReLU(inplace=True), + + nn.Conv2d(inp, oup, 1, 1, 0, bias=False), + nn.BatchNorm2d(oup), + nn.ReLU(inplace=True), + ) + + +class MobileNet(nn.Module): + def __init__(self, n_class, profile='normal'): + super(MobileNet, self).__init__() + + # original + if profile == 'normal': + in_planes = 32 + cfg = [64, (128, 2), 128, (256, 2), 256, (512, 2), 512, 512, 512, 512, 512, (1024, 2), 1024] + # 0.5 AMC + elif profile == '0.5flops': + in_planes = 24 + cfg = [48, (96, 2), 80, (192, 2), 200, (328, 2), 352, 368, 360, 328, 400, (736, 2), 752] + else: + raise NotImplementedError + + self.conv1 = conv_bn(3, in_planes, stride=2) + + self.features = self._make_layers(in_planes, cfg, conv_dw) + + self.classifier = nn.Sequential( + nn.Linear(cfg[-1], n_class), + ) + + self._initialize_weights() + + def forward(self, x): + x = self.conv1(x) + x = self.features(x) + x = x.mean(3).mean(2) # global average pooling + + x = self.classifier(x) + return x + + def _make_layers(self, in_planes, cfg, layer): + layers = [] + for x in cfg: + out_planes = x if isinstance(x, int) else x[0] + stride = 1 if isinstance(x, int) else x[1] + layers.append(layer(in_planes, out_planes, stride)) + in_planes = out_planes + return nn.Sequential(*layers) + + def _initialize_weights(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + if m.bias is not None: + m.bias.data.zero_() + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + n = m.weight.size(1) + m.weight.data.normal_(0, 0.01) + m.bias.data.zero_() diff --git a/src/sdk/pynni/tests/test_pruners.py b/src/sdk/pynni/tests/test_pruners.py index 9bba85e3e4..7157d9bc08 100644 --- a/src/sdk/pynni/tests/test_pruners.py +++ b/src/sdk/pynni/tests/test_pruners.py @@ -5,11 +5,14 @@ import torch import torch.nn as nn import torch.nn.functional as F +import torch.utils.data import math from unittest import TestCase, main from nni.compression.torch import LevelPruner, SlimPruner, FPGMPruner, L1FilterPruner, \ L2FilterPruner, AGPPruner, ActivationMeanRankFilterPruner, ActivationAPoZRankFilterPruner, \ - TaylorFOWeightFilterPruner, NetAdaptPruner, SimulatedAnnealingPruner, ADMMPruner, AutoCompressPruner + TaylorFOWeightFilterPruner, NetAdaptPruner, SimulatedAnnealingPruner, ADMMPruner, \ + AutoCompressPruner, AMCPruner +from models.pytorch_models.mobilenet import MobileNet def validate_sparsity(wrapper, sparsity, bias=False): masks = [wrapper.weight_mask] @@ -154,6 +157,12 @@ def validate_sparsity(wrapper, sparsity, bias=False): 'evaluator': lambda model: 0.9, 'dummy_input': torch.randn([64, 1, 28, 28]), 'validators': [] + }, + 'amc': { + 'pruner_class': AMCPruner, + 'config_list':[{ + 'op_types': ['Conv2d', 'Linear'] + }] } } @@ -244,6 +253,13 @@ def test_agp(pruning_algorithm): # set abs_tol = 0.2, considering the sparsity error for channel pruning when number of channels is small. assert math.isclose(actual_sparsity, target_sparsity, abs_tol=0.2) +class SimpleDataset: + def __getitem__(self, index): + return torch.randn(3, 32, 32), 1. + + def __len__(self): + return 1000 + class PrunerTestCase(TestCase): def test_pruners(self): pruners_test(bias=True) @@ -259,5 +275,15 @@ def test_agp_pruner(self): prune_config['agp']['config_list'][0]['op_types'] = ['default'] test_agp(pruning_algorithm) + def testAMC(self): + model = MobileNet(n_class=10) + + def validate(val_loader, model): + return 80. + val_loader = torch.utils.data.DataLoader(SimpleDataset(), batch_size=16, shuffle=False, drop_last=True) + config_list = prune_config['amc']['config_list'] + pruner = AMCPruner(model, config_list, validate, val_loader, train_episode=1) + pruner.compress() + if __name__ == '__main__': main() From 0c4e84bb61b30eb753effd5abdc2a293037b178a Mon Sep 17 00:00:00 2001 From: Yuge Zhang Date: Wed, 12 Aug 2020 20:32:17 +0800 Subject: [PATCH 25/28] Upgrade Dockerfile to Ubuntu 18.04 (#2783) * Upgrade dockerfile to ubuntu 18.04 * Upgrade to cuda 9.2 * Set frontend noninteractive * Upgrade tensorflow to 1.15.2 * Fix pip version --- deployment/docker/Dockerfile | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/deployment/docker/Dockerfile b/deployment/docker/Dockerfile index 493cdba17b..b5418ec783 100644 --- a/deployment/docker/Dockerfile +++ b/deployment/docker/Dockerfile @@ -1,12 +1,13 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -FROM nvidia/cuda:9.0-cudnn7-runtime-ubuntu16.04 +FROM nvidia/cuda:9.2-cudnn7-runtime-ubuntu18.04 LABEL maintainer='Microsoft NNI Team' -RUN DEBIAN_FRONTEND=noninteractive && \ - apt-get -y update && \ +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get -y update && \ apt-get -y install sudo \ apt-utils \ git \ @@ -21,7 +22,7 @@ RUN DEBIAN_FRONTEND=noninteractive && \ openssh-client \ openssh-server \ lsof \ - python3.5 \ + python3.6 \ python3-dev \ python3-pip \ python3-tk \ @@ -37,7 +38,7 @@ RUN cp /usr/bin/python3 /usr/bin/python # # update pip # -RUN python3 -m pip install --upgrade pip setuptools==39.1.0 +RUN python3 -m pip install --upgrade pip==20.0.2 setuptools==39.1.0 # numpy 1.14.3 scipy 1.1.0 RUN python3 -m pip --no-cache-dir install \ @@ -46,7 +47,7 @@ RUN python3 -m pip --no-cache-dir install \ # # Tensorflow 1.15 # -RUN python3 -m pip --no-cache-dir install tensorflow-gpu==1.15 +RUN python3 -m pip --no-cache-dir install tensorflow-gpu==1.15.0 # # Keras 2.1.6 From f6991e8a3901ca277e33ac83f29212244da23c1b Mon Sep 17 00:00:00 2001 From: lin bin <756691769@qq.com> Date: Wed, 12 Aug 2020 20:32:45 +0800 Subject: [PATCH 26/28] fix break windows local and remote pipelines (#2785) * fix break windows local and remote pipelines * python os to separate platform * Update integration_tests.yml Co-authored-by: Yuge Zhang --- test/config/integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/config/integration_tests.yml b/test/config/integration_tests.yml index 4078321479..dfd34e9ae2 100644 --- a/test/config/integration_tests.yml +++ b/test/config/integration_tests.yml @@ -158,7 +158,7 @@ testCases: configFile: test/config/examples/sklearn-regression.yml setExperimentIdtoVar: $resumeExpId # for subfolder in codedir test - launchCommand: mkdir -p ../examples/trials/sklearn/regression/subfolder && touch ../examples/trials/sklearn/regression/subfolder/subfile && nnictl create --config $configFile --debug + launchCommand: python3 -c "import os; os.makedirs('../examples/trials/sklearn/regression/subfolder', exist_ok=True); open('../examples/trials/sklearn/regression/subfolder/subfile', 'a').close()" && nnictl create --config $configFile --debug # Experiment resume test part 2 - name: nnictl-resume-2 From bbda6a8a6553202b8738523f74fc58658ff6ec4e Mon Sep 17 00:00:00 2001 From: chicm-ms <38930155+chicm-ms@users.noreply.github.com> Date: Thu, 13 Aug 2020 09:59:10 +0800 Subject: [PATCH 27/28] Replace tensorboardX with torch.utils.tensorboard (#2786) --- azure-pipelines.yml | 4 ---- .../pynni/nni/compression/torch/pruning/amc/amc_pruner.py | 5 ++--- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 2889a8df67..78917d879f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -28,7 +28,6 @@ jobs: set -e sudo apt-get install -y pandoc python3 -m pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html --user - python3 -m pip install tensorboardX==1.9 python3 -m pip install tensorflow==2.2.0 --user python3 -m pip install keras==2.4.2 --user python3 -m pip install gym onnx peewee thop --user @@ -69,7 +68,6 @@ jobs: - script: | set -e python3 -m pip install torch==1.3.1+cpu torchvision==0.4.2+cpu -f https://download.pytorch.org/whl/torch_stable.html --user - python3 -m pip install tensorboardX==1.9 python3 -m pip install tensorflow==1.15.2 --user python3 -m pip install keras==2.1.6 --user python3 -m pip install gym onnx peewee --user @@ -119,7 +117,6 @@ jobs: set -e # pytorch Mac binary does not support CUDA, default is cpu version python3 -m pip install torchvision==0.6.0 torch==1.5.0 --user - python3 -m pip install tensorboardX==1.9 python3 -m pip install tensorflow==1.15.2 --user brew install swig@3 rm -f /usr/local/bin/swig @@ -147,7 +144,6 @@ jobs: python -m pip install scikit-learn==0.23.2 --user python -m pip install keras==2.1.6 --user python -m pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html --user - python -m pip install tensorboardX==1.9 python -m pip install tensorflow==1.15.2 --user displayName: 'Install dependencies' - script: | diff --git a/src/sdk/pynni/nni/compression/torch/pruning/amc/amc_pruner.py b/src/sdk/pynni/nni/compression/torch/pruning/amc/amc_pruner.py index 5f3e1ce6be..2852fe6266 100644 --- a/src/sdk/pynni/nni/compression/torch/pruning/amc/amc_pruner.py +++ b/src/sdk/pynni/nni/compression/torch/pruning/amc/amc_pruner.py @@ -6,6 +6,7 @@ from argparse import Namespace import numpy as np import torch +from torch.utils.tensorboard import SummaryWriter from nni.compression.torch.compressor import Pruner from .channel_pruning_env import ChannelPruningEnv @@ -148,8 +149,6 @@ def __init__( epsilon=50000, seed=None): - from tensorboardX import SummaryWriter - self.job = job self.searched_model_path = searched_model_path self.export_path = export_path @@ -189,7 +188,7 @@ def __init__( if self.job == 'train_export': print('=> Saving logs to {}'.format(self.output_dir)) - self.tfwriter = SummaryWriter(logdir=self.output_dir) + self.tfwriter = SummaryWriter(log_dir=self.output_dir) self.text_writer = open(os.path.join(self.output_dir, 'log.txt'), 'w') print('=> Output path: {}...'.format(self.output_dir)) From 3fdbbdb3afbfe9564222a8e8a2381e1651a08a62 Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Thu, 13 Aug 2020 16:52:52 +0800 Subject: [PATCH 28/28] Fix remote pipeline (#2787) --- test/nni_test/nnitest/remote_docker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/nni_test/nnitest/remote_docker.py b/test/nni_test/nnitest/remote_docker.py index d0252e3feb..2c89c34374 100644 --- a/test/nni_test/nnitest/remote_docker.py +++ b/test/nni_test/nnitest/remote_docker.py @@ -37,7 +37,7 @@ def start_container(image, name, nnimanager_os): '''Start docker container, generate a port in /tmp/nnitest/{name}/port file''' port = find_port() source_dir = '/tmp/nnitest/' + name - run_cmds = ['docker', 'run', '-d', '-p', str(port) + ':22', '--name', name, '--mount', 'type=bind,source=' + source_dir + ',target=/tmp/nni', image] + run_cmds = ['docker', 'run', '-d', '-t', '-p', str(port) + ':22', '--name', name, '--mount', 'type=bind,source=' + source_dir + ',target=/tmp/nni', image] output = check_output(run_cmds) commit_id = output.decode('utf-8') @@ -57,7 +57,7 @@ def get_dist(wheel_name): else: return '/tmp/nni/dist/{0}'.format(wheel_name) - pip_cmds = ['docker', 'exec', name, 'python3', '-m', 'pip', 'install', '--upgrade', 'pip', 'setuptools==39.1.0'] + pip_cmds = ['docker', 'exec', name, 'python3', '-m', 'pip', 'install', '--upgrade', 'pip', 'setuptools==41.0.0'] check_call(pip_cmds) sdk_cmds = ['docker', 'exec', name, 'python3', '-m', 'pip', 'install', get_dist(wheel_name)] check_call(sdk_cmds)