From ec662e4e9f8e5f4b0b8c17196acd4a5be096b7e7 Mon Sep 17 00:00:00 2001 From: John St John Date: Tue, 4 Mar 2025 01:20:12 +0000 Subject: [PATCH] Update the brca notebook with a run on an fp8 supporting machine --- .../examples/bionemo-evo2/.gitignore | 2 + .../bionemo-evo2/evo2_zeroshot_brca.ipynb | 985 +++++++++--------- 2 files changed, 512 insertions(+), 475 deletions(-) diff --git a/docs/docs/user-guide/examples/bionemo-evo2/.gitignore b/docs/docs/user-guide/examples/bionemo-evo2/.gitignore index fa7159094..465fe2329 100644 --- a/docs/docs/user-guide/examples/bionemo-evo2/.gitignore +++ b/docs/docs/user-guide/examples/bionemo-evo2/.gitignore @@ -10,3 +10,5 @@ nemo2_evo2_1b_8k/ preprocessed_data/ pretraining_demo/ +brca1_fasta_files/ +brca1/ diff --git a/docs/docs/user-guide/examples/bionemo-evo2/evo2_zeroshot_brca.ipynb b/docs/docs/user-guide/examples/bionemo-evo2/evo2_zeroshot_brca.ipynb index 02e1bbc20..7574ebc5d 100644 --- a/docs/docs/user-guide/examples/bionemo-evo2/evo2_zeroshot_brca.ipynb +++ b/docs/docs/user-guide/examples/bionemo-evo2/evo2_zeroshot_brca.ipynb @@ -13,19 +13,19 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[33mDEPRECATION: Loading egg at /usr/local/lib/python3.12/dist-packages/opt_einsum-3.4.0-py3.12.egg is deprecated. pip 25.1 will enforce this behaviour change. A possible replacement is to use pip for package installation. Discussion can be found at https://github.com/pypa/pip/issues/12330\u001b[0m\u001b[33m\n", - "\u001b[0m\u001b[33mDEPRECATION: Loading egg at /usr/local/lib/python3.12/dist-packages/nvfuser-0.2.23a0+6627725-py3.12-linux-x86_64.egg is deprecated. pip 25.1 will enforce this behaviour change. A possible replacement is to use pip for package installation. Discussion can be found at https://github.com/pypa/pip/issues/12330\u001b[0m\u001b[33m\n", - "\u001b[0m\u001b[33mDEPRECATION: Loading egg at /usr/local/lib/python3.12/dist-packages/lightning_utilities-0.12.0.dev0-py3.12.egg is deprecated. pip 25.1 will enforce this behaviour change. A possible replacement is to use pip for package installation. Discussion can be found at https://github.com/pypa/pip/issues/12330\u001b[0m\u001b[33m\n", - "\u001b[0m\u001b[33mDEPRECATION: Loading egg at /usr/local/lib/python3.12/dist-packages/dill-0.3.9-py3.12.egg is deprecated. pip 25.1 will enforce this behaviour change. A possible replacement is to use pip for package installation. Discussion can be found at https://github.com/pypa/pip/issues/12330\u001b[0m\u001b[33m\n", + "\u001b[33mDEPRECATION: Loading egg at /usr/local/lib/python3.12/dist-packages/lightning_utilities-0.12.0.dev0-py3.12.egg is deprecated. pip 25.1 will enforce this behaviour change. A possible replacement is to use pip for package installation. Discussion can be found at https://github.com/pypa/pip/issues/12330\u001b[0m\u001b[33m\n", "\u001b[0m\u001b[33mDEPRECATION: Loading egg at /usr/local/lib/python3.12/dist-packages/looseversion-1.3.0-py3.12.egg is deprecated. pip 25.1 will enforce this behaviour change. A possible replacement is to use pip for package installation. Discussion can be found at https://github.com/pypa/pip/issues/12330\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mDEPRECATION: Loading egg at /usr/local/lib/python3.12/dist-packages/nvfuser-0.2.23a0+6627725-py3.12-linux-x86_64.egg is deprecated. pip 25.1 will enforce this behaviour change. A possible replacement is to use pip for package installation. Discussion can be found at https://github.com/pypa/pip/issues/12330\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mDEPRECATION: Loading egg at /usr/local/lib/python3.12/dist-packages/opt_einsum-3.4.0-py3.12.egg is deprecated. pip 25.1 will enforce this behaviour change. A possible replacement is to use pip for package installation. Discussion can be found at https://github.com/pypa/pip/issues/12330\u001b[0m\u001b[33m\n", "\u001b[0m\u001b[33mDEPRECATION: Loading egg at /usr/local/lib/python3.12/dist-packages/lightning_thunder-0.2.0.dev0-py3.12.egg is deprecated. pip 25.1 will enforce this behaviour change. A possible replacement is to use pip for package installation. Discussion can be found at https://github.com/pypa/pip/issues/12330\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mDEPRECATION: Loading egg at /usr/local/lib/python3.12/dist-packages/dill-0.3.9-py3.12.egg is deprecated. pip 25.1 will enforce this behaviour change. A possible replacement is to use pip for package installation. Discussion can be found at https://github.com/pypa/pip/issues/12330\u001b[0m\u001b[33m\n", "\u001b[0mLooking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com\n", "Requirement already satisfied: biopython in /usr/local/lib/python3.12/dist-packages (1.85)\n", "Requirement already satisfied: openpyxl in /usr/local/lib/python3.12/dist-packages (3.1.5)\n", @@ -43,7 +43,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -71,7 +71,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -98,7 +98,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -239,7 +239,7 @@ "9 17 41276132 A T -0.207552 FUNC/INT" ] }, - "execution_count": 5, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -279,7 +279,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -316,28 +316,28 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "To make things run faster, we'll just look at a balanced sample of our data." + "To make things run faster, we'll just look at a balanced sample of our data. If you want to run on the full dataset, set `disable_sample=True`" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "(84, 6)" + "(330, 6)" ] }, - "execution_count": 7, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "disable_sample = False\n", - "SAMPLE_FRAC = 0.05\n", + "SAMPLE_FRAC = 0.2\n", "balanced_sample = True\n", "\n", "random_state = 42\n", @@ -365,15 +365,15 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Total unique reference sequences: 79\n", - "Total unique variant sequences: 84\n" + "Total unique reference sequences: 296\n", + "Total unique variant sequences: 330\n" ] } ], @@ -447,13 +447,13 @@ "\n", "Then, we load Evo 2 1B model, loading the Evo 2 weights from hugging face.\n", "\n", - "*Note - for better performance, load the 7b model by replacing all occurrences of `1b` below with `7b`.*\n", + "*Note - for better performance, load the 7b model by setting `MODEL_SIZE=\"7b\"` which also works well GPUs that do not support FP8.*\n", "\n" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -465,12 +465,13 @@ } ], "source": [ + "MODEL_SIZE = \"1b\" # also try 7b if you have a GPU with more than 32GB of memory\n", "# Define checkpoint path\n", - "checkpoint_path = Path(\"nemo2_evo2_1b_8k\")\n", + "checkpoint_path = Path(f\"nemo2_evo2_{MODEL_SIZE}_8k\")\n", "\n", "# Check if the directory does not exist or is empty\n", "if not checkpoint_path.exists() or not any(checkpoint_path.iterdir()):\n", - " !evo2_convert_to_nemo2 --model-path hf://arcinstitute/savanna_evo2_1b_base --model-size 1b --output-dir nemo2_evo2_1b_8k\n", + " !evo2_convert_to_nemo2 --model-path hf://arcinstitute/savanna_evo2_1b_base --model-size {MODEL_SIZE} --output-dir nemo2_evo2_{MODEL_SIZE}_8k\n", "else:\n", " print(\"Checkpoint directory is not empty. Skipping command.\")\n" ] @@ -484,40 +485,80 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "FP8 Support: True\n", + "Device: NVIDIA RTX 6000 Ada Generation, Compute Capability: 8.9\n" + ] + } + ], "source": [ "# Define output directories for prediction results\n", "predict_ref_dir = output_dir / \"reference_predictions\"\n", "predict_var_dir = output_dir / \"variant_predictions\"\n", "predict_ref_dir.mkdir(parents=True, exist_ok=True)\n", "predict_var_dir.mkdir(parents=True, exist_ok=True)\n", + "# Check if FP8 is supported on the current GPU\n", + "import torch\n", + "\n", + "def check_fp8_support():\n", + " \"\"\"\n", + " Check if FP8 is supported on the current GPU.\n", + " FP8 requires compute capability 8.9+ (Ada Lovelace/Hopper architecture or newer).\n", + " \"\"\"\n", + " if not torch.cuda.is_available():\n", + " return False, \"CUDA not available\"\n", + " \n", + " device_props = torch.cuda.get_device_properties(0)\n", + " compute_capability = f\"{device_props.major}.{device_props.minor}\"\n", + " device_name = device_props.name\n", + " \n", + " # FP8 is supported on compute capability 8.9+ (Ada Lovelace/Hopper architecture)\n", + " is_supported = (device_props.major > 8) or (device_props.major == 8 and device_props.minor >= 9)\n", + " \n", + " return is_supported, f\"Device: {device_name}, Compute Capability: {compute_capability}\"\n", + "\n", + "fp8_supported, gpu_info = check_fp8_support()\n", + "print(f\"FP8 Support: {fp8_supported}\")\n", + "print(gpu_info)\n", + "\n", + "# Note: If FP8 is not supported, you may want to disable it in the model config\n", + "# The Evo2 config has 'use_fp8_input_projections: True' by default\n", + "\n", + "fp8_option = \"--fp8\" if fp8_supported else \"\"\n", "\n", "# Update predict commands to run on the full dataset\n", "predict_ref_command = (\n", " f\"predict_evo2 --fasta {ref_fasta_path} --ckpt-dir {checkpoint_path} \"\n", - " f\"--output-dir {predict_ref_dir} --model-size 1b --tensor-parallel-size 1 \"\n", - " \"--pipeline-model-parallel-size 1 --context-parallel-size 1 --output-log-prob-seqs\"\n", + " f\"--output-dir {predict_ref_dir} --model-size {MODEL_SIZE} --tensor-parallel-size 1 \"\n", + " f\"--pipeline-model-parallel-size 1 --context-parallel-size 1 --output-log-prob-seqs {fp8_option}\"\n", ")\n", "\n", "predict_var_command = (\n", " f\"predict_evo2 --fasta {var_fasta_path} --ckpt-dir {checkpoint_path} \"\n", - " f\"--output-dir {predict_var_dir} --model-size 1b --tensor-parallel-size 1 \"\n", - " \"--pipeline-model-parallel-size 1 --context-parallel-size 1 --output-log-prob-seqs\"\n", + " f\"--output-dir {predict_var_dir} --model-size {MODEL_SIZE} --tensor-parallel-size 1 \"\n", + " f\"--pipeline-model-parallel-size 1 --context-parallel-size 1 --output-log-prob-seqs {fp8_option}\"\n", ")" ] }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "[NeMo W 2025-03-03 23:36:30 nemo_logging:405] /usr/local/lib/python3.12/dist-packages/pydub/utils.py:170: RuntimeWarning: Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work\n", + "Running command: predict_evo2 --fasta brca1_fasta_files/brca1_reference_sequences.fasta --ckpt-dir nemo2_evo2_1b_8k --output-dir brca1_fasta_files/reference_predictions --model-size 1b --tensor-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 1 --output-log-prob-seqs --fp8\n", + "[WARNING | bitsandbytes.cextension]: Could not find the bitsandbytes CUDA binary at PosixPath('/usr/local/lib/python3.12/dist-packages/bitsandbytes/libbitsandbytes_cuda128.so')\n", + "[WARNING | bitsandbytes.cextension]: The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers, 8-bit multiplication, and GPU quantization are unavailable.\n", + "[NeMo W 2025-03-04 01:01:10 nemo_logging:405] /usr/local/lib/python3.12/dist-packages/pydub/utils.py:170: RuntimeWarning: Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work\n", " warn(\"Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work\", RuntimeWarning)\n", " \n", "[WARNING | py.warnings ]: /usr/local/lib/python3.12/dist-packages/mamba_ssm/ops/selective_scan_interface.py:163: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.\n", @@ -547,318 +588,319 @@ "[INFO | pytorch_lightning.utilities.rank_zero]: GPU available: True (cuda), used: True\n", "[INFO | pytorch_lightning.utilities.rank_zero]: TPU available: False, using: 0 TPU cores\n", "[INFO | pytorch_lightning.utilities.rank_zero]: HPU available: False, using: 0 HPUs\n", - "[NeMo W 2025-03-03 23:36:31 nemo_logging:405] No version folders would be created under the log folder as 'resume_if_exists' is enabled.\n", - "[NeMo I 2025-03-03 23:36:31 nemo_logging:393] Experiments will be logged at /tmp/tmpsn4mexa6/default\n", - "[NeMo W 2025-03-03 23:36:31 nemo_logging:405] \"update_logger_directory\" is True. Overwriting tensorboard logger \"save_dir\" to /tmp/tmpsn4mexa6\n", - "[NeMo I 2025-03-03 23:36:31 nemo_logging:393] Using byte-level tokenization\n", - "[NeMo I 2025-03-03 23:36:31 nemo_logging:393] Rank 0 has data parallel group : [0]\n", - "[NeMo I 2025-03-03 23:36:31 nemo_logging:393] Rank 0 has combined group of data parallel and context parallel : [0]\n", - "[NeMo I 2025-03-03 23:36:31 nemo_logging:393] All data parallel group ranks with context parallel combined: [[0]]\n", - "[NeMo I 2025-03-03 23:36:31 nemo_logging:393] Ranks 0 has data parallel rank: 0\n", - "[NeMo I 2025-03-03 23:36:31 nemo_logging:393] Rank 0 has context parallel group: [0]\n", - "[NeMo I 2025-03-03 23:36:31 nemo_logging:393] All context parallel group ranks: [[0]]\n", - "[NeMo I 2025-03-03 23:36:31 nemo_logging:393] Ranks 0 has context parallel rank: 0\n", - "[NeMo I 2025-03-03 23:36:31 nemo_logging:393] Rank 0 has model parallel group: [0]\n", - "[NeMo I 2025-03-03 23:36:31 nemo_logging:393] All model parallel group ranks: [[0]]\n", - "[NeMo I 2025-03-03 23:36:31 nemo_logging:393] Rank 0 has tensor model parallel group: [0]\n", - "[NeMo I 2025-03-03 23:36:31 nemo_logging:393] All tensor model parallel group ranks: [[0]]\n", - "[NeMo I 2025-03-03 23:36:31 nemo_logging:393] Rank 0 has tensor model parallel rank: 0\n", - "[NeMo I 2025-03-03 23:36:31 nemo_logging:393] Rank 0 has pipeline model parallel group: [0]\n", - "[NeMo I 2025-03-03 23:36:31 nemo_logging:393] Rank 0 has embedding group: [0]\n", - "[NeMo I 2025-03-03 23:36:31 nemo_logging:393] All pipeline model parallel group ranks: [[0]]\n", - "[NeMo I 2025-03-03 23:36:31 nemo_logging:393] Rank 0 has pipeline model parallel rank 0\n", - "[NeMo I 2025-03-03 23:36:31 nemo_logging:393] All embedding group ranks: [[0]]\n", - "[NeMo I 2025-03-03 23:36:31 nemo_logging:393] Rank 0 has embedding rank: 0\n", + "[NeMo W 2025-03-04 01:01:11 nemo_logging:405] No version folders would be created under the log folder as 'resume_if_exists' is enabled.\n", + "[NeMo I 2025-03-04 01:01:11 nemo_logging:393] Experiments will be logged at /tmp/tmpupzx4lk1/default\n", + "[NeMo W 2025-03-04 01:01:11 nemo_logging:405] \"update_logger_directory\" is True. Overwriting tensorboard logger \"save_dir\" to /tmp/tmpupzx4lk1\n", + "[NeMo I 2025-03-04 01:01:11 nemo_logging:393] Using byte-level tokenization\n", + "[NeMo I 2025-03-04 01:01:11 nemo_logging:393] Rank 0 has data parallel group : [0]\n", + "[NeMo I 2025-03-04 01:01:11 nemo_logging:393] Rank 0 has combined group of data parallel and context parallel : [0]\n", + "[NeMo I 2025-03-04 01:01:11 nemo_logging:393] All data parallel group ranks with context parallel combined: [[0]]\n", + "[NeMo I 2025-03-04 01:01:11 nemo_logging:393] Ranks 0 has data parallel rank: 0\n", + "[NeMo I 2025-03-04 01:01:11 nemo_logging:393] Rank 0 has context parallel group: [0]\n", + "[NeMo I 2025-03-04 01:01:11 nemo_logging:393] All context parallel group ranks: [[0]]\n", + "[NeMo I 2025-03-04 01:01:11 nemo_logging:393] Ranks 0 has context parallel rank: 0\n", + "[NeMo I 2025-03-04 01:01:11 nemo_logging:393] Rank 0 has model parallel group: [0]\n", + "[NeMo I 2025-03-04 01:01:11 nemo_logging:393] All model parallel group ranks: [[0]]\n", + "[NeMo I 2025-03-04 01:01:11 nemo_logging:393] Rank 0 has tensor model parallel group: [0]\n", + "[NeMo I 2025-03-04 01:01:11 nemo_logging:393] All tensor model parallel group ranks: [[0]]\n", + "[NeMo I 2025-03-04 01:01:11 nemo_logging:393] Rank 0 has tensor model parallel rank: 0\n", + "[NeMo I 2025-03-04 01:01:11 nemo_logging:393] Rank 0 has pipeline model parallel group: [0]\n", + "[NeMo I 2025-03-04 01:01:11 nemo_logging:393] Rank 0 has embedding group: [0]\n", + "[NeMo I 2025-03-04 01:01:11 nemo_logging:393] All pipeline model parallel group ranks: [[0]]\n", + "[NeMo I 2025-03-04 01:01:11 nemo_logging:393] Rank 0 has pipeline model parallel rank 0\n", + "[NeMo I 2025-03-04 01:01:11 nemo_logging:393] All embedding group ranks: [[0]]\n", + "[NeMo I 2025-03-04 01:01:11 nemo_logging:393] Rank 0 has embedding rank: 0\n", "Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/1\n", "[INFO | pytorch_lightning.utilities.rank_zero]: ----------------------------------------------------------------------------------------------------\n", "distributed_backend=nccl\n", "All distributed processes registered. Starting with 1 processes\n", "----------------------------------------------------------------------------------------------------\n", "\n", - "[NeMo I 2025-03-03 23:36:31 num_microbatches_calculator:228] setting number of microbatches to constant 1\n", - "[NeMo I 2025-03-03 23:36:32 nemo_logging:393] Padded vocab_size: 512, original vocab_size: 512, dummy tokens: 0.\n", - "[NeMo W 2025-03-03 23:36:32 random:220] CPU RNG state changed within GPU RNG context\n", - "[NeMo W 2025-03-03 23:36:32 random:220] CPU RNG state changed within GPU RNG context\n", - "[NeMo W 2025-03-03 23:36:32 random:220] CPU RNG state changed within GPU RNG context\n", - "[NeMo W 2025-03-03 23:36:32 random:220] CPU RNG state changed within GPU RNG context\n", - "[NeMo W 2025-03-03 23:36:32 random:220] CPU RNG state changed within GPU RNG context\n", - "[NeMo W 2025-03-03 23:36:32 random:220] CPU RNG state changed within GPU RNG context\n", - "[NeMo W 2025-03-03 23:36:32 random:220] CPU RNG state changed within GPU RNG context\n", - "[NeMo W 2025-03-03 23:36:32 random:220] CPU RNG state changed within GPU RNG context\n", - "[NeMo W 2025-03-03 23:36:32 random:220] CPU RNG state changed within GPU RNG context\n", - "[NeMo W 2025-03-03 23:36:32 random:220] CPU RNG state changed within GPU RNG context\n", - "[NeMo W 2025-03-03 23:36:32 random:220] CPU RNG state changed within GPU RNG context\n", - "[NeMo W 2025-03-03 23:36:32 random:220] CPU RNG state changed within GPU RNG context\n", - "[NeMo W 2025-03-03 23:36:32 random:220] CPU RNG state changed within GPU RNG context\n", - "[NeMo W 2025-03-03 23:36:32 random:220] CPU RNG state changed within GPU RNG context\n", - "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2]\n", - "[NeMo W 2025-03-03 23:36:32 nemo_logging:405] Could not copy Trainer's 'max_steps' to LR scheduler's 'max_steps'. If you are not using an LR scheduler, this warning can safely be ignored.\n", - "[NeMo I 2025-03-03 23:36:32 nemo_logging:393] > number of parameters on (tensor, pipeline) model parallel rank (0 ,0): 1108204800\n", - "[NeMo I 2025-03-03 23:36:32 utils:302] Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=False, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=False, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, average_in_collective=False, fp8_param_gather=False)\n", - "[NeMo I 2025-03-03 23:36:32 utils:323] Number of buckets for gradient all-reduce / reduce-scatter: 1\n", + "[NeMo I 2025-03-04 01:01:11 num_microbatches_calculator:228] setting number of microbatches to constant 1\n", + "[NeMo I 2025-03-04 01:01:11 nemo_logging:393] Padded vocab_size: 512, original vocab_size: 512, dummy tokens: 0.\n", + "[NeMo W 2025-03-04 01:01:11 random:220] CPU RNG state changed within GPU RNG context\n", + "[NeMo W 2025-03-04 01:01:11 random:220] CPU RNG state changed within GPU RNG context\n", + "[NeMo W 2025-03-04 01:01:11 random:220] CPU RNG state changed within GPU RNG context\n", + "[NeMo W 2025-03-04 01:01:11 random:220] CPU RNG state changed within GPU RNG context\n", + "[NeMo W 2025-03-04 01:01:11 random:220] CPU RNG state changed within GPU RNG context\n", + "[NeMo W 2025-03-04 01:01:11 random:220] CPU RNG state changed within GPU RNG context\n", + "[NeMo W 2025-03-04 01:01:11 random:220] CPU RNG state changed within GPU RNG context\n", + "[NeMo W 2025-03-04 01:01:11 random:220] CPU RNG state changed within GPU RNG context\n", + "[NeMo W 2025-03-04 01:01:11 random:220] CPU RNG state changed within GPU RNG context\n", + "[NeMo W 2025-03-04 01:01:11 random:220] CPU RNG state changed within GPU RNG context\n", + "[NeMo W 2025-03-04 01:01:11 random:220] CPU RNG state changed within GPU RNG context\n", + "[NeMo W 2025-03-04 01:01:11 random:220] CPU RNG state changed within GPU RNG context\n", + "[NeMo W 2025-03-04 01:01:11 random:220] CPU RNG state changed within GPU RNG context\n", + "[NeMo W 2025-03-04 01:01:11 random:220] CPU RNG state changed within GPU RNG context\n", + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n", + "[NeMo W 2025-03-04 01:01:11 nemo_logging:405] Could not copy Trainer's 'max_steps' to LR scheduler's 'max_steps'. If you are not using an LR scheduler, this warning can safely be ignored.\n", + "[NeMo I 2025-03-04 01:01:11 nemo_logging:393] > number of parameters on (tensor, pipeline) model parallel rank (0 ,0): 1108204800\n", + "[NeMo I 2025-03-04 01:01:11 utils:302] Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=False, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=False, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, average_in_collective=False, fp8_param_gather=False)\n", + "[NeMo I 2025-03-04 01:01:11 utils:323] Number of buckets for gradient all-reduce / reduce-scatter: 1\n", " Params for bucket 1 (1108204800 elements):\n", - " \tmodule.decoder.layers.18.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.16.mixer.dense_projection.weight\n", - " \tmodule.decoder.layers.13.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.8.mixer.dense_projection.weight\n", - " \tmodule.decoder.layers.6.mixer.mixer.filter.gamma\n", - " \tmodule.decoder.layers.3.mlp.linear_fc1.layer_norm_weight\n", - " \tmodule.decoder.layers.3.self_attention.linear_proj.bias\n", - " \tmodule.decoder.layers.0.mixer.dense.bias\n", - " \tmodule.decoder.layers.22.mixer.mixer.conv_bias\n", - " \tmodule.decoder.layers.20.mixer.dense_projection.layer_norm_weight\n", - " \tmodule.decoder.layers.14.mlp.linear_fc1.weight\n", - " \tmodule.decoder.layers.9.mlp.linear_fc1.weight\n", - " \tmodule.decoder.layers.7.mixer.dense_projection.layer_norm_weight\n", - " \tmodule.decoder.layers.1.mixer.mixer.conv_bias\n", - " \tmodule.decoder.layers.23.mixer.mixer.filter.gamma\n", - " \tmodule.decoder.layers.18.mixer.dense_projection.weight\n", - " \tmodule.decoder.layers.15.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.13.mixer.mixer.filter.p\n", - " \tmodule.decoder.layers.11.mixer.hyena_proj_conv.short_conv_weight\n", - " \tmodule.decoder.layers.7.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.1.mlp.linear_fc1.layer_norm_weight\n", - " \tmodule.embedding.word_embeddings.weight\n", - " \tmodule.decoder.layers.24.mlp.linear_fc1.weight\n", - " \tmodule.decoder.layers.24.self_attention.linear_proj.weight\n", - " \tmodule.decoder.layers.21.mixer.mixer.short_conv.short_conv_weight\n", - " \tmodule.decoder.layers.14.mixer.dense.weight\n", - " \tmodule.decoder.layers.12.mixer.dense.bias\n", - " \tmodule.decoder.layers.9.mlp.linear_fc1.layer_norm_weight\n", - " \tmodule.decoder.layers.9.mixer.dense_projection.layer_norm_weight\n", - " \tmodule.decoder.layers.4.mixer.hyena_proj_conv.short_conv_weight\n", - " \tmodule.decoder.layers.0.mixer.mixer.short_conv.short_conv_weight\n", - " \tmodule.decoder.layers.0.mixer.dense_projection.layer_norm_weight\n", - " \tmodule.decoder.layers.20.mixer.dense_projection.weight\n", - " \tmodule.decoder.layers.17.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.13.mixer.hyena_proj_conv.short_conv_weight\n", - " \tmodule.decoder.layers.10.self_attention.linear_qkv.weight\n", - " \tmodule.decoder.layers.7.mixer.dense_projection.weight\n", - " \tmodule.decoder.layers.5.mixer.hyena_proj_conv.short_conv_weight\n", - " \tmodule.decoder.layers.1.mixer.mixer.filter.h\n", - " \tmodule.decoder.layers.24.self_attention.linear_qkv.layer_norm_weight\n", - " \tmodule.decoder.layers.23.mixer.mixer.conv_bias\n", - " \tmodule.decoder.layers.18.mlp.linear_fc1.weight\n", - " \tmodule.decoder.layers.16.mixer.dense.bias\n", - " \tmodule.decoder.layers.13.mlp.linear_fc1.weight\n", - " \tmodule.decoder.layers.12.mixer.dense_projection.layer_norm_weight\n", - " \tmodule.decoder.layers.9.mixer.mixer.filter.R\n", - " \tmodule.decoder.layers.8.mixer.dense.bias\n", - " \tmodule.decoder.layers.3.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.2.mixer.dense_projection.weight\n", - " \tmodule.decoder.layers.22.mixer.hyena_proj_conv.short_conv_weight\n", - " \tmodule.decoder.layers.19.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.17.mlp.linear_fc1.layer_norm_weight\n", - " \tmodule.decoder.layers.15.mlp.linear_fc1.layer_norm_weight\n", - " \tmodule.decoder.layers.12.mixer.dense.weight\n", - " \tmodule.decoder.layers.11.mixer.dense.bias\n", - " \tmodule.decoder.layers.6.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.18.mixer.dense.weight\n", - " \tmodule.decoder.layers.15.mlp.linear_fc1.weight\n", - " \tmodule.decoder.layers.13.mlp.linear_fc1.layer_norm_weight\n", - " \tmodule.decoder.layers.7.mlp.linear_fc1.weight\n", - " \tmodule.decoder.layers.1.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.23.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.16.mixer.dense.weight\n", - " \tmodule.decoder.layers.14.mlp.linear_fc1.layer_norm_weight\n", - " \tmodule.decoder.layers.12.mixer.dense_projection.weight\n", - " \tmodule.decoder.layers.8.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.6.mixer.mixer.filter.p\n", - " \tmodule.decoder.layers.4.mixer.dense.weight\n", - " \tmodule.decoder.layers.20.mixer.dense.bias\n", - " \tmodule.decoder.layers.17.mlp.linear_fc1.weight\n", - " \tmodule.decoder.layers.17.self_attention.linear_proj.bias\n", - " \tmodule.decoder.layers.15.mixer.mixer.filter.h\n", - " \tmodule.decoder.layers.13.mixer.mixer.filter.R\n", - " \tmodule.decoder.layers.11.mixer.dense_projection.layer_norm_weight\n", - " \tmodule.decoder.layers.7.mixer.dense.weight\n", - " \tmodule.decoder.layers.6.mixer.mixer.conv_bias\n", - " \tmodule.decoder.layers.24.mlp.linear_fc1.layer_norm_weight\n", - " \tmodule.decoder.layers.23.mixer.mixer.filter.p\n", - " \tmodule.decoder.layers.21.mixer.hyena_proj_conv.short_conv_weight\n", - " \tmodule.decoder.layers.19.mlp.linear_fc1.layer_norm_weight\n", - " \tmodule.decoder.layers.11.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.6.mixer.hyena_proj_conv.short_conv_weight\n", - " \tmodule.decoder.layers.3.self_attention.linear_qkv.weight\n", - " \tmodule.decoder.layers.0.mixer.hyena_proj_conv.short_conv_weight\n", - " \tmodule.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight\n", " \tmodule.decoder.layers.22.mixer.dense.bias\n", - " \tmodule.decoder.layers.19.mlp.linear_fc1.weight\n", - " \tmodule.decoder.layers.15.mixer.mixer.conv_bias\n", - " \tmodule.decoder.layers.13.mixer.dense_projection.layer_norm_weight\n", + " \tmodule.decoder.layers.19.mixer.mixer.conv_bias\n", + " \tmodule.decoder.layers.16.mixer.mixer.conv_bias\n", + " \tmodule.decoder.layers.11.mlp.linear_fc1.weight\n", + " \tmodule.decoder.layers.9.mixer.dense.bias\n", " \tmodule.decoder.layers.6.mlp.linear_fc1.weight\n", - " \tmodule.decoder.layers.5.mixer.dense_projection.layer_norm_weight\n", - " \tmodule.decoder.layers.1.mixer.hyena_proj_conv.short_conv_weight\n", + " \tmodule.decoder.layers.3.mlp.linear_fc2.weight\n", + " \tmodule.decoder.layers.0.mixer.dense_projection.weight\n", " \tmodule.decoder.layers.23.mixer.hyena_proj_conv.short_conv_weight\n", " \tmodule.decoder.layers.20.mixer.dense.weight\n", - " \tmodule.decoder.layers.18.mlp.linear_fc1.layer_norm_weight\n", - " \tmodule.decoder.layers.16.mixer.mixer.filter.gamma\n", - " \tmodule.decoder.layers.11.mixer.dense_projection.weight\n", - " \tmodule.decoder.layers.8.mixer.mixer.conv_bias\n", + " \tmodule.decoder.layers.15.mixer.hyena_proj_conv.short_conv_weight\n", + " \tmodule.decoder.layers.12.mlp.linear_fc2.weight\n", + " \tmodule.decoder.layers.10.mlp.linear_fc1.layer_norm_weight\n", + " \tmodule.decoder.layers.8.mlp.linear_fc1.layer_norm_weight\n", " \tmodule.decoder.layers.5.mixer.dense.weight\n", - " \tmodule.decoder.layers.1.mlp.linear_fc1.weight\n", - " \tmodule.decoder.layers.17.self_attention.linear_proj.weight\n", - " \tmodule.decoder.layers.24.self_attention.linear_proj.bias\n", + " \tmodule.decoder.layers.2.mixer.mixer.filter.R\n", + " \tmodule.decoder.layers.1.mixer.dense_projection.weight\n", " \tmodule.decoder.layers.23.mlp.linear_fc1.weight\n", " \tmodule.decoder.layers.22.mixer.dense_projection.layer_norm_weight\n", - " \tmodule.decoder.layers.19.mixer.mixer.filter.h\n", - " \tmodule.decoder.layers.14.mixer.mixer.short_conv.short_conv_weight\n", + " \tmodule.decoder.layers.18.mixer.mixer.short_conv.short_conv_weight\n", + " \tmodule.decoder.layers.11.mixer.dense.weight\n", + " \tmodule.decoder.layers.10.self_attention.linear_proj.bias\n", " \tmodule.decoder.layers.8.mlp.linear_fc1.weight\n", " \tmodule.decoder.layers.6.mlp.linear_fc1.layer_norm_weight\n", - " \tmodule.decoder.layers.4.mixer.dense_projection.weight\n", + " \tmodule.decoder.layers.3.mlp.linear_fc1.layer_norm_weight\n", " \tmodule.decoder.layers.22.mixer.dense.weight\n", - " \tmodule.decoder.layers.13.mixer.dense_projection.weight\n", - " \tmodule.decoder.layers.10.mlp.linear_fc2.weight\n", + " \tmodule.decoder.layers.16.mlp.linear_fc2.weight\n", + " \tmodule.decoder.layers.9.mixer.dense.weight\n", " \tmodule.decoder.layers.7.mlp.linear_fc1.layer_norm_weight\n", " \tmodule.decoder.layers.5.mixer.dense_projection.weight\n", - " \tmodule.decoder.layers.2.mixer.dense.weight\n", + " \tmodule.decoder.layers.4.mlp.linear_fc1.layer_norm_weight\n", + " \tmodule.decoder.layers.0.mlp.linear_fc2.weight\n", " \tmodule.decoder.layers.23.mlp.linear_fc1.layer_norm_weight\n", - " \tmodule.decoder.layers.19.mixer.mixer.conv_bias\n", - " \tmodule.decoder.layers.16.mixer.mixer.conv_bias\n", - " \tmodule.decoder.layers.11.mlp.linear_fc1.weight\n", - " \tmodule.decoder.layers.9.mixer.dense.bias\n", + " \tmodule.decoder.layers.13.mixer.dense.bias\n", + " \tmodule.decoder.layers.10.mlp.linear_fc1.weight\n", " \tmodule.decoder.layers.8.mixer.mixer.filter.h\n", " \tmodule.decoder.layers.6.mixer.mixer.filter.R\n", - " \tmodule.decoder.layers.4.mixer.dense_projection.layer_norm_weight\n", - " \tmodule.decoder.layers.0.mixer.dense.weight\n", + " \tmodule.decoder.layers.3.self_attention.linear_proj.bias\n", + " \tmodule.decoder.layers.2.mlp.linear_fc2.weight\n", + " \tmodule.decoder.layers.1.mixer.dense.weight\n", + " \tmodule.decoder.layers.24.self_attention.linear_qkv.weight\n", " \tmodule.decoder.layers.22.mixer.dense_projection.weight\n", - " \tmodule.decoder.layers.20.mixer.mixer.filter.gamma\n", - " \tmodule.decoder.layers.15.mixer.hyena_proj_conv.short_conv_weight\n", - " \tmodule.decoder.layers.12.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.10.mlp.linear_fc1.layer_norm_weight\n", - " \tmodule.decoder.layers.7.mixer.dense.bias\n", - " \tmodule.decoder.layers.4.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.2.mixer.dense.bias\n", - " \tmodule.decoder.layers.1.mixer.dense.bias\n", - " \tmodule.decoder.layers.23.mixer.mixer.filter.R\n", - " \tmodule.decoder.layers.21.mixer.dense_projection.layer_norm_weight\n", - " \tmodule.decoder.layers.18.mixer.mixer.short_conv.short_conv_weight\n", - " \tmodule.decoder.layers.11.mixer.dense.weight\n", - " \tmodule.decoder.layers.6.mixer.dense_projection.layer_norm_weight\n", - " \tmodule.decoder.layers.3.self_attention.linear_proj.weight\n", - " \tmodule.decoder.layers.2.mixer.mixer.filter.gamma\n", - " \tmodule.decoder.layers.21.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.16.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.9.mixer.dense.weight\n", - " \tmodule.decoder.layers.9.mixer.mixer.filter.gamma\n", - " \tmodule.decoder.layers.4.mlp.linear_fc1.layer_norm_weight\n", - " \tmodule.decoder.layers.1.mixer.dense_projection.layer_norm_weight\n", - " \tmodule.decoder.layers.23.mixer.dense_projection.layer_norm_weight\n", - " \tmodule.decoder.layers.20.mixer.mixer.conv_bias\n", - " \tmodule.decoder.layers.13.mixer.dense.bias\n", - " \tmodule.decoder.layers.10.mlp.linear_fc1.weight\n", - " \tmodule.decoder.layers.7.mixer.mixer.short_conv.short_conv_weight\n", - " \tmodule.decoder.layers.5.mixer.dense.bias\n", - " \tmodule.decoder.layers.21.mixer.dense_projection.weight\n", " \tmodule.decoder.layers.19.mixer.hyena_proj_conv.short_conv_weight\n", " \tmodule.decoder.layers.16.mixer.mixer.filter.p\n", " \tmodule.decoder.layers.14.mixer.hyena_proj_conv.short_conv_weight\n", " \tmodule.decoder.layers.12.mlp.linear_fc1.layer_norm_weight\n", - " \tmodule.decoder.layers.6.mixer.dense_projection.weight\n", " \tmodule.decoder.layers.3.mlp.linear_fc1.weight\n", - " \tmodule.decoder.layers.0.mixer.dense_projection.weight\n", - " \tmodule.decoder.layers.21.mixer.dense.bias\n", + " \tmodule.decoder.layers.0.mlp.linear_fc1.layer_norm_weight\n", + " \tmodule.decoder.layers.23.mixer.mixer.filter.R\n", + " \tmodule.decoder.layers.21.mixer.dense_projection.layer_norm_weight\n", " \tmodule.decoder.layers.15.mixer.dense.bias\n", " \tmodule.decoder.layers.12.mlp.linear_fc1.weight\n", " \tmodule.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight\n", - " \tmodule.decoder.layers.4.mlp.linear_fc1.weight\n", - " \tmodule.decoder.layers.2.mixer.mixer.filter.R\n", - " \tmodule.decoder.layers.1.mixer.dense_projection.weight\n", - " \tmodule.decoder.layers.23.mixer.dense_projection.weight\n", - " \tmodule.decoder.layers.20.mlp.linear_fc2.weight\n", + " \tmodule.decoder.layers.8.mixer.mixer.conv_bias\n", + " \tmodule.decoder.layers.6.mixer.dense_projection.layer_norm_weight\n", + " \tmodule.decoder.layers.2.mixer.mixer.filter.p\n", + " \tmodule.decoder.final_norm.weight\n", + " \tmodule.decoder.layers.21.mlp.linear_fc2.weight\n", " \tmodule.decoder.layers.16.mixer.hyena_proj_conv.short_conv_weight\n", " \tmodule.decoder.layers.13.mixer.dense.weight\n", " \tmodule.decoder.layers.11.mlp.linear_fc1.layer_norm_weight\n", - " \tmodule.decoder.layers.10.self_attention.linear_proj.bias\n", - " \tmodule.decoder.layers.8.mixer.hyena_proj_conv.short_conv_weight\n", - " \tmodule.decoder.layers.5.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.21.mlp.linear_fc1.weight\n", + " \tmodule.decoder.layers.9.mixer.mixer.filter.gamma\n", + " \tmodule.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight\n", + " \tmodule.decoder.layers.2.mixer.hyena_proj_conv.short_conv_weight\n", + " \tmodule.decoder.layers.24.self_attention.linear_proj.bias\n", + " \tmodule.decoder.layers.23.mixer.dense_projection.layer_norm_weight\n", + " \tmodule.decoder.layers.20.mixer.mixer.conv_bias\n", + " \tmodule.decoder.layers.19.mlp.linear_fc2.weight\n", " \tmodule.decoder.layers.16.mlp.linear_fc1.weight\n", " \tmodule.decoder.layers.15.mixer.dense_projection.layer_norm_weight\n", " \tmodule.decoder.layers.12.mixer.mixer.filter.h\n", " \tmodule.decoder.layers.10.self_attention.linear_proj.weight\n", - " \tmodule.decoder.layers.9.mixer.hyena_proj_conv.short_conv_weight\n", - " \tmodule.decoder.layers.5.mlp.linear_fc1.layer_norm_weight\n", - " \tmodule.decoder.layers.0.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.22.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.20.mixer.mixer.filter.p\n", + " \tmodule.decoder.layers.7.mixer.mixer.short_conv.short_conv_weight\n", + " \tmodule.decoder.layers.5.mixer.dense.bias\n", + " \tmodule.decoder.layers.21.mixer.dense_projection.weight\n", " \tmodule.decoder.layers.18.mixer.hyena_proj_conv.short_conv_weight\n", " \tmodule.decoder.layers.15.mixer.dense.weight\n", - " \tmodule.decoder.layers.2.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.1.mixer.dense.weight\n", - " \tmodule.decoder.layers.21.mixer.dense.weight\n", + " \tmodule.decoder.layers.6.mixer.dense_projection.weight\n", + " \tmodule.decoder.layers.2.mlp.linear_fc1.weight\n", + " \tmodule.decoder.layers.2.mixer.dense_projection.layer_norm_weight\n", + " \tmodule.decoder.layers.20.mixer.dense_projection.layer_norm_weight\n", " \tmodule.decoder.layers.19.mixer.dense.bias\n", " \tmodule.decoder.layers.16.mlp.linear_fc1.layer_norm_weight\n", " \tmodule.decoder.layers.12.mixer.mixer.conv_bias\n", " \tmodule.decoder.layers.9.mixer.mixer.conv_bias\n", - " \tmodule.decoder.layers.8.mlp.linear_fc1.layer_norm_weight\n", + " \tmodule.decoder.layers.5.mixer.dense_projection.layer_norm_weight\n", " \tmodule.decoder.layers.4.mixer.dense.bias\n", - " \tmodule.decoder.layers.0.mlp.linear_fc1.layer_norm_weight\n", - " \tmodule.decoder.layers.20.mixer.hyena_proj_conv.short_conv_weight\n", + " \tmodule.decoder.layers.2.mixer.mixer.conv_bias\n", " \tmodule.decoder.layers.17.self_attention.linear_qkv.weight\n", + " \tmodule.decoder.layers.23.mixer.dense_projection.weight\n", + " \tmodule.decoder.layers.20.mlp.linear_fc2.weight\n", " \tmodule.decoder.layers.15.mixer.dense_projection.weight\n", " \tmodule.decoder.layers.13.mixer.mixer.filter.gamma\n", - " \tmodule.decoder.layers.7.mixer.hyena_proj_conv.short_conv_weight\n", - " \tmodule.decoder.layers.5.mixer.mixer.conv_bias\n", - " \tmodule.decoder.layers.2.mixer.mixer.filter.p\n", - " \tmodule.decoder.layers.23.mixer.dense.bias\n", - " \tmodule.decoder.layers.20.mlp.linear_fc1.weight\n", - " \tmodule.decoder.layers.19.mixer.dense_projection.layer_norm_weight\n", - " \tmodule.decoder.layers.16.mixer.mixer.filter.R\n", - " \tmodule.decoder.layers.14.mixer.dense_projection.layer_norm_weight\n", + " \tmodule.decoder.layers.8.mixer.hyena_proj_conv.short_conv_weight\n", + " \tmodule.decoder.layers.5.mlp.linear_fc2.weight\n", + " \tmodule.decoder.layers.2.mlp.linear_fc1.layer_norm_weight\n", + " \tmodule.decoder.layers.21.mlp.linear_fc1.weight\n", + " \tmodule.decoder.layers.19.mixer.dense_projection.layer_norm_weight\n", + " \tmodule.decoder.layers.16.mixer.mixer.filter.R\n", + " \tmodule.decoder.layers.14.mixer.dense_projection.layer_norm_weight\n", " \tmodule.decoder.layers.11.mixer.mixer.short_conv.short_conv_weight\n", - " \tmodule.decoder.layers.5.mlp.linear_fc1.weight\n", - " \tmodule.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight\n", - " \tmodule.decoder.layers.2.mixer.hyena_proj_conv.short_conv_weight\n", - " \tmodule.decoder.layers.24.self_attention.linear_qkv.weight\n", - " \tmodule.decoder.layers.22.mlp.linear_fc1.layer_norm_weight\n", + " \tmodule.decoder.layers.0.mixer.dense.bias\n", + " \tmodule.decoder.layers.22.mlp.linear_fc2.weight\n", + " \tmodule.decoder.layers.20.mixer.mixer.filter.p\n", " \tmodule.decoder.layers.19.mixer.dense.weight\n", " \tmodule.decoder.layers.18.mixer.dense.bias\n", " \tmodule.decoder.layers.14.mlp.linear_fc2.weight\n", " \tmodule.decoder.layers.9.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.9.mixer.dense_projection.weight\n", - " \tmodule.decoder.layers.6.mixer.dense.weight\n", + " \tmodule.decoder.layers.5.mlp.linear_fc1.layer_norm_weight\n", " \tmodule.decoder.layers.4.mixer.mixer.short_conv.short_conv_weight\n", - " \tmodule.decoder.layers.0.mlp.linear_fc1.weight\n", - " \tmodule.decoder.layers.22.mlp.linear_fc1.weight\n", - " \tmodule.decoder.layers.20.mlp.linear_fc1.layer_norm_weight\n", + " \tmodule.decoder.layers.1.mixer.mixer.conv_bias\n", + " \tmodule.decoder.layers.21.mixer.dense.weight\n", " \tmodule.decoder.layers.16.mixer.dense_projection.layer_norm_weight\n", " \tmodule.decoder.layers.13.mixer.mixer.conv_bias\n", - " \tmodule.decoder.layers.8.mixer.dense_projection.layer_norm_weight\n", - " \tmodule.decoder.layers.5.mixer.mixer.filter.h\n", - " \tmodule.decoder.layers.2.mlp.linear_fc1.weight\n", - " \tmodule.decoder.layers.2.mixer.dense_projection.layer_norm_weight\n", + " \tmodule.decoder.layers.6.mixer.dense.bias\n", + " \tmodule.decoder.layers.1.mlp.linear_fc1.layer_norm_weight\n", " \tmodule.decoder.layers.24.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.23.mixer.dense.weight\n", - " \tmodule.decoder.layers.21.mlp.linear_fc1.layer_norm_weight\n", + " \tmodule.decoder.layers.21.mixer.dense.bias\n", " \tmodule.decoder.layers.19.mixer.dense_projection.weight\n", " \tmodule.decoder.layers.14.mixer.dense_projection.weight\n", " \tmodule.decoder.layers.12.mixer.hyena_proj_conv.short_conv_weight\n", " \tmodule.decoder.layers.9.mixer.mixer.filter.p\n", + " \tmodule.decoder.layers.7.mixer.hyena_proj_conv.short_conv_weight\n", + " \tmodule.decoder.layers.4.mlp.linear_fc1.weight\n", + " \tmodule.decoder.layers.0.mixer.mixer.short_conv.short_conv_weight\n", + " \tmodule.decoder.layers.23.mixer.dense.bias\n", + " \tmodule.decoder.layers.20.mlp.linear_fc1.weight\n", + " \tmodule.decoder.layers.18.mixer.dense_projection.layer_norm_weight\n", + " \tmodule.decoder.layers.14.mixer.dense.bias\n", + " \tmodule.decoder.layers.8.mixer.dense.bias\n", + " \tmodule.decoder.layers.5.mlp.linear_fc1.weight\n", + " \tmodule.decoder.layers.1.mixer.mixer.filter.h\n", + " \tmodule.decoder.layers.24.mlp.linear_fc1.layer_norm_weight\n", + " \tmodule.decoder.layers.22.mlp.linear_fc1.layer_norm_weight\n", + " \tmodule.decoder.layers.18.mlp.linear_fc2.weight\n", + " \tmodule.decoder.layers.16.mixer.dense_projection.weight\n", + " \tmodule.decoder.layers.13.mlp.linear_fc2.weight\n", + " \tmodule.decoder.layers.9.mixer.hyena_proj_conv.short_conv_weight\n", + " \tmodule.decoder.layers.6.mixer.dense.weight\n", + " \tmodule.decoder.layers.2.mixer.dense_projection.weight\n", + " \tmodule.decoder.layers.0.mixer.dense_projection.layer_norm_weight\n", + " \tmodule.decoder.layers.22.mlp.linear_fc1.weight\n", + " \tmodule.decoder.layers.20.mlp.linear_fc1.layer_norm_weight\n", + " \tmodule.decoder.layers.20.mixer.dense_projection.weight\n", + " \tmodule.decoder.layers.14.mlp.linear_fc1.weight\n", + " \tmodule.decoder.layers.9.mlp.linear_fc1.weight\n", + " \tmodule.decoder.layers.8.mixer.dense_projection.layer_norm_weight\n", + " \tmodule.decoder.layers.5.mixer.mixer.filter.h\n", + " \tmodule.decoder.layers.23.mixer.dense.weight\n", + " \tmodule.decoder.layers.21.mlp.linear_fc1.layer_norm_weight\n", + " \tmodule.decoder.layers.20.mixer.mixer.filter.gamma\n", + " \tmodule.decoder.layers.18.mixer.dense_projection.weight\n", + " \tmodule.decoder.layers.15.mlp.linear_fc2.weight\n", + " \tmodule.decoder.layers.13.mixer.mixer.filter.p\n", + " \tmodule.decoder.layers.11.mixer.hyena_proj_conv.short_conv_weight\n", " \tmodule.decoder.layers.8.mixer.dense.weight\n", - " \tmodule.decoder.layers.6.mixer.dense.bias\n", - " \tmodule.decoder.layers.2.mixer.mixer.conv_bias\n", - " \tmodule.decoder.final_norm.weight\n", + " \tmodule.decoder.layers.1.mlp.linear_fc2.weight\n", + " \tmodule.decoder.layers.24.mlp.linear_fc1.weight\n", " \tmodule.decoder.layers.22.mixer.mixer.filter.h\n", " \tmodule.decoder.layers.20.mixer.mixer.filter.R\n", - " \tmodule.decoder.layers.18.mixer.dense_projection.layer_norm_weight\n", - " \tmodule.decoder.layers.14.mixer.dense.bias\n", - " \tmodule.decoder.layers.2.mlp.linear_fc1.layer_norm_weight\n", - "[NeMo I 2025-03-03 23:36:32 nemo_logging:393] Doing selective restore from RestoreConfig(path='nemo2_evo2_1b_8k', adapter_path=None, load_model_state=True, load_optim_state=False, load_artifacts=False)\n", - "[NeMo I 2025-03-03 23:36:32 nemo_logging:393] Using dist-ckpt load strategy.\n", - "[WARNING | py.warnings ]: /workspace/bionemo2/3rdparty/Megatron-LM/megatron/core/dist_checkpointing/strategies/torch.py:847: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead.\n", + " \tmodule.decoder.layers.14.mixer.dense.weight\n", + " \tmodule.decoder.layers.12.mixer.dense.bias\n", + " \tmodule.decoder.layers.9.mlp.linear_fc1.layer_norm_weight\n", + " \tmodule.decoder.layers.5.mixer.mixer.conv_bias\n", + " \tmodule.decoder.layers.4.mixer.hyena_proj_conv.short_conv_weight\n", + " \tmodule.decoder.layers.17.self_attention.linear_proj.bias\n", + " \tmodule.decoder.layers.17.mlp.linear_fc2.weight\n", + " \tmodule.decoder.layers.13.mixer.hyena_proj_conv.short_conv_weight\n", + " \tmodule.decoder.layers.10.self_attention.linear_qkv.weight\n", + " \tmodule.decoder.layers.8.mixer.dense_projection.weight\n", + " \tmodule.decoder.layers.6.mixer.mixer.filter.gamma\n", + " \tmodule.decoder.layers.24.self_attention.linear_qkv.layer_norm_weight\n", + " \tmodule.decoder.layers.22.mixer.mixer.conv_bias\n", + " \tmodule.decoder.layers.18.mlp.linear_fc1.weight\n", + " \tmodule.decoder.layers.16.mixer.dense.bias\n", + " \tmodule.decoder.layers.13.mlp.linear_fc1.weight\n", + " \tmodule.decoder.layers.12.mixer.dense_projection.layer_norm_weight\n", + " \tmodule.decoder.layers.9.mixer.mixer.filter.R\n", + " \tmodule.decoder.layers.7.mixer.dense_projection.layer_norm_weight\n", + " \tmodule.decoder.layers.3.self_attention.linear_qkv.weight\n", + " \tmodule.decoder.layers.0.mixer.hyena_proj_conv.short_conv_weight\n", + " \tmodule.decoder.layers.23.mixer.mixer.filter.gamma\n", + " \tmodule.decoder.layers.17.mlp.linear_fc1.layer_norm_weight\n", + " \tmodule.decoder.layers.15.mlp.linear_fc1.layer_norm_weight\n", + " \tmodule.decoder.layers.12.mixer.dense.weight\n", + " \tmodule.decoder.layers.11.mixer.dense.bias\n", + " \tmodule.decoder.layers.7.mlp.linear_fc2.weight\n", + " \tmodule.decoder.layers.1.mixer.hyena_proj_conv.short_conv_weight\n", + " \tmodule.decoder.layers.24.self_attention.linear_proj.weight\n", + " \tmodule.decoder.layers.21.mixer.mixer.short_conv.short_conv_weight\n", + " \tmodule.decoder.layers.18.mixer.dense.weight\n", + " \tmodule.decoder.layers.15.mlp.linear_fc1.weight\n", + " \tmodule.decoder.layers.13.mlp.linear_fc1.layer_norm_weight\n", + " \tmodule.decoder.layers.9.mixer.dense_projection.layer_norm_weight\n", + " \tmodule.decoder.layers.6.mixer.mixer.conv_bias\n", + " \tmodule.decoder.layers.1.mlp.linear_fc1.weight\n", + " \tmodule.decoder.layers.20.mixer.hyena_proj_conv.short_conv_weight\n", + " \tmodule.decoder.layers.16.mixer.dense.weight\n", + " \tmodule.decoder.layers.14.mlp.linear_fc1.layer_norm_weight\n", + " \tmodule.decoder.layers.12.mixer.dense_projection.weight\n", + " \tmodule.decoder.layers.7.mixer.dense_projection.weight\n", + " \tmodule.decoder.layers.5.mixer.hyena_proj_conv.short_conv_weight\n", + " \tmodule.decoder.layers.4.mixer.dense.weight\n", + " \tmodule.decoder.layers.23.mixer.mixer.conv_bias\n", + " \tmodule.decoder.layers.17.mlp.linear_fc1.weight\n", + " \tmodule.decoder.layers.15.mixer.mixer.filter.h\n", + " \tmodule.decoder.layers.13.mixer.mixer.filter.R\n", + " \tmodule.decoder.layers.11.mixer.dense_projection.layer_norm_weight\n", + " \tmodule.decoder.layers.7.mixer.dense.bias\n", + " \tmodule.decoder.layers.2.mixer.dense.weight\n", + " \tmodule.embedding.word_embeddings.weight\n", + " \tmodule.decoder.layers.22.mixer.hyena_proj_conv.short_conv_weight\n", + " \tmodule.decoder.layers.19.mlp.linear_fc1.layer_norm_weight\n", + " \tmodule.decoder.layers.11.mlp.linear_fc2.weight\n", + " \tmodule.decoder.layers.9.mixer.dense_projection.weight\n", + " \tmodule.decoder.layers.6.mlp.linear_fc2.weight\n", + " \tmodule.decoder.layers.4.mixer.dense_projection.layer_norm_weight\n", + " \tmodule.decoder.layers.0.mixer.dense.weight\n", + " \tmodule.decoder.layers.19.mlp.linear_fc1.weight\n", + " \tmodule.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight\n", + " \tmodule.decoder.layers.15.mixer.mixer.conv_bias\n", + " \tmodule.decoder.layers.13.mixer.dense_projection.layer_norm_weight\n", + " \tmodule.decoder.layers.7.mlp.linear_fc1.weight\n", + " \tmodule.decoder.layers.4.mlp.linear_fc2.weight\n", + " \tmodule.decoder.layers.2.mixer.dense.bias\n", + " \tmodule.decoder.layers.1.mixer.dense.bias\n", + " \tmodule.decoder.layers.23.mlp.linear_fc2.weight\n", + " \tmodule.decoder.layers.18.mlp.linear_fc1.layer_norm_weight\n", + " \tmodule.decoder.layers.16.mixer.mixer.filter.gamma\n", + " \tmodule.decoder.layers.11.mixer.dense_projection.weight\n", + " \tmodule.decoder.layers.8.mlp.linear_fc2.weight\n", + " \tmodule.decoder.layers.6.mixer.mixer.filter.p\n", + " \tmodule.decoder.layers.3.self_attention.linear_proj.weight\n", + " \tmodule.decoder.layers.2.mixer.mixer.filter.gamma\n", + " \tmodule.decoder.layers.0.mlp.linear_fc1.weight\n", + " \tmodule.decoder.layers.20.mixer.dense.bias\n", + " \tmodule.decoder.layers.19.mixer.mixer.filter.h\n", + " \tmodule.decoder.layers.17.self_attention.linear_proj.weight\n", + " \tmodule.decoder.layers.14.mixer.mixer.short_conv.short_conv_weight\n", + " \tmodule.decoder.layers.7.mixer.dense.weight\n", + " \tmodule.decoder.layers.4.mixer.dense_projection.weight\n", + " \tmodule.decoder.layers.1.mixer.dense_projection.layer_norm_weight\n", + " \tmodule.decoder.layers.23.mixer.mixer.filter.p\n", + " \tmodule.decoder.layers.21.mixer.hyena_proj_conv.short_conv_weight\n", + " \tmodule.decoder.layers.13.mixer.dense_projection.weight\n", + " \tmodule.decoder.layers.10.mlp.linear_fc2.weight\n", + " \tmodule.decoder.layers.6.mixer.hyena_proj_conv.short_conv_weight\n", + "[NeMo I 2025-03-04 01:01:11 nemo_logging:393] Doing selective restore from RestoreConfig(path='nemo2_evo2_1b_8k', adapter_path=None, load_model_state=True, load_optim_state=False, load_artifacts=False)\n", + "[NeMo I 2025-03-04 01:01:11 nemo_logging:393] Using dist-ckpt load strategy.\n", + "[WARNING | py.warnings ]: /workspaces/bionemo-framework/3rdparty/Megatron-LM/megatron/core/dist_checkpointing/strategies/torch.py:847: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead.\n", " checkpoint.load_state_dict(\n", "\n", "[WARNING | py.warnings ]: /usr/local/lib/python3.12/dist-packages/torch/distributed/checkpoint/planner_helpers.py:316: FutureWarning: Please use DTensor instead and we are deprecating ShardedTensor.\n", " device = getattr(value, \"device\", None)\n", "\n", - "[NeMo I 2025-03-03 23:36:33 nemo_logging:393] Global Checkpoint Load : Rank : 0 : Start time : 1741044992.504s : Time spent in load_checkpoint: 1.046s\n", - "[NeMo I 2025-03-03 23:36:33 nemo_logging:393] Restoring model weights from RestoreConfig(path='nemo2_evo2_1b_8k', adapter_path=None, load_model_state=True, load_optim_state=False, load_artifacts=False)\n", - "[NeMo I 2025-03-03 23:36:33 nemo_logging:393] Finished restoring from RestoreConfig(path='nemo2_evo2_1b_8k', adapter_path=None, load_model_state=True, load_optim_state=False, load_artifacts=False), cleaning up.\n" + "[NeMo I 2025-03-04 01:01:12 nemo_logging:393] Global Checkpoint Load : Rank : 0 : Start time : 1741050071.495s : Time spent in load_checkpoint: 0.932s\n", + "[NeMo I 2025-03-04 01:01:12 nemo_logging:393] Restoring model weights from RestoreConfig(path='nemo2_evo2_1b_8k', adapter_path=None, load_model_state=True, load_optim_state=False, load_artifacts=False)\n", + "[NeMo I 2025-03-04 01:01:12 nemo_logging:393] Finished restoring from RestoreConfig(path='nemo2_evo2_1b_8k', adapter_path=None, load_model_state=True, load_optim_state=False, load_artifacts=False), cleaning up.\n" ] } ], "source": [ + "print(f\"Running command: {predict_ref_command}\")\n", "!{predict_ref_command}" ] }, @@ -871,14 +913,17 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "[NeMo W 2025-03-03 23:37:15 nemo_logging:405] /usr/local/lib/python3.12/dist-packages/pydub/utils.py:170: RuntimeWarning: Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work\n", + "Running command: predict_evo2 --fasta brca1_fasta_files/brca1_variant_sequences.fasta --ckpt-dir nemo2_evo2_1b_8k --output-dir brca1_fasta_files/variant_predictions --model-size 1b --tensor-parallel-size 1 --pipeline-model-parallel-size 1 --context-parallel-size 1 --output-log-prob-seqs --fp8\n", + "[WARNING | bitsandbytes.cextension]: Could not find the bitsandbytes CUDA binary at PosixPath('/usr/local/lib/python3.12/dist-packages/bitsandbytes/libbitsandbytes_cuda128.so')\n", + "[WARNING | bitsandbytes.cextension]: The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers, 8-bit multiplication, and GPU quantization are unavailable.\n", + "[NeMo W 2025-03-04 01:02:34 nemo_logging:405] /usr/local/lib/python3.12/dist-packages/pydub/utils.py:170: RuntimeWarning: Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work\n", " warn(\"Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work\", RuntimeWarning)\n", " \n", "[WARNING | py.warnings ]: /usr/local/lib/python3.12/dist-packages/mamba_ssm/ops/selective_scan_interface.py:163: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead.\n", @@ -908,69 +953,102 @@ "[INFO | pytorch_lightning.utilities.rank_zero]: GPU available: True (cuda), used: True\n", "[INFO | pytorch_lightning.utilities.rank_zero]: TPU available: False, using: 0 TPU cores\n", "[INFO | pytorch_lightning.utilities.rank_zero]: HPU available: False, using: 0 HPUs\n", - "[NeMo W 2025-03-03 23:37:17 nemo_logging:405] No version folders would be created under the log folder as 'resume_if_exists' is enabled.\n", - "[NeMo I 2025-03-03 23:37:17 nemo_logging:393] Experiments will be logged at /tmp/tmpcu9581ff/default\n", - "[NeMo W 2025-03-03 23:37:17 nemo_logging:405] \"update_logger_directory\" is True. Overwriting tensorboard logger \"save_dir\" to /tmp/tmpcu9581ff\n", - "[NeMo I 2025-03-03 23:37:17 nemo_logging:393] Using byte-level tokenization\n", - "[NeMo I 2025-03-03 23:37:17 nemo_logging:393] Rank 0 has data parallel group : [0]\n", - "[NeMo I 2025-03-03 23:37:17 nemo_logging:393] Rank 0 has combined group of data parallel and context parallel : [0]\n", - "[NeMo I 2025-03-03 23:37:17 nemo_logging:393] All data parallel group ranks with context parallel combined: [[0]]\n", - "[NeMo I 2025-03-03 23:37:17 nemo_logging:393] Ranks 0 has data parallel rank: 0\n", - "[NeMo I 2025-03-03 23:37:17 nemo_logging:393] Rank 0 has context parallel group: [0]\n", - "[NeMo I 2025-03-03 23:37:17 nemo_logging:393] All context parallel group ranks: [[0]]\n", - "[NeMo I 2025-03-03 23:37:17 nemo_logging:393] Ranks 0 has context parallel rank: 0\n", - "[NeMo I 2025-03-03 23:37:17 nemo_logging:393] Rank 0 has model parallel group: [0]\n", - "[NeMo I 2025-03-03 23:37:17 nemo_logging:393] All model parallel group ranks: [[0]]\n", - "[NeMo I 2025-03-03 23:37:17 nemo_logging:393] Rank 0 has tensor model parallel group: [0]\n", - "[NeMo I 2025-03-03 23:37:17 nemo_logging:393] All tensor model parallel group ranks: [[0]]\n", - "[NeMo I 2025-03-03 23:37:17 nemo_logging:393] Rank 0 has tensor model parallel rank: 0\n", - "[NeMo I 2025-03-03 23:37:17 nemo_logging:393] Rank 0 has pipeline model parallel group: [0]\n", - "[NeMo I 2025-03-03 23:37:17 nemo_logging:393] Rank 0 has embedding group: [0]\n", - "[NeMo I 2025-03-03 23:37:17 nemo_logging:393] All pipeline model parallel group ranks: [[0]]\n", - "[NeMo I 2025-03-03 23:37:17 nemo_logging:393] Rank 0 has pipeline model parallel rank 0\n", - "[NeMo I 2025-03-03 23:37:17 nemo_logging:393] All embedding group ranks: [[0]]\n", - "[NeMo I 2025-03-03 23:37:17 nemo_logging:393] Rank 0 has embedding rank: 0\n", + "[NeMo W 2025-03-04 01:02:35 nemo_logging:405] No version folders would be created under the log folder as 'resume_if_exists' is enabled.\n", + "[NeMo I 2025-03-04 01:02:35 nemo_logging:393] Experiments will be logged at /tmp/tmpf9avvfzw/default\n", + "[NeMo W 2025-03-04 01:02:35 nemo_logging:405] \"update_logger_directory\" is True. Overwriting tensorboard logger \"save_dir\" to /tmp/tmpf9avvfzw\n", + "[NeMo I 2025-03-04 01:02:35 nemo_logging:393] Using byte-level tokenization\n", + "[NeMo I 2025-03-04 01:02:35 nemo_logging:393] Rank 0 has data parallel group : [0]\n", + "[NeMo I 2025-03-04 01:02:35 nemo_logging:393] Rank 0 has combined group of data parallel and context parallel : [0]\n", + "[NeMo I 2025-03-04 01:02:35 nemo_logging:393] All data parallel group ranks with context parallel combined: [[0]]\n", + "[NeMo I 2025-03-04 01:02:35 nemo_logging:393] Ranks 0 has data parallel rank: 0\n", + "[NeMo I 2025-03-04 01:02:35 nemo_logging:393] Rank 0 has context parallel group: [0]\n", + "[NeMo I 2025-03-04 01:02:35 nemo_logging:393] All context parallel group ranks: [[0]]\n", + "[NeMo I 2025-03-04 01:02:35 nemo_logging:393] Ranks 0 has context parallel rank: 0\n", + "[NeMo I 2025-03-04 01:02:35 nemo_logging:393] Rank 0 has model parallel group: [0]\n", + "[NeMo I 2025-03-04 01:02:35 nemo_logging:393] All model parallel group ranks: [[0]]\n", + "[NeMo I 2025-03-04 01:02:35 nemo_logging:393] Rank 0 has tensor model parallel group: [0]\n", + "[NeMo I 2025-03-04 01:02:35 nemo_logging:393] All tensor model parallel group ranks: [[0]]\n", + "[NeMo I 2025-03-04 01:02:35 nemo_logging:393] Rank 0 has tensor model parallel rank: 0\n", + "[NeMo I 2025-03-04 01:02:35 nemo_logging:393] Rank 0 has pipeline model parallel group: [0]\n", + "[NeMo I 2025-03-04 01:02:35 nemo_logging:393] Rank 0 has embedding group: [0]\n", + "[NeMo I 2025-03-04 01:02:35 nemo_logging:393] All pipeline model parallel group ranks: [[0]]\n", + "[NeMo I 2025-03-04 01:02:35 nemo_logging:393] Rank 0 has pipeline model parallel rank 0\n", + "[NeMo I 2025-03-04 01:02:35 nemo_logging:393] All embedding group ranks: [[0]]\n", + "[NeMo I 2025-03-04 01:02:35 nemo_logging:393] Rank 0 has embedding rank: 0\n", "Initializing distributed: GLOBAL_RANK: 0, MEMBER: 1/1\n", "[INFO | pytorch_lightning.utilities.rank_zero]: ----------------------------------------------------------------------------------------------------\n", "distributed_backend=nccl\n", "All distributed processes registered. Starting with 1 processes\n", "----------------------------------------------------------------------------------------------------\n", "\n", - "[NeMo I 2025-03-03 23:37:17 num_microbatches_calculator:228] setting number of microbatches to constant 1\n", - "[NeMo I 2025-03-03 23:37:17 nemo_logging:393] Padded vocab_size: 512, original vocab_size: 512, dummy tokens: 0.\n", - "[NeMo W 2025-03-03 23:37:17 random:220] CPU RNG state changed within GPU RNG context\n", - "[NeMo W 2025-03-03 23:37:17 random:220] CPU RNG state changed within GPU RNG context\n", - "[NeMo W 2025-03-03 23:37:17 random:220] CPU RNG state changed within GPU RNG context\n", - "[NeMo W 2025-03-03 23:37:17 random:220] CPU RNG state changed within GPU RNG context\n", - "[NeMo W 2025-03-03 23:37:17 random:220] CPU RNG state changed within GPU RNG context\n", - "[NeMo W 2025-03-03 23:37:17 random:220] CPU RNG state changed within GPU RNG context\n", - "[NeMo W 2025-03-03 23:37:17 random:220] CPU RNG state changed within GPU RNG context\n", - "[NeMo W 2025-03-03 23:37:17 random:220] CPU RNG state changed within GPU RNG context\n", - "[NeMo W 2025-03-03 23:37:17 random:220] CPU RNG state changed within GPU RNG context\n", - "[NeMo W 2025-03-03 23:37:17 random:220] CPU RNG state changed within GPU RNG context\n", - "[NeMo W 2025-03-03 23:37:17 random:220] CPU RNG state changed within GPU RNG context\n", - "[NeMo W 2025-03-03 23:37:17 random:220] CPU RNG state changed within GPU RNG context\n", - "[NeMo W 2025-03-03 23:37:17 random:220] CPU RNG state changed within GPU RNG context\n", - "[NeMo W 2025-03-03 23:37:17 random:220] CPU RNG state changed within GPU RNG context\n", - "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2]\n", - "[NeMo W 2025-03-03 23:37:17 nemo_logging:405] Could not copy Trainer's 'max_steps' to LR scheduler's 'max_steps'. If you are not using an LR scheduler, this warning can safely be ignored.\n", - "[NeMo I 2025-03-03 23:37:17 nemo_logging:393] > number of parameters on (tensor, pipeline) model parallel rank (0 ,0): 1108204800\n", - "[NeMo I 2025-03-03 23:37:17 utils:302] Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=False, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=False, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, average_in_collective=False, fp8_param_gather=False)\n", - "[NeMo I 2025-03-03 23:37:17 utils:323] Number of buckets for gradient all-reduce / reduce-scatter: 1\n", + "[NeMo I 2025-03-04 01:02:35 num_microbatches_calculator:228] setting number of microbatches to constant 1\n", + "[NeMo I 2025-03-04 01:02:35 nemo_logging:393] Padded vocab_size: 512, original vocab_size: 512, dummy tokens: 0.\n", + "[NeMo W 2025-03-04 01:02:35 random:220] CPU RNG state changed within GPU RNG context\n", + "[NeMo W 2025-03-04 01:02:35 random:220] CPU RNG state changed within GPU RNG context\n", + "[NeMo W 2025-03-04 01:02:35 random:220] CPU RNG state changed within GPU RNG context\n", + "[NeMo W 2025-03-04 01:02:35 random:220] CPU RNG state changed within GPU RNG context\n", + "[NeMo W 2025-03-04 01:02:35 random:220] CPU RNG state changed within GPU RNG context\n", + "[NeMo W 2025-03-04 01:02:35 random:220] CPU RNG state changed within GPU RNG context\n", + "[NeMo W 2025-03-04 01:02:35 random:220] CPU RNG state changed within GPU RNG context\n", + "[NeMo W 2025-03-04 01:02:35 random:220] CPU RNG state changed within GPU RNG context\n", + "[NeMo W 2025-03-04 01:02:35 random:220] CPU RNG state changed within GPU RNG context\n", + "[NeMo W 2025-03-04 01:02:35 random:220] CPU RNG state changed within GPU RNG context\n", + "[NeMo W 2025-03-04 01:02:35 random:220] CPU RNG state changed within GPU RNG context\n", + "[NeMo W 2025-03-04 01:02:35 random:220] CPU RNG state changed within GPU RNG context\n", + "[NeMo W 2025-03-04 01:02:35 random:220] CPU RNG state changed within GPU RNG context\n", + "[NeMo W 2025-03-04 01:02:35 random:220] CPU RNG state changed within GPU RNG context\n", + "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n", + "[NeMo W 2025-03-04 01:02:35 nemo_logging:405] Could not copy Trainer's 'max_steps' to LR scheduler's 'max_steps'. If you are not using an LR scheduler, this warning can safely be ignored.\n", + "[NeMo I 2025-03-04 01:02:35 nemo_logging:393] > number of parameters on (tensor, pipeline) model parallel rank (0 ,0): 1108204800\n", + "[NeMo I 2025-03-04 01:02:35 utils:302] Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=False, overlap_grad_reduce=False, overlap_param_gather=False, align_param_gather=False, use_distributed_optimizer=False, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=None, average_in_collective=False, fp8_param_gather=False)\n", + "[NeMo I 2025-03-04 01:02:35 utils:323] Number of buckets for gradient all-reduce / reduce-scatter: 1\n", " Params for bucket 1 (1108204800 elements):\n", + " \tmodule.decoder.layers.24.self_attention.linear_qkv.weight\n", + " \tmodule.decoder.layers.22.mixer.dense_projection.weight\n", + " \tmodule.decoder.layers.20.mixer.mixer.filter.gamma\n", + " \tmodule.decoder.layers.15.mixer.hyena_proj_conv.short_conv_weight\n", + " \tmodule.decoder.layers.12.mixer.mixer.conv_bias\n", + " \tmodule.decoder.layers.9.mixer.dense.bias\n", + " \tmodule.decoder.layers.7.mixer.dense.bias\n", + " \tmodule.decoder.layers.4.mlp.linear_fc2.weight\n", + " \tmodule.decoder.layers.2.mixer.dense.bias\n", + " \tmodule.decoder.layers.1.mixer.dense.bias\n", + " \tmodule.decoder.layers.23.mixer.mixer.filter.R\n", + " \tmodule.decoder.layers.21.mixer.dense_projection.layer_norm_weight\n", + " \tmodule.decoder.layers.18.mixer.mixer.short_conv.short_conv_weight\n", + " \tmodule.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight\n", + " \tmodule.decoder.layers.6.mixer.dense_projection.layer_norm_weight\n", + " \tmodule.decoder.layers.3.mlp.linear_fc1.layer_norm_weight\n", + " \tmodule.decoder.layers.2.mixer.mixer.filter.gamma\n", + " \tmodule.decoder.layers.21.mlp.linear_fc2.weight\n", + " \tmodule.decoder.layers.16.mlp.linear_fc2.weight\n", + " \tmodule.decoder.layers.11.mlp.linear_fc1.layer_norm_weight\n", + " \tmodule.decoder.layers.9.mixer.mixer.filter.gamma\n", + " \tmodule.decoder.layers.4.mlp.linear_fc1.layer_norm_weight\n", + " \tmodule.decoder.layers.1.mixer.dense_projection.layer_norm_weight\n", + " \tmodule.decoder.final_norm.weight\n", + " \tmodule.decoder.layers.23.mixer.dense_projection.layer_norm_weight\n", + " \tmodule.decoder.layers.20.mixer.mixer.conv_bias\n", + " \tmodule.decoder.layers.13.mixer.dense.bias\n", + " \tmodule.decoder.layers.12.mixer.mixer.filter.h\n", + " \tmodule.decoder.layers.10.self_attention.linear_proj.weight\n", + " \tmodule.decoder.layers.7.mixer.mixer.short_conv.short_conv_weight\n", + " \tmodule.decoder.layers.5.mixer.dense.bias\n", " \tmodule.decoder.layers.21.mixer.dense_projection.weight\n", " \tmodule.decoder.layers.19.mixer.hyena_proj_conv.short_conv_weight\n", " \tmodule.decoder.layers.16.mixer.mixer.filter.p\n", " \tmodule.decoder.layers.14.mixer.hyena_proj_conv.short_conv_weight\n", " \tmodule.decoder.layers.11.mixer.dense.bias\n", + " \tmodule.decoder.layers.10.self_attention.linear_proj.bias\n", " \tmodule.decoder.layers.6.mixer.dense_projection.weight\n", " \tmodule.decoder.layers.3.mlp.linear_fc1.weight\n", - " \tmodule.decoder.layers.0.mlp.linear_fc1.layer_norm_weight\n", + " \tmodule.decoder.layers.0.mixer.dense_projection.weight\n", " \tmodule.decoder.layers.21.mixer.dense.bias\n", " \tmodule.decoder.layers.15.mixer.dense.bias\n", - " \tmodule.decoder.layers.12.mlp.linear_fc1.weight\n", + " \tmodule.decoder.layers.13.mixer.dense_projection.layer_norm_weight\n", " \tmodule.decoder.layers.4.mlp.linear_fc1.weight\n", - " \tmodule.decoder.layers.2.mixer.mixer.filter.p\n", + " \tmodule.decoder.layers.2.mixer.mixer.filter.R\n", + " \tmodule.decoder.layers.1.mixer.dense_projection.weight\n", " \tmodule.decoder.layers.23.mixer.dense_projection.weight\n", " \tmodule.decoder.layers.20.mlp.linear_fc2.weight\n", " \tmodule.decoder.layers.16.mixer.hyena_proj_conv.short_conv_weight\n", @@ -978,47 +1056,44 @@ " \tmodule.decoder.layers.8.mixer.hyena_proj_conv.short_conv_weight\n", " \tmodule.decoder.layers.5.mlp.linear_fc2.weight\n", " \tmodule.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight\n", - " \tmodule.decoder.layers.2.mixer.hyena_proj_conv.short_conv_weight\n", - " \tmodule.decoder.layers.1.mixer.dense_projection.layer_norm_weight\n", " \tmodule.decoder.layers.24.self_attention.linear_proj.bias\n", " \tmodule.decoder.layers.21.mlp.linear_fc1.weight\n", " \tmodule.decoder.layers.16.mlp.linear_fc1.weight\n", " \tmodule.decoder.layers.15.mixer.dense_projection.layer_norm_weight\n", " \tmodule.decoder.layers.11.mixer.mixer.short_conv.short_conv_weight\n", " \tmodule.decoder.layers.6.mixer.mixer.conv_bias\n", - " \tmodule.decoder.layers.0.mlp.linear_fc1.weight\n", + " \tmodule.decoder.layers.0.mlp.linear_fc2.weight\n", " \tmodule.decoder.layers.22.mlp.linear_fc2.weight\n", " \tmodule.decoder.layers.20.mixer.mixer.filter.p\n", " \tmodule.decoder.layers.18.mixer.hyena_proj_conv.short_conv_weight\n", " \tmodule.decoder.layers.15.mixer.dense.weight\n", - " \tmodule.decoder.layers.10.self_attention.linear_proj.bias\n", " \tmodule.decoder.layers.9.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.2.mlp.linear_fc1.weight\n", - " \tmodule.decoder.layers.2.mixer.dense_projection.layer_norm_weight\n", + " \tmodule.decoder.layers.2.mlp.linear_fc2.weight\n", + " \tmodule.decoder.layers.1.mixer.dense.weight\n", " \tmodule.decoder.layers.21.mixer.dense.weight\n", " \tmodule.decoder.layers.19.mixer.dense.bias\n", " \tmodule.decoder.layers.16.mlp.linear_fc1.layer_norm_weight\n", - " \tmodule.decoder.layers.12.mixer.dense_projection.layer_norm_weight\n", " \tmodule.decoder.layers.8.mlp.linear_fc1.layer_norm_weight\n", " \tmodule.decoder.layers.4.mixer.dense.bias\n", - " \tmodule.decoder.layers.2.mixer.mixer.conv_bias\n", + " \tmodule.decoder.layers.3.self_attention.linear_proj.bias\n", + " \tmodule.decoder.layers.0.mlp.linear_fc1.layer_norm_weight\n", " \tmodule.decoder.layers.24.mlp.linear_fc2.weight\n", " \tmodule.decoder.layers.20.mixer.hyena_proj_conv.short_conv_weight\n", " \tmodule.decoder.layers.17.self_attention.linear_qkv.weight\n", " \tmodule.decoder.layers.15.mixer.dense_projection.weight\n", - " \tmodule.decoder.layers.13.mixer.mixer.filter.gamma\n", + " \tmodule.decoder.layers.12.mixer.hyena_proj_conv.short_conv_weight\n", " \tmodule.decoder.layers.9.mixer.mixer.filter.p\n", " \tmodule.decoder.layers.7.mixer.hyena_proj_conv.short_conv_weight\n", " \tmodule.decoder.layers.5.mixer.mixer.conv_bias\n", - " \tmodule.decoder.layers.2.mlp.linear_fc1.layer_norm_weight\n", + " \tmodule.decoder.layers.2.mixer.mixer.filter.p\n", " \tmodule.decoder.layers.23.mixer.dense.bias\n", " \tmodule.decoder.layers.20.mlp.linear_fc1.weight\n", " \tmodule.decoder.layers.19.mixer.dense_projection.layer_norm_weight\n", " \tmodule.decoder.layers.16.mixer.mixer.filter.R\n", " \tmodule.decoder.layers.14.mixer.dense_projection.layer_norm_weight\n", + " \tmodule.decoder.layers.13.mixer.mixer.filter.gamma\n", " \tmodule.decoder.layers.5.mlp.linear_fc1.weight\n", - " \tmodule.decoder.layers.3.self_attention.linear_proj.bias\n", - " \tmodule.decoder.layers.0.mixer.dense.bias\n", + " \tmodule.decoder.layers.2.mixer.hyena_proj_conv.short_conv_weight\n", " \tmodule.decoder.layers.24.mlp.linear_fc1.layer_norm_weight\n", " \tmodule.decoder.layers.22.mlp.linear_fc1.layer_norm_weight\n", " \tmodule.decoder.layers.19.mixer.dense.weight\n", @@ -1027,63 +1102,68 @@ " \tmodule.decoder.layers.9.mixer.hyena_proj_conv.short_conv_weight\n", " \tmodule.decoder.layers.6.mixer.dense.weight\n", " \tmodule.decoder.layers.4.mixer.mixer.short_conv.short_conv_weight\n", - " \tmodule.decoder.layers.1.mixer.mixer.conv_bias\n", + " \tmodule.decoder.layers.0.mlp.linear_fc1.weight\n", " \tmodule.decoder.layers.22.mlp.linear_fc1.weight\n", " \tmodule.decoder.layers.20.mlp.linear_fc1.layer_norm_weight\n", " \tmodule.decoder.layers.16.mixer.dense_projection.layer_norm_weight\n", " \tmodule.decoder.layers.13.mixer.mixer.conv_bias\n", + " \tmodule.decoder.layers.12.mlp.linear_fc1.weight\n", " \tmodule.decoder.layers.9.mlp.linear_fc1.weight\n", " \tmodule.decoder.layers.8.mixer.dense_projection.layer_norm_weight\n", " \tmodule.decoder.layers.5.mixer.mixer.filter.h\n", - " \tmodule.decoder.layers.1.mlp.linear_fc1.layer_norm_weight\n", + " \tmodule.decoder.layers.2.mlp.linear_fc1.weight\n", + " \tmodule.decoder.layers.2.mixer.dense_projection.layer_norm_weight\n", " \tmodule.decoder.layers.23.mixer.dense.weight\n", " \tmodule.decoder.layers.21.mlp.linear_fc1.layer_norm_weight\n", " \tmodule.decoder.layers.19.mixer.dense_projection.weight\n", " \tmodule.decoder.layers.14.mixer.dense_projection.weight\n", - " \tmodule.decoder.layers.12.mixer.mixer.conv_bias\n", " \tmodule.decoder.layers.11.mixer.hyena_proj_conv.short_conv_weight\n", " \tmodule.decoder.layers.8.mixer.dense.weight\n", " \tmodule.decoder.layers.6.mixer.dense.bias\n", - " \tmodule.decoder.layers.0.mixer.mixer.short_conv.short_conv_weight\n", + " \tmodule.decoder.layers.2.mixer.mixer.conv_bias\n", " \tmodule.decoder.layers.24.mlp.linear_fc1.weight\n", " \tmodule.decoder.layers.22.mixer.mixer.filter.h\n", " \tmodule.decoder.layers.20.mixer.mixer.filter.R\n", " \tmodule.decoder.layers.18.mixer.dense_projection.layer_norm_weight\n", " \tmodule.decoder.layers.14.mixer.dense.bias\n", " \tmodule.decoder.layers.9.mlp.linear_fc1.layer_norm_weight\n", - " \tmodule.decoder.layers.1.mixer.mixer.filter.h\n", + " \tmodule.decoder.layers.2.mlp.linear_fc1.layer_norm_weight\n", " \tmodule.decoder.layers.18.mlp.linear_fc2.weight\n", " \tmodule.decoder.layers.16.mixer.dense_projection.weight\n", " \tmodule.decoder.layers.13.mlp.linear_fc2.weight\n", " \tmodule.decoder.layers.10.self_attention.linear_qkv.weight\n", " \tmodule.decoder.layers.8.mixer.dense_projection.weight\n", " \tmodule.decoder.layers.6.mixer.mixer.filter.gamma\n", - " \tmodule.decoder.layers.2.mixer.dense_projection.weight\n", - " \tmodule.decoder.layers.0.mixer.dense_projection.layer_norm_weight\n", + " \tmodule.decoder.layers.0.mixer.dense.bias\n", " \tmodule.decoder.layers.24.self_attention.linear_qkv.layer_norm_weight\n", " \tmodule.decoder.layers.22.mixer.mixer.conv_bias\n", " \tmodule.decoder.layers.20.mixer.dense_projection.layer_norm_weight\n", " \tmodule.decoder.layers.14.mlp.linear_fc1.weight\n", - " \tmodule.decoder.layers.12.mixer.dense_projection.weight\n", + " \tmodule.decoder.layers.12.mixer.dense_projection.layer_norm_weight\n", " \tmodule.decoder.layers.9.mixer.mixer.filter.R\n", " \tmodule.decoder.layers.7.mixer.dense_projection.layer_norm_weight\n", + " \tmodule.decoder.layers.1.mixer.mixer.conv_bias\n", " \tmodule.decoder.layers.23.mixer.mixer.filter.gamma\n", " \tmodule.decoder.layers.18.mixer.dense_projection.weight\n", " \tmodule.decoder.layers.15.mlp.linear_fc2.weight\n", " \tmodule.decoder.layers.13.mixer.mixer.filter.p\n", + " \tmodule.decoder.layers.12.mixer.dense.weight\n", " \tmodule.decoder.layers.7.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.1.mlp.linear_fc2.weight\n", + " \tmodule.decoder.layers.1.mlp.linear_fc1.layer_norm_weight\n", + " \tmodule.embedding.word_embeddings.weight\n", " \tmodule.decoder.layers.24.self_attention.linear_proj.weight\n", " \tmodule.decoder.layers.21.mixer.mixer.short_conv.short_conv_weight\n", " \tmodule.decoder.layers.14.mixer.dense.weight\n", " \tmodule.decoder.layers.9.mixer.dense_projection.layer_norm_weight\n", " \tmodule.decoder.layers.4.mixer.hyena_proj_conv.short_conv_weight\n", + " \tmodule.decoder.layers.0.mixer.mixer.short_conv.short_conv_weight\n", " \tmodule.decoder.layers.20.mixer.dense_projection.weight\n", " \tmodule.decoder.layers.17.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.17.self_attention.linear_proj.bias\n", " \tmodule.decoder.layers.13.mixer.hyena_proj_conv.short_conv_weight\n", + " \tmodule.decoder.layers.12.mixer.dense_projection.weight\n", " \tmodule.decoder.layers.7.mixer.dense_projection.weight\n", " \tmodule.decoder.layers.5.mixer.hyena_proj_conv.short_conv_weight\n", + " \tmodule.decoder.layers.1.mixer.mixer.filter.h\n", " \tmodule.decoder.layers.23.mixer.mixer.conv_bias\n", " \tmodule.decoder.layers.18.mlp.linear_fc1.weight\n", " \tmodule.decoder.layers.16.mixer.dense.bias\n", @@ -1091,55 +1171,53 @@ " \tmodule.decoder.layers.11.mixer.dense_projection.layer_norm_weight\n", " \tmodule.decoder.layers.8.mixer.dense.bias\n", " \tmodule.decoder.layers.3.self_attention.linear_qkv.weight\n", - " \tmodule.decoder.layers.0.mixer.hyena_proj_conv.short_conv_weight\n", + " \tmodule.decoder.layers.2.mixer.dense_projection.weight\n", + " \tmodule.decoder.layers.0.mixer.dense_projection.layer_norm_weight\n", " \tmodule.decoder.layers.22.mixer.hyena_proj_conv.short_conv_weight\n", " \tmodule.decoder.layers.19.mlp.linear_fc2.weight\n", " \tmodule.decoder.layers.17.mlp.linear_fc1.layer_norm_weight\n", " \tmodule.decoder.layers.15.mlp.linear_fc1.layer_norm_weight\n", - " \tmodule.decoder.layers.12.mixer.dense.weight\n", " \tmodule.decoder.layers.11.mlp.linear_fc2.weight\n", " \tmodule.decoder.layers.9.mixer.dense_projection.weight\n", " \tmodule.decoder.layers.6.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.1.mixer.hyena_proj_conv.short_conv_weight\n", - " \tmodule.decoder.layers.1.mixer.dense_projection.weight\n", " \tmodule.decoder.layers.18.mixer.dense.weight\n", " \tmodule.decoder.layers.15.mlp.linear_fc1.weight\n", " \tmodule.decoder.layers.13.mlp.linear_fc1.layer_norm_weight\n", + " \tmodule.decoder.layers.12.mlp.linear_fc2.weight\n", + " \tmodule.decoder.layers.12.mlp.linear_fc1.layer_norm_weight\n", " \tmodule.decoder.layers.7.mlp.linear_fc1.weight\n", - " \tmodule.decoder.layers.1.mlp.linear_fc1.weight\n", + " \tmodule.decoder.layers.1.mlp.linear_fc2.weight\n", " \tmodule.decoder.layers.23.mlp.linear_fc2.weight\n", + " \tmodule.decoder.layers.17.self_attention.linear_proj.bias\n", " \tmodule.decoder.layers.16.mixer.dense.weight\n", " \tmodule.decoder.layers.14.mlp.linear_fc1.layer_norm_weight\n", - " \tmodule.decoder.layers.12.mixer.hyena_proj_conv.short_conv_weight\n", " \tmodule.decoder.layers.11.mixer.dense_projection.weight\n", " \tmodule.decoder.layers.8.mlp.linear_fc2.weight\n", " \tmodule.decoder.layers.6.mixer.mixer.filter.p\n", " \tmodule.decoder.layers.4.mixer.dense.weight\n", - " \tmodule.embedding.word_embeddings.weight\n", " \tmodule.decoder.layers.20.mixer.dense.bias\n", " \tmodule.decoder.layers.17.mlp.linear_fc1.weight\n", " \tmodule.decoder.layers.15.mixer.mixer.filter.h\n", " \tmodule.decoder.layers.13.mixer.mixer.filter.R\n", + " \tmodule.decoder.layers.12.mixer.dense.bias\n", " \tmodule.decoder.layers.9.mixer.mixer.conv_bias\n", " \tmodule.decoder.layers.7.mixer.dense.weight\n", - " \tmodule.decoder.layers.2.mixer.dense.weight\n", " \tmodule.decoder.layers.23.mixer.mixer.filter.p\n", " \tmodule.decoder.layers.21.mixer.hyena_proj_conv.short_conv_weight\n", " \tmodule.decoder.layers.19.mlp.linear_fc1.layer_norm_weight\n", " \tmodule.decoder.layers.10.mlp.linear_fc2.weight\n", " \tmodule.decoder.layers.6.mixer.hyena_proj_conv.short_conv_weight\n", " \tmodule.decoder.layers.4.mixer.dense_projection.layer_norm_weight\n", - " \tmodule.decoder.layers.0.mixer.dense.weight\n", + " \tmodule.decoder.layers.0.mixer.hyena_proj_conv.short_conv_weight\n", + " \tmodule.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight\n", " \tmodule.decoder.layers.22.mixer.dense.bias\n", " \tmodule.decoder.layers.19.mlp.linear_fc1.weight\n", - " \tmodule.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight\n", " \tmodule.decoder.layers.15.mixer.mixer.conv_bias\n", - " \tmodule.decoder.layers.13.mixer.dense_projection.layer_norm_weight\n", " \tmodule.decoder.layers.11.mlp.linear_fc1.weight\n", " \tmodule.decoder.layers.6.mlp.linear_fc1.weight\n", + " \tmodule.decoder.layers.5.mlp.linear_fc1.layer_norm_weight\n", " \tmodule.decoder.layers.5.mixer.dense_projection.layer_norm_weight\n", - " \tmodule.decoder.layers.2.mixer.dense.bias\n", - " \tmodule.decoder.layers.1.mixer.dense.bias\n", + " \tmodule.decoder.layers.1.mixer.hyena_proj_conv.short_conv_weight\n", " \tmodule.decoder.layers.23.mixer.hyena_proj_conv.short_conv_weight\n", " \tmodule.decoder.layers.20.mixer.dense.weight\n", " \tmodule.decoder.layers.18.mlp.linear_fc1.layer_norm_weight\n", @@ -1148,23 +1226,22 @@ " \tmodule.decoder.layers.8.mixer.mixer.conv_bias\n", " \tmodule.decoder.layers.5.mixer.dense.weight\n", " \tmodule.decoder.layers.3.self_attention.linear_proj.weight\n", - " \tmodule.decoder.layers.2.mixer.mixer.filter.gamma\n", + " \tmodule.decoder.layers.1.mlp.linear_fc1.weight\n", " \tmodule.decoder.layers.23.mlp.linear_fc1.weight\n", " \tmodule.decoder.layers.22.mixer.dense_projection.layer_norm_weight\n", " \tmodule.decoder.layers.19.mixer.mixer.filter.h\n", " \tmodule.decoder.layers.17.self_attention.linear_proj.weight\n", " \tmodule.decoder.layers.14.mixer.mixer.short_conv.short_conv_weight\n", - " \tmodule.decoder.layers.12.mixer.dense.bias\n", " \tmodule.decoder.layers.11.mixer.dense.weight\n", " \tmodule.decoder.layers.8.mlp.linear_fc1.weight\n", " \tmodule.decoder.layers.6.mlp.linear_fc1.layer_norm_weight\n", " \tmodule.decoder.layers.4.mixer.dense_projection.weight\n", - " \tmodule.decoder.final_norm.weight\n", " \tmodule.decoder.layers.22.mixer.dense.weight\n", " \tmodule.decoder.layers.13.mixer.dense_projection.weight\n", " \tmodule.decoder.layers.9.mixer.dense.weight\n", " \tmodule.decoder.layers.7.mlp.linear_fc1.layer_norm_weight\n", " \tmodule.decoder.layers.5.mixer.dense_projection.weight\n", + " \tmodule.decoder.layers.2.mixer.dense.weight\n", " \tmodule.decoder.layers.23.mlp.linear_fc1.layer_norm_weight\n", " \tmodule.decoder.layers.19.mixer.mixer.conv_bias\n", " \tmodule.decoder.layers.16.mixer.mixer.conv_bias\n", @@ -1172,54 +1249,23 @@ " \tmodule.decoder.layers.8.mixer.mixer.filter.h\n", " \tmodule.decoder.layers.6.mixer.mixer.filter.R\n", " \tmodule.decoder.layers.3.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.0.mixer.dense_projection.weight\n", - " \tmodule.decoder.layers.24.self_attention.linear_qkv.weight\n", - " \tmodule.decoder.layers.22.mixer.dense_projection.weight\n", - " \tmodule.decoder.layers.20.mixer.mixer.filter.gamma\n", - " \tmodule.decoder.layers.15.mixer.hyena_proj_conv.short_conv_weight\n", - " \tmodule.decoder.layers.12.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.12.mixer.mixer.filter.h\n", - " \tmodule.decoder.layers.9.mixer.dense.bias\n", - " \tmodule.decoder.layers.7.mixer.dense.bias\n", - " \tmodule.decoder.layers.4.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.2.mixer.mixer.filter.R\n", - " \tmodule.decoder.layers.23.mixer.mixer.filter.R\n", - " \tmodule.decoder.layers.21.mixer.dense_projection.layer_norm_weight\n", - " \tmodule.decoder.layers.18.mixer.mixer.short_conv.short_conv_weight\n", - " \tmodule.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight\n", - " \tmodule.decoder.layers.6.mixer.dense_projection.layer_norm_weight\n", - " \tmodule.decoder.layers.5.mlp.linear_fc1.layer_norm_weight\n", - " \tmodule.decoder.layers.3.mlp.linear_fc1.layer_norm_weight\n", - " \tmodule.decoder.layers.21.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.16.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.12.mlp.linear_fc1.layer_norm_weight\n", - " \tmodule.decoder.layers.11.mlp.linear_fc1.layer_norm_weight\n", - " \tmodule.decoder.layers.9.mixer.mixer.filter.gamma\n", - " \tmodule.decoder.layers.4.mlp.linear_fc1.layer_norm_weight\n", - " \tmodule.decoder.layers.0.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.23.mixer.dense_projection.layer_norm_weight\n", - " \tmodule.decoder.layers.20.mixer.mixer.conv_bias\n", - " \tmodule.decoder.layers.13.mixer.dense.bias\n", - " \tmodule.decoder.layers.10.self_attention.linear_proj.weight\n", - " \tmodule.decoder.layers.7.mixer.mixer.short_conv.short_conv_weight\n", - " \tmodule.decoder.layers.5.mixer.dense.bias\n", - " \tmodule.decoder.layers.2.mlp.linear_fc2.weight\n", - " \tmodule.decoder.layers.1.mixer.dense.weight\n", - "[NeMo I 2025-03-03 23:37:17 nemo_logging:393] Doing selective restore from RestoreConfig(path='nemo2_evo2_1b_8k', adapter_path=None, load_model_state=True, load_optim_state=False, load_artifacts=False)\n", - "[NeMo I 2025-03-03 23:37:17 nemo_logging:393] Using dist-ckpt load strategy.\n", - "[WARNING | py.warnings ]: /workspace/bionemo2/3rdparty/Megatron-LM/megatron/core/dist_checkpointing/strategies/torch.py:847: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead.\n", + " \tmodule.decoder.layers.0.mixer.dense.weight\n", + "[NeMo I 2025-03-04 01:02:35 nemo_logging:393] Doing selective restore from RestoreConfig(path='nemo2_evo2_1b_8k', adapter_path=None, load_model_state=True, load_optim_state=False, load_artifacts=False)\n", + "[NeMo I 2025-03-04 01:02:35 nemo_logging:393] Using dist-ckpt load strategy.\n", + "[WARNING | py.warnings ]: /workspaces/bionemo-framework/3rdparty/Megatron-LM/megatron/core/dist_checkpointing/strategies/torch.py:847: FutureWarning: `load_state_dict` is deprecated and will be removed in future versions. Please use `load` instead.\n", " checkpoint.load_state_dict(\n", "\n", "[WARNING | py.warnings ]: /usr/local/lib/python3.12/dist-packages/torch/distributed/checkpoint/planner_helpers.py:316: FutureWarning: Please use DTensor instead and we are deprecating ShardedTensor.\n", " device = getattr(value, \"device\", None)\n", "\n", - "[NeMo I 2025-03-03 23:37:18 nemo_logging:393] Global Checkpoint Load : Rank : 0 : Start time : 1741045037.679s : Time spent in load_checkpoint: 1.103s\n", - "[NeMo I 2025-03-03 23:37:18 nemo_logging:393] Restoring model weights from RestoreConfig(path='nemo2_evo2_1b_8k', adapter_path=None, load_model_state=True, load_optim_state=False, load_artifacts=False)\n", - "[NeMo I 2025-03-03 23:37:18 nemo_logging:393] Finished restoring from RestoreConfig(path='nemo2_evo2_1b_8k', adapter_path=None, load_model_state=True, load_optim_state=False, load_artifacts=False), cleaning up.\n" + "[NeMo I 2025-03-04 01:02:36 nemo_logging:393] Global Checkpoint Load : Rank : 0 : Start time : 1741050155.807s : Time spent in load_checkpoint: 0.618s\n", + "[NeMo I 2025-03-04 01:02:36 nemo_logging:393] Restoring model weights from RestoreConfig(path='nemo2_evo2_1b_8k', adapter_path=None, load_model_state=True, load_optim_state=False, load_artifacts=False)\n", + "[NeMo I 2025-03-04 01:02:36 nemo_logging:393] Finished restoring from RestoreConfig(path='nemo2_evo2_1b_8k', adapter_path=None, load_model_state=True, load_optim_state=False, load_artifacts=False), cleaning up.\n" ] } ], "source": [ + "print(f\"Running command: {predict_var_command}\")\n", "!{predict_var_command}" ] }, @@ -1240,7 +1286,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -1268,7 +1314,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -1309,108 +1355,101 @@ " \n", " 0\n", " 17\n", - " 41199726\n", - " T\n", + " 41199729\n", " C\n", - " 0.159762\n", - " FUNC/INT\n", - " BRCA1_ref_pos_41199726_T_class_FUNC/INT\n", - " BRCA1_var_pos_41199726_TtoC_class_FUNC/INT\n", - " -1.048409\n", - " -1.048462\n", - " -0.000054\n", + " T\n", + " -2.646816\n", + " LOF\n", + " BRCA1_ref_pos_41199729_C_class_LOF\n", + " BRCA1_var_pos_41199729_CtoT_class_LOF\n", + " -0.952360\n", + " -0.953044\n", + " -0.000684\n", " \n", " \n", " 1\n", " 17\n", - " 41209074\n", + " 41215381\n", " T\n", - " A\n", - " -2.065569\n", + " G\n", + " -2.352741\n", " LOF\n", - " BRCA1_ref_pos_41209074_T_class_LOF\n", - " BRCA1_var_pos_41209074_TtoA_class_LOF\n", - " -0.826655\n", - " -0.826915\n", - " -0.000260\n", + " BRCA1_ref_pos_41215381_T_class_LOF\n", + " BRCA1_var_pos_41215381_TtoG_class_LOF\n", + " -0.848368\n", + " -0.848730\n", + " -0.000361\n", " \n", " \n", " 2\n", " 17\n", - " 41256913\n", - " A\n", + " 41215390\n", " C\n", - " -0.847753\n", - " FUNC/INT\n", - " BRCA1_ref_pos_41256913_A_class_FUNC/INT\n", - " BRCA1_var_pos_41256913_AtoC_class_FUNC/INT\n", - " -0.864035\n", - " -0.864014\n", - " 0.000021\n", + " A\n", + " -1.371155\n", + " LOF\n", + " BRCA1_ref_pos_41215390_C_class_LOF\n", + " BRCA1_var_pos_41215390_CtoA_class_LOF\n", + " -0.848341\n", + " -0.847456\n", + " 0.000885\n", " \n", " \n", " 3\n", " 17\n", - " 41219631\n", + " 41219688\n", " T\n", " A\n", - " -2.053739\n", + " -2.053136\n", " LOF\n", - " BRCA1_ref_pos_41219631_T_class_LOF\n", - " BRCA1_var_pos_41219631_TtoA_class_LOF\n", - " -1.091372\n", - " -1.091227\n", - " 0.000145\n", + " BRCA1_ref_pos_41219688_T_class_LOF\n", + " BRCA1_var_pos_41219688_TtoA_class_LOF\n", + " -1.027623\n", + " -1.028068\n", + " -0.000445\n", " \n", " \n", " 4\n", " 17\n", - " 41215965\n", + " 41219652\n", + " C\n", " G\n", - " A\n", - " -1.671525\n", + " -2.026390\n", " LOF\n", - " BRCA1_ref_pos_41215965_G_class_LOF\n", - " BRCA1_var_pos_41215965_GtoA_class_LOF\n", - " -0.930776\n", - " -0.930750\n", - " 0.000026\n", + " BRCA1_ref_pos_41219652_C_class_LOF\n", + " BRCA1_var_pos_41219652_CtoG_class_LOF\n", + " -1.032667\n", + " -1.032678\n", + " -0.000011\n", " \n", " \n", "\n", "" ], "text/plain": [ - " chrom pos ref alt score class \\\n", - "0 17 41199726 T C 0.159762 FUNC/INT \n", - "1 17 41209074 T A -2.065569 LOF \n", - "2 17 41256913 A C -0.847753 FUNC/INT \n", - "3 17 41219631 T A -2.053739 LOF \n", - "4 17 41215965 G A -1.671525 LOF \n", - "\n", - " ref_fasta_name \\\n", - "0 BRCA1_ref_pos_41199726_T_class_FUNC/INT \n", - "1 BRCA1_ref_pos_41209074_T_class_LOF \n", - "2 BRCA1_ref_pos_41256913_A_class_FUNC/INT \n", - "3 BRCA1_ref_pos_41219631_T_class_LOF \n", - "4 BRCA1_ref_pos_41215965_G_class_LOF \n", + " chrom pos ref alt score class \\\n", + "0 17 41199729 C T -2.646816 LOF \n", + "1 17 41215381 T G -2.352741 LOF \n", + "2 17 41215390 C A -1.371155 LOF \n", + "3 17 41219688 T A -2.053136 LOF \n", + "4 17 41219652 C G -2.026390 LOF \n", "\n", - " var_fasta_name ref_log_probs var_log_probs \\\n", - "0 BRCA1_var_pos_41199726_TtoC_class_FUNC/INT -1.048409 -1.048462 \n", - "1 BRCA1_var_pos_41209074_TtoA_class_LOF -0.826655 -0.826915 \n", - "2 BRCA1_var_pos_41256913_AtoC_class_FUNC/INT -0.864035 -0.864014 \n", - "3 BRCA1_var_pos_41219631_TtoA_class_LOF -1.091372 -1.091227 \n", - "4 BRCA1_var_pos_41215965_GtoA_class_LOF -0.930776 -0.930750 \n", + " ref_fasta_name var_fasta_name \\\n", + "0 BRCA1_ref_pos_41199729_C_class_LOF BRCA1_var_pos_41199729_CtoT_class_LOF \n", + "1 BRCA1_ref_pos_41215381_T_class_LOF BRCA1_var_pos_41215381_TtoG_class_LOF \n", + "2 BRCA1_ref_pos_41215390_C_class_LOF BRCA1_var_pos_41215390_CtoA_class_LOF \n", + "3 BRCA1_ref_pos_41219688_T_class_LOF BRCA1_var_pos_41219688_TtoA_class_LOF \n", + "4 BRCA1_ref_pos_41219652_C_class_LOF BRCA1_var_pos_41219652_CtoG_class_LOF \n", "\n", - " evo2_delta_score \n", - "0 -0.000054 \n", - "1 -0.000260 \n", - "2 0.000021 \n", - "3 0.000145 \n", - "4 0.000026 " + " ref_log_probs var_log_probs evo2_delta_score \n", + "0 -0.952360 -0.953044 -0.000684 \n", + "1 -0.848368 -0.848730 -0.000361 \n", + "2 -0.848341 -0.847456 0.000885 \n", + "3 -1.027623 -1.028068 -0.000445 \n", + "4 -1.032667 -1.032678 -0.000011 " ] }, - "execution_count": 26, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -1440,12 +1479,12 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 14, "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAC+CAYAAAAx3qiRAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAPWVJREFUeJzt3XdYU9f/B/B3AknYoGwQZSniwgW4RcXiXq0TrVrr3iiO2oq1tjjqqFupirWOr3vhRAFHpW6LljpwoCwVIYDs5Pz+0NwfkYCJJATw83oensq9N+e+uQ35cO899xweY4yBEEIIeY+v7QCEEEIqFioMhBBC5FBhIIQQIocKAyGEEDlUGAghhMihwkAIIUQOFQZCCCFyqDAQQgiRo6vtAKR0UqkUiYmJMDY2Bo/H03YcQkglxRhDZmYm7OzswOeXfk5AhaGCS0xMhIODg7ZjEEKqiOfPn6NGjRqlbkOFoYIzNjYG8O5/pomJiZbTEEIqq4yMDDg4OHCfKaWhwlDByS4fmZiYUGEghJSZMpek6eYzIYQQOVQYCCGEyKHCQAghRI7K9xhu3rwJgUCAhg0bAgCOHDmCbdu2oV69eliwYAGEQqHaQxJCVLNixQpkZGTAxMQEAQEB2o5DKhmVzxjGjh2LBw8eAAAeP36MQYMGwcDAAPv27cOsWbPUHpAQoroVK1bgxx9/xIoVK7QdhVRCKheGBw8eoHHjxgCAffv2oV27dti1axdCQ0Nx4MABdecjhBBSzlQuDIwxSKVSAEB4eDi6desGAHBwcMDr16/Vm44QQki5U7kwNG/eHIsWLcKOHTsQFRWF7t27AwCePHkCa2trtQckhBBSvlQuDKtWrcLNmzcxadIkzJs3D66urgCA/fv3o1WrVmoPSAgpm+zsbBw6dAiRkZHajkIqCZV7JTVq1AgxMTHFli9btgw6OjpqCUWIqmJjY/HixQu0bdsWenp62o6jdQUFBQDeXfqNjo7G33//DQCoXbs27O3ttRmNVAIqF4bnz5+Dx+NxgzBdvXoVu3btQr169TBmzBi1ByTkY7KysrBjxw5IpVLk5uaiZ8+e2o6kVW/fvkVOTg4AIDc3FzVr1oSuri6MjY1RrVo1LacjlYHKl5KGDBmCiIgIAEBycjI6d+6Mq1evYt68eVi4cKHaAxLyMUKhkBsYzNLSUstptE8kEnHj4ejo6MDV1RU//PADZs6cCQMDg3LPExMTgy1btuD+/fvlvu+q6urVq1i+fDl3JqhuKp8x3L17F15eXgCAvXv3okGDBrh8+TLOnDmDcePGYf78+WoPSUhphEIhpk2bhoyMDOoAAeDNmzdgjAH4/wHTRCKR1vKEhYUhPT0dmZmZcHNz01qOqiQqKgqpqam4cOECvL291d6+ymcMBQUF3JssPDwcvXr1AgDUrVsXSUlJ6k1HiJL09fUrXFG4evUqzp49y13vLy8fm4SlvDVq1Ag6OjrcaAmk7Nq0aYPq1aujdevWGmlf5TOG+vXrY+PGjejevTvOnj2Ln376CcC7CWXMzc3VHpCQyighIQEHDx4E8O6v9Xbt2pXbvi0sLGBoaIjMzEwIBAKF27x+/Rr6+vowNDTUeJ5u3bpxzzsR9WjZsiVatmypsfZV/tNiyZIl2LRpE3x8fDB48GB4eHgAAI4ePcpdYiLkc2diYgJDQ0Pw+XytnMmU1kMwJiYGy5cvx/Lly5GVlaVwm+joaOzatYseWv1MqXzG4OPjg9evXyMjI0Ouh8OYMWO0cmOLVG3p6el4+vQp3N3dtXqdXFXGxsaYMWMG8vPzYWZmpu04cl6/fg3GGLKzs5GdnQ0jIyO59bm5uTh8+DCAdwVm4MCBWkhJtOmTZnDT0dEp1u3N0dFRHXkIkRMSEoLU1FR4eHhg8ODB2o6jEqFQiGfPnkFXV7fYh682tWnTBowxmJubw8rKqth6oVAIJycnPH36FHXq1NFCQqJtn1QY9u/fj7179yI+Ph75+fly627evKmWYIRUVq9evUJISAjy8vKQl5cHS0tLzJgxQ9uxOAKBAB07dixxPZ/Px9ixY1FYWAhdXZr993Ok8j2G1atXY+TIkbC2tsatW7fg5eUFc3NzPH78GF27dtVERvIZGz16NAYNGoR+/fppO4rSHj9+jIyMDOTl5QFAsT+eNOXu3bu4efMm11W1rKgofL5U/j+/fv16bN68GYMHD0ZoaChmzZoFZ2dnzJ8/H2/evNFERvIZMzMz44Z5rywaNWqEuLg46OjowMHBoVwuxzx9+hR//vknAODhw4coLCxUa/vJycnYsmULTExMMHr0aBp2pIpTuTDEx8dzg+Xp6+sjMzMTADBs2DC0aNECa9euVW9CQioZfX19DBkypFz3qaenBz6fD6lUilu3biE7O1ut7d+/fx+ZmZnIzMzE5s2boaOjg2HDhsHExESt+yEVg8qXkmxsbLgzg5o1ayI6OhrAu2G31XUKS6q2+Ph4JCYmajtGlWJjY4Np06ahQ4cOAP7/iWd1adq0Kdzd3VG7dm0kJibi+fPniI2NLbbd27dvceHCBfr/W8mpXBg6duyIo0ePAgBGjhyJ6dOno3Pnzhg4cCD69u2r9oCkarl//z7Wr1+PtWvX0oeHmllZWcHPzw8TJ05Uey8oY2NjDB8+HP7+/qhduzZq1aoFd3f3YtsdPnwYJ06cwJYtW8q8z1u3biE8PLzc7tGQ/6fypaTNmzdzM7hNnDgR5ubm+Ouvv9CrVy+MHTtW7QFJ1SK7ISuVSukXXkMcHBzUfsYgo6enh1GjRpW4XlaQyvpE9dOnT/G///0PwLteUqX1oiLqp/IZA5/Pl+utMGjQIKxevRqTJ0+GUChUqa0RI0aAx+MV+3r06BF8fHwwbdq0Yq8JDQ2Ve2BowYIF4PF4GDdunNx2t2/fBo/Hw9OnT+WWHzhwAD4+PjA1NYWRkREaNWqEhQsXFrtxvn37drRp0wYAimXx8fEBj8fDnj175F6zatUq7nkO2TYlffn4+Kh0rKqKRo0aYciQIRgxYgQ9+6JBssu65X15t2fPnhgzZkyx30dV5OXlcTfSARoxVxuUOmP4559/lG6wUaNGKgXo0qULtm3bJrdM1TeCnp4etmzZghkzZqB27dolbjdv3jwsWbIE06dPxy+//AI7Ozs8fPgQGzduxI4dOzB16lRu2yNHjnADBJa0z++//x5ffvmlwvFoDh48yP1F/Pz5c3h5eSE8PBz169cHAJWLaFWi6ntE2woKCpCWlgZLS0uN/SVeVF5eHjIzM2FhYfHJbbx9+xYAuHkZ1EUqlUIqlZbYlZXP58PZ2blM+ygoKOBunnt6etLge1qgVGFo3LgxeDzeR//64PF4kEgkKgUQiUSwsbFR6TUfcnNzg5WVFebNm4e9e/cq3Obq1av45ZdfsGrVKrkC4OjoiM6dOyM9PZ1blpubizNnzuCXX34pcZ+DBw/G0aNHERISggkTJhRbX716dbn2AMDc3LzMPyspf1u2bMHTp0/Rrl07jQ8GV1hYiDVr1uD169fo1atXhZouNy8vD+vWrcObN28wYsQIblpfdTMyMsKIESOQmJio0YHiSMmUKgxPnjzRdI4yW7x4MTw9PXH9+nU0b9682PqdO3fCyMhI4Yc4ALnLU+fOnYO9vT3q1q1b4v5MTEy4yYmGDx+utlEqZU/LymRkZKilXfLpXr58CQBISUnR+L4KCgq4y5pl2Z+hoSHEYrFaxy9LS0vjjkVcXJzGCgMAmJqawtTUtFKNj1WVKFUYatWqpbEAx48fl+tB0bVrV+zbt0/ldpo2bYoBAwZg9uzZOHfuXLH1Dx8+hLOzc4nDEBf1sctIMhMmTMBvv/2GFStW4IcfflA5syLBwcH48ccf1dIWUY9hw4YhNjYWLVq00Pi+9PX1MXToUMTHx6Nt27af3I6iS15isRgCgeCTi4WNjQ2++OILvHr1SmPzAADAs2fPsHHjRvB4PIwfPx4ODg4a2xdRTOVeScHBwbC2tsY333wjt3zr1q149eoVZs+erVJ7HTp0wIYNG7jvy/KX96JFi+Du7o4zZ84UGxxM2ZtwjDEcO3asxEtSRYlEIixcuBCTJ0/G+PHjPynzh+bOnYuAgADu+4yMDPrF0DInJyc4OTmV2/7q1auHevXqqbXNhw8fYtu2bdDT08O0adM++cG08ugd9PbtWzDGuBFgSflTuVfSpk2bFF5ikU3goypDQ0O4urpyX7a2tgDeXaoRi8XFtk9PT4epqanCtlxcXDB69GjMmTOnWCGoU6cOHj9+/NHZtK5evYrCwkKlr+0OHToUtWrVwqJFi5Ta/mNEIhFMTEzkvsjn7cGDBzh79myZPiRTUlIglUqRnZ2t8PdKVW/fvsWBAwcQGRlZ5rY+VK9ePQwcOBCDBg2iqUC1ROXCkJyczH14F2VpaanWqT3d3NwUjtR68+bNUseemT9/Ph48eFCsK+mQIUOQlZWF9evXK3yd7ObzkSNH0L1791InOimKz+cjODgYGzZsKNY1lpCyys3Nxfbt23Hu3DmcPn36k9vx9vZGp06d0LdvX7WcgV65cgXXrl3DqVOnNPKgYpMmTSrdGFlVicqFwcHBAZcvXy62/PLly7Czs1NLKAAYP348Hjx4gClTpuCff/7B/fv3sWLFCuzevbvUIYytra0REBCA1atXyy339vbGrFmzMGPGDMyaNQtXrlzBs2fPcO7cOfTv3x/bt28H8G4mOmXuLxTVvXt3eHt7Y9OmTar/oISUQldXlztDLsvUuQKBAJ07d1bbxPGOjo7Q1dWFubm5XA88UjWofI9h9OjRmDZtGgoKCrjrjefOneM+dNXF2dkZFy5cwLx58+Dr64v8/HzUrVsX+/btQ5cuXUp97cyZM7Fhwwaum6jMkiVL0KxZM6xbtw4bN26EVCqFi4sLvvrqKwwfPhxxcXF49OgR/Pz8VM67ZMmSCtW1kFQNurq6mDJlCtLT01Xq6ix776t7lFUZV1dXzJ8/Hzo6OkqfXZNKhKlIKpWyWbNmMT09Pcbn8xmfz2cGBgbsxx9/VLWpCmf58uWsa9eu2o4hRywWMwBMLBZrO0q5EIvF7NGjR0wqlWo7SoV1//59tmHDBhYdHa1w/b59+5iRkREDwKpVq1bO6crH3bt32alTp9jbt2+1HaXSUOWzROUzBh6PhyVLluCHH35AbGws9PX1Ubt27SrR37hGjRqYO3eutmN8tgoKCrB69WpkZWXhiy++oPFxSnDu3Dk8e/YMycnJCi8NxcTEcP8u7Qn7u3fv4saNG2jbtm2Zn1YuT5mZmfjzzz/BGEN+fj569uyp7UhVjsr3GGSMjIzg6emJBg0aVImiAAADBgwoU99xUjYSiYR7uI+6KZasadOm0NPTQ7NmzRSu79mzJzdkRWmzsB04cACxsbE4ceKERnJqikgk4u67WFtbazlN1URz95EKQ09PD6NHj0ZCQkKJH3qfgwcPHuDy5cvw8vLixtYqytvbG97e3pBIJAgJCUFiYiKGDh0KFxcXAO+6ZhsYGMgN8/Kh2NhYbhwldXYaKQ9CoRDTpk1DVlZWmcaTIiWjwkAqlJo1a6JmzZrajqFVYWFhSElJwcuXLxUWBpn09HTExcUBAO7duwcXFxe8ePECGzdu5GZWLInsyWgej6dwCJmKTk9Pj6YX1SAqDAQAkJqaimvXrqFBgwaoUaOGtuNUaNevX0dBQQG8vb3B53/y1dgSNWjQ4KNFAXjXfbV9+/ZITEzkesS9efMGhYWFH33Sv27duhg1ahQEAsFnX4hJcTz2sXfQe8ePH0e3bt008otASpaRkQFTU1OIxWKVnoJOT0/HtWvX4O7urtQH/ebNm/H48WOYmprSDfhSxMXFISQkBAAwcOBANGnSRCP7kUqln/S7JpVKcenSJXz11Vd49eoV7O3t8eLFCw0kJJWNKp8lSr/z+vTpAwcHB8ybNw+PHj0qc0iiWQcPHsS5c+cQGhqq1PayOTBoUpTSGRsbQ1dXF3w+v8ShWdThU/8A4/P5aNeu3Wc93wcpO6UvJT158gTbtm3D9u3bsXjxYrRp0wbffvstvvrqK+jr62syI/kEsqdklX1atnfv3vD29qbC8BFWVlaYMWMGJBIJ3fgkVZbSl5KKioiIQGhoKA4cOABdXV0MGjQIo0aNgqenpyYyftY+9VKSVCpFUlISLC0t6a/Hz1CNGjWQkJCg8qWkvLw8SCQStc7jQCoGjVxKKqpDhw7Yvn07kpKSsGzZMsTExKBFixbw8PD4pMBE/fh8Puzt7akoEKWlp6djyZIl+OWXXxAfH6/tOESLynQn2djYGJ06dUKHDh1gZmaGf//9V125CCHlLDU1FdnZ2SgsLNTIiKmk8vikwpCTk4M//vgDPj4+qF27Nvbs2YOAgAAadpqQSszZ2RldunRB+/btP+sHDImKzzFER0dj69at2Lt3L/Lz89GvXz+Eh4ejQ4cOmspHCCknPB4PPj4+2o5BKgClC0O9evVw//59NGnSBMHBwRgyZIhGu+sRQgjRDqULg6+vL3bv3k03mAkhpIpTujB8OCMaIYSQqknpwqDM2Pg8Hg/nzp0rUyBCCCHapXRhKO0SUmZmJnbt2sWNpU8IIaTyUrowrFy5stiywsJCrFu3Dj///DPs7e3x008/qTUcIeTTBAQEICMjQ6Wn5QmR+aQhMQBg586dmD9/PnJycvD9999jzJgxpc4WRT7Npw6JQQghRanyWaLyJ/mpU6cwZ84cPHnyBDNnzkRAQAAMDQ0/OSwhhJCKRenCcPXqVcyePRvR0dEYN24cwsPDaXRJQgipgpS+lMTn86Gvr48xY8bAycmpxO2mTJmitnCELiURQtRDlc8SpQuDo6MjN09siY3xeHj8+LHySclHUWEghKiDRu4x0AB5hBDyeaAJnAkhhMhRujBcuXIFx48fl1v2xx9/wMnJCVZWVhgzZgw94EYIIVWA0oVh4cKFuHfvHvd9TEwMRo0aBV9fX8yZMwfHjh1DcHCwRkISQggpP0oXhtu3b6NTp07c93v27IG3tzdCQkIQEBCA1atXY+/evRoJSQghpPwoXRjS0tJgbW3NfR8VFYWuXbty33t6euL58+fqTUcIIaTcKV0YrK2t8eTJEwBAfn4+bt68iRYtWnDrMzMzIRAI1J+QEEJIuVK6u2q3bt0wZ84cLFmyBIcPH4aBgQHatm3Lrf/nn3/g4uKikZCEkPK3YsUKbiC+gIAAbcch5UjpB9xev36Nfv364dKlSzAyMsL27dvRt29fbn2nTp3QokUL/PzzzxoL+zmiB9yIttSoUQMJCQmwt7fHixcvtB2HlJFGHnCzsLDAhQsXIBaLYWRkBB0dHbn1+/btg5GR0aclJoQQUmGoPLqqqampwuXVq1cvcxhCCCHaR08+E0IIkUOFgWhU4rx5eNCyFTJOndZ2FKIBrLBQ2xGIBlBhIBrDCgogPnAQkrQ0iI8e1XYcjch78gSpW7agICmp3Pdd+OYNJGJxue9XRnzsGP7zaIxnI0dC1YkgxceO4753CyT98IOG0pGyoMLwmZNkZKDw1SuNtM0TCGAZEAD9xo1h/u0ojexD215MmoyXy35F4qzZ5brfnJgYPPLpgEedfJH/7Fm57lsmK+oCIJEg+0o0pG+zVXqt+NhRSMVipB84qHJRIZqntkma09LScOzYMXz99dfqapJoWEFyMp707gNJdjZq/v47DL291L4PizGjYTFmtNrbrSh0zc2RHxcHHQvzct1v3qM4sPx8sPx85MfHQ1irVrnuHwAsJowHy8uDYauW0DEyhEQsRmZEBAxbtoLA2qr0144eDWnWWxj7+n50nhdS/tRWGOLj4zFy5EgqDJVIQWIidyki79FDjRSGqs5hw3rk3L0H/SaN1dpu9rVr4OnpQb9hQ4XrTXt0R0FCAvj6ejBs00at+1aWyNkZNdas5r5PnDUbWVFRELm5wfnI4VJfa+DpCcddOzWckHwqpQtDRkZGqeszMzPLHIaUL4OmTWE9bx4kaW9g9tVX2o5TKfENDdVeULOiovB87DiAz4fj3r3Qb1C/2DY8gQCWkyaqdb9l8WbXLuS9HzKH8Xl4tW4dpNnZsJw6FXyhkNvu5fLlyLpwEdbffUd/iFRgShcGMzOzUk/5GGN0SlgJVR82VNsRyAek+fnv/yEFCgu0G0YJOffuIWXhTwAAkx49YOTjg8SZMwEAIicn7o8OaW4uUkN+BwCk7dxJhaECU7owGBsbY968efD29la4/uHDhxg7dqzaghHyuTLp3Bm8dWvfXUpq3FjbcT5KYGMDHXNzSMRimPbpA5GTI3RMTSHNz4fI3Z3bjq+nB7PBg/D2wkWY9acz1IpM6cLQtGlTAED79u0VrjczM6PeBYSoiXGRuU8qOl1zc7icPg2Wkw1dS0sAgGvEeTCpFDofDJNjGxSkjYhERUp3Vx0yZAj09PRKXG9jY4Mg+p9OSsGkUqTt2QPx8TBtR6mUJFlZyDhzBoVpadqOUoyOkSFXFACAb2BQrCiQykPpM4bRo0vvcmhtbU2FgZRKfOQokhf8CAAQ2NnC4P1ZKFFO4oyZyIqKgr6HBxz/t0dj+2FSKfIePdJY+6TiU9sDbunp6Vi7dq26miNVkK6VJcDjgScUQsesmrbjVDqs4N1Naen7/2pKctACPOnVG5L0dI3t48327Xg6cBDeRkdrbB/k05X5OYZz585hy5YtOHToEAwMDDBp0iR15CJVkFHr1nA+dhQ8PX0Ia9hrO06lY7d8ObLOn9f4cwt5jx8D0Ow4SC9/XQ5WUIDXmzbBsMhMkKRi+KQzhufPn2PhwoVwcnLCF198AR6Ph0OHDiE5OVnd+UgVI3J1paLwiXSrVYPZl19CUGTudU2wXfQTzL8dBZ0ShthXB7MBA8A3Noaodh3qtFIBKV0YCgoKsG/fPvj5+cHNzQ23b9/GsmXLwOfzMW/ePHTp0oXmfCakChA5OcFq5kzwNPj7bD52DFh+PtL++AOpoaFqa5cVFGht7KiqROnCYG9vjzVr1uDLL79EQkICDh48iK/oaVlCiBJYYSEyw8OR/yKhyMJ3Zwqvli5D4vffq2U/z8eORZxfF6QsXaaW9j5XSheGwsJC8Hg88Hi8YtN6VlUjRoxAnz59FK7LyclBUFAQ6tSpA5FIBAsLC/Tv3x/37t2T227BggXccSv6FR4eXg4/ASEVw8sVK/Fi0mQ8HTAALD8fAisrOP5vD4QuLgBjyDx9Ri37yf3vPgAg779YtbT3uVK6MCQmJmLMmDHYvXs3bGxs8OWXX+LQoUOf5TAYeXl58PX1xdatW7Fo0SI8ePAAJ06cQGFhIby9vRH9QU+L+vXrIykpSe6rXbt2WkpPSPljubkAAGlWFqQF74b50HN3h+3CH2HYri1sflDPGUON31ah2tfDYDN/vlra+1wpXRj09PTg7++P8+fPIyYmBu7u7pgyZQoKCwvx888/4+zZs5BIJJrMWmGsWrUKV65cwfHjxzFgwADUqlULXl5eOHDgANzd3TFq1Ci5G2q6urqwsbGR+xIWGViMkKpOv0kTAADLz8fbqChuuUGzZqi5eTNMe/VSy34MPD1h8913EDo6qqW9z9Un9UpycXHBokWL8OzZM4SFhSEvLw89evSAlVXpY7BXFbt27ULnzp3h4eEht5zP52P69On4999/cefOnU9qOy8vDxkZGXJfhFR2evXrgW9kBL6hIURubiq9Vpqbi4TAWXgxZSokNIpzuSjTcwx8Ph9du3ZF165d8fr1a/zxxx/qylWhPXjwAB06dFC4zv39oGEPHjxA4/cDoMXExMCoyPAA9erVw9WrVxW+Pjg4GD/++KN6AxOiZSJnZ9SOigRj74bPUEXWxYvIOHbs3Td8HmwXLICOmZn6QxKO0mcMaWlpWLNmjcK/YMViMXbv3o1vv/1WreEqMlX6Xsu698q+Dhw4UOK2c+fOhVgs5r6eP3+ujriEaB3f0FDlogAABo0bQ+DsDPB4yDx1GinBwRpIV7kwxjQ6ZpbShWHt2rW4cOECTExMiq0zNTXFxYsXP5shMerUqYPYWMW9HmTL69Spwy0TCoVwdXXlvhwcHEpsWyQSwcTERO6LEG0qfP0aOR/0titPupaWcAk7DsH73xsdCwu1tS3Ny4MkK0tt7ZWXhClT8bBlK7xavUYj7StdGA4cOIBx48aVuH7s2LHYt2+fWkJVdIMGDUJ4eHix+whSqRQrV65EvXr1it1/IKTSkXUmkUiQUcYRcfOfPkXCjJlI37+/1O0yTp9B0vwg5L94Ibecx+PBae//UOvPHbCaMaNMWWQK37xB3Bd+eNiqNbKvXVNLm+Ul+/2laE3lVvoeQ1xcHGrXrl3i+tq1ayMuLk4toSoSsViM27dvyy0bOnQojhw5gp49e2L58uXw9vZGSkoKfvnlF8TGxiI8PPyz7MZLVCfNy0NWZBT0PRpBYGOj7Tjy3j+vxBMIYNq3T5maerV2HTLCwpBx4gRMunYF37D4JSWWn4+EGTOAwkJIMjNQY+VK+ThmZjBo3rxMOYoqeP4chSkpAICcf2Jg4OmptrY1zXbJYmSePIXqI4ZrpH2lC4OOjg4SExNRs2ZNhesTExPB56ttsNYKIzIyEk3ed7WTGTVqFM6fP49ffvkF3333HZ49ewZjY2N06NAB0dHRaNCggZbSli/xsWN4uXwFTPv0htW0aeW+//T9+5F5NhwWkyZBv2HlPOYpixYhfd9+6NrZovb589qOA0nWWyTPn/+uKLy/j6ZTvTr0ilwa/RSGLVsiIywM+s2agmdgoHAbnlAI/fr1kXPnDgw++J3TBH0PD1jOCEBhykuYDRyo8f2pk7GPD4x9fDTWPo8peRe1Q4cO8Pb2xuLFixWunz17Nq5evYqIiAi1BvzcZWRkwNTUFGKxuMLdb3g6dChyrt8A39AQbjeul+u+GWP4r2EjoLAQhu3aoubmzeW6f3VJnDMX4sOHoWtpidoXL2g7DsRHjiBx9hwAgO/rV0hMTYW9vT1efHBpR1mFaWl4PnoMpNnZsF/9GyQZGUiYPAUiV1c4hGwG/4PneVhhISTp6dBV430E8o4qnyVKnzFMmjQJgwYNQo0aNTB+/HhuWAyJRIL169dj5cqV2LVrV9mSk0rF/Jtv8CozS20PJ6ki9fffIbCxRkFSMow7Vp5pMD9kM/8HGHh7w6BZxZi0yKB5cwjs7QEdHfAyxGVuL+fWbeTevfvu39evI//5c0hSU5GdmoqC+HiIXF3ltufp6lJRqACUPmMAgHnz5iE4OBjGxsZwdnYGADx+/BhZWVkIDAws8WyCfLqKfMagLbkPHuBJr94AAPPx42A1daqWE1VNNWrUQEJCQpnOGKQ5OUic+x2k2W9ht3gxpG/fIjloAUS1XWE1Zw7diytHqnyWqFQYAODq1avYuXMnHj16BMYY6tSpgyFDhsDLy6tMoYliVa0wZF2+jFerV8O0ew9U/3rYJ7UhyXqLp19+iYLERNTYuAFGrVurOSUB1FMYSMWhkUtJMl5eXgqLQFJSEn7++efP5lmGiir3/n2kHzgA0549od+wobbjFJO6OQS5d/5B3n/3P7kw6BgZwvlEGFh+Pvj6+mpOSAhRqRvRvXv3sHbtWoSEhCD9/Xywr1+/xvTp0+Hs7Ew3niuApO/mIe2PHUgMnKXtKAqZ9ukDHTMzmA0cUKZ2eDo6n2VRePv3VSQEzkL2deVv9kuy3iJxzlwkL/wJ7P3IpsC7PvAvf1uNgpSXmohKKjGlzxiOHj2Kr776CoXv54FdsmQJQkJCMGDAADRr1gyHDh1Cly5dNBaUKEdU1w259+5BVLeutqMoZNa3D8zK2Cf+c5YcFIT8p0+R918snGXjB31ExvHjEB8+DAAQh4XB0MsT1YYNQ/zX7/rA5/57DzU3bdJUZFIJKX3GsGjRIkycOBEZGRlYsWIFHj9+jClTpuDEiRM4deoUFYUKwvann+By+hTsl/+q7ShEAwy8vd/918tb+dc0bwYdMzPwDAwgFYuReTYcWUWGvtYxrvz3roh6KX3z2dTUFDdu3ICrqyskEglEIhFOnToFX19fTWf8rFW1m8+k7CRiMXRMTVV6DZNKkfvgIVIWLoR+w4YwHzcWKcHB4Ovpw+bHBQp7B9HN56pFIzefMzMzucZ0dHSgr6/PdVklhJQfVYsCAPD4fOjXdYPjrp3cMvulS9UZi1QhKvVKOn36NEzfvymlUinOnTuHu+8fXpHppYWHnQghJcu+dQuSN29g3KnyPghIypdKhWH4cPkBm8aOHSv3PY/H+2ym9ySkMsiLi8OzocMAiQS2wcF0458oRenCIJVKNZmDEKIJvP/vX8LTqXqDXBLNKNPUnoSQik3k7ATHPXsgSXsDo3bttB2HVBIqF4bU1FSYm5sDAJ4/f46QkBDk5OSgZ8+eaEdvPEIqnMo6JDnRHqXPLWNiYuDo6AgrKyvUrVsXt2/fhqenJ1auXInNmzejY8eOOPz+IRpCCCGVl9KFYdasWWjYsCEuXLgAHx8f9OjRA927d4dYLEZaWhrGjh1Lo6sSQkgVoPSlpGvXruH8+fNo1KgRPDw8sHnzZkyYMIGbtW3y5Mlo0aKFxoISQggpH0qfMbx58wY27+ekNTIygqGhIapVq8atr1atGjIzM9WfkBBCSLlS6ebzh4/N0yQbhFRdAQEByMjIoKFYPkMqFYYRI0ZAJBIBAHJzczFu3DgYGhoCAPLy8tSfjhCiNQEBAdqOQLRE6cLw4VPPQ4cOLbbN119/XfZEhBBCtErpwrBt2zZN5iCEEFJB0DPyhBBC5FBhIIQQIofGSqrgZPMoZWRkaDkJIaQyk32GKDM3GxWGCk72bIiDg4OWkxBCqoLMzExuXp2SKD21J9EOqVSKxMREGBsbq/TcSEZGBhwcHPD8+fNK1Q+dcpcvyl2+tJmbMYbMzEzY2dlxI1aUhM4YKjg+n48aNWp88utNTEwq1S+ODOUuX5S7fGkr98fOFGTo5jMhhBA5VBgIIYTIocJQRYlEIgQFBXFDmFQWlLt8Ue7yVVly081nQgghcuiMgRBCiBwqDIQQQuRQYSCEECKHCkMF8ebNG/j7+8PExARmZmYYNWoUsrKySn1Nbm4uJk6cCHNzcxgZGeHLL79ESkqK3Dbx8fHo3r07DAwMYGVlhcDAQBQWFsptExkZiaZNm0IkEsHV1RWhoaFy6x0dHcHj8Yp9TZw4kcutq6tbbP24ceO0mnvBggXFMtWtW7fY8RaJRBCJRDA0NFSYpbxzBwcHw9PTE8bGxjA2NoaBgQFEIhG8vb1x9epVAICPj4/C471v3z7UrVsXenp6aNiwIU6cOCHXNmMM8+fPh62tLfT19eHr64uHDx/KbaPMe/Gff/5B27ZtoaenBwcHByxdulRu/bp162BlZQU+nw8+nw8XF5diWYoqj9yRkZHo3bs3bG1tYWhoiMaNG2Pnzp1ybQwbNqzYcdXT09Nq7qdPnyr8/YuOjlYpi0oYqRC6dOnCPDw8WHR0NLt48SJzdXVlgwcPLvU148aNYw4ODuzcuXPs+vXrrEWLFqxVq1bc+sLCQtagQQPm6+vLbt26xU6cOMEsLCzY3LlzuW0eP37MDAwMWEBAAPv333/ZmjVrmI6ODjt16hS3zcuXL1lSUhL3dfbsWQaARUREcLmbNGnCevbsyZycnFjfvn1ZUlISE4vFWs0dFBTE6tevL5f91atXcse7b9++zMrKitnb2zM/P79iWbSR28/Pj23bto0tW7aMCQQC1rBhQ2Zra8tGjBjBzMzMWEpKCmvfvj0bPXq03M925swZpqOjw5YuXcr+/fdf9v333zOBQMBiYmK4thcvXsxMTU3Z4cOH2Z07d1ivXr2Yk5MTy8nJ4bb52HtRLBYza2tr5u/vz+7evct2797N9PX12aZNmxhjjO3Zs4cJBALG4/HYjBkzWP/+/Zmenh7T1dWVyyJz+fLlcsn9888/s++//55dvnyZPXr0iK1atYrx+Xx27NgxLreuri7T09NjkZGRzN/fn5mamirMXJ65nzx5wgCw8PBwuf/f+fn5KmVRBRWGCuDff/9lANi1a9e4ZSdPnmQ8Ho8lJCQofE16ejoTCARs37593LLY2FgGgF25coUxxtiJEycYn89nycnJ3DYbNmxgJiYmLC8vjzHG2KxZs1j9+vXl2h44cCDz8/MrMe/UqVOZi4sLu3fvHpe7ffv2bOrUqRUqd1BQEPPw8CiWQXa8IyIiuCyy3FFRUXJZtJFbxsvLi02cOJG9fPmSy2tnZ8eCg4O5413UgAEDWPfu3eWWeXt7s7FjxzLGGJNKpczGxoYtW7ZM7ucSiURs9+7dcsemtPfi+vXrWbVq1bifiTHGZs+ezdzc3Ljcrq6uXBaJRMLs7OyYg4MDl0UbuRXp1q0bGzlyJJe7Y8eOzNTUVC53cHCwwteWV25ZYbh161aJP8fHsqiKLiVVAFeuXIGZmRmaN2/OLfP19QWfz8fff/+t8DU3btxAQUEBfH19uWV169ZFzZo1ceXKFa7dhg0bwtramtvGz88PGRkZuHfvHrdN0TZk28ja+FB+fj7+/PNPfPPNN4iOjpbLvXPnTvj7+4MxhkmTJiE7O1vruR8+fAg7Ozs4OzvD398f8fHx3PGWSqVcFtnxTk1Nlcuirdz5+fm4ceMGfH19IRaLAQAWFhbw9fXltt25cycsLCzQoEEDzJ07F3/99VepbT958gTJycly25iamsLb21vuZ/jYe/HKlSto164dhEKh3H7u37+PlJQU3LhxA2lpadx++Hw+fH19oa+vr/B99bFjoq7ciojFYlSvXp073vXr10dWVhZq1aqFWrVqgc/n48yZMwpfW965e/XqBSsrK7Rp0wZHjx5VKYuqqDBUAMnJybCyspJbpquri+rVqyM5ObnE1wiFQpiZmcktt7a25l6TnJws9yElWy9bV9o2GRkZyMnJKbbfw4cPIz09HSNGjJDLPWTIEPz555+IjIyEiYkJzp8/r3D61/LM7e3tjdDQUJw6dQobNmzAkydP0LZtWzx79gxWVlZyWYoe76JZtJEbAF6/fg2JRAJLS0tMmzYNrVu3RoMGDbj9yY53REQE5s6dix07diAhIUFh20X3XTRTSdt87L1Y2s/533//QSKRID09XW4ba2tr5OXlKXw/l9SeunN/aO/evbh27RpGjhzJHe8GDRpg69atOHLkCP78808IhUJERUXhxYsXWsttZGSE5cuXY9++fQgLC0ObNm3Qp08fueLwsSyqokH0NGjOnDlYsmRJqdvExsaWUxrlyaZxNTAwKHGbD+eHGDNmDPdvkUiEIUOG4LfffkNcXBxcXFw0E/QDyuQ2NDRETExMueRRh19//RV3797FpUuX5JYXPd4NGzaEra0tOnXqVOKNcyIvIiICI0eOREhICOrXr4/ExEQA745ly5Ytue369OmDdevWYdOmTfjpp5+0ktXCwgIBAQHc956enkhMTMSyZcvQq1cvjeyTCoMGzZgxAyNGjCh1G2dnZ9jY2ODly5dyywsLC/HmzRvY2NgofJ2NjQ3y8/ORnp4u91dsSkoK9xobGxuuF0vR9bJ1sv9++GHSpk0bnDx5EtevX5dbnpCQgM6dO2PNmjUfze3t7Q0AePTokVxh0Ebuovz9/ZGbm4uXL1/KZTEyMuKOd9Es5ZE7JSUFJiYm0NfX55ZZWFiAx+Ph4sWLuHbtGjfCrqJsALjj/d9//xVru+i+ZctsbW3ltmncuDG3zcfeiyX9DMC7y2s6OjowMzOT2yYlJQUikUjh6J4ltafu3DJRUVHo2bMnVq5cia+//hrAu+Oto6NTLMfr169RvXp1PHr0SOu5i/L29sbZs2eVzqKyT7ozQdRKdgPq+vXr3LLTp08rdRN3//793LL//vtP4c3QlJQUbptNmzYxExMTlpubyxh7dzO0QYMGcm0PHjxY4c3QoKAgZmNjwwoKCj6a+/DhwwwAu3PnjtZzy2RmZrJq1aqxuXPnMgAsMjKSyyLL/bGbz+WRWyqVsokTJzKBQMCGDh3KLZdIJMze3l7hzdBLly4xAKxdu3Zyy1u2bFnsZuivv/7KrReLxQpvhpb2XpTdfC7aK2bu3LnFbj736NFDLnfNmjVLvPks21aTuRljLCIighkaGrK1a9cWy+Hl5cUmTZrEfS/LbWFhwaZPn67V3B/69ttvWZMmTZTOoioqDBVEly5dWJMmTdjff//NLl26xGrXri3XZe3FixfMzc2N/f3339yycePGsZo1a7Lz58+z69evs5YtW7KWLVty62XdJ7/44gt2+/ZtdurUKWZpaamw+2RgYCCLjY1l69atK9Z9krF3vyQ1a9Zks2fPLpa7Xr16bMyYMez3339njo6OrF27dszZ2Zm1a9dOq7lnzJjBIiMj2ZMnT9jly5eZr68vs7CwYC9fvuSOd9++fZm1tTWrUaMG8/Pz47JoM/f48eOZqakpmz9/PhOJROy3335jUVFR7JtvvmFmZmYsOjqaLVy4kHXr1o2NHz+eHTlyhDk7OzMPDw+mq6vLfv31VxYbG8uCgoIUdp80MzNjR44cYf/88w/r3bu3wu6Tpb0X09PTmbW1NRs2bBi7e/cu27NnDzMwMJDrrioUChmfz2eBgYFswIABct1Vhw0bxubMmcO1d/ny5XLJff78eWZgYMDmzp0r1+0zNTWVy62rq8tmzpzJTp8+zfr168cEAgETiUTs3r17WssdGhrKdu3axWJjY1lsbCz7+eefGZ/PZ1u3blUpiyqoMFQQqampbPDgwczIyIiZmJiwkSNHsszMTG69rMtaREQEtywnJ4dNmDCBVatWjRkYGHDPDxT19OlT1rVrV6avr88sLCzYjBkzuL/4ZSIiIljjxo2ZUChkzs7ObNu2bcXynT59mgFg9+/fL5a7d+/eTEdHh/F4PMbn85mzszMLDAxkYrFYq7kHDhzIbG1tmVAoZPb29mzgwIHs0aNHxY63UChkQqGQ6evrc1m0mRuAwi9nZ2cWHR3N4uPjWbt27Ziuri7j8/nM1dWVO9579+5lderUYUKhkNWvX5+FhYXJtS2VStkPP/zArK2tmUgkYp06dVL4/7S09yJjjN25c4e1adOGiUQiZm9vzxYvXiy3fs2aNczCwoLxeDzG4/GYk5MTl6V9+/Zs+PDhctuXR+7hw4crPK7t27fntvHx8WE6OjoMANPV1WWtWrViN2/e1Gru0NBQ5u7uzgwMDJiJiQnz8vKS6zatbBZV0OiqhBBC5FB3VUIIIXKoMBBCCJFDhYEQQogcKgyEEELkUGEghBAihwoDIYQQOVQYCCGEyKHCQAghRA4VBlKhLFiwgBtgTJ1k0yPevn0bwLtpHnk8HtLT0wEAoaGhxYbUVsXH2tPUz6UMHx8fTJs2TSv7JpUTFQZSZiNGjODmoRUIBLC2tkbnzp2xdetWSKXSMrfdp08f9QQtolWrVkhKSlI42qc6DBw4EA8ePNBI2+Sd0NBQhXMhlzZHszqEhISgbdu2qFatGqpVqwZfX99io+pWdlQYiFp06dIFSUlJePr0KU6ePIkOHTpg6tSp6NGjBwoLC7UdrxihUAgbGxvweDyNtK+vr19sAhYCMMbU+n4wMTFBUlKS3NezZ8/U1r4ikZGRGDx4MCIiInDlyhU4ODjgiy++QEJCgkb3W56oMBC1EIlEsLGxgb29PZo2bYrvvvsOR44cwcmTJxEaGsptl56ejm+//RaWlpYwMTFBx44dcefOHYVtLliwANu3b8eRI0e4vwYjIyMBALNnz0adOnVgYGAAZ2dn/PDDDygoKFA674eXfj706tUrNG/eHH379kVeXh6kUimCg4Ph5OQEfX19eHh4YP/+/SW2X9KlqR07dsDR0RGmpqYYNGgQMjMzuXV5eXmYMmUKrKysoKenhzZt2uDatWtyr4+KioKXlxdEIhFsbW0xZ84cuQ/at2/f4uuvv4aRkRFsbW2xfPnyjx6LO3fuoEOHDjA2NoaJiQmaNWsmN6fF5cuX4ePjAwMDA1SrVg1+fn5IS0tTKrPsOJ88eRLNmjWDSCTCpUuXVD6eJeHxeLCxsZH7ks1ktnnzZtjZ2RU7a+3duze++eYb7vsNGzbAxcUFQqEQbm5u2LFjR6n73LlzJyZMmIDGjRujbt26+P333yGVSnHu3DmV81dUVBiIxnTs2BEeHh44ePAgt6x///54+fIlTp48iRs3bqBp06bo1KkT3rx5U+z1M2fOxIABA7izkaSkJLRq1QoAYGxsjNDQUPz777/47bffEBISgpUrV6ol9/Pnz9G2bVs0aNAA+/fvh0gkQnBwMP744w9s3LgR9+7dw/Tp0zF06FBERUUp3W5cXBwOHz6M48eP4/jx44iKisLixYu59bNmzcKBAwewfft23Lx5E66urvDz8+OOTUJCArp16wZPT0/cuXMHGzZswJYtW7Bo0SKujcDAQERFReHIkSM4c+YMIiMjcfPmzVJz+fv7o0aNGrh27Rpu3LiBOXPmQCAQAABu376NTp06oV69erhy5QouXbqEnj17QiKRKJVZZs6cOVi8eDFiY2PRqFEjtRzPj+nfvz9SU1MRERHBLXvz5g1OnToFf39/AMChQ4cwdepUzJgxA3fv3sXYsWMxcuRIudd8THZ2NgoKClC9enW1Zde6Tx6XlZD3hg8fznr37q1w3cCBA5m7uztjjLGLFy/KTVoj4+Liwo3lHxQUxDw8PJRqu6hly5axZs2albheNoz2rVu3GGPvhr4GwNLS0hhjjG3bto2Zmpqy//77jzk4OLApU6YwqVTKGGMsNzeXGRgYsL/++kuuzVGjRnHj5pfUnkxQUBAzMDBgGRkZ3LLAwEDm7e3NGGMsKyuLCQQCtnPnTm59fn4+s7OzY0uXLmWMMfbdd98xNzc3LhdjjK1bt44ZGRkxiUTCMjMzmVAoZHv37uXWp6amMn19fTZ16tQSj42xsTELDQ1VuG7w4MGsdevWCtcpk1l2XA4fPsxto8zxVMa2bdsYAGZoaCj31aVLF26b3r17s2+++Yb7ftOmTczOzo5JJBLGGGOtWrVio0ePlmu3f//+rFu3bkrnGD9+PHN2dpabY6Gyo6k9iUYxxrjr+Hfu3EFWVhbMzc3ltsnJyUFcXJxK7f7vf//D6tWrERcXh6ysLBQWFsLExKRMWXNyctC2bVsMGTIEq1at4pY/evQI2dnZ6Ny5s9z2+fn5aNKkidLtOzo6wtjYmPve1taWm9YxLi4OBQUFaN26NbdeIBDAy8uLmxc8NjYWLVu2lLsv0rp1a2RlZeHFixdIS0tDfn4+N80nAFSvXh1ubm6l5goICMC3336LHTt2wNfXF/379+emY719+zb69++v8HXKZJZp3rw59291HU/g3Znjh2dERadI9ff3x+jRo7F+/XqIRCLs3LkTgwYNAp//7mJJbGys3PzZwLtj+ttvvym1/8WLF2PPnj2IjIzU+E3v8kSFgWhUbGwsnJycAABZWVmwtbXl7hMUpUpX0StXrsDf3x8//vgj/Pz8YGpqij179ih1Pb00IpEIvr6+OH78OAIDA2Fvb8/lBoCwsDBuWdHXKEt2eUaGx+OVudeWOixYsABDhgxBWFgYTp48iaCgIOzZswd9+/aV+5AtC0NDQ+7f6jqeAMDn8+Hq6lri+p49e4IxhrCwMHh6euLixYtqu+T466+/YvHixQgPD0ejRo3U0mZFQfcYiMacP38eMTEx+PLLLwEATZs2RXJyMnR1deHq6ir3ZWFhobANoVDIXc+W+euvv1CrVi3MmzcPzZs3R+3atdXSE4XP52PHjh1o1qwZOnTogMTERABAvXr1IBKJEB8fXyy3g4NDmfcLgLv5efnyZW5ZQUEBrl27hnr16gEA3N3dceXKFbAic2tdvnwZxsbGqFGjBlxcXCAQCPD3339z69PS0pTqNlunTh1Mnz4dZ86cQb9+/bBt2zYAQKNGjUq8qapMZkXK43jK6OnpoV+/fti5cyd2794NNzc3NG3alFvv7u4ulx94d0xLyw8AS5cuxU8//YRTp07JnQ1VFXTGQNQiLy8PycnJkEgkSElJwalTpxAcHIwePXrg66+/BgD4+vqiZcuW6NOnD5YuXYo6deogMTERYWFh6Nu3r8JfMEdHR5w+fRr379+Hubk5TE1NUbt2bcTHx2PPnj3w9PREWFgYDh06pJafQ0dHBzt37sTgwYPRsWNHREZGwsbGBjNnzsT06dMhlUrRpk0biMViXL58GSYmJhg+fHiZ92toaIjx48cjMDAQ1atXR82aNbF06VJkZ2dj1KhRAIAJEyZg1apVmDx5MiZNmoT79+8jKCgIAQEB4PP5MDIywqhRoxAYGAhzc3NYWVlh3rx53GUTRXJychAYGIivvvoKTk5OePHiBa5du8YV87lz56Jhw4aYMGECxo0bB6FQiIiICPTv3x8WFhYfzayIsbGx2o4nYwzJycnFlltZWXE/t7+/P3r06IF79+5h6NChctsFBgZiwIABaNKkCXx9fXHs2DEcPHgQ4eHhJe5zyZIlmD9/Pnbt2gVHR0du/0ZGRjAyMlI6e4Wm3VscpCooOpeurq4us7S0ZL6+vmzr1q3cTT6ZjIwMNnnyZGZnZ8cEAgFzcHBg/v7+LD4+njFW/Obzy5cvWefOnZmRkZHcHMyBgYHM3NycGRkZsYEDB7KVK1fK3ez9kLI3n2UKCgpYv379mLu7O0tJSWFSqZStWrWKubm5MYFAwCwtLZmfnx+LiopSqr0Pfy7GGFu5ciWrVasW931OTg6bPHkys7CwYCKRiLVu3ZpdvXpV7jWRkZHM09OTCYVCZmNjw2bPni03p3RmZiYbOnQoMzAwYNbW1mzp0qWsffv2Jd58zsvLY4MGDWIODg5MKBQyOzs7NmnSJLkbqZGRkaxVq1ZMJBIxMzMz5ufnx/2cH8v84XGR+djxZIyxWrVqsaCgIIW5ZccYJcyPXXQubolEwmxtbRkAFhcXV6yd9evXM2dnZyYQCFidOnXYH3/8UeI+ZbkU7bO0rJUNzflMCKlwsrOzYW5ujpMnT8LHx0fbcT47dI+BEFLhREREoGPHjlQUtITOGAghhMihMwZCCCFyqDAQQgiRQ4WBEEKIHCoMhBBC5FBhIIQQIocKAyGEEDlUGAghhMihwkAIIUQOFQZCCCFyqDAQQgiR83+R2L2SJJujbwAAAABJRU5ErkJggg==", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAC+CAYAAAAx3qiRAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAVAVJREFUeJzt3XdYU9cbB/BvBgkjhA0BZIvgxAmOWhWx7l33rnuvuvtzVavWtlo71bqrUtS6Z3EiRXGCIqAiskFkhpWQ5Pz+wNwSCZhIGOL5PA+P5q6ce8X73nvGe1iEEAKKoiiKeoNd0wWgKIqiahcaGCiKoigVNDBQFEVRKmhgoCiKolTQwEBRFEWpoIGBoiiKUkEDA0VRFKWCBgaKoihKBbemC0BVTKFQIDk5GcbGxmCxWDVdHIqiPlCEEIjFYtjZ2YHNrvidgAaGWi45ORkODg41XQyKouqIhIQE1KtXr8JtaGCo5YyNjQGU/GMKhcIaLg1FUR+q3NxcODg4MPeUitDAUMspq4+EQiENDBRFVZomVdK08ZmiKIpSQQMDRVEUpYIGBoqiKEqF1m0M9+/fh56eHpo2bQoAOHnyJPbs2YNGjRph9erV4PF4Oi8kRVFURX744Qfk5uZCKBRiwYIFNV2cD57WbwxTp07F06dPAQAvXrzA8OHDYWhoiCNHjmDx4sU6LyBFUdS7/PDDD1izZg1++OGHmi5KnaB1YHj69CmaN28OADhy5Ag+/fRTHDp0CHv37sWxY8d0XT6KoiiqmmkdGAghUCgUAIDAwED06tULAODg4IDXr1/rtnQURVFUtdM6MLRu3Rrr1q3DgQMHcP36dfTu3RsAEBsbCxsbG50XkKIoiqpeWgeGrVu34v79+5g1axZWrFiB+vXrAwCOHj2K9u3b67yAFEVRVPViEUKILg5UVFQEDocDPT09XRyOeiM3NxcmJibIycmhI58pqhz16tVDUlISbIVCXGvcBOYTxsNy8uSaLlatos29ROs3hoSEBCQmJjKfQ0NDMW/ePOzfv58GBYqiqkzxq1dQFBWpLJPn5SHlfyuhyM8HACgKCiDPzET2XwFqjyF7/RrZfx+HLCOjysv7IdN6HMPIkSMxZcoUjBkzBqmpqejWrRsaN26MgwcPIjU1FStXrqyKclIU9RHLOXUKyUuWgufoCJeTJ8Di85H29TqIAwMhe/UKirw8Zlu2kRFYhoZ4OXwE7LdugZ5IhJQ1a1Bw6zaIXI7i+HhwzMzA9/SEkY8PzMeOATgc5AcFwcDLC1wrqxo809pB68Dw+PFjeHt7AwACAgLQpEkTBAcH49KlS5g2bRoNDBT1EVBIpWBXcjArkUrxauuPYHHYsJo7Fyxu+bejwsePAUIgjY/Hq+9/QN716yiOj1e7reXsWXi1cRMAQPxPIEz69UX2YX+VbeRZWSgICUFBSAgU+fmQvXqFnJMnoefkBMEnn0BRWAib5cvBERhV6hw/VFpXJRUXF4PP5wMo6a7ar18/AICnpydSUlJ0WzqKomqdrCNHEO3VHPETJ6ksL4qORlZAABSFhRodJ/fSP8jcvRsZO/+AOPByhdtaTpsGs7FjYL1iObIOHEBxfDxYAgG4IhFMR40CSmUMFXz6KYzat4N+kyYwaNEc4qtXYTJ4MHguLkDpYPZmH66tLRRSCQBALs5F1sGDyPn7b+SePavRedRFWgeGxo0b4/fff0dQUBD++ecf9OjRA0DJhDIWFhY6LyBFUbVL/o0ggBDkh4SASKUAAIVEgriRo5C6chVeffe9RsfRb+gJtrEx2CYm0G/oWeG2XHNziJYvh2GTJswyUliIelu3ADIZ8KYPDYvPB9/VFY67d8Pl6BGkrv0aKUuXoTgpCW7nz8H+++/+CyJv9uGamcJ27VpYL1kMjuC/uQpYBvoaX5PqVvjoMVLXrkXho0dVcnytq5I2bdqEgQMHYvPmzRg3bhy8vLwAAKdOnWKqmCiKqrus5swG2GwIPu0I1psncBaLBZaBAZCfD7ahYZl9JM+egWtlBY6pKbOM7+YG96AbAAC2vmY3YYPmzWE6bBiy//oLkMtRGB6OvKtXmfVELgcAyMViEJkMrDdTWCr/FHbrBvYfO1H0JBLpW7cCCgU4pqbgGBuDra+vUj3FMaq91Ugpy5dB8uw5Cu7cgevp0zo/vtaBoXPnznj9+jVyc3NhZmbGLJ8yZQoM1fxCUBSlXua+fci7EQSrBfNh0LhxhdsW3L8PPVtb6Nnaav098uxspG36Flwb65K6/ErOHc53d0e9H7cyn2WZmeAIhXA5EgDJ06cw+uQTle2z/P9C6urV4NrYwO38OZXA8XZAyD52DAUPHqDocQQMmjeH7epVZb5ftHoVuNZWkGdkwnTwYGTu3//fSpkMWQEBSPt6HQDA/tdfQMRiGHXoAAAoTnuFosePYTpsGIx9u4BIpZDnilEYHo6C+w/AMTEB28wMFhMmwLhr10pdp6qk37gJJM+eQ79Rxb837+u9ZnDjcDgqQQEAnJ2ddVEeiqqTFIWFSF2/Hiw2BzYrlgMA0jZsBACwDQ1R76dt5e6befAg0r5eB7aJCepfugiOiYlW350VcAQ5x48DAIw7d4bBm1xnupDl74/U1Wug37QpnP/yVxu4pG+ewmWvX0NRUKD2jQIAiqKfImXFV8xnSVQUBB0/KXODZrFYsJo5k/lss2wZcPMmIJOBLRAgdc1a4M2bg/TFCwg++YS5Zi/69IFCLEbmn3+iQVAQci9eQtLcuQCbDbxJ9WO96EuYfv55Ja5K1bP9Zj0sZ82Cnp32DwqaeK/AcPToUQQEBCA+Ph7SN3WMSvfv39dJwSiqLhFfuoScoyVJJo3at4OwRw8Yf/YZ8q5fh3H37hXuK8/MAgAo8vOhkEjA0fK7Ddu0BsvQEFxLy5IG2HLIXr8G29Cw3Bu3OgVv/r8XPXkCUlQElpp9LWdMB1tgBP2GDcG1tCz3WFwrS7AEApBSXU9lr9893kCenc0EAqJQgO/pCUlEBAx9vJHxxy682rARorVrIOzenWkYJ4Ul4yGI5M24CELAtbICIQQGrVppdO41icVmg1fPvsqOr3Vg2LZtG1asWIHx48fj5MmTmDBhAmJiYnDnzh3MLBXFKYr6j0GLFuBYWYLFYsOgWTMAQL1tP2q0r8XUKeCYm4HvVh961tZaf7dhixbwuH0L4HCYuva3ia9cReLs2eBaW8P15AlwKhgZK0tPh/jaNRh37gzruXPB1jeAUVufcgMKRyCA1YwZ7ywn19wcQj8/5Jw4AQAwnzwZpgMHlNmuKDoauWfOwKR/f/Dr10fBgwfMOlJQAP0mTSB5/hySuHjI3yT2lMbFIXHevJK3ClNTOO76AwBg0q8fWDw+OKYmMGrbFoSQSle11QVaB4Zff/0VO3bswIgRI7B3714sXrwYrq6uWLlyJTIzM6uijBT1weM5OsL9RklDq7Y3HjaPB/NRoyr1/ax3ZCUoiogA5HLIUlIge/26wsCQOHceCu/fR06LFnA+fAi2a9dUqmylWc2bC3A5MGzRAqaDB6vdJmnhQkifxyDvRhBcT56AYRvv/3oacTjI+esvAIA8NRUsPh+mw4bBcvp0JE4vCU58Z2eVNh1hj5I3Nnl2NgojImDUpg3TqP6x0jowxMfHM8nyDAwMIBaLAQBjxoxB27Zt8fPPP+u2hBRVR1T2SVSek4PXv2+HvqcHTPr311GpSpiPHwdFfj54zs7gu7pWuC3TE+nNeCZd0hOJYLduXcXb2IggfR4DybNnKAwPR3ZAANP1lPmTxQIIAZFIIPj0U3AEAtj/uBV5129A0LGkcVyel4e0DRvAMTWF9cKFiBs/AZKoKAh794btN+vBroLz+1BoHRhEIhEyMzPh5OQER0dH3Lp1C15eXoiNjYWO8vFRFKVGxs6dyNyzBwBg2Lo19Ox1V8fMMTaGzdIlGm1bb9uPyL91C0Zt2+rs+zWR/++/SF6+AlzleSsUkMS8gOmggUDAX8wyALDbvBmS58/ANjSCUYeSB1muublK1VTO8RPIOfY3AEDQsSPkOTkAgNxz51Dw4D5c//5bpXvtx0TrAW6+vr44deoUAGDChAmYP38+unXrhmHDhmHgwIE6LyBFUSX4DRsCLBb07O1r9IbFEQoh/OwzgMVC4uzZSJwzF/K8/Cr/3pxTpyFLTUXRvXsAAJaBAYTdP4Pp4MElYyiUWCyweDxYz5sHyymTIX3xAnFjxyFl9WqQN4EDAAxbtQRbIADXzhZ8d3c47voDRp0+BQiBLDkFxR9xJget024rFAooFApw3+Q18ff3x7///gt3d3dMnToVvI+8bk7XaNptqrTitLSSwVi1YMxQzqlTSF5c8pZh9/13MHkzaVdVKXz0GKnrvgaRFkMSGQkAMJ8wHjZLlsDe2hrJ6emw4XJx1a0+zCdMgM2SxYifMgX5QTeZKibX06fAd3dnjklkMoDNZhrl5WIx0n/cBp5DPZiPG1el51PdqjTtNpvNZoICAAwfPhzbtm3D7NmztQ4K48ePLxkx+dbP8+fP0blzZ8ybN6/MPnv37oVpqael1atXg8ViYdq0aSrbPXz4ECwWCy9fvlRZfuzYMXTu3BkmJiYQCARo1qwZ1q5dW6bhfN++ffjkzUCdt8vSuXNnsFgs+PurJubaunUrM55DuU15P507d9bqWlEUAOjZ2FQqKMjS05G+7ScU3L1b6bIYenuD5+YGXn03GLZuU+njlSaNj0fmgT8hKzVdsEHTJnD56y84Hz4Ejrk5ACD/ZjCAUo3rLBaMfLvAbPw4EKm0ZD0hYPH5MGrfHjwnJ5XvYXG5Kj21OMbGEH21os4FBW1p1MYQHh6u8QGbvemKp6kePXpgz5t6UyUrLdPe6uvrY9euXVi4cCHcSz0NvG3FihXYtGkT5s+fj2+++QZ2dnZ49uwZfv/9dxw4cABz585ltj158iSTILC87/zqq68wePBgtfNQ/P3338wYj4SEBHh7eyMwMBCN3/SGoG9WHzcilyNl+XJIYl/CbuOGdzb46kraho3IPXcOGXv3wuNOKFgcbUdFlMjy98erbzfDZNAgiL5aoeNSAglTpkL68iXyrl+H4x87VdYVPXoEQx9vKPILYPHFF5AmJjLjE7jm5ii6dx9xw0fA2d8ftmvXIO/6dVjOng2OiQmKX70Cr149FIaHI/fsWZgOGQL+m1koqf9oFBiaN28OFov1zsZlFosF+ZuBJpri8/kQiURa7fM2Dw8PWFtbY8WKFQgIUD9BR2hoKL755hts3bpVJQA4OzujW7duyM7OZpYVFRXh0qVL+Oabb8r9zhEjRuDUqVPYuXMnZqjpo23+5olGeTwAsLCwqPS5UnWD5Plz5JwsaavLOX4C1gsXVMv36jk5AgB49nbvHRQAIOf0GSgKCpBz4kSVBAb2m6oOjtBYZTkhBAlTp0FRUACD1q1R+OgRUtaugeJN70hFcXFJI3JODmI//xxcMzOI1q6FODAQr3/+BQDgtH8fkpcuQ3FiIgofhsH5L9U3f0rDwBAbG1vV5ai0jRs3ok2bNrh79y5at25dZv3BgwchEAjU3sQBqFRPXb58Gfb29vD0LD/jo1AoxIoVK7B27VqMGzcORjpKuCWRSCCRSJjPubm5OjkuVbvwXVwg6NoV0hcvIOzVs1q+89WPP0J86R/YfLUCppXsKGI1cwZe//obhP36arUfUSiQ/OWXKHj4EHYbN8KonMSbjjt3oDA8HIZvrWexWOC5uKAoIgKFd++i8O5dlZTbbH19mAweBElUFIoinkCeno64MWNKRka/aXiWJiSC7+mB4sRE8D08tDzzj4NGgcHprXo5XTpz5gwEAgHzuWfPnjhy5IjWx2nZsiWGDh2KJUuW4PLlsrndnz17BldXV42mH31XNZLSjBkz8OOPP+KHH37A//73P63LrM6GDRuwZo3uBgxRtROLx4PDL9U35ocQgoztOwCFAnnXb8B89OhKHc+ofXsYvRnPpA1Z+mvknjsPAMg9fbrcwMAxMYGgY0e165z+PIAXgwahOPZlyYLSNRksFuzWr4ckJgbxEyZA9iodKC4uWcdmw9jPDyb9+sKkbx9IExKYNgd5Xj4K79+DYatWYNfirKrVRevG5w0bNmD37t1llu/evRubNm3SugBdunTBw4cPmZ9t28pPJvYu69atQ1BQEC5dulRmnaadrwghOH36tEaBgc/nY+3atfjuu+/wulQjWWUsW7YMOTk5zE9CQoJOjkt93FgsFiy+mAA9R0eYjRiu1b6S2FhkHjpUkpOokvRsrGE2dgz0mzWD6XDNy5F94gSedvgEr77/HmwDA1ir6ZhSGt/NDfWvXYPj3r0w+rQjwGZDtGoV6m37ESwOBywuF3wXF6bhOWnOHCRMmYrEefMrc3p1htaBYfv27WqrWJQT+GjLyMgI9evXZ35s32RnFAqFyHkz4KS07OxsmJSTXdLNzQ2TJ0/G0qVLywSCBg0a4MWLFyhWPj2UIzQ0FDKZjBnd/S6jR4+Gk5MT1r1jtKam+Hw+hEKhyg9Vc+R5+UjbvBlZ/n/VdFEqzfrLL1H/0kUY+/pqtV/8xIlIW/s1UnT0VixavhwuAX+9M9V4aa9/+hnyjAxk7NoNolBA0KULDFu3BsvAAHqOjmCXqnVQYrHZMGrrA8cdO+AZHgazYUPLPb5c2UZBq24BvEdgSE1NZW7epVlZWel0ak8PDw+1mVrv37+PBg0alLvfypUr8fTp0zJdSUeOHIm8vDz8+uuvavdTNj6fPHkSvXv3BkfDhjk2m40NGzbgt99+K9M1lvrwZf15AJm7diN19WoUPX1a08WpERyjkpsu20j15qsoKtJ4Gk9NFae9guJNZ43S9OrVY/5OJBKweTw4/XkAng/uw+XoEaYhnUileNa5C5IWfqnycFjRfNIAUO/HrbBZsQL2peaZ+JhpHRgcHBwQHBxcZnlwcDDs7Ox0UigAmD59Op4+fYo5c+YgPDwc0dHR+OGHH3D48GEsXLiw3P1sbGywYMGCMlVSPj4+WLx4MRYuXIjFixcjJCQEcXFxuHz5MoYMGYJ9+/YBKJmJTpNqpNJ69+4NHx8fbN++XfsTpWo1vqcnwOGAa23NZDYtuH8fKWvWoOjNIKu6iMjlyPjjD2Ts3gOHvXvgsP13iEoly5PGx+N5p8549mknSJ4/r9R3ybOzkThvPl6OGYvnnTohdsBAKEp1wAAA++82w/yLL2D3/fcoTklVWZe8aDGTzkJRWARZaipyz56FPKskXbn05UtmTojy6NnZwXzMaOjRXoMA3iNX0uTJkzFv3jwUFxfD980r6eXLl5mbrq64urrixo0bWLFiBfz8/CCVSuHp6YkjR44w80yX58svv8Rvv/3GdBNV2rRpE1q1aoVffvkFv//+OxQKBdzc3PD5559j3LhxiImJwfPnz9H9Hfnx1dm0aZPG1U/Uh8O4c2e4X7taMk/Bm0bJ5MVLUJyYiKLHEXA5or579IdOfOkSM3czz8mxzGQ5RdHRzM246MmTSo0FyDlzFuILF5jP0rg4KMRilSR2XCsrWM+bi5i+fVEcFw+b5ctgPnYsipOSmGogAGBx2OCYm4HI5JA8fw4pV6+kVxKbDacD+6HfsCHYfD4KHjyA5OkzmAwcADYdU1SG1ikxCCFYunQptm3bxgzg0tfXx5IlS7By5coqKWR1+eGHHxAYGIhz587VdFEYNCVG7ZM4fz7E5y/AbOQIiD7w3/nyFD15gpcjRgIsFpwD/oL+W9W3RCZD+o8/gsjkJd1HHz+G5cyZ0LPRfr4IyfPniJswAWxDIxj6eMOojTdM+vZBcUoKsg77Q9C5EwxbtoRcLMbTdu0Bmawklfa0qYjp3QekoABdXsQgrbgYNvr6uOrkDKBkzIaBV3PkvsntxtLXB9vAAA67/kDc8BEgUikspk+DdalxTXWZNvcSrQODUl5eHiIjI2FgYAB3d3fw60CK2oCAANja2qJjOd3kagINDLUPUSggS00F19a2Tk/qIktPB9hscC0syt8mIwPPPukIEALT4cNgu3r1e31X5v4DyDxwAObjxsF8dMncEwkzZiLvyhWwjY3hcScUACC+dg2F9x/AfMJ4yF+/xou+JdW+XV7GIk0igQ2fj+tNmwIsNtOQbDlrFqTx8UyAsPv+e6SuXQtFTg6sFy2CxcQv3qvMHxpt7iXvNbUnAAgEArRpo9v8KDVt6NDyey1QlBKLzYaeDtvTaiuuBqlpOMbG4Lu7Q/LsGQxbtHjv78rctw/FSUlIW7cO+bdvw+GnbeC5OAOAynSkxp07w/hNnjGumRnq/fpLSRbUL97c3OVyKMR54DdqBMmTJwCAoqgoFD2NLimvlRWKk5Ph+vcxSBMSqj11+Ifivd8YqOpB3xioqpT27WaILwdCtHw5BJ06vdcxSHEx5Hl54JqZvXc5Mg8dQtqmb4E3jc42X30Fs1EjIXn6DDxHB7BLp9Uu/d0yGQDAztQUqfn5sOFyEdyjBwy8miPn6NH/NnwzcY+S4759MPJRP7iurqrS7KoURdUNRC5H5p49KI6LR9bh988XxNLTq1RQAADzkSPhtHsXk96i8P49sFgs6Hs0KDcovP59O6KaNEVUMy8mQIDFgvOBA7BZuhRG3bqVHI/NBtehnsq+bBP6kFURGhgo6gOWe+EC0n/6+b0mymFxOLCYNAk8NzeYVXJOaSKXI3PfPmQfO8Ysk8bHI/Xrdci7WbZ7uzqGrVrB/vvvYDJ4EKzm/zcCmRQXI/eff1DwZoIeJfHVKyV/UShKciEB4JialqT/2LGjpCqJEEChgCy+JIMAy0AfDnt2g+/gAPGVK0zPKkqVxm0MZ86cQa9evcBm01hCUbVBcVISkuYvKJnbWC57Z5oIdawXLtBJZteck6eQtmEjAEDPwQFG3t5I+2YD8q5dQ/bx4/C8f+8dRygh7NULwl69AJQEm7hx41B4/0HJzZ/NhuvJE+C7u0N85SqEPXtCUSQBm8sFOy0VyM6GPCsLz7r4AmoGyQGAsEdPQKHAi379UZyUBIMWLeB8+FClz7+u0fguP2DAADg4OGDFihV4XskBLRRFVR5bKATX0hIAqnw+h6LopxUOEtOztwc4HLAMDJhGa/1GjQCUJMR7nxHS8txcFN69x2RFBSEQ37iBV1u2IHHGDLzauAnS6GgoCgpUq5tKBQW9NxNnAYDZ2DGwXfc1kmbPQXFSEgAw5ZLExuLl8BFIXrYcRMupA+oijQNDbGwspk6dCn9/f3h4eKBTp044cOAACnU8JJ6iKM1wjI3heuY03C6ch4mWo/W1kRccjNgBA/Cidx9IYmLUbmPk4w23C+fhduE8+G96ERl3/wwAIEtJQdbBg2X2EV++jPyQkHK/l2tmBrOxY5jPAj8/pG/+riRLbCmKoiKmYZltaAjTYUMhHDwI5hMnwuHnn8C1tgb09GDcuTNYHA703+RoMur0KRx+LZmjIefv4yh8+BA5x49XeiR3XaBxYHBwcMDKlSsRExODwMBAODs7Y/r06bC1tcW0adNw586dqiwnRVFqcExMwCv1VFwV5BkZJdVVyklwysFzcICejQ3zmWtpCY6ZGcBmg+fmprJt7sVLSJw5C/ETvkBhBTNEipYvh/3WrbBdvx4mfd/M/aCnB9H6dTAZOhRcOzsI+/SB7E36C0VBAbL/CkBB8L9gsVnQs7eH28ULcL9xnUkT7rh7F9wCA+G4fTvT7VjYswf0HB0h6NSJCWwfs0p1VxWLxfD398fevXtx69YtNGnSBGFhYbos30ePdletvYhMhpRVq1CclAy79etKqlPqIKJQIPvYMXAEAgh7ajepkDwnB4r8/DLjPsRXriJxxgyAzYbL0SNMtdO7FNx/AI7QGPz69RHVtBnIm2zJXWKeI00mgw2Xi6tu/6XnUKbOoKppgBsAGBsbo2vXroiLi0NUVBSevBlQQlEfg6InT5Bz7G8AJfMFWM2cWcMlqhosNhtmQ4a8174cExNw1KTJN/btAsf9+8A2MNA4KACAYcuSQXSKoiKwhcKStxngv1ncWCzw6ruh+FU6SGEh+B7lz8JIle+9AkNhYSGOHDmC3bt3IygoCC4uLliwYAHGjx+v4+JRVO3Fd3eHYZs2KE5OLpNkjnq38mZv00RxQgITFAzb+oCTlQm8egWutTXczpyBPC8fRCoBt9Tc65TmtAoMt27dwu7duxEQEACpVIpBgwYhMDAQXbp0qaryUVStxTYwgNOB/TVdjI8S390dFlOmQPIiBqIVK8AKDFRZzxEYAaBTdL4vjQNDo0aNEB0djRYtWmDDhg0YOXJkuTOpURRFVTXrBXQazqqicWDw8/PD4cOH4eXlVZXloSiKomqYxoHh7RnRKIqiqLpJ48Dgq8EE4iwWC5cvX65UgSiKoqiapXFgqKgKSSwW49ChQ5C8NU8rRVEU9eHRODBs2bKlzDKZTIZffvkF69evh729Pb7++mudFo6iKEoTCxYsQG5uLh0EqiPvPfL54MGDWLlyJQoLC/HVV19hypQp4HIrNV6OUoOOfKYoSheqdOTzhQsXsHTpUsTGxuLLL7/EggULYGRE+wtTFEXVFRoHhtDQUCxZsgS3bt3CtGnTEBgYCMs3KX8piqKoukPjqiQ2mw0DAwNMmTIFLhVkH5wzZ47OCkfRqiSKonRDm3uJxoHB2dkZLGWiqvIOxmLhxYsXmpeUeicaGCiK0oUqaWN4+fJlZctFURRFfQDoBM4URVGUCo0DQ0hICM6cOaOybP/+/XBxcYG1tTWmTJlCB7hRFEXVARoHhrVr1yIiIoL5/OjRI0ycOBF+fn5YunQpTp8+jQ0bNlRJISmKoqjqo3FgePjwIbqWmozE398fPj4+2LlzJxYsWIBt27YhICCgSgpJURRFVR+NA0NWVhZsSk30ff36dfQsNf9rmzZtkJCQoNvSURRFUdVO48BgY2OD2NhYAIBUKsX9+/fRtm1bZr1YLIaenp7uS0hRFEVVK427q/bq1QtLly7Fpk2bcOLECRgaGqJjx47M+vDwcLi5uVVJISmK+vD98MMPTKK7BQsW1HRxqApoPMDt9evXGDRoEG7evAmBQIB9+/Zh4MCBzPquXbuibdu2WL9+fZUV9mNEB7hRdUW9evWQlJQEe3t7JCYm1nRxPjpVMsDN0tISN27cQE5ODgQCATgcjsr6I0eOQCAQvF+JKYqiqFpD6+yqJiYmapebm5tXujAURVFUzaMjnymK+uDl5OSgoKCgpotRZ9DA8BFIT0+no9LruIiICDx//lxlmVwux5MnT5CVlVVDpapYVFQUnj59+s7tcnNzKzyH6OhobNq0Cd999x3EYrEui/jRooGhjgsKCsL333+Pn376CXK5vKaLUylyuRxnzpzBiRMnIJVKa7o4tUZ4eDgOHDiAP/74Q6VR99KlS9i/fz9+/vlnyGSyGixhWVFRUdi7dy92797NZGSWSqV49uyZyr9teno6Nm/ejM2bN5ebyPPVq1dQKBQoKChAbm5udRS/zqNzcdZxr169AlAyQFEmk5XpNPAhiYqKws2bNwGU9HBp3bp1mW0UCgUuXLiAnJwc9OvXr07MLpiXlwcjI6Ny097Hx8czf2ez/3vWKy4uBlAyN7smnQ9zcnKwb98+cDgcjB8/vsy1S0pKgoGBgU7aE5W/hywWi/n7wYMHER0dDQ8PD0yYMIEpk/I8MjMz4ezsXOZYbdu2RUxMDBQKBSwsLMqcU2BgIJycnNT+vlDq6SwwZGVl4fTp0xg7dqyuDknpQPfu3WFkZARnZ2fw+fyaLk6l2Nvbw9jYGDKZDI6Ojmq3iY+Px40bNwAAtra26Ny5c7WU7dSpU3j27Bn69++P+vXr6+y4p0+fRnBwMFq0aIFhw4aVWf/y5UsmWPr5+cHOzo5Z17NnT9jZ2aFevXoaDT599uwZkpOTAQAvXrxA06ZNmXWPHj3CwYMHoaenhwULFsDMzKxS5+Xu7o6pU6eCzWbDyckJAJCfn6/yJwDUr18fAwcOhFQqRfPmzdUe68mTJ4iKigIA3Lx5E35+fsy6y5cv486dO7h79y4aNmxYJx4UqoPOAkN8fDwmTJhAA0MtIxAI0KNHj5ouhk6Ymppi2bJlIISU++ZjbW0NKysr5ObmVtuAS6lUin///RcAcPv27QoDQ0JCAq5evYpmzZqVe6MrTVl9Ul41iqGhIbhcLuRyOXODVdLT09PqKblhw4Zwd3cHh8OBu7u7yrq8vDwAJW8f72qvkkgk73wIKSoqQmhoKPT19VGvXj1wOByMHDkSYWFhSExMxJEjR9CvXz9IJBJ4e3szb0sSiQTXr19HcXExoqOj0bhxY5W2FVtbW5XvcXJywp07dyASiaCvr6/xtfjYaRwY3lV3Rxt9KG2JxWIYGRmpVH+8y7u2NTQ0xMKFC6FQKLQ6bmXweDx06NABT58+hY+PD7P83Llz+Pfff+Hn58e8uVy4cAExMTF4/vy5RoFhwIABuHXrFlq2bKl2vbW1NebPnw+ZTKaSy+x9GBkZYeLEiWrXeXt7g81mw9jYGCKRqNxjXL16FRcvXkTTpk0xatSocrd78OABHjx4AKAkIDVo0ADm5uYwMzPDxYsXAZRUHcXGxqJRo0YQiURwcXHB4cOHVXofKatKAaBx48Zo3Lgx5HI5Xr9+jYiICBgZGeF///sf+Hz+B12NWt00DgympqYVTu1JCHnn1J9U9ZLL5VAoFLUyh1VgYCACAwPRoEEDfPHFFzo/fnUFBaW+ffuWWRYWFgaZTIawsDAmMHh6eiImJgYNGzbU6LgODg5wcHBQWXb9+nUEBwejc+fOaN++fZl69coqKirCrVu34OjoCFdXVwAlbQKlg155nj17pvJneVxcXGBoaAh9fX2V6i8nJycYGxuDxWIhJycHQElV0ZMnT2BhYVGmSyqXy4VMJoOdnR3GjBmDsLAwXLp0CRkZGcw2IpGozNsUVTGNA4OxsTFWrFhR7i/Hs2fPMHXqVJ0VjKqcnJwc/Pzzz5BKpZg6darKf77aIC4uTuVPbRQUFCAkJATOzs61Oj9Xr169cOfOHXzyySfMso4dO6J9+/aVenoNCQlBbm4ubt++jfbt2+uiqCouXLiAW7dugcvlMk/bmurduzdu3LiBZs2aVbidSCTCsmXLEB4ejuzsbCZrwr179yAWi+Hs7IyePXviwIEDTDWWk5MThEIheDwe0tPTIRaL0bhxY2RkZKBPnz7IyMjA4cOHVb6Hy+WqDMoNDQ3FqVOn0Lx5c3z++ecan9fHRuPAoHyV7dSpk9r1pqamGvV8oKpHamoqU72XkJBQ6wJD3759ERQUhMaNG2u0fW5uLoqKimBtbY3z58/jzp074HK5WLlyJXg8XhWX9v14eXnBy8urzHJ1QSE0NBRhYWHo2rUr85ReHl9fX4SEhODTTz/VWVlLU95I1aW+eRd7e3uMGDFCo22vXbuGy5cvg8vlYtmyZTAyMkJSUhIAIDExEQUFBfDz88Pp06fh6emJzz//nHkTVCgUiIuLw/bt2wEAycnJaN68OYyNjSEWi8FisUAIgbe3N0xNTZnvDA8Ph0wmw8OHD2lgqIDGgWHkyJEoLCwsd71IJMKqVat0Uiiq8tzd3dGpUydIJBK0aNGipotThrW1NQYPHqzRttnZ2diyZQukUinGjBnD9IgRCoUfZL2xQqEo04B++vRpFBcXgxCCKVOmVLi/t7c3vL291a4rKCiodH16ly5d4ObmBgsLC3C5VdejXVnFyeVymRt+nz59AACRkZHYt28fJk+erDYxJ5vNhpmZGfh8PqRSKSwsLGBgYID58+cjKSkJxcXFOHz4MG7duoVGjRoxHQKUPZZK97iiytL4X33y5MkVrrexsaGBoRZhs9kqEyl9yAoKCpieMFlZWfD19YWHhwfMzc1rPDDk5uYiICAARkZGGDJkyDtvpPn5+fj555+Rk5MDU1NTjB49GnZ2dmjRogUePHig9g1DUw8ePEBAQABEIhFmzZqlcm0UCgXu378PoVCIBg0aID4+HkKhUOVpurTyugPrUqdOnWBrawtLS0sYGBgAACwsLPDJJ58gMjISLBYLXC4XRUVFuHbtGkQiEeRyOa5cuQJvb2906tQJixYtglQqZcZWRERE4NixYzA0NGTGP6SnpzOBQSKRoGnTpnRMwzvo7HEgOzsbf/75J2bNmqWrQ1IUAMDOzg4jRoxAbm4u08Zlb29fw6Uq8fDhQ6a7ZJs2bd45hiEtLY1J75CZmYkHDx7Azs4OgwYNwqBBgypVltjYWBBCkJqaisLCQpVsx6GhoThx4gRYLBa6du2KwMBA6Ovr48svv6ySrMhZWVnIz89HvXr1AJSkrXi7ZyOLxYKHh0eZfd3c3DB9+nSwWCw4Ojri/PnzuH79OlgsFqytrZGRkYGgoCB06tSpTNlfv34NACgsLETPnj0hl8shEokQFBQEJycn7N27F4QQREdHo3fv3jpvuK8rKh0YLl++jF27duH48eMwNDSkgYHSKblcjj///BMpKSkYOXKk1lUbBQUFyM7OrrI2loYNGyI0NBRGRkZleg+p4+zsjE6dOuHRo0fQ09Mrtxvq++jatSsUCgW4XC4OHTqEDh06MG04yidyDofDpJyQSqXMU7Uu5eTkMFV/I0aMgJeXFyIiIpg2SE3aIkv3IlJ2jxUKhejQoQMuX74MoVCI5ORklX/X/Px85OTkwNXVFe3bt0eTJk0gk8mwdu1aSKVSeHl5gcvlori4mMkhNXfuXB2ffd3wXoEhISEBe/bswZ49exAfH4/hw4fj+PHj6Nq1q67LR33ksrKyEBkZCaCk+6c2VRwymQzbtm1DdnY2evfurTLjoK5YWVnhyy+/1GhbsViM33//HVKpFFOmTIGVlZVOy2JiYoLPP/8cmzdvRkZGBtNrByhpCDc1NYWhoSHMzMxgZmYGS0vLSo9gLk0qlYLH40EikTABRywWo6ioCMnJyUx39tLd2pVVXPHx8eBwOOjWrRsMDQ1Vjmtvb48ZM2YgMTEROTk5yMvLQ05ODg4cOIAlS5YAKOlsceLECWYgYO/evQGUVKkaGhoy1U3z5s3D0aNHERsbW2XzxxBC8O+//4IQgvbt21d712ld0DgwFBcX48SJE/jjjz8QFBSEHj16YPPmzRgxYgRWrFiBRo0aVWU5qY+UhYUFfHx8kJKSUm6Da3lkMhnTMyszM7MqiqeVhIQEpn/9ixcvdB4YlJo3b44bN24wA+hkMhnCw8MhEomY72zXrp1Ov/PMmTO4efMm2rdvj379+mHcuHHIzs5GmzZtEBUVhcTERLVvCnfv3sXff//NfDYwMMBnn33GfH78+DH+/PNP8Hi8MokTS99wDx48iPT0dLBYLFhYWDBVRGw2G7NmzUJ6ejqcnZ3BYrHwxRdfID4+XqM3vPfx5MkTnD59GkBJb80mTZpUyfdUJY0Dg729PTw9PTF69Gj4+/szTxqadk2jKE1IpVJkZWUxo3hZLJbKFLLa0NfXx/jx4xEfH18l/f211aBBA7Rp04ap1qgq3bp1Q7du3ZjPly9fxtWrV8Hj8bB8+fIqSQ0RHR0NAEwabU9PT2adm5sb3N3d1VYDln47ULYplKZsj5FKpdDX14dEIkG7du2QmZmJXr16MdtZWFggPT0djRs3Rvfu3VXOUSAQqLwd6OnpVen4FzMzM+ZcdflGVp00nvPZ3NwcTZs2xejRozFs2DBmzlA9PT2EhYXVyTeG8ePHIzs7GydOnCizrrCwEBs3bsThw4cRFxcHY2NjdOnSBatXr1bpm7969WqsWbOmzP7//POPSrKv8nxscz5v3boVqamp6Nq1q8rN7WOjUCiwb98+xMbGYtiwYRqP91BHOcpcX18fy5Ytq5Jkis+fP8ft27fh7e1dJs+S0ttzPhNCIJVKER8fjwcPHkAulyMlJQU+Pj7o0KEDLl68iKioKDg5OcHV1RX169eHVCpV25NKJpMhOTkZf/31FzIyMjBgwAC0bduWWZ+fn4+kpCSYmZlV2ZtaacpR2+XNeFkTqmTO5+TkZBw7dgy7du3C3Llz0bNnT4wePfqjTIMhkUjg5+eH+Ph4fP/99/Dx8UFaWho2bNgAHx8fBAYGqvxSNm7cGIGBgSrHqKqpUAkhzD/8h1a3qVAomKqW9PT0Gi7Nu8nlcly9ehUsFgtdunTR6fUuLCxknsIjIiIqFRh8fX1hZ2cHa2vrKsuwW79+fY2zykokEoSEhCA8PBwvX76Ep6cn044ElKT88Pb2xtWrVwGUzDevHEn9dvvDq1evYGpqCh6PB0tLS+YNIzU1ldkmKCgIZ8+eBVDyILto0aIqf8iqTQHhfWgcGPT19TFq1CiMGjUKMTEx2LNnD+bMmQOZTIb169dj/Pjx8PX1rfF+5dVh69atCAkJUel37uTkhGPHjsHHxwcTJ07E48ePmaDJ5XIrTDymSydOnMDt27fRvHlzDB8+vFq+U1fYbDbGjx+PZ8+eVbrqp7i4GLt27UJ6ejrGjh1bJblyIiIimIBvY2Oj07pkIyMjdO/eHbGxseVmG9AUm82uVW/0RUVFOHnyJDM6ubCwEGw2G3w+HzweD+3bt4eenh58fHwQHR2NVq1aqT2O8k3I1tYWc+fOhaGhIby9vZGUlKQyKjw2Npb5uzJ/GFWx9+qV5ObmhnXr1mHt2rW4ePEidu3ahT59+kAgEKgkr6qrDh06hG7dupWpJ2az2Zg/fz5GjRqFsLAwjbJnvk0ikaikNdZ2RqqEhASVPz80bm5uKvW/UVFRyM/PR4sWLbR6Is/IyGB6qERGRlZJYLC2tmbScVhbWwMo+ffbs2cPxGIxxo4dW6mMp126dEGXLl10UlZNZGdn48aNG3B3d9c4yd/7YLFYMDIygq+vLxISEtC1a1cYGBiAx+OppDd5V9uSMrPq69evIZfLkZGRgVu3bgEA7t+/z1TV9uzZE/r6+uDz+UzvLKpilRrHoBxd27NnT7x+/Rr79+/XVblqtadPn5b7H1b5H+rp06dMYHj06JFK41ejRo0QGhqqdv8NGzaobZPQ1ODBgxEaGqrT/vE1JTk5GXv37gVQUs3Upk0bjfe1sbFBhw4d8Pr1a42ygr4PkUiEpUuXAviviiM5OVklIFU2FXZ1On/+PMLCwnD79m2sWbOmytJhGBsbY9GiRZVuBO/Tpw/MzMyYOSTi4uLA4XCgUChUxjdYWVlh6NChlS32R0Xjf/msrCz8+eefGDduXJn6uZycHBw+fBiTJk3SeQFrK20SBnp4eODUqVPM54rqeZctW4YFCxYwn3Nzc7XqVmdvb691L578/HwEBASAx+NhyJAhtSYpHY/HA4fDgVwuZwZoaYrFYqlNha1rb9d5Ozo6okWLFhCLxVrlqCoqKsLOnTuRk5ODXr16wdHREZaWlrouboXs7OwQFhYGkUhUpTmSWCxWuUFBKpUiMzMTNjY272y/FAqFTNqX0NBQXLx4EXK5HGZmZrWq6uxDpPG//s8//4zw8HDMnj27zDoTExMEBQVBLBZj+fLlOi1gbdSgQQOVxrLSlMsbNGjALOPxeBo3zPH5/GqfgvPRo0dMQ2fLli2rtBpBG5aWlpg3bx6KioqqrM+5rnE4HLVTcL5LWloak1k0ICAAXC4XvXr1QmhoKFq3bq2SuruqdOrUCc2aNYOxsXGVf1d5duzYgcTERHTq1KlMri+ZTIbY2Fg4ODioBJYXL14wYyG4XC7Ng6QDGlfaHjt2DNOmTSt3/dSpU3HkyBGdFKq2Gz58OAIDAxEWFqayXKFQYMuWLWjUqFGV9lPXNR6PBzabDSMjo2pJnqYNKyurDyYoVIaDgwPatm3LdFKQy+W4ffs2UlNTmd4576OgoAAHDhzAkSNHIJPJ3rm9sg++TCZDYmKiRvvokrI3mrpeabt378auXbuwYcMGlQZkExMT5nd4woQJNAODDmj8xhATE1Nu/2SgJM1zTEyMTgpVm+Tk5ODhw4cqy0aPHo2TJ0+ib9++Kt1Vv/nmG0RGRiIwMPCD6sYbFRUFhUKB/Pz8Kq1CqCpisRh6enof9Jy+bDYbAwYMgEKhwL1792BsbIy8vDz8888/GrWtKBQKHDp0CElJSRg+fDjT2B4WFoaIiAgAQLNmzdQmrVPn8OHDiIiIQJMmTTB69Oj3PzEtjR8/HtHR0WpHZisT5EkkEhQWFsLIyAhAyeC2RYsWobi4mOkGLpfLkZycDJFIVCtnMKztNL4LcDgcJCcnl/tEmZyc/MH1m9fEtWvXytQVT5w4EVeuXME333yD5cuXqwxwu3Xr1gc3BL5t27Z49eoV6tevX+3VWJV1584dHDt2DDweD4sXL66y/DfVhc1mqwQCTatFsrOz8fjxYwAlGV+VgaF+/fowMTGBvr4+k+m0NIVCgfDwcJiamsLZ2ZlZrkwhUt2pRFxcXODi4lJmeUxMDBo3bozo6GiIRKIybTtvV38dPXoUDx48gLu7e7nzWEskErBYrFrTplabaDzyuUuXLvDx8cHGjRvVrl+yZAlCQ0Mr9dpLlfWxjXzW1k8//cTUzU+ePLlWT/WpqfT0dDx9+hReXl4aBzpCCE6ePImkpCQMHjxY43EzwcHBOH36NNhsNhYuXMjkGHr16hUePnwIPp+PmzdvokWLFiopKN7H2yOfNVVQUID169dDLpczYx/69u2LDh06lLvP77//jpcvX8LS0lJtksPU1FT8+uuv4HA4mDVr1keRflube4nGj/izZs3C999/j59//hlyuZxZLpfL8dNPP2HLli2YOXPm+5eaqnaEEI3TLp89exY7duxg+o7XFh06dACLxYKlpaXaJ01dk8vluHXrVrmdD3Rh165dOH36NI4eParxPiwWCwMGDMDMmTO1GkypfFpms9kqg1Otra3x2WefITo6GmKxGCEhIZqfgI5xuVzmDUFZRVtQUFDu76JcLoeXlxfatWtXbjVYcnIypFIpCgsLVUZJUyU0rkoaPHgwFi9ejDlz5mDFihXMvLQvXrxAXl4eFi1aROdQ/YDI5XJs374diYmJGD58eIWTtysnRgFKJqLv379/dRXznVq2bInmzZurVGNmZmYiKCgIDRs2VOkdpgshISE4c+YMWCwW5s2bVyXjFAwNDZGdnV2muqQqtGnTBqampkhOTkZgYCB8fX1V0rUop4etqc4UBQUFOH78OFxcXJgR0fHx8Th37hyuXLmCCRMmlPk3vnTpEq5fvw4DA4Ny33KaNWuGtLQ0cDicWtMLrzbRqqVx/fr16N+/Pw4ePIjnz5+DEIJOnTph5MiRWqdEpmpWYWEh4uPjAZQMxqsoMJiamqJBgwZITk6udXPlymQy7Nu3D+np6Rg9ejTq1auHM2fO4MmTJ7h79y7Wrl2r044AyrrsqmzsnjRpEpKSkrR6AyoqKgKbzX6v+nInJyfs3r0bhBDIZDKVVCoeHh4aN1hXJDExEXl5eVrvFx4ejkePHgEoeQhwdnaGXC5n3nSVyepKKz0hUHk15Vwut85MfVsVtO6CUt5E5CkpKVi/fj1+/vlnnRSMqloCgQD9+vVDXFwcfH19K9yWw+Hgiy++qKaSaSc9PR3Pnj0DUJK7v169erCzs8OTJ09gZ2encVBIS0vD/fv34eXlVeFsb15eXrCwsICRkVGVJUozNDQstwegRCLBrVu3YG9vz4yNSUhIwI4dO8Dj8TBnzpwy5SoqKsLz58/h6uqq9i2Ey+WiXr16SEhIUGmABkrevoKDg9GwYcMyY3GUDby2trYYPHgwbG1tyz2nW7duMVXQ2gwOdXV1BZvNhkKhQGJiIjw9PeHo6Ijhw4ejoKBAJY9SUVERgoOD4eDggGHDhsHOzk4lUL548YI5JlUxrQJDREQErl69Cj6fjyFDhsDU1BSvX7/G+vXr8fvvv9ML/oFp3759rZinoDJsbGzg7e2N9PR0pgePn58fWrZsqVVjfUBAAJKSkhAVFYX58+dXuK263j1VoaCgAPfu3YOrqyszx/U///yDmzdvgsPhYPny5TAyMkJSUhKKi4tRXFyM9PT0MoHh0KFDePr0KZydndWORWKz2Zg2bRqKioqYLqBKp0+fRmRkJO7cuYO1a9eqrFOmyk5MTMT58+fVPjwkJCTg6tWrEIlETHUfi8VCcHAwcnNz0bVrV5Wbt3IWOCUzMzMmuJfOw6YuD9mVK1dw48YNsNlsLFu2TKWnUkxMDHbu3Amg7nRSqEoaB4ZTp07h888/Zwa8bNq0CTt37sTQoUPRqlUrHD9+HD169KiyglIfl/z8/DI3KXXYbDYGDRpUZrm2ac2trKyQlJTEJMOrDU6dOoWHDx/CwMAAK1euBIvFYm76hoaGTP/8Vq1aISMjA/r6+mpveMpqF5lMBqlUioSEBDg6Oqr07+dwOGqvt62tLSIjI9W+DfTt2xeXLl1CQUFBuW84Fy9exPPnz/Hs2TMYGxsjNzcXcrmcmeFMIBAwU67++eefePz4MXr16gVjY2NERUWha9euGD16NGJiYlQypr7t+fPniIuLY475dpVa6bcUbd5YPlYaB4Z169Zh5syZ+Prrr/HHH39gwYIFmDNnDs6dO6dVcjOKepdjx47hzp07aNeuXbU1dA8ZMgSdO3eulklcNKW8URsZGTFPzR07doSzszPMzMyYm5+enh4zx7E6o0aNQmRkJNzd3fHjjz8iIyND44Frn332GVq2bKk2I2nbtm3Rtm3bMk/5pXl6eiImJkYldxGbzYZAIEBhYaFKwFGmZYmOjkZsbCwUCgVkMhnGjBmDhg0bIi4uDmfPnkXLli3LtHv89ddfEIvFsLOzw6RJk8qMx6lfvz4znkHT9DQfM427q0ZHR2PmzJkQCASYPXs22Gw2tmzZQoPCR0ihUCAqKooZiapryvz5Dx48YBrIy5OYmIh169bh119/LTMnsDY4HA5EIlGtmk+kd+/emDJlCmbMmKGy3MHBQauBfAKBAG3atEFiYiJTHZOWlqbx/paWlhWOiK+owfuTTz7BunXrVKYAlslkKCgogJOTk8obzpAhQ9C0aVP06tULLi4uYLFYcHZ2RlBQEOLi4nD69GmEhYVh//79ZX4vlNVGQqGw3N5c7u7uFWZvoP6jcWAQi8VMnS2Hw4GBgQFtU/hIXb9+HXv37sVPP/2EwsJCnR9/0KBB0NPTQ1FREVPlUJ7IyEjk5eUhPj6+1o2xqCw2m11ug/H7KN1fvzrzCb0dbGUyGRQKBV68eKESzJs1a4ZRo0bB3t4ekyZNwurVq5GZmYmzZ8/ijz/+YLIuyOVynD17Frdv32ZyJil7PGVnZ1fPSdVxWjU+X7x4kanjVCgUuHz5MjMMX6lfv366Kx1VKynbmRQKhc5mw5LJZHjx4gXs7e1hYWHB1AO/a5yAt7c3EhMTYW5uXmFvIuq/GzSLxVJJbVNYWAiFQqFRm87bwsPD8eDBA3Tq1KlMj6by8Pl8pnfRP//8A5FIxHQcuHnzJuLi4tCjRw+m9xcAZkxCfn4+nj59iri4OMTFxYEQgrZt26Jdu3b4999/0aZNG6SlpcHKyqpOpuipLloFhnHjxql8njp1qspnFoulMiqaqpt8fX1hYWEBGxub97qZqHPixAncvXsXNjY26NmzJxN83pXt1cTEBBMmTNBJGT4kEokEx44dAyEEgwcP1mhMRceOHaGvrw8LCwumcT4zMxPbtm2DTCbDlClTtM6ue+LECRQUFKCgoADTp0/XaB/lFK7nz5/HzZs3AZR0IeXxeDhz5gyAkkAwaNAgdO3aFW5ubrCysgKXy8WIESOQmZmJLVu2oLi4mKlS69KlC7y9vbFp0yacPn0aHh4eH+Xvha5oHBjoPKmUEofD0fkMccoqqcLCQri7u6Njx46QSqXvNT3qxyAqKgrh4eEASmYN1OTfg8vllslampGRgaKiIgAlVU3aBoZGjRrh7t277zV62N7eHiwWC2ZmZjAyMgKXy4WTkxMSEhKYtgAWi1VmoJ+5uTkzT4eyGy/wX5oLoGRcFfX+NE6iR9WMjyWJXn5+Ph4+fAh3d/da1WW0tsrNzcXOnTtBCMGkSZPeex5jQgiCgoJQVFQEX1/f90q7LpfLNWq0V5dELzc3F/r6+ioN2JoeT105/P39kZqaikGDBlVL7qwPiTb3Eq0DQ0ZGBpOJMCEhATt37kRhYSH69u1bYT9j6v18LIGBqvveN7sqpRtVkl310aNHcHZ2hrW1NTw9PfHw4UO0adMGW7ZswY4dO+Dr64sTJ05UtuwURVFUDdM4MCxevBhNmzbFjRs30LlzZ/Tp0we9e/dGTk4OsrKyMHXq1HLnaqAoiqI+HBpXKN65cwdXrlxBs2bN4OXlhR07dmDGjBlMl7DZs2ejbdu2VVZQiqIoqnpo/MaQmZnJTAAiEAhgZGQEMzMzZr2ZmRnEYrHuS0hRFEVVK626ILydwvhDmvCeoqiatWDBAuTm5tJOFB8ArQLD+PHjmeRURUVFmDZtGjPASSKR6L50FEXVGQsWLKjpIlAa0jgwvD3qWV1mxrFjx1a+RBRFUVSN0jgw7NmzpyrLQVEURdUSNMsURVEUpYIGBoqiKEqF9olRqGqlzFiSm5tbwyWhKOpDpryHaJIFiQaGWk45NsTBwaGGS0JRVF0gFouZeXXKQ7Or1nIKhQLJyckwNjb+qMeN5ObmwsHBAQkJCbQf/FvotVGPXhdVhBBmXux3TWJE3xhqOTabjXr16tV0MWoNoVBI/5OXg14b9eh1+c+73hSUaOMzRVEUpYIGBoqiKEoFDQzUB4HP52PVqlVMShbqP/TaqEevy/ujjc8URVGUCvrGQFEURamggYGiKIpSQQMDRVEUpYIGBqpGZGZmYtSoURAKhTA1NcXEiRORl5dX4T5FRUWYOXMmLCwsIBAIMHjwYKSlpalsEx8fj969e8PQ0BDW1tZYtGgRZDKZ2uMFBweDy+WiefPmujotnaipa/P333+jW7dusLKyglAoRLt27XDx4sUqOUdN/PLLL3B2doa+vj58fHwQGhpa4fZHjhyBp6cn9PX10bRpU5w7d05lPSEEK1euhK2tLQwMDODn54dnz56pbPM+175OIhRVA3r06EG8vLzIrVu3SFBQEKlfvz4ZMWJEhftMmzaNODg4kMuXL5O7d++Stm3bkvbt2zPrZTIZadKkCfHz8yMPHjwg586dI5aWlmTZsmVljpWVlUVcXV3JZ599Rry8vHR9epVSU9dm7ty5ZNOmTSQ0NJQ8ffqULFu2jOjp6ZH79+9X2bmWx9/fn/B4PLJ7924SERFBJk+eTExNTUlaWpra7YODgwmHwyHffvstefLkCfnqq6+Inp4eefToEbPNxo0biYmJCTlx4gQJCwsj/fr1Iy4uLqSwsJDZ5n2ufV1EAwNV7Z48eUIAkDt37jDLzp8/T1gsFklKSlK7T3Z2NtHT0yNHjhxhlkVGRhIAJCQkhBBCyLlz5wibzSapqanMNr/99hsRCoVEIpGoHG/YsGHkq6++IqtWrapVgaE2XJvSGjVqRNasWVPZ09Kat7c3mTlzJvNZLpcTOzs7smHDBrXbDx06lPTu3VtlmY+PD5k6dSohhBCFQkFEIhHZvHkzsz47O5vw+Xxy+PBhQsj7Xfu6ilYlUdUuJCQEpqamaN26NbPMz88PbDYbt2/fVrvPvXv3UFxcDD8/P2aZp6cnHB0dERISwhy3adOmsLGxYbbp3r07cnNzERERwSzbs2cPXrx4gVWrVun61Cqtpq9NaQqFAmKxGObm5ro4NY1JpVLcu3dP5XzYbDb8/PyY83lbSEiIyvZAyfkpt4+NjUVqaqrKNiYmJvDx8VG5Rtpe+7qK5kqiql1qaiqsra1VlnG5XJibmyM1NbXcfXg8HkxNTVWW29jYMPukpqaq3PiU65XrAODZs2dYunQpgoKCwOXWvl//mrw2b/vuu++Ql5eHoUOHvs+pvLfXr19DLperLW9UVJTafco7v9Lnr1xW0TbaXvu6ir4xUDqzdOlSsFisCn/K+49dHeRyOUaOHIk1a9agQYMG1frdtf3avO3QoUNYs2YNAgICytwsqbqv9j0yUR+shQsXYvz48RVu4+rqCpFIhFevXqksl8lkyMzMhEgkUrufSCSCVCpFdna2ypNxWloas49IJCrTc0XZM0ckEkEsFuPu3bt48OABZs2aBaCkuoQQAi6Xi0uXLsHX11ebU9ZYbb82pfn7+2PSpEk4cuRImeqZ6mBpaQkOh1OmV1Xp83mbSCSqcHvln2lpabC1tVXZRtkr7X2ufZ1V040c1MdH2ch39+5dZtnFixc1amA9evQosywqKkptA2vpnivbt28nQqGQFBUVEblcTh49eqTyM336dOLh4UEePXpE8vLyquiMNVdT10bp0KFDRF9fn5w4cULXp6YVb29vMmvWLOazXC4n9vb2FTY+9+nTR2VZu3btyjQ+f/fdd8z6nJwctY3P2lz7uooGBqpG9OjRg7Ro0YLcvn2b3Lx5k7i7u6t0C0xMTCQeHh7k9u3bzLJp06YRR0dHcuXKFXL37l3Srl070q5dO2a9skvmZ599Rh4+fEguXLhArKys1HZXVaptvZIIqblrc/DgQcLlcskvv/xCUlJSmJ/s7OzqOfFS/P39CZ/PJ3v37iVPnjwhU6ZMIaampkyvqjFjxpClS5cy2wcHBxMul0u+++47EhkZSVatWqW2u6qpqSk5efIkCQ8PJ/3791fbXbWia/+xoIGBqhEZGRlkxIgRRCAQEKFQSCZMmEDEYjGzPjY2lgAgV69eZZYVFhaSGTNmEDMzM2JoaEgGDhxIUlJSVI778uVL0rNnT2JgYEAsLS3JwoULSXFxcbnlqI2BoaauTadOnQiAMj/jxo2r6lNW66effiKOjo6Ex+MRb29vcuvWLZWyvl2ugIAA0qBBA8Lj8Ujjxo3J2bNnVdYrFAryv//9j9jY2BA+n0+6du1KoqOjVbZ517X/WNDsqhRFUZQK2iuJoiiKUkEDA0VRFKWCBgaKoihKBQ0MFEVRlAoaGCiKoigVNDBQFEVRKmhgoCiKolTQwEBRFEWpoIGBqlVWr15dJVNtvnz5EiwWCw8fPgQAXLt2DSwWC9nZ2QCAvXv3lklbrY13Ha+qzksTnTt3xrx582rku6kPEw0MVKWNHz+eSR2tp6cHGxsbdOvWDbt374ZCoaj0sQcMGKCbgpbSvn17pKSkwMTEROfHBoBhw4bh6dOnVXJsqsTevXvVpi/X19ev0u/duXMnOnbsCDMzM5iZmcHPz++d81F/aGhgoHSiR48eSElJwcuXL3H+/Hl06dIFc+fORZ8+fVQmnK8teDweRCIRWCxWlRzfwMCAzmOgBiFEp78PQqEQKSkpKj9xcXE6O746165dw4gRI3D16lWEhITAwcEBn332GZKSkqr0e6sTDQyUTvD5fIhEItjb26Nly5ZYvnw5Tp48ifPnz2Pv3r3MdtnZ2Zg0aRKsrKwgFArh6+uLsLAwtcdcvXo19u3bh5MnTzJPg9euXQMALFmyBA0aNIChoSFcXV3xv//9D8XFxRqX9+2qn7elp6ejdevWGDhwICQSCRQKBTZs2AAXFxcYGBjAy8sLR48eLff45VVNHThwAM7OzjAxMcHw4cMhFouZdRKJBHPmzIG1tTX09fXxySef4M6dOyr7X79+Hd7e3uDz+bC1tcXSpUtVbrT5+fkYO3YsBAIBbG1t8f3337/zWoSFhaFLly4wNjaGUChEq1atcPfuXWZ9cHAwOnfuDENDQ5iZmaF79+7IysrSqMzK63z+/Hm0atUKfD4fN2/e1Pp6lofFYkEkEqn8KGdp27FjB+zs7Mq8tfbv3x9ffPEF8/m3336Dm5sbeDwePDw8cODAgQq/8+DBg5gxYwaaN28OT09P/PHHH1AoFLh8+bLW5a+taGCgqoyvry+8vLzw999/M8uGDBmCV69e4fz587h37x5atmyJrl27IjMzs8z+X375JYYOHcq8jaSkpKB9+/YAAGNjY+zduxdPnjzBjz/+iJ07d2LLli06KXdCQgI6duyIJk2a4OjRo+Dz+diwYQP279+P33//HREREZg/fz5Gjx6N69eva3zcmJgYnDhxAmfOnMGZM2dw/fp1bNy4kVm/ePFiHDt2DPv27cP9+/dRv359dO/enbk2SUlJ6NWrF9q0aYOwsDD89ttv2LVrF9atW8ccY9GiRbh+/TpOnjyJS5cu4dq1a7h//36F5Ro1ahTq1auHO3fu4N69e1i6dCn09PQAAA8fPkTXrl3RqFEjhISE4ObNm+jbty/kcrlGZVZaunQpNm7ciMjISDRr1kwn1/NdhgwZgoyMDFy9epVZlpmZiQsXLmDUqFEAgOPHj2Pu3LlYuHAhHj9+jKlTp2LChAkq+7xLQUEBiouLq31u7CpVw9ldqTpg3LhxpH///mrXDRs2jDRs2JAQQkhQUFCZiWEIIcTNzY1s376dEFI2DXZFxy5t8+bNpFWrVuWuV6aqfvDgASGEkKtXrxIAJCsrixBCyJ49e4iJiQmJiooiDg4OZM6cOUShUBBCCCkqKiKGhobk33//VTnmxIkTmVz95R1PadWqVcTQ0JDk5uYyyxYtWkR8fHwIIYTk5eURPT09cvDgQWa9VColdnZ25NtvvyWEELJ8+XLi4eHBlIsQQn755RciEAiIXC4nYrGY8Hg8EhAQwKzPyMggBgYGZO7cueVeG2NjY7J3716160aMGEE6dOigdp0mZVZel9IT/2hyPTWxZ88eAoAYGRmp/PTo0YPZpn///uSLL75gPm/fvp3Y2dkRuVxOCCGkffv2ZPLkySrHHTJkCOnVq5fG5Zg+fTpxdXVVmdfhQ0en9qSqFCGEqccPCwtDXl4eLCwsVLYpLCxETEyMVsf966+/sG3bNsTExCAvLw8ymQxCobBSZS0sLETHjh0xcuRIbN26lVn+/PlzFBQUoFu3birbS6VStGjRQuPjOzs7w9jYmPlsa2vLTCUZExOD4uJidOjQgVmvp6cHb29vREZGAgAiIyPRrl07lXaRDh06IC8vD4mJicjKyoJUKoWPjw+z3tzcHB4eHhWWa8GCBZg0aRIOHDgAPz8/DBkyBG5ubgBK3hiGDBmidj9NyqzUunVr5u+6up5AyZvj229EBgYGzN9HjRqFyZMn49dffwWfz8fBgwcxfPhwsNkllSWRkZGYMmWKyv4dOnTAjz/+qNH3b9y4Ef7+/rh27VqVN3pXJxoYqCoVGRkJFxcXAEBeXh5sbW2ZdoLStOkqGhISglGjRmHNmjXo3r07TExM4O/vr1F9ekX4fD78/Pxw5swZLFq0CPb29ky5AeDs2bPMstL7aEpZPaPEYrEq3WtLF1avXo2RI0fi7NmzOH/+PFatWgV/f38MHDhQ5SZbGUZGRszfdXU9AYDNZqN+/frlru/bty8IITh79izatGmDoKAgnVU5fvfdd9i4cSMCAwPRrFkznRyztqBtDFSVuXLlCh49eoTBgwcDAFq2bInU1FRwuVzUr19f5cfS0lLtMXg8HlOfrfTvv//CyckJK1asQOvWreHu7q6TnihsNhsHDhxAq1at0KVLFyQnJwMAGjVqBD6fj/j4+DLldnBwqPT3AmAaP4ODg5llxcXFuHPnDho1agQAaNiwIUJCQkBKza0VHBwMY2Nj1KtXD25ubtDT08Pt27eZ9VlZWRp1m23QoAHmz5+PS5cuYdCgQdizZw8AoFmzZuU2qmpSZnWq43oq6evrY9CgQTh48CAOHz4MDw8PtGzZklnfsGFDlfIDJde0ovIDwLfffouvv/4aFy5cUHkbqivoGwOlExKJBKmpqZDL5UhLS8OFCxewYcMG9OnTB2PHjgUA+Pn5oV27dhgwYAC+/fZbNGjQAMnJyTh79iwGDhyo9j+Ys7MzLl68iOjoaFhYWMDExATu7u6Ij4+Hv78/2rRpg7Nnz+L48eM6OQ8Oh4ODBw9ixIgR8PX1xbVr1yASifDll19i/vz5UCgU+OSTT5CTk4Pg4GAIhUKMGzeu0t9rZGSE6dOnY9GiRTA3N4ejoyO+/fZbFBQUYOLEiQCAGTNmYOvWrZg9ezZmzZqF6OhorFq1CgsWLACbzYZAIMDEiROxaNEiWFhYwNraGitWrGCqTdQpLCzEokWL8Pnnn8PFxQWJiYm4c+cOE8yXLVuGpk2bYsaMGZg2bRp4PB6uXr2KIUOGwNLS8p1lVsfY2Fhn15MQgtTU1DLLra2tmfMeNWoU+vTpg4iICIwePVplu0WLFmHo0KFo0aIF/Pz8cPr0afz9998IDAws9zs3bdqElStX4tChQ3B2dma+XyAQQCAQaFz2Wq1mmzioumDcuHHM/MBcLpdYWVkRPz8/snv3bqaRTyk3N5fMnj2b2NnZET09PeLg4EBGjRpF4uPjCSFlG59fvXpFunXrRgQCgco8x4sWLSIWFhZEIBCQYcOGkS1btqg09r5N08ZnpeLiYjJo0CDSsGFDkpaWRhQKBdm6dSvx8PAgenp6xMrKinTv3p1cv35do+Opm1t6y5YtxMnJiflcWFhIZs+eTSwtLQmfzycdOnQgoaGhKvtcu3aNtGnThvB4PCISiciSJUtU5m0Wi8Vk9OjRxNDQkNjY2JBvv/2WdOrUqdzGZ4lEQoYPH04cHBwIj8cjdnZ2ZNasWSoNqdeuXSPt27cnfD6fmJqaku7duzPn+a4yv31dlN51PQkhxMnJiaxatUptuZXXGGrmqAagMt+1XC4ntra2BACJiYkpc5xff/2VuLq6Ej09PdKgQQOyf//+cr9TWS5131lRWT80dM5niqJqnYKCAlhYWOD8+fPo3LlzTRfno0PbGCiKqnWuXr0KX19fGhRqCH1joCiKolTQNwaKoihKBQ0MFEVRlAoaGCiKoigVNDBQFEVRKmhgoCiKolTQwEBRFEWpoIGBoiiKUkEDA0VRFKWCBgaKoihKBQ0MFEVRlIr/A2URopgVAQ1AAAAAAElFTkSuQmCC", "text/plain": [ "
" ] @@ -1493,20 +1532,23 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We can also calculate the area under the receiver operating characteristic curve (AUROC) of this zero-shot prediction method.\n", + "We can also calculate the area under the receiver operating characteristic curve (AUROC) of this zero-shot prediction method. Note that the results are nearly random unless you are on one of the following configurations:\n", + "* `--fp8` on an fp8 enabled GPU with either the 1b or 7b models. The 40b likely works as well.\n", + "* the 7b model uniquely seems to work well without `--fp8` so if you are on an older device, the 7b model should produce\n", + " robust results. Change the `MODEL_SIZE` earlier in this tutorial and rerun for good results in that case.\n", "\n" ] }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 15, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Zero-shot prediction AUROC: 0.4\n" + "Zero-shot prediction AUROC: 0.77\n" ] } ], @@ -1517,13 +1559,6 @@ "auroc = roc_auc_score(y_true, -brca1_df['evo2_delta_score'])\n", "print(f'Zero-shot prediction AUROC: {auroc:.2}')" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": {