From 38b691afc4ea682415f21082ca637b93c3b921c3 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 22 Dec 2021 00:16:22 +0800 Subject: [PATCH 001/153] Initial code for s3 io datapipes with successful pybind11 build --- CMakeLists.txt | 43 +++ setup.cfg | 6 + setup.py | 7 + test/test_remote_io.py | 20 +- tools/__init__.py | 0 tools/setup_helpers/__init__.py | 1 + tools/setup_helpers/extension.py | 57 +++ torchdata/csrc/pybind/s3_io/pybind.cpp | 32 ++ torchdata/csrc/pybind/s3_io/s3_io.cpp | 475 +++++++++++++++++++++++++ torchdata/csrc/pybind/s3_io/s3_io.h | 57 +++ torchdata/datapipes/iter/__init__.py | 10 +- torchdata/datapipes/iter/load/s3io.py | 74 ++++ 12 files changed, 777 insertions(+), 5 deletions(-) create mode 100644 CMakeLists.txt create mode 100644 setup.cfg create mode 100644 tools/__init__.py create mode 100644 tools/setup_helpers/__init__.py create mode 100644 tools/setup_helpers/extension.py create mode 100644 torchdata/csrc/pybind/s3_io/pybind.cpp create mode 100644 torchdata/csrc/pybind/s3_io/s3_io.cpp create mode 100644 torchdata/csrc/pybind/s3_io/s3_io.h create mode 100644 torchdata/datapipes/iter/load/s3io.py diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 000000000..c3f0801b6 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,43 @@ +cmake_minimum_required(VERSION 3.13) +project(_torchdata) + +set(CMAKE_CXX_STANDARD 17) +SET(TORCH_MIN_VERSION "1.5.1") + +find_package(Python3 COMPONENTS Interpreter Development) + +find_package(AWSSDK REQUIRED COMPONENTS s3 transfer) + +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +set(INCLUDE_DIRS "torchdata/csrc/pybind/s3_io") + +set(SOURCES "${INCLUDE_DIRS}/s3_io.cpp" ) + +include_directories(${INCLUDE_DIRS}) +find_package(pybind11 REQUIRED) +pybind11_add_module(_torchdata ${SOURCES} "${INCLUDE_DIRS}/pybind.cpp") + +Message(STATUS "All linked libs: ${AWSSDK_LINK_LIBRARIES}") + +target_link_libraries(_torchdata PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS}) + + +# project(torchdata) + +# find_package(Python3 COMPONENTS Interpreter Development) +# find_package(AWSSDK RUNTIME COMPONENTS s3 transfer) +# find_package(pybind11 REQUIRED) + +# set(INCLUDE_DIRS "torchdata/csrc/s3_io") +# set(SOURCES "${INCLUDE_DIRS}/s3_io.cpp") + +# include_directories(${INCLUDE_DIRS}) +# # add_subdirectory(torchdata/csrc) + +# pybind11_add_module(_torchdata ${SOURCES} "${INCLUDE_DIRS}/pybind.cpp") + +# target_link_libraries(_torchdata PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS}) + +# # EXAMPLE_VERSION_INFO is defined by setup.py and passed into the C++ code as a +# # define (VERSION_INFO) here. +# # target_compile_definitions(_torchdata PRIVATE ) \ No newline at end of file diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..8e87bf939 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,6 @@ +[bdist_wheel] +universal=1 + +[metadata] +license_file = LICENSE + diff --git a/setup.py b/setup.py index 464afc45b..d3b1f092c 100644 --- a/setup.py +++ b/setup.py @@ -9,6 +9,8 @@ from torchdata.datapipes.gen_pyi import gen_pyi +from tools import setup_helpers + ROOT_DIR = Path(__file__).parent.resolve() @@ -85,5 +87,10 @@ def _export_version(version, sha): # Package Info packages=find_packages(exclude=["test*", "examples*"]), zip_safe=False, + extras_require={ + "scipy": ["scipy"], + }, + ext_modules=setup_helpers.get_ext_modules(), + cmdclass=dict(build_ext=setup_helpers.CMakeBuild), ) gen_pyi() diff --git a/test/test_remote_io.py b/test/test_remote_io.py index 5bf3b92e4..fab4d2f8c 100644 --- a/test/test_remote_io.py +++ b/test/test_remote_io.py @@ -4,9 +4,14 @@ import unittest import warnings -import expecttest - -from _utils._common_utils_for_test import check_hash_fn, create_temp_dir +from torchdata.datapipes.iter import ( + EndOnDiskCacheHolder, + FileLoader, + HttpReader, + IterableWrapper, + OnDiskCacheHolder, + S3FileLister, +) from torchdata.datapipes.iter import EndOnDiskCacheHolder, FileOpener, HttpReader, IterableWrapper, OnDiskCacheHolder @@ -161,6 +166,15 @@ def _read_and_decode(x): self.assertTrue(os.path.exists(expected_csv_path)) self.assertEqual(expected_csv_path, csv_path) + def test_s3_lister_iterdatapipe(self): + + file_url = "s3://pt-s3plugin-test-data-west2/images/test" + s3_lister_dp = S3FileLister(IterableWrapper([file_url])) + count = 0 + for item in s3_lister_dp: + count = count + 1 + print("number of items:", count) + if __name__ == "__main__": unittest.main() diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/setup_helpers/__init__.py b/tools/setup_helpers/__init__.py new file mode 100644 index 000000000..7afa3f31c --- /dev/null +++ b/tools/setup_helpers/__init__.py @@ -0,0 +1 @@ +from .extension import * # noqa diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py new file mode 100644 index 000000000..1a9603295 --- /dev/null +++ b/tools/setup_helpers/extension.py @@ -0,0 +1,57 @@ +from distutils.version import LooseVersion +import os +import platform +from pathlib import Path +import re +import subprocess +import sys +from setuptools import Extension +from setuptools.command.build_ext import build_ext + + +__all__ = [ + 'get_ext_modules', + # 'CMakeExtension', + 'CMakeBuild', +] + +_THIS_DIR = Path(__file__).parent.resolve() +_ROOT_DIR = _THIS_DIR.parent.parent.resolve() + +def get_ext_modules(): + return [ + Extension(name='torchdata._torchdata', sources=[]) + ] + + +class CMakeBuild(build_ext): + def build_extension(self, ext): + extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) + # required for auto-detection of auxiliary "native" libs + if not extdir.endswith(os.path.sep): + extdir += os.path.sep + + cmake_args = ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir, + '-DPYTHON_EXECUTABLE=' + sys.executable, + '-DCMAKE_PREFIX_PATH=' + os.environ['CMAKE_PREFIX_PATH'], + '-DCMAKE_CXX_FLAGS=' + "-fPIC"] + + cfg = 'Debug' if self.debug else 'Release' + build_args = ['--config', cfg] + + if platform.system() == "Windows": + cmake_args += ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}'.format(cfg.upper(), extdir)] + if sys.maxsize > 2**32: + cmake_args += ['-A', 'x64'] + build_args += ['--', '/m'] + else: + cmake_args += ['-DCMAKE_BUILD_TYPE=' + cfg] + build_args += ['--', '-j2'] + + env = os.environ.copy() + env['CXXFLAGS'] = '{} -DVERSION_INFO=\\"{}\\"'.format(env.get('CXXFLAGS', ''), + self.distribution.get_version()) + if not os.path.exists(self.build_temp): + os.makedirs(self.build_temp) + subprocess.check_call(['cmake', str(_ROOT_DIR)] + cmake_args, cwd=self.build_temp, env=env) + subprocess.check_call(['cmake', '--build', '.'] + build_args, cwd=self.build_temp) diff --git a/torchdata/csrc/pybind/s3_io/pybind.cpp b/torchdata/csrc/pybind/s3_io/pybind.cpp new file mode 100644 index 000000000..13fdb4ed9 --- /dev/null +++ b/torchdata/csrc/pybind/s3_io/pybind.cpp @@ -0,0 +1,32 @@ +#include +#include + +#include +#include + +#include "s3_io.h" + +namespace py = pybind11; +// TODO: change to S3Client +using torchdata::S3Init; +PYBIND11_MODULE(_torchdata, m) { + py::class_(m, "S3Init") + // TODO: pass in timeout + .def(py::init<>()) + .def("s3_read", + [](S3Init* self, const std::string& file_url) { + std::string result; + self->s3_read(file_url, &result); + return py::bytes(result); + }) + .def("list_files", + [](S3Init* self, const std::string& file_url) { + std::vector filenames; + self->list_files(file_url, &filenames); + return filenames; + }) + .def("file_exists", + [](S3Init* self, const std::string& file_url) { + return self->file_exists(file_url); + }); +} diff --git a/torchdata/csrc/pybind/s3_io/s3_io.cpp b/torchdata/csrc/pybind/s3_io/s3_io.cpp new file mode 100644 index 000000000..15ce9ca49 --- /dev/null +++ b/torchdata/csrc/pybind/s3_io/s3_io.cpp @@ -0,0 +1,475 @@ +#include "s3_io.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace torchdata +{ + namespace + { + static const size_t s3ReadBufferSize = 120 * 1024 * 1024; // 16 MB + static const uint64_t s3MultiPartDownloadChunkSize = 50 * 1024 * 1024; // 50 MB + static const int downloadRetries = 3; + static const int64_t s3TimeoutMsec = 300000; + static const int executorPoolSize = 25; + static const int S3GetFilesMaxKeys = 100; + + Aws::Client::ClientConfiguration &setUpS3Config() + { + static Aws::Client::ClientConfiguration cfg; + Aws::String config_file; + const char *config_file_env = getenv("AWS_CONFIG_FILE"); + if (config_file_env) + { + config_file = config_file_env; + } + else + { + const char *home_env = getenv("HOME"); + if (home_env) + { + config_file = home_env; + config_file += "/.aws/config"; + } + } + Aws::Config::AWSConfigFileProfileConfigLoader loader(config_file); + loader.Load(); + + const char *use_https = getenv("S3_USE_HTTPS"); + if (use_https) + { + if (use_https[0] == '0') + { + cfg.scheme = Aws::Http::Scheme::HTTP; + } + else + { + cfg.scheme = Aws::Http::Scheme::HTTPS; + } + } + const char *verify_ssl = getenv("S3_VERIFY_SSL"); + if (verify_ssl) + { + if (verify_ssl[0] == '0') + { + cfg.verifySSL = false; + } + else + { + cfg.verifySSL = true; + } + } + + const char *region = getenv("AWS_REGION"); + if (region) + { + cfg.region = region; + } + else + { + cfg.region = "us-west-2"; + } + + const char *endpoint_url = getenv("S3_ENDPOINT_URL"); + if (endpoint_url) + { + cfg.endpointOverride = endpoint_url; + } + return cfg; + } + + void ShutdownClient(std::shared_ptr *s3_client) + { + if (s3_client != nullptr) + { + delete s3_client; + Aws::SDKOptions options; + Aws::ShutdownAPI(options); + } + } + + void ShutdownTransferManager( + std::shared_ptr *transfer_manager) + { + if (transfer_manager != nullptr) + { + delete transfer_manager; + } + } + + void ShutdownExecutor(Aws::Utils::Threading::PooledThreadExecutor *executor) + { + if (executor != nullptr) + { + delete executor; + } + } + + void parseS3Path(const std::string &fname, std::string *bucket, + std::string *object) + { + if (fname.empty()) + { + throw std::invalid_argument{"The filename cannot be an empty string."}; + } + + if (fname.size() < 5 || fname.substr(0, 5) != "s3://") + { + throw std::invalid_argument{ + "The filename must start with the S3 scheme."}; + } + + std::string path = fname.substr(5); + + if (path.empty()) + { + throw std::invalid_argument{"The filename cannot be an empty string."}; + } + + auto pos = path.find_first_of('/'); + if (pos == 0) + { + throw std::invalid_argument{ + "The filename does not contain a bucket name."}; + } + + *bucket = path.substr(0, pos); + *object = path.substr(pos + 1); + if (pos == std::string::npos) + { + *object = ""; + } + } + + class S3FS + { + public: + S3FS(const std::string &bucket, const std::string &object, + const bool multi_part_download, + std::shared_ptr transfer_manager, + std::shared_ptr s3_client) + : bucket_name_(bucket), + object_name_(object), + multi_part_download_(multi_part_download), + transfer_manager_(transfer_manager), + s3_client_(s3_client) {} + + size_t read(uint64_t offset, size_t n, char *buffer) + { + if (multi_part_download_) + { + return readS3TransferManager(offset, n, buffer); + } + else + { + return readS3Client(offset, n, buffer); + } + } + + size_t readS3Client(uint64_t offset, size_t n, char *buffer) + { + Aws::S3::Model::GetObjectRequest getObjectRequest; + + getObjectRequest.WithBucket(this->bucket_name_.c_str()) + .WithKey(this->object_name_.c_str()); + + std::string bytes = "bytes="; + bytes += std::to_string(offset) + "-" + std::to_string(offset + n - 1); + + getObjectRequest.SetRange(bytes.c_str()); + + // When you don’t want to load the entire file into memory, + // you can use IOStreamFactory in AmazonWebServiceRequest to pass a + // lambda to create a string stream. + getObjectRequest.SetResponseStreamFactory( + []() + { return Aws::New("S3IOAllocationTag"); }); + // get the object + auto getObjectOutcome = this->s3_client_->GetObject(getObjectRequest); + + if (!getObjectOutcome.IsSuccess()) + { + auto error = getObjectOutcome.GetError(); + std::cout << "ERROR: " << error.GetExceptionName() << ": " + << error.GetMessage() << std::endl; + return 0; + } + else + { + n = getObjectOutcome.GetResult().GetContentLength(); + // read data as a block: + getObjectOutcome.GetResult().GetBody().read(buffer, n); + return n; + } + } + + size_t readS3TransferManager(uint64_t offset, size_t n, char *buffer) + { + auto create_stream_fn = [&]() { // create stream lambda fn + return Aws::New( + "S3ReadStream", + Aws::New( + "S3ReadStream", reinterpret_cast(buffer), + n)); + }; // This buffer is what we used to initialize streambuf and is in memory + + std::shared_ptr downloadHandle = + this->transfer_manager_.get()->DownloadFile( + this->bucket_name_.c_str(), this->object_name_.c_str(), offset, + n, create_stream_fn); + downloadHandle->WaitUntilFinished(); + + Aws::OFStream storeFile(object_name_.c_str(), + Aws::OFStream::out | Aws::OFStream::trunc); + + if (downloadHandle->GetStatus() != + Aws::Transfer::TransferStatus::COMPLETED) + { + auto error = downloadHandle->GetLastError(); + std::cout << "ERROR: " << error.GetExceptionName() << ": " + << error.GetMessage() << std::endl; + return 0; + } + else + { + return downloadHandle->GetBytesTransferred(); + } + } + + private: + std::string bucket_name_; + std::string object_name_; + bool multi_part_download_; + std::shared_ptr s3_client_; + std::shared_ptr transfer_manager_; + }; + } // namespace + + S3Init::S3Init() + : s3_client_(nullptr, ShutdownClient), + transfer_manager_(nullptr, ShutdownTransferManager), + executor_(nullptr, ShutdownExecutor), + initialization_lock_() + { + // Load reading parameters + buffer_size_ = s3ReadBufferSize; + const char *bufferSizeStr = getenv("S3_BUFFER_SIZE"); + if (bufferSizeStr) + { + buffer_size_ = std::stoull(bufferSizeStr); + } + multi_part_download_ = true; + const char *multi_download_disable_char = + getenv("S3_DISABLE_MULTI_PART_DOWNLOAD"); + if (multi_download_disable_char) + { + std::string multi_download_disable_str(multi_download_disable_char); + if (multi_download_disable_str == "ON") + { + multi_part_download_ = false; + } + } + initializeS3Client(); + } + + S3Init::~S3Init() {} + + std::shared_ptr S3Init::initializeS3Client() + { + std::lock_guard lock(this->initialization_lock_); + if (this->s3_client_.get() == nullptr) + { + Aws::SDKOptions options; + Aws::InitAPI(options); + + // Set up the request + this->s3_client_ = + std::shared_ptr(new Aws::S3::S3Client( + setUpS3Config(), + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, + false)); + } + return this->s3_client_; + } + + std::shared_ptr + S3Init::initializeExecutor() + { + if (this->executor_.get() == nullptr) + { + this->executor_ = + Aws::MakeShared( + "executor", executorPoolSize); + } + return this->executor_; + } + + std::shared_ptr + S3Init::initializeTransferManager() + { + std::shared_ptr s3_client = initializeS3Client(); + std::lock_guard lock(this->initialization_lock_); + + if (this->transfer_manager_.get() == nullptr) + { + Aws::Transfer::TransferManagerConfiguration transfer_config( + initializeExecutor().get()); + transfer_config.s3Client = s3_client; + // This buffer is what we used to initialize streambuf and is in memory + transfer_config.bufferSize = s3MultiPartDownloadChunkSize; + transfer_config.transferBufferMaxHeapSize = + (executorPoolSize + 1) * s3MultiPartDownloadChunkSize; + this->transfer_manager_ = + Aws::Transfer::TransferManager::Create(transfer_config); + } + return this->transfer_manager_; + } + + void S3Init::s3_read(const std::string &file_url, std::string *result) + { + std::string bucket, object; + parseS3Path(file_url, &bucket, &object); + S3FS s3handler(bucket, object, multi_part_download_, + initializeTransferManager(), initializeS3Client()); + + uint64_t offset = 0; + uint64_t result_size = 0; + uint64_t file_size = this->get_file_size(bucket, object); + std::size_t part_count = (std::max)( + static_cast((file_size + buffer_size_ - 1) / buffer_size_), + static_cast(1)); + result->resize(file_size); + + for (int i = 0; i < part_count; i++) + { + + offset = result_size; + + size_t buf_len = std::min(buffer_size_, file_size - result_size); + + size_t read_len = + s3handler.read(offset, buf_len, (char *)(result->data()) + offset); + + result_size += read_len; + + if (result_size == file_size) + { + break; + } + + if (read_len != buf_len) + { + std::cout << "Result size and buffer size did not match"; + break; + } + } + } + + bool S3Init::file_exists(const std::string &file_url) + { + std::string bucket, object; + parseS3Path(file_url, &bucket, &object); + Aws::S3::Model::HeadObjectRequest headObjectRequest; + headObjectRequest.WithBucket(bucket.c_str()).WithKey(object.c_str()); + auto headObjectOutcome = + this->initializeS3Client()->HeadObject(headObjectRequest); + if (headObjectOutcome.IsSuccess()) + { + return true; + } + return false; + } + + size_t S3Init::get_file_size(const std::string &bucket, + const std::string &object) + { + Aws::S3::Model::HeadObjectRequest headObjectRequest; + headObjectRequest.WithBucket(bucket.c_str()).WithKey(object.c_str()); + auto headObjectOutcome = + this->initializeS3Client()->HeadObject(headObjectRequest); + if (headObjectOutcome.IsSuccess()) + { + return headObjectOutcome.GetResult().GetContentLength(); + } + Aws::String const &error_aws = headObjectOutcome.GetError().GetMessage(); + std::string error_str(error_aws.c_str(), error_aws.size()); + throw std::invalid_argument(error_str); + return 0; + } + + size_t S3Init::get_file_size(const std::string &file_url) + { + std::string bucket, object; + parseS3Path(file_url, &bucket, &object); + return this->get_file_size(bucket, object); + } + + void S3Init::list_files(const std::string &file_url, + std::vector *filenames) + { + std::string bucket, prefix; + parseS3Path(file_url, &bucket, &prefix); + Aws::String default_key = ""; + if (prefix.empty()) + { + default_key = "/"; + } + + Aws::S3::Model::ListObjectsRequest listObjectsRequest; + listObjectsRequest.WithBucket(bucket.c_str()) + .WithPrefix(prefix.c_str()) + .WithMaxKeys(S3GetFilesMaxKeys); + + Aws::S3::Model::ListObjectsResult listObjectsResult; + do + { + auto listObjectsOutcome = + this->initializeS3Client()->ListObjects(listObjectsRequest); + if (!listObjectsOutcome.IsSuccess()) + { + Aws::String const &error_aws = + listObjectsOutcome.GetError().GetMessage(); + std::string error_str(error_aws.c_str(), error_aws.size()); + throw std::invalid_argument(error_str); + } + + listObjectsResult = listObjectsOutcome.GetResult(); + Aws::Vector objects = listObjectsResult.GetContents(); + if (!objects.empty()) + { + for (const auto &object : objects) + { + Aws::String key = default_key + object.GetKey(); + if (key.back() == '/') + { + continue; + } + Aws::String bucket_aws(bucket.c_str(), bucket.size()); + Aws::String entry = "s3://" + bucket_aws + "/" + object.GetKey(); + filenames->push_back(entry.c_str()); + } + listObjectsRequest.SetMarker(listObjectsResult.GetContents().back().GetKey()); + } + } while (listObjectsResult.GetIsTruncated()); + } + +} // namespace torchdata diff --git a/torchdata/csrc/pybind/s3_io/s3_io.h b/torchdata/csrc/pybind/s3_io/s3_io.h new file mode 100644 index 000000000..3c556256e --- /dev/null +++ b/torchdata/csrc/pybind/s3_io/s3_io.h @@ -0,0 +1,57 @@ +#ifndef TORCHDATA_S3_IO_H +#define TORCHDATA_S3_IO_H + +#include +#include +#include +#include + +#include + +namespace torchdata +{ + // In memory stream implementation + class S3UnderlyingStream : public Aws::IOStream + { + public: + using Base = Aws::IOStream; + + // provide a customer controlled streambuf, so as to put all transferred + // data into this in memory buffer. + S3UnderlyingStream(std::streambuf *buf) : Base(buf) {} + + virtual ~S3UnderlyingStream() = default; + }; + + class S3Init + { + private: + std::shared_ptr s3_client_; + std::shared_ptr executor_; + std::shared_ptr transfer_manager_; + size_t buffer_size_; + bool multi_part_download_; + + size_t get_file_size(const std::string &bucket, const std::string &object); + + public: + S3Init(); + + ~S3Init(); + + std::mutex initialization_lock_; + + std::shared_ptr initializeS3Client(); + std::shared_ptr + initializeExecutor(); + std::shared_ptr initializeTransferManager(); + + void s3_read(const std::string &file_url, std::string *result); + size_t get_file_size(const std::string &file_url); + bool file_exists(const std::string &file_url); + void list_files(const std::string &file_url, + std::vector *filenames); + }; +} // namespace torchdata + +#endif // TORCHDATA_S3_IO_H diff --git a/torchdata/datapipes/iter/__init__.py b/torchdata/datapipes/iter/__init__.py index 02085fa45..9efce8d71 100644 --- a/torchdata/datapipes/iter/__init__.py +++ b/torchdata/datapipes/iter/__init__.py @@ -19,8 +19,8 @@ ShardingFilter, Shuffler, StreamReader, - UnBatcher, - Zipper, + IterableWrapper, + ShardingFilter, ) from torchdata.datapipes.iter.load.fsspec import ( FSSpecFileListerIterDataPipe as FSSpecFileLister, @@ -38,6 +38,10 @@ HTTPReaderIterDataPipe as HttpReader, OnlineReaderIterDataPipe as OnlineReader, ) +from torchdata.datapipes.iter.load.s3io import ( + S3FileListerIterDataPipe as S3FileLister, + S3FileLoaderIterDataPipe as S3FileLoader, +) from torchdata.datapipes.iter.transform.bucketbatcher import ( BucketBatcherIterDataPipe as BucketBatcher, InBatchShufflerIterDataPipe as InBatchShuffler, @@ -148,6 +152,8 @@ "RarArchiveLoader", "RoutedDecoder", "Rows2Columnar", + "S3FileLister", + "S3FileLoader", "SampleMultiplexer", "Sampler", "Saver", diff --git a/torchdata/datapipes/iter/load/s3io.py b/torchdata/datapipes/iter/load/s3io.py new file mode 100644 index 000000000..d30b5ea13 --- /dev/null +++ b/torchdata/datapipes/iter/load/s3io.py @@ -0,0 +1,74 @@ +from typing import Iterator, Tuple +from itertools import chain + +# import torchdata._torchdata +from torchdata import _torchdata + +from torchdata.datapipes import functional_datapipe +from torchdata.datapipes.iter import IterDataPipe +from torchdata.datapipes.utils import StreamWrapper + + +@functional_datapipe("list_file_by_s3") +class S3FileListerIterDataPipe(IterDataPipe[str]): + r""":class:`S3FileListerIterDataPipe`. + + Iterable DataPipe that lists URLs with the given prefixes. + + Args: + source_datapipe: a DataPipe that contains URLs/URL prefixes to s3 files + length: Nominal length of the datapipe + + Note: + AWS_CPP_SDK is necessary to use the S3 DataPipe(s). + """ + + def __init__(self, source_datapipe: IterDataPipe[str], length: int = -1) -> None: + # TODO: check whether AWS_CPP_SDK available + self.source_datapipe: IterDataPipe[str] = source_datapipe + self.length: int = length + + def __iter__(self) -> Iterator[str]: + # TODO: timeout + handler = _torchdata.S3Init() + for prefix in self.source_datapipe: + # if handler.file_exists(prefix): + # yield prefix + # else: + # urls = handler.list_files(prefix) + # for url in urls: + # yield url + urls = handler.list_files(prefix) + for url in urls: + yield url + + def __len__(self) -> int: + if self.length == -1: + raise TypeError("{} instance doesn't have valid length".format(type(self).__name__)) + return self.length + + +@functional_datapipe("load_file_by_s3") +class S3FileLoaderIterDataPipe(IterDataPipe[Tuple[str, StreamWrapper]]): + r""":class:`S3FileListerIterDataPipe`. + + Iterable DataPipe that loads S3 files given S3 URLs. + + Args: + source_datapipe: a DataPipe that contains URLs to s3 files + + Note: + AWS_CPP_SDK is necessary to use the S3 DataPipe(s). + """ + + def __init__(self, source_datapipe: IterDataPipe[str]) -> None: + self.source_datapipe: IterDataPipe[str] = source_datapipe + + def __iter__(self) -> Iterator[Tuple[str, StreamWrapper]]: + # TODO: timeout + handler = _torchdata.S3Init() + for url in self.source_datapipe: + yield url, handler.s3_read(url) + + def __len__(self) -> int: + return len(self.source_datapipe) From da92961679f3eeaa811b281d259bb33e43dfbfe0 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 12 Jan 2022 13:07:42 +0800 Subject: [PATCH 002/153] change from S3Init to S3Handler --- torchdata/csrc/pybind/s3_io/pybind.cpp | 16 ++++++++++------ torchdata/csrc/pybind/s3_io/s3_io.cpp | 24 ++++++++++++------------ torchdata/csrc/pybind/s3_io/s3_io.h | 6 +++--- torchdata/datapipes/iter/load/s3io.py | 4 ++-- 4 files changed, 27 insertions(+), 23 deletions(-) diff --git a/torchdata/csrc/pybind/s3_io/pybind.cpp b/torchdata/csrc/pybind/s3_io/pybind.cpp index 13fdb4ed9..fafba5f76 100644 --- a/torchdata/csrc/pybind/s3_io/pybind.cpp +++ b/torchdata/csrc/pybind/s3_io/pybind.cpp @@ -8,25 +8,29 @@ namespace py = pybind11; // TODO: change to S3Client -using torchdata::S3Init; -PYBIND11_MODULE(_torchdata, m) { - py::class_(m, "S3Init") +using torchdata::S3Handler; +PYBIND11_MODULE(_torchdata, m) +{ + py::class_(m, "S3Handler") // TODO: pass in timeout .def(py::init<>()) .def("s3_read", - [](S3Init* self, const std::string& file_url) { + [](S3Handler *self, const std::string &file_url) + { std::string result; self->s3_read(file_url, &result); return py::bytes(result); }) .def("list_files", - [](S3Init* self, const std::string& file_url) { + [](S3Handler *self, const std::string &file_url) + { std::vector filenames; self->list_files(file_url, &filenames); return filenames; }) .def("file_exists", - [](S3Init* self, const std::string& file_url) { + [](S3Handler *self, const std::string &file_url) + { return self->file_exists(file_url); }); } diff --git a/torchdata/csrc/pybind/s3_io/s3_io.cpp b/torchdata/csrc/pybind/s3_io/s3_io.cpp index 15ce9ca49..e18354632 100644 --- a/torchdata/csrc/pybind/s3_io/s3_io.cpp +++ b/torchdata/csrc/pybind/s3_io/s3_io.cpp @@ -264,7 +264,7 @@ namespace torchdata }; } // namespace - S3Init::S3Init() + S3Handler::S3Handler() : s3_client_(nullptr, ShutdownClient), transfer_manager_(nullptr, ShutdownTransferManager), executor_(nullptr, ShutdownExecutor), @@ -291,9 +291,9 @@ namespace torchdata initializeS3Client(); } - S3Init::~S3Init() {} + S3Handler::~S3Handler() {} - std::shared_ptr S3Init::initializeS3Client() + std::shared_ptr S3Handler::initializeS3Client() { std::lock_guard lock(this->initialization_lock_); if (this->s3_client_.get() == nullptr) @@ -312,7 +312,7 @@ namespace torchdata } std::shared_ptr - S3Init::initializeExecutor() + S3Handler::initializeExecutor() { if (this->executor_.get() == nullptr) { @@ -324,7 +324,7 @@ namespace torchdata } std::shared_ptr - S3Init::initializeTransferManager() + S3Handler::initializeTransferManager() { std::shared_ptr s3_client = initializeS3Client(); std::lock_guard lock(this->initialization_lock_); @@ -344,7 +344,7 @@ namespace torchdata return this->transfer_manager_; } - void S3Init::s3_read(const std::string &file_url, std::string *result) + void S3Handler::s3_read(const std::string &file_url, std::string *result) { std::string bucket, object; parseS3Path(file_url, &bucket, &object); @@ -384,7 +384,7 @@ namespace torchdata } } - bool S3Init::file_exists(const std::string &file_url) + bool S3Handler::file_exists(const std::string &file_url) { std::string bucket, object; parseS3Path(file_url, &bucket, &object); @@ -399,8 +399,8 @@ namespace torchdata return false; } - size_t S3Init::get_file_size(const std::string &bucket, - const std::string &object) + size_t S3Handler::get_file_size(const std::string &bucket, + const std::string &object) { Aws::S3::Model::HeadObjectRequest headObjectRequest; headObjectRequest.WithBucket(bucket.c_str()).WithKey(object.c_str()); @@ -416,15 +416,15 @@ namespace torchdata return 0; } - size_t S3Init::get_file_size(const std::string &file_url) + size_t S3Handler::get_file_size(const std::string &file_url) { std::string bucket, object; parseS3Path(file_url, &bucket, &object); return this->get_file_size(bucket, object); } - void S3Init::list_files(const std::string &file_url, - std::vector *filenames) + void S3Handler::list_files(const std::string &file_url, + std::vector *filenames) { std::string bucket, prefix; parseS3Path(file_url, &bucket, &prefix); diff --git a/torchdata/csrc/pybind/s3_io/s3_io.h b/torchdata/csrc/pybind/s3_io/s3_io.h index 3c556256e..d262dea92 100644 --- a/torchdata/csrc/pybind/s3_io/s3_io.h +++ b/torchdata/csrc/pybind/s3_io/s3_io.h @@ -23,7 +23,7 @@ namespace torchdata virtual ~S3UnderlyingStream() = default; }; - class S3Init + class S3Handler { private: std::shared_ptr s3_client_; @@ -35,9 +35,9 @@ namespace torchdata size_t get_file_size(const std::string &bucket, const std::string &object); public: - S3Init(); + S3Handler(); - ~S3Init(); + ~S3Handler(); std::mutex initialization_lock_; diff --git a/torchdata/datapipes/iter/load/s3io.py b/torchdata/datapipes/iter/load/s3io.py index d30b5ea13..f83fb3869 100644 --- a/torchdata/datapipes/iter/load/s3io.py +++ b/torchdata/datapipes/iter/load/s3io.py @@ -30,7 +30,7 @@ def __init__(self, source_datapipe: IterDataPipe[str], length: int = -1) -> None def __iter__(self) -> Iterator[str]: # TODO: timeout - handler = _torchdata.S3Init() + handler = _torchdata.S3Handler() for prefix in self.source_datapipe: # if handler.file_exists(prefix): # yield prefix @@ -66,7 +66,7 @@ def __init__(self, source_datapipe: IterDataPipe[str]) -> None: def __iter__(self) -> Iterator[Tuple[str, StreamWrapper]]: # TODO: timeout - handler = _torchdata.S3Init() + handler = _torchdata.S3Handler() for url in self.source_datapipe: yield url, handler.s3_read(url) From 837575906734b185add8c799e08224de59711f3c Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 12 Jan 2022 16:32:31 +0800 Subject: [PATCH 003/153] include pybind11 at compilation automatically --- tools/setup_helpers/extension.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index 1a9603295..c9d632ac9 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -2,12 +2,11 @@ import os import platform from pathlib import Path -import re import subprocess import sys from setuptools import Extension from setuptools.command.build_ext import build_ext - +import torch __all__ = [ 'get_ext_modules', @@ -18,6 +17,7 @@ _THIS_DIR = Path(__file__).parent.resolve() _ROOT_DIR = _THIS_DIR.parent.parent.resolve() + def get_ext_modules(): return [ Extension(name='torchdata._torchdata', sources=[]) @@ -31,10 +31,12 @@ def build_extension(self, ext): if not extdir.endswith(os.path.sep): extdir += os.path.sep - cmake_args = ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir, - '-DPYTHON_EXECUTABLE=' + sys.executable, - '-DCMAKE_PREFIX_PATH=' + os.environ['CMAKE_PREFIX_PATH'], - '-DCMAKE_CXX_FLAGS=' + "-fPIC"] + cmake_args = [ + '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir, + '-DPYTHON_EXECUTABLE=' + sys.executable, + f"-DCMAKE_PREFIX_PATH={torch.utils.cmake_prefix_path}", + '-DCMAKE_CXX_FLAGS=' + "-fPIC", + ] cfg = 'Debug' if self.debug else 'Release' build_args = ['--config', cfg] From fa258e418566e3d461f50e13b66daf7d06b73993 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 12 Jan 2022 17:20:24 +0800 Subject: [PATCH 004/153] new torchdata._torchdata to avoid circular import --- CMakeLists.txt | 69 +++++++++++++++------------ tools/setup_helpers/extension.py | 22 +++++++++ torchdata/datapipes/iter/load/s3io.py | 6 +-- 3 files changed, 64 insertions(+), 33 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c3f0801b6..f0bd02a61 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,43 +1,52 @@ -cmake_minimum_required(VERSION 3.13) -project(_torchdata) - -set(CMAKE_CXX_STANDARD 17) -SET(TORCH_MIN_VERSION "1.5.1") - -find_package(Python3 COMPONENTS Interpreter Development) +cmake_minimum_required(VERSION 3.18 FATAL_ERROR) -find_package(AWSSDK REQUIRED COMPONENTS s3 transfer) +# Most of the configurations are taken from PyTorch +# https://github.com/pytorch/pytorch/blob/0c9fb4aff0d60eaadb04e4d5d099fb1e1d5701a9/CMakeLists.txt -set(CMAKE_POSITION_INDEPENDENT_CODE ON) -set(INCLUDE_DIRS "torchdata/csrc/pybind/s3_io") +# Use compiler ID "AppleClang" instead of "Clang" for XCode. +# Not setting this sometimes makes XCode C compiler gets detected as "Clang", +# even when the C++ one is detected as "AppleClang". +cmake_policy(SET CMP0010 NEW) +cmake_policy(SET CMP0025 NEW) -set(SOURCES "${INCLUDE_DIRS}/s3_io.cpp" ) +# Suppress warning flags in default MSVC configuration. It's not +# mandatory that we do this (and we don't if cmake is old), but it's +# nice when it's possible, and it's possible on our Windows configs. +if(NOT CMAKE_VERSION VERSION_LESS 3.15.0) + cmake_policy(SET CMP0092 NEW) +endif() -include_directories(${INCLUDE_DIRS}) -find_package(pybind11 REQUIRED) -pybind11_add_module(_torchdata ${SOURCES} "${INCLUDE_DIRS}/pybind.cpp") - -Message(STATUS "All linked libs: ${AWSSDK_LINK_LIBRARIES}") +project(_torchdata) -target_link_libraries(_torchdata PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS}) +# check and set CMAKE_CXX_STANDARD +string(FIND "${CMAKE_CXX_FLAGS}" "-std=c++" env_cxx_standard) +if(env_cxx_standard GREATER -1) + message( + WARNING "C++ standard version definition detected in environment variable." + "PyTorch requires -std=c++14. Please remove -std=c++ settings in your environment.") +endif() +set(CMAKE_CXX_STANDARD 14) +set(CMAKE_C_STANDARD 11) -# project(torchdata) +# Options +option(BUILD_S3 "Build s3 io functionality" ON) -# find_package(Python3 COMPONENTS Interpreter Development) -# find_package(AWSSDK RUNTIME COMPONENTS s3 transfer) -# find_package(pybind11 REQUIRED) +find_package(Torch REQUIRED) -# set(INCLUDE_DIRS "torchdata/csrc/s3_io") -# set(SOURCES "${INCLUDE_DIRS}/s3_io.cpp") +if(BUILD_S3) + find_package(Python3 COMPONENTS Interpreter Development) + find_package(AWSSDK REQUIRED COMPONENTS s3 transfer) + find_package(pybind11 REQUIRED) + set(CMAKE_POSITION_INDEPENDENT_CODE ON) + set(INCLUDE_DIRS "torchdata/csrc/pybind/s3_io") -# include_directories(${INCLUDE_DIRS}) -# # add_subdirectory(torchdata/csrc) + set(SOURCES "${INCLUDE_DIRS}/s3_io.cpp" ) -# pybind11_add_module(_torchdata ${SOURCES} "${INCLUDE_DIRS}/pybind.cpp") + include_directories(${INCLUDE_DIRS}) + pybind11_add_module(_torchdata ${SOURCES} "${INCLUDE_DIRS}/pybind.cpp") -# target_link_libraries(_torchdata PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS}) + Message(STATUS "All linked libs: ${AWSSDK_LINK_LIBRARIES}") -# # EXAMPLE_VERSION_INFO is defined by setup.py and passed into the C++ code as a -# # define (VERSION_INFO) here. -# # target_compile_definitions(_torchdata PRIVATE ) \ No newline at end of file + target_link_libraries(_torchdata PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS}) +endif() diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index c9d632ac9..ed86661d1 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -18,6 +18,25 @@ _ROOT_DIR = _THIS_DIR.parent.parent.resolve() +def _get_build(var, default=False): + if var not in os.environ: + return default + + val = os.environ.get(var, '0') + trues = ['1', 'true', 'TRUE', 'on', 'ON', 'yes', 'YES'] + falses = ['0', 'false', 'FALSE', 'off', 'OFF', 'no', 'NO'] + if val in trues: + return True + if val not in falses: + print( + f'WARNING: Unexpected environment variable value `{var}={val}`. ' + f'Expected one of {trues + falses}') + return False + + +_BUILD_S3 = _get_build("BUILD_S3", False) + + def get_ext_modules(): return [ Extension(name='torchdata._torchdata', sources=[]) @@ -31,11 +50,14 @@ def build_extension(self, ext): if not extdir.endswith(os.path.sep): extdir += os.path.sep + print("BUILD_S3:", 'ON' if _BUILD_S3 else 'OFF') + cmake_args = [ '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir, '-DPYTHON_EXECUTABLE=' + sys.executable, f"-DCMAKE_PREFIX_PATH={torch.utils.cmake_prefix_path}", '-DCMAKE_CXX_FLAGS=' + "-fPIC", + f"-DBUILD_S3:BOOL={'ON' if _BUILD_S3 else 'OFF'}", ] cfg = 'Debug' if self.debug else 'Release' diff --git a/torchdata/datapipes/iter/load/s3io.py b/torchdata/datapipes/iter/load/s3io.py index f83fb3869..66dac6532 100644 --- a/torchdata/datapipes/iter/load/s3io.py +++ b/torchdata/datapipes/iter/load/s3io.py @@ -2,7 +2,7 @@ from itertools import chain # import torchdata._torchdata -from torchdata import _torchdata +import torchdata from torchdata.datapipes import functional_datapipe from torchdata.datapipes.iter import IterDataPipe @@ -30,7 +30,7 @@ def __init__(self, source_datapipe: IterDataPipe[str], length: int = -1) -> None def __iter__(self) -> Iterator[str]: # TODO: timeout - handler = _torchdata.S3Handler() + handler = torchdata._torchdata.S3Handler() for prefix in self.source_datapipe: # if handler.file_exists(prefix): # yield prefix @@ -66,7 +66,7 @@ def __init__(self, source_datapipe: IterDataPipe[str]) -> None: def __iter__(self) -> Iterator[Tuple[str, StreamWrapper]]: # TODO: timeout - handler = _torchdata.S3Handler() + handler = torchdata._torchdata.S3Handler() for url in self.source_datapipe: yield url, handler.s3_read(url) From 7016deb8a20a2cf6521feff08387d372fc30bbae Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 12 Jan 2022 18:43:31 +0800 Subject: [PATCH 005/153] new build flag, new extension import, new clean command --- CMakeLists.txt | 6 ++- setup.py | 26 +++++++++++- torchdata/__init__.py | 1 + torchdata/_extension.py | 39 +++++++++++++++++ torchdata/_internal/__init__.py | 7 +++ torchdata/_internal/module_utils.py | 66 +++++++++++++++++++++++++++++ 6 files changed, 142 insertions(+), 3 deletions(-) create mode 100644 torchdata/_extension.py create mode 100644 torchdata/_internal/__init__.py create mode 100644 torchdata/_internal/module_utils.py diff --git a/CMakeLists.txt b/CMakeLists.txt index f0bd02a61..4eddff00f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,11 +30,13 @@ set(CMAKE_CXX_STANDARD 14) set(CMAKE_C_STANDARD 11) # Options -option(BUILD_S3 "Build s3 io functionality" ON) +option(BUILD_S3 "Build s3 io functionality" OFF) find_package(Torch REQUIRED) +message(STATUS "BUILD_S3: ${BUILD_S3}") if(BUILD_S3) + message(STATUS "BUILD_S3: ${BUILD_S3}") find_package(Python3 COMPONENTS Interpreter Development) find_package(AWSSDK REQUIRED COMPONENTS s3 transfer) find_package(pybind11 REQUIRED) @@ -46,7 +48,7 @@ if(BUILD_S3) include_directories(${INCLUDE_DIRS}) pybind11_add_module(_torchdata ${SOURCES} "${INCLUDE_DIRS}/pybind.cpp") - Message(STATUS "All linked libs: ${AWSSDK_LINK_LIBRARIES}") + message(STATUS "All linked libs: ${AWSSDK_LINK_LIBRARIES}") target_link_libraries(_torchdata PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS}) endif() diff --git a/setup.py b/setup.py index d3b1f092c..a43916cef 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # Copyright (c) Facebook, Inc. and its affiliates. +import distutils.command.clean import os +import shutil import subprocess from pathlib import Path @@ -54,6 +56,25 @@ def _export_version(version, sha): ] +class clean(distutils.command.clean.clean): + def run(self): + # Run default behavior first + distutils.command.clean.clean.run(self) + + # Remove torchaudio extension + for path in (ROOT_DIR / "torchaudio").glob("**/*.so"): + print(f"removing '{path}'") + path.unlink() + # Remove build directory + build_dirs = [ + ROOT_DIR / "build", + ] + for path in build_dirs: + if path.exists(): + print(f"removing '{path}' (and everything under it)") + shutil.rmtree(str(path), ignore_errors=True) + + if __name__ == "__main__": VERSION, SHA = _get_version() _export_version(VERSION, SHA) @@ -91,6 +112,9 @@ def _export_version(version, sha): "scipy": ["scipy"], }, ext_modules=setup_helpers.get_ext_modules(), - cmdclass=dict(build_ext=setup_helpers.CMakeBuild), + cmdclass={ + "build_ext": setup_helpers.CMakeBuild, + "clean": clean, + }, ) gen_pyi() diff --git a/torchdata/__init__.py b/torchdata/__init__.py index 18a0daa88..8b99d848f 100644 --- a/torchdata/__init__.py +++ b/torchdata/__init__.py @@ -1,4 +1,5 @@ # Copyright (c) Facebook, Inc. and its affiliates. +from torchdata import _extension # noqa: F401 from . import datapipes try: diff --git a/torchdata/_extension.py b/torchdata/_extension.py new file mode 100644 index 000000000..daac5f59d --- /dev/null +++ b/torchdata/_extension.py @@ -0,0 +1,39 @@ +import os +import warnings +from pathlib import Path + +import torch +from torchdata._internal import module_utils as _mod_utils # noqa: F401 + +_LIB_DIR = Path(__file__).parent / "lib" + + +def _get_lib_path(lib: str): + suffix = "pyd" if os.name == "nt" else "so" + path = _LIB_DIR / f"{lib}.{suffix}" + return path + + +def _load_lib(lib: str): + path = _get_lib_path(lib) + # In case `torchdata` is deployed with `pex` format, this file does not exist. + # In this case, we expect that `libtorchdata` is available somewhere + # in the search path of dynamic loading mechanism, and importing `_torchdata`, + # which depends on `libtorchdata` and dynamic loader will handle it for us. + if path.exists(): + torch.ops.load_library(path) + torch.classes.load_library(path) + + +def _init_extension(): + if not _mod_utils.is_module_available("torchdata._torchdata"): + warnings.warn("torchdata C++ extension is not available.") + return + + _load_lib("libtorchdata") + # This import is for initializing the methods registered via PyBind11 + # This has to happen after the base library is loaded + from torchdata import _torchdata # noqa + + +_init_extension() diff --git a/torchdata/_internal/__init__.py b/torchdata/_internal/__init__.py new file mode 100644 index 000000000..654425399 --- /dev/null +++ b/torchdata/_internal/__init__.py @@ -0,0 +1,7 @@ +from torch.hub import load_state_dict_from_url, download_url_to_file + + +__all__ = [ + "load_state_dict_from_url", + "download_url_to_file", +] diff --git a/torchdata/_internal/module_utils.py b/torchdata/_internal/module_utils.py new file mode 100644 index 000000000..fff7e96be --- /dev/null +++ b/torchdata/_internal/module_utils.py @@ -0,0 +1,66 @@ +import importlib.util +import warnings +from functools import wraps +from typing import Optional + +import torch + + +def is_module_available(*modules: str) -> bool: + r"""Returns if a top-level module with :attr:`name` exists *without** + importing it. This is generally safer than try-catch block around a + `import X`. It avoids third party libraries breaking assumptions of some of + our tests, e.g., setting multiprocessing start method when imported + (see librosa/#747, torchvision/#544). + """ + return all(importlib.util.find_spec(m) is not None for m in modules) + + +def requires_module(*modules: str): + """Decorate function to give error message if invoked without required optional modules. + + This decorator is to give better error message to users rather + than raising ``NameError: name 'module' is not defined`` at random places. + """ + missing = [m for m in modules if not is_module_available(m)] + + if not missing: + # fall through. If all the modules are available, no need to decorate + def decorator(func): + return func + + else: + req = f"module: {missing[0]}" if len(missing) == 1 else f"modules: {missing}" + + def decorator(func): + @wraps(func) + def wrapped(*args, **kwargs): + raise RuntimeError(f"{func.__module__}.{func.__name__} requires {req}") + + return wrapped + + return decorator + + +def deprecated(direction: str, version: Optional[str] = None): + """Decorator to add deprecation message + + Args: + direction (str): Migration steps to be given to users. + version (str or int): The version when the object will be removed + """ + + def decorator(func): + @wraps(func) + def wrapped(*args, **kwargs): + message = ( + f"{func.__module__}.{func.__name__} has been deprecated " + f'and will be removed from {"future" if version is None else version} release. ' + f"{direction}" + ) + warnings.warn(message, stacklevel=2) + return func(*args, **kwargs) + + return wrapped + + return decorator From 2be7b873f2092ee72a585c589da76ec2a8519b60 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 12 Jan 2022 19:08:08 +0800 Subject: [PATCH 006/153] structured CMakeLists following torchaudio style --- CMakeLists.txt | 43 ++++++++++++---------- tools/setup_helpers/extension.py | 62 +++++++++++++++++++++++--------- torchdata/csrc/CMakeLists.txt | 15 ++++++++ 3 files changed, 85 insertions(+), 35 deletions(-) create mode 100644 torchdata/csrc/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 4eddff00f..dec4f0fc7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,7 +16,7 @@ if(NOT CMAKE_VERSION VERSION_LESS 3.15.0) cmake_policy(SET CMP0092 NEW) endif() -project(_torchdata) +project(torchdata) # check and set CMAKE_CXX_STANDARD string(FIND "${CMAKE_CXX_FLAGS}" "-std=c++" env_cxx_standard) @@ -29,26 +29,33 @@ endif() set(CMAKE_CXX_STANDARD 14) set(CMAKE_C_STANDARD 11) -# Options -option(BUILD_S3 "Build s3 io functionality" OFF) - -find_package(Torch REQUIRED) -message(STATUS "BUILD_S3: ${BUILD_S3}") +# https://developercommunity.visualstudio.com/t/VS-16100-isnt-compatible-with-CUDA-11/1433342 +if(MSVC) + if(USE_CUDA) + set(CMAKE_CXX_STANDARD 17) + endif() +endif() -if(BUILD_S3) - message(STATUS "BUILD_S3: ${BUILD_S3}") - find_package(Python3 COMPONENTS Interpreter Development) - find_package(AWSSDK REQUIRED COMPONENTS s3 transfer) - find_package(pybind11 REQUIRED) - set(CMAKE_POSITION_INDEPENDENT_CODE ON) - set(INCLUDE_DIRS "torchdata/csrc/pybind/s3_io") - set(SOURCES "${INCLUDE_DIRS}/s3_io.cpp" ) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +set(CMAKE_POSITION_INDEPENDENT_CODE ON) - include_directories(${INCLUDE_DIRS}) - pybind11_add_module(_torchdata ${SOURCES} "${INCLUDE_DIRS}/pybind.cpp") +# Apple specific +if(APPLE) + # Get clang version on macOS + execute_process( COMMAND ${CMAKE_CXX_COMPILER} --version OUTPUT_VARIABLE clang_full_version_string ) + string(REGEX REPLACE "Apple LLVM version ([0-9]+\\.[0-9]+).*" "\\1" CLANG_VERSION_STRING ${clang_full_version_string}) + message( STATUS "CLANG_VERSION_STRING: " ${CLANG_VERSION_STRING} ) - message(STATUS "All linked libs: ${AWSSDK_LINK_LIBRARIES}") + # RPATH stuff + set(CMAKE_MACOSX_RPATH ON) - target_link_libraries(_torchdata PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS}) + set(CMAKE_SHARED_LIBRARY_SUFFIX ".so") endif() + +# Options +option(BUILD_S3 "Build s3 io functionality" OFF) + +find_package(Torch REQUIRED) + +add_subdirectory(torchdata/csrc) diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index ed86661d1..74ae2fff0 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -1,16 +1,15 @@ -from distutils.version import LooseVersion import os import platform -from pathlib import Path import subprocess import sys +from pathlib import Path + +import torch from setuptools import Extension from setuptools.command.build_ext import build_ext -import torch __all__ = [ 'get_ext_modules', - # 'CMakeExtension', 'CMakeBuild', ] @@ -43,39 +42,68 @@ def get_ext_modules(): ] +# Based off of pybiind cmake_example +# https://github.com/pybind/cmake_example/blob/2440893c60ed2578fb127dc16e6b348fa0be92c1/setup.py +# and torchaudio CMakeBuild() +# https://github.com/pytorch/audio/blob/ece03edc3fc28a1ce2c28ef438d2898ed0a78d3f/tools/setup_helpers/extension.py#L65 class CMakeBuild(build_ext): def build_extension(self, ext): + # Because the following `cmake` command will build all of `ext_modules`` at the same time, + # we would like to prevent multiple calls to `cmake`. + # Therefore, we call `cmake` only for `torchdata._torchdata`, + # in case `ext_modules` contains more than one module. + if ext.name != 'torchdata._torchdata': + return + extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) + # required for auto-detection of auxiliary "native" libs if not extdir.endswith(os.path.sep): extdir += os.path.sep - print("BUILD_S3:", 'ON' if _BUILD_S3 else 'OFF') + debug = int(os.environ.get("DEBUG", 0)) if self.debug is None else self.debug + cfg = "Debug" if debug else "Release" cmake_args = [ - '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir, - '-DPYTHON_EXECUTABLE=' + sys.executable, + f"-DCMAKE_BUILD_TYPE={cfg}", # not used on MSVC, but no harm f"-DCMAKE_PREFIX_PATH={torch.utils.cmake_prefix_path}", - '-DCMAKE_CXX_FLAGS=' + "-fPIC", + f"-DCMAKE_INSTALL_PREFIX={extdir}", + f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}", + "-DCMAKE_CXX_FLAGS=-fPIC", f"-DBUILD_S3:BOOL={'ON' if _BUILD_S3 else 'OFF'}", ] - cfg = 'Debug' if self.debug else 'Release' - build_args = ['--config', cfg] + build_args = [ + '--config', cfg + ] - if platform.system() == "Windows": - cmake_args += ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}'.format(cfg.upper(), extdir)] + if 'CMAKE_GENERATOR' not in os.environ or platform.system() == 'Windows': + cmake_args += ["-GNinja"] + if platform.system() == 'Windows': + import sys + python_version = sys.version_info + cmake_args += [ + "-DCMAKE_C_COMPILER=cl", + "-DCMAKE_CXX_COMPILER=cl", + f"-DPYTHON_VERSION={python_version.major}.{python_version.minor}", + '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}'.format(cfg.upper(), extdir), + ] if sys.maxsize > 2**32: cmake_args += ['-A', 'x64'] build_args += ['--', '/m'] else: - cmake_args += ['-DCMAKE_BUILD_TYPE=' + cfg] build_args += ['--', '-j2'] - env = os.environ.copy() - env['CXXFLAGS'] = '{} -DVERSION_INFO=\\"{}\\"'.format(env.get('CXXFLAGS', ''), - self.distribution.get_version()) + # Set CMAKE_BUILD_PARALLEL_LEVEL to control the parallel build level + # across all generators. + if "CMAKE_BUILD_PARALLEL_LEVEL" not in os.environ: + # self.parallel is a Python 3 only way to set parallel jobs by hand + # using -j in the build_ext call, not supported by pip or PyPA-build. + if hasattr(self, "parallel") and self.parallel: + # CMake 3.12+ only. + build_args += ["-j{}".format(self.parallel)] + if not os.path.exists(self.build_temp): os.makedirs(self.build_temp) - subprocess.check_call(['cmake', str(_ROOT_DIR)] + cmake_args, cwd=self.build_temp, env=env) + subprocess.check_call(['cmake', str(_ROOT_DIR)] + cmake_args, cwd=self.build_temp) subprocess.check_call(['cmake', '--build', '.'] + build_args, cwd=self.build_temp) diff --git a/torchdata/csrc/CMakeLists.txt b/torchdata/csrc/CMakeLists.txt new file mode 100644 index 000000000..ced337721 --- /dev/null +++ b/torchdata/csrc/CMakeLists.txt @@ -0,0 +1,15 @@ +if(BUILD_S3) + message(STATUS "Building S3 IO functionality") + find_package(Python3 COMPONENTS Interpreter Development) + find_package(AWSSDK REQUIRED COMPONENTS s3 transfer) + find_package(pybind11 REQUIRED) + set(CMAKE_POSITION_INDEPENDENT_CODE ON) + + set(SOURCES "pybind/s3_io/s3_io.cpp" ) + + pybind11_add_module(_torchdata ${SOURCES} "pybind/s3_io/pybind.cpp") + + message(STATUS "All linked libs: ${AWSSDK_LINK_LIBRARIES}") + + target_link_libraries(_torchdata PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS}) +endif() From d7e0b386d2b0571137a46130264db32783294e41 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 12 Jan 2022 19:21:55 +0800 Subject: [PATCH 007/153] clean up cpp naming style --- torchdata/csrc/pybind/s3_io/pybind.cpp | 9 ++---- torchdata/csrc/pybind/s3_io/s3_io.cpp | 45 +++++++++----------------- torchdata/csrc/pybind/s3_io/s3_io.h | 23 ++++++------- 3 files changed, 29 insertions(+), 48 deletions(-) diff --git a/torchdata/csrc/pybind/s3_io/pybind.cpp b/torchdata/csrc/pybind/s3_io/pybind.cpp index fafba5f76..1ee733ffb 100644 --- a/torchdata/csrc/pybind/s3_io/pybind.cpp +++ b/torchdata/csrc/pybind/s3_io/pybind.cpp @@ -18,19 +18,14 @@ PYBIND11_MODULE(_torchdata, m) [](S3Handler *self, const std::string &file_url) { std::string result; - self->s3_read(file_url, &result); + self->S3Read(file_url, &result); return py::bytes(result); }) .def("list_files", [](S3Handler *self, const std::string &file_url) { std::vector filenames; - self->list_files(file_url, &filenames); + self->ListFiles(file_url, &filenames); return filenames; - }) - .def("file_exists", - [](S3Handler *self, const std::string &file_url) - { - return self->file_exists(file_url); }); } diff --git a/torchdata/csrc/pybind/s3_io/s3_io.cpp b/torchdata/csrc/pybind/s3_io/s3_io.cpp index e18354632..b34649d38 100644 --- a/torchdata/csrc/pybind/s3_io/s3_io.cpp +++ b/torchdata/csrc/pybind/s3_io/s3_io.cpp @@ -288,12 +288,12 @@ namespace torchdata multi_part_download_ = false; } } - initializeS3Client(); + InitializeS3Client(); } S3Handler::~S3Handler() {} - std::shared_ptr S3Handler::initializeS3Client() + std::shared_ptr S3Handler::InitializeS3Client() { std::lock_guard lock(this->initialization_lock_); if (this->s3_client_.get() == nullptr) @@ -312,7 +312,7 @@ namespace torchdata } std::shared_ptr - S3Handler::initializeExecutor() + S3Handler::InitializeExecutor() { if (this->executor_.get() == nullptr) { @@ -324,15 +324,15 @@ namespace torchdata } std::shared_ptr - S3Handler::initializeTransferManager() + S3Handler::InitializeTransferManager() { - std::shared_ptr s3_client = initializeS3Client(); + std::shared_ptr s3_client = InitializeS3Client(); std::lock_guard lock(this->initialization_lock_); if (this->transfer_manager_.get() == nullptr) { Aws::Transfer::TransferManagerConfiguration transfer_config( - initializeExecutor().get()); + InitializeExecutor().get()); transfer_config.s3Client = s3_client; // This buffer is what we used to initialize streambuf and is in memory transfer_config.bufferSize = s3MultiPartDownloadChunkSize; @@ -344,16 +344,16 @@ namespace torchdata return this->transfer_manager_; } - void S3Handler::s3_read(const std::string &file_url, std::string *result) + void S3Handler::S3Read(const std::string &file_url, std::string *result) { std::string bucket, object; parseS3Path(file_url, &bucket, &object); S3FS s3handler(bucket, object, multi_part_download_, - initializeTransferManager(), initializeS3Client()); + InitializeTransferManager(), InitializeS3Client()); uint64_t offset = 0; uint64_t result_size = 0; - uint64_t file_size = this->get_file_size(bucket, object); + uint64_t file_size = this->GetFileSize(bucket, object); std::size_t part_count = (std::max)( static_cast((file_size + buffer_size_ - 1) / buffer_size_), static_cast(1)); @@ -384,28 +384,13 @@ namespace torchdata } } - bool S3Handler::file_exists(const std::string &file_url) - { - std::string bucket, object; - parseS3Path(file_url, &bucket, &object); - Aws::S3::Model::HeadObjectRequest headObjectRequest; - headObjectRequest.WithBucket(bucket.c_str()).WithKey(object.c_str()); - auto headObjectOutcome = - this->initializeS3Client()->HeadObject(headObjectRequest); - if (headObjectOutcome.IsSuccess()) - { - return true; - } - return false; - } - - size_t S3Handler::get_file_size(const std::string &bucket, + size_t S3Handler::GetFileSize(const std::string &bucket, const std::string &object) { Aws::S3::Model::HeadObjectRequest headObjectRequest; headObjectRequest.WithBucket(bucket.c_str()).WithKey(object.c_str()); auto headObjectOutcome = - this->initializeS3Client()->HeadObject(headObjectRequest); + this->InitializeS3Client()->HeadObject(headObjectRequest); if (headObjectOutcome.IsSuccess()) { return headObjectOutcome.GetResult().GetContentLength(); @@ -416,14 +401,14 @@ namespace torchdata return 0; } - size_t S3Handler::get_file_size(const std::string &file_url) + size_t S3Handler::GetFileSize(const std::string &file_url) { std::string bucket, object; parseS3Path(file_url, &bucket, &object); - return this->get_file_size(bucket, object); + return this->GetFileSize(bucket, object); } - void S3Handler::list_files(const std::string &file_url, + void S3Handler::ListFiles(const std::string &file_url, std::vector *filenames) { std::string bucket, prefix; @@ -443,7 +428,7 @@ namespace torchdata do { auto listObjectsOutcome = - this->initializeS3Client()->ListObjects(listObjectsRequest); + this->InitializeS3Client()->ListObjects(listObjectsRequest); if (!listObjectsOutcome.IsSuccess()) { Aws::String const &error_aws = diff --git a/torchdata/csrc/pybind/s3_io/s3_io.h b/torchdata/csrc/pybind/s3_io/s3_io.h index d262dea92..fb221682a 100644 --- a/torchdata/csrc/pybind/s3_io/s3_io.h +++ b/torchdata/csrc/pybind/s3_io/s3_io.h @@ -29,27 +29,28 @@ namespace torchdata std::shared_ptr s3_client_; std::shared_ptr executor_; std::shared_ptr transfer_manager_; + std::mutex initialization_lock_; size_t buffer_size_; bool multi_part_download_; - size_t get_file_size(const std::string &bucket, const std::string &object); + std::shared_ptr InitializeS3Client(); + std::shared_ptr InitializeTransferManager(); + std::shared_ptr + InitializeExecutor(); + size_t GetFileSize(const std::string &bucket, const std::string &object); + size_t GetFileSize(const std::string &file_url); public: S3Handler(); - ~S3Handler(); - std::mutex initialization_lock_; - - std::shared_ptr initializeS3Client(); + std::shared_ptr GetS3Client() { return this->s3_client_; }; + std::shared_ptr GetTransferManager() { return this->transfer_manager_; }; std::shared_ptr - initializeExecutor(); - std::shared_ptr initializeTransferManager(); + GetExecutor() { return this->executor_; }; - void s3_read(const std::string &file_url, std::string *result); - size_t get_file_size(const std::string &file_url); - bool file_exists(const std::string &file_url); - void list_files(const std::string &file_url, + void S3Read(const std::string &file_url, std::string *result); + void ListFiles(const std::string &file_url, std::vector *filenames); }; } // namespace torchdata From f1354a2c3b8c8ef98a69191a0973b61216c26dde Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 12 Jan 2022 19:42:38 +0800 Subject: [PATCH 008/153] separate get and initilize cpp functions --- torchdata/csrc/pybind/s3_io/s3_io.cpp | 83 +++++++++++++++++---------- torchdata/csrc/pybind/s3_io/s3_io.h | 10 ++-- 2 files changed, 57 insertions(+), 36 deletions(-) diff --git a/torchdata/csrc/pybind/s3_io/s3_io.cpp b/torchdata/csrc/pybind/s3_io/s3_io.cpp index b34649d38..e0f4b722a 100644 --- a/torchdata/csrc/pybind/s3_io/s3_io.cpp +++ b/torchdata/csrc/pybind/s3_io/s3_io.cpp @@ -296,50 +296,71 @@ namespace torchdata std::shared_ptr S3Handler::InitializeS3Client() { std::lock_guard lock(this->initialization_lock_); + Aws::SDKOptions options; + Aws::InitAPI(options); + + // Set up the request + this->s3_client_ = + std::shared_ptr(new Aws::S3::S3Client( + setUpS3Config(), + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, + false)); + return this->s3_client_; + } + + std::shared_ptr + S3Handler::InitializeExecutor() + { + std::lock_guard lock(this->initialization_lock_); + this->executor_ = + Aws::MakeShared( + "executor", executorPoolSize); + return this->executor_; + } + + std::shared_ptr + S3Handler::InitializeTransferManager() + { + std::shared_ptr s3_client = GetS3Client(); + std::lock_guard lock(this->initialization_lock_); + + Aws::Transfer::TransferManagerConfiguration transfer_config( + InitializeExecutor().get()); + transfer_config.s3Client = s3_client; + // This buffer is what we used to initialize streambuf and is in memory + transfer_config.bufferSize = s3MultiPartDownloadChunkSize; + transfer_config.transferBufferMaxHeapSize = + (executorPoolSize + 1) * s3MultiPartDownloadChunkSize; + this->transfer_manager_ = + Aws::Transfer::TransferManager::Create(transfer_config); + return this->transfer_manager_; + } + + std::shared_ptr S3Handler::GetS3Client() + { if (this->s3_client_.get() == nullptr) { - Aws::SDKOptions options; - Aws::InitAPI(options); - - // Set up the request - this->s3_client_ = - std::shared_ptr(new Aws::S3::S3Client( - setUpS3Config(), - Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, - false)); + this->InitializeS3Client(); } return this->s3_client_; } std::shared_ptr - S3Handler::InitializeExecutor() + S3Handler::GetExecutor() { if (this->executor_.get() == nullptr) { - this->executor_ = - Aws::MakeShared( - "executor", executorPoolSize); + this->InitializeExecutor(); } return this->executor_; } std::shared_ptr - S3Handler::InitializeTransferManager() + S3Handler::GetTransferManager() { - std::shared_ptr s3_client = InitializeS3Client(); - std::lock_guard lock(this->initialization_lock_); - if (this->transfer_manager_.get() == nullptr) { - Aws::Transfer::TransferManagerConfiguration transfer_config( - InitializeExecutor().get()); - transfer_config.s3Client = s3_client; - // This buffer is what we used to initialize streambuf and is in memory - transfer_config.bufferSize = s3MultiPartDownloadChunkSize; - transfer_config.transferBufferMaxHeapSize = - (executorPoolSize + 1) * s3MultiPartDownloadChunkSize; - this->transfer_manager_ = - Aws::Transfer::TransferManager::Create(transfer_config); + this->InitializeTransferManager(); } return this->transfer_manager_; } @@ -349,7 +370,7 @@ namespace torchdata std::string bucket, object; parseS3Path(file_url, &bucket, &object); S3FS s3handler(bucket, object, multi_part_download_, - InitializeTransferManager(), InitializeS3Client()); + InitializeTransferManager(), GetS3Client()); uint64_t offset = 0; uint64_t result_size = 0; @@ -385,12 +406,12 @@ namespace torchdata } size_t S3Handler::GetFileSize(const std::string &bucket, - const std::string &object) + const std::string &object) { Aws::S3::Model::HeadObjectRequest headObjectRequest; headObjectRequest.WithBucket(bucket.c_str()).WithKey(object.c_str()); auto headObjectOutcome = - this->InitializeS3Client()->HeadObject(headObjectRequest); + this->GetS3Client()->HeadObject(headObjectRequest); if (headObjectOutcome.IsSuccess()) { return headObjectOutcome.GetResult().GetContentLength(); @@ -409,7 +430,7 @@ namespace torchdata } void S3Handler::ListFiles(const std::string &file_url, - std::vector *filenames) + std::vector *filenames) { std::string bucket, prefix; parseS3Path(file_url, &bucket, &prefix); @@ -428,7 +449,7 @@ namespace torchdata do { auto listObjectsOutcome = - this->InitializeS3Client()->ListObjects(listObjectsRequest); + this->GetS3Client()->ListObjects(listObjectsRequest); if (!listObjectsOutcome.IsSuccess()) { Aws::String const &error_aws = diff --git a/torchdata/csrc/pybind/s3_io/s3_io.h b/torchdata/csrc/pybind/s3_io/s3_io.h index fb221682a..a73200caa 100644 --- a/torchdata/csrc/pybind/s3_io/s3_io.h +++ b/torchdata/csrc/pybind/s3_io/s3_io.h @@ -34,9 +34,9 @@ namespace torchdata bool multi_part_download_; std::shared_ptr InitializeS3Client(); - std::shared_ptr InitializeTransferManager(); std::shared_ptr InitializeExecutor(); + std::shared_ptr InitializeTransferManager(); size_t GetFileSize(const std::string &bucket, const std::string &object); size_t GetFileSize(const std::string &file_url); @@ -44,14 +44,14 @@ namespace torchdata S3Handler(); ~S3Handler(); - std::shared_ptr GetS3Client() { return this->s3_client_; }; - std::shared_ptr GetTransferManager() { return this->transfer_manager_; }; + std::shared_ptr GetS3Client(); std::shared_ptr - GetExecutor() { return this->executor_; }; + GetExecutor(); + std::shared_ptr GetTransferManager(); void S3Read(const std::string &file_url, std::string *result); void ListFiles(const std::string &file_url, - std::vector *filenames); + std::vector *filenames); }; } // namespace torchdata From 058154cc5729401d732bfa10fe5e035fc1aa9f54 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 12 Jan 2022 22:11:14 +0800 Subject: [PATCH 009/153] change default region to the default Region configured in the applicable AWS credentials --- torchdata/csrc/pybind/s3_io/s3_io.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/torchdata/csrc/pybind/s3_io/s3_io.cpp b/torchdata/csrc/pybind/s3_io/s3_io.cpp index e0f4b722a..c0c78bce1 100644 --- a/torchdata/csrc/pybind/s3_io/s3_io.cpp +++ b/torchdata/csrc/pybind/s3_io/s3_io.cpp @@ -84,10 +84,6 @@ namespace torchdata { cfg.region = region; } - else - { - cfg.region = "us-west-2"; - } const char *endpoint_url = getenv("S3_ENDPOINT_URL"); if (endpoint_url) From baea3b68e49220b080811b76ed116c5d1cbeb198 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 12 Jan 2022 22:28:15 +0800 Subject: [PATCH 010/153] clean up get and initialize cpp functions --- torchdata/csrc/pybind/s3_io/s3_io.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/torchdata/csrc/pybind/s3_io/s3_io.cpp b/torchdata/csrc/pybind/s3_io/s3_io.cpp index c0c78bce1..77aef97e5 100644 --- a/torchdata/csrc/pybind/s3_io/s3_io.cpp +++ b/torchdata/csrc/pybind/s3_io/s3_io.cpp @@ -78,13 +78,11 @@ namespace torchdata cfg.verifySSL = true; } } - const char *region = getenv("AWS_REGION"); if (region) { cfg.region = region; } - const char *endpoint_url = getenv("S3_ENDPOINT_URL"); if (endpoint_url) { @@ -321,7 +319,7 @@ namespace torchdata std::lock_guard lock(this->initialization_lock_); Aws::Transfer::TransferManagerConfiguration transfer_config( - InitializeExecutor().get()); + this->GetExecutor().get()); transfer_config.s3Client = s3_client; // This buffer is what we used to initialize streambuf and is in memory transfer_config.bufferSize = s3MultiPartDownloadChunkSize; @@ -366,7 +364,7 @@ namespace torchdata std::string bucket, object; parseS3Path(file_url, &bucket, &object); S3FS s3handler(bucket, object, multi_part_download_, - InitializeTransferManager(), GetS3Client()); + GetTransferManager(), GetS3Client()); uint64_t offset = 0; uint64_t result_size = 0; From aff8ed0f1e301714318c1dbb9bcdfd756c590e74 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 12 Jan 2022 22:31:08 +0800 Subject: [PATCH 011/153] remove unused constants --- torchdata/csrc/pybind/s3_io/s3_io.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/torchdata/csrc/pybind/s3_io/s3_io.cpp b/torchdata/csrc/pybind/s3_io/s3_io.cpp index 77aef97e5..e9ca8fca6 100644 --- a/torchdata/csrc/pybind/s3_io/s3_io.cpp +++ b/torchdata/csrc/pybind/s3_io/s3_io.cpp @@ -28,8 +28,6 @@ namespace torchdata { static const size_t s3ReadBufferSize = 120 * 1024 * 1024; // 16 MB static const uint64_t s3MultiPartDownloadChunkSize = 50 * 1024 * 1024; // 50 MB - static const int downloadRetries = 3; - static const int64_t s3TimeoutMsec = 300000; static const int executorPoolSize = 25; static const int S3GetFilesMaxKeys = 100; From 619d86c26315b9847b8e1c11db55394988431d13 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 12 Jan 2022 23:21:19 +0800 Subject: [PATCH 012/153] create a class-level static configuration file --- torchdata/csrc/pybind/s3_io/s3_io.cpp | 21 ++++++++++++--------- torchdata/csrc/pybind/s3_io/s3_io.h | 2 ++ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/torchdata/csrc/pybind/s3_io/s3_io.cpp b/torchdata/csrc/pybind/s3_io/s3_io.cpp index e9ca8fca6..5c8e9586b 100644 --- a/torchdata/csrc/pybind/s3_io/s3_io.cpp +++ b/torchdata/csrc/pybind/s3_io/s3_io.cpp @@ -31,9 +31,9 @@ namespace torchdata static const int executorPoolSize = 25; static const int S3GetFilesMaxKeys = 100; - Aws::Client::ClientConfiguration &setUpS3Config() + std::shared_ptr setUpS3Config() { - static Aws::Client::ClientConfiguration cfg; + auto cfg = std::shared_ptr(new Aws::Client::ClientConfiguration()); Aws::String config_file; const char *config_file_env = getenv("AWS_CONFIG_FILE"); if (config_file_env) @@ -57,11 +57,11 @@ namespace torchdata { if (use_https[0] == '0') { - cfg.scheme = Aws::Http::Scheme::HTTP; + cfg->scheme = Aws::Http::Scheme::HTTP; } else { - cfg.scheme = Aws::Http::Scheme::HTTPS; + cfg->scheme = Aws::Http::Scheme::HTTPS; } } const char *verify_ssl = getenv("S3_VERIFY_SSL"); @@ -69,22 +69,22 @@ namespace torchdata { if (verify_ssl[0] == '0') { - cfg.verifySSL = false; + cfg->verifySSL = false; } else { - cfg.verifySSL = true; + cfg->verifySSL = true; } } const char *region = getenv("AWS_REGION"); if (region) { - cfg.region = region; + cfg->region = region; } const char *endpoint_url = getenv("S3_ENDPOINT_URL"); if (endpoint_url) { - cfg.endpointOverride = endpoint_url; + cfg->endpointOverride = endpoint_url; } return cfg; } @@ -256,6 +256,8 @@ namespace torchdata }; } // namespace + std::shared_ptr S3Handler::s3_handler_cfg_; + S3Handler::S3Handler() : s3_client_(nullptr, ShutdownClient), transfer_manager_(nullptr, ShutdownTransferManager), @@ -292,9 +294,10 @@ namespace torchdata Aws::InitAPI(options); // Set up the request + S3Handler::s3_handler_cfg_ = setUpS3Config(); this->s3_client_ = std::shared_ptr(new Aws::S3::S3Client( - setUpS3Config(), + *S3Handler::s3_handler_cfg_, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, false)); return this->s3_client_; diff --git a/torchdata/csrc/pybind/s3_io/s3_io.h b/torchdata/csrc/pybind/s3_io/s3_io.h index a73200caa..4cfe2fbbe 100644 --- a/torchdata/csrc/pybind/s3_io/s3_io.h +++ b/torchdata/csrc/pybind/s3_io/s3_io.h @@ -26,6 +26,8 @@ namespace torchdata class S3Handler { private: + static std::shared_ptr s3_handler_cfg_; + std::shared_ptr s3_client_; std::shared_ptr executor_; std::shared_ptr transfer_manager_; From 9f853a5d8497ed2f0044d340367b30a243d51ad3 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 12 Jan 2022 23:24:50 +0800 Subject: [PATCH 013/153] update indentation of the test --- test/test_remote_io.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/test/test_remote_io.py b/test/test_remote_io.py index fab4d2f8c..052f5859e 100644 --- a/test/test_remote_io.py +++ b/test/test_remote_io.py @@ -166,14 +166,13 @@ def _read_and_decode(x): self.assertTrue(os.path.exists(expected_csv_path)) self.assertEqual(expected_csv_path, csv_path) - def test_s3_lister_iterdatapipe(self): - - file_url = "s3://pt-s3plugin-test-data-west2/images/test" - s3_lister_dp = S3FileLister(IterableWrapper([file_url])) - count = 0 - for item in s3_lister_dp: - count = count + 1 - print("number of items:", count) + def test_s3_lister_iterdatapipe(self): + file_urls = ["s3://ai2-public-datasets/charades"] + s3_lister_dp = S3FileLister(IterableWrapper(file_urls)) + count = 0 + for item in s3_lister_dp: + count = count + 1 + print("number of items:", count) if __name__ == "__main__": From 4bae528189590c682d701ab4ac572e5fcdee4c52 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Thu, 13 Jan 2022 18:23:00 +0800 Subject: [PATCH 014/153] clean up linting and wrong modifications --- test/test_remote_io.py | 11 +++-------- tools/setup_helpers/extension.py | 1 - torchdata/_internal/module_utils.py | 2 -- torchdata/datapipes/iter/__init__.py | 4 ++-- torchdata/datapipes/iter/load/s3io.py | 3 --- 5 files changed, 5 insertions(+), 16 deletions(-) diff --git a/test/test_remote_io.py b/test/test_remote_io.py index 052f5859e..ed8a434f2 100644 --- a/test/test_remote_io.py +++ b/test/test_remote_io.py @@ -4,14 +4,9 @@ import unittest import warnings -from torchdata.datapipes.iter import ( - EndOnDiskCacheHolder, - FileLoader, - HttpReader, - IterableWrapper, - OnDiskCacheHolder, - S3FileLister, -) +import expecttest + +from _utils._common_utils_for_test import check_hash_fn, create_temp_dir from torchdata.datapipes.iter import EndOnDiskCacheHolder, FileOpener, HttpReader, IterableWrapper, OnDiskCacheHolder diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index 74ae2fff0..52dfa4e74 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -1,7 +1,6 @@ import os import platform import subprocess -import sys from pathlib import Path import torch diff --git a/torchdata/_internal/module_utils.py b/torchdata/_internal/module_utils.py index fff7e96be..47ff06215 100644 --- a/torchdata/_internal/module_utils.py +++ b/torchdata/_internal/module_utils.py @@ -3,8 +3,6 @@ from functools import wraps from typing import Optional -import torch - def is_module_available(*modules: str) -> bool: r"""Returns if a top-level module with :attr:`name` exists *without** diff --git a/torchdata/datapipes/iter/__init__.py b/torchdata/datapipes/iter/__init__.py index 9efce8d71..7fbe01a4f 100644 --- a/torchdata/datapipes/iter/__init__.py +++ b/torchdata/datapipes/iter/__init__.py @@ -19,8 +19,8 @@ ShardingFilter, Shuffler, StreamReader, - IterableWrapper, - ShardingFilter, + UnBatcher, + Zipper, ) from torchdata.datapipes.iter.load.fsspec import ( FSSpecFileListerIterDataPipe as FSSpecFileLister, diff --git a/torchdata/datapipes/iter/load/s3io.py b/torchdata/datapipes/iter/load/s3io.py index 66dac6532..d4513dce6 100644 --- a/torchdata/datapipes/iter/load/s3io.py +++ b/torchdata/datapipes/iter/load/s3io.py @@ -1,9 +1,6 @@ from typing import Iterator, Tuple -from itertools import chain -# import torchdata._torchdata import torchdata - from torchdata.datapipes import functional_datapipe from torchdata.datapipes.iter import IterDataPipe from torchdata.datapipes.utils import StreamWrapper From 6599ce931e82aded086733487fec1ddaf0d7ef00 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Thu, 13 Jan 2022 19:06:06 +0800 Subject: [PATCH 015/153] clean up cpp api's, styling, include necessary headers --- torchdata/csrc/pybind/s3_io/s3_io.cpp | 48 +++++++++++---------------- torchdata/csrc/pybind/s3_io/s3_io.h | 16 ++++----- 2 files changed, 26 insertions(+), 38 deletions(-) diff --git a/torchdata/csrc/pybind/s3_io/s3_io.cpp b/torchdata/csrc/pybind/s3_io/s3_io.cpp index 5c8e9586b..6389321e1 100644 --- a/torchdata/csrc/pybind/s3_io/s3_io.cpp +++ b/torchdata/csrc/pybind/s3_io/s3_io.cpp @@ -1,7 +1,9 @@ #include "s3_io.h" +#include #include #include +#include #include #include #include @@ -10,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -33,7 +36,8 @@ namespace torchdata std::shared_ptr setUpS3Config() { - auto cfg = std::shared_ptr(new Aws::Client::ClientConfiguration()); + std::shared_ptr cfg = + std::shared_ptr(new Aws::Client::ClientConfiguration()); Aws::String config_file; const char *config_file_env = getenv("AWS_CONFIG_FILE"); if (config_file_env) @@ -121,27 +125,25 @@ namespace torchdata { if (fname.empty()) { - throw std::invalid_argument{"The filename cannot be an empty string."}; + throw std::invalid_argument("The filename cannot be an empty string."); } if (fname.size() < 5 || fname.substr(0, 5) != "s3://") { - throw std::invalid_argument{ - "The filename must start with the S3 scheme."}; + throw std::invalid_argument("The filename must start with the S3 scheme."); } std::string path = fname.substr(5); if (path.empty()) { - throw std::invalid_argument{"The filename cannot be an empty string."}; + throw std::invalid_argument("The filename cannot be an empty string."); } - auto pos = path.find_first_of('/'); + size_t pos = path.find_first_of('/'); if (pos == 0) { - throw std::invalid_argument{ - "The filename does not contain a bucket name."}; + throw std::invalid_argument("The filename does not contain a bucket name."); } *bucket = path.substr(0, pos); @@ -196,11 +198,11 @@ namespace torchdata []() { return Aws::New("S3IOAllocationTag"); }); // get the object - auto getObjectOutcome = this->s3_client_->GetObject(getObjectRequest); + Aws::S3::Model::GetObjectOutcome getObjectOutcome = this->s3_client_->GetObject(getObjectRequest); if (!getObjectOutcome.IsSuccess()) { - auto error = getObjectOutcome.GetError(); + Aws::S3::S3Error error = getObjectOutcome.GetError(); std::cout << "ERROR: " << error.GetExceptionName() << ": " << error.GetMessage() << std::endl; return 0; @@ -236,7 +238,7 @@ namespace torchdata if (downloadHandle->GetStatus() != Aws::Transfer::TransferStatus::COMPLETED) { - auto error = downloadHandle->GetLastError(); + const Aws::Client::AWSError error = downloadHandle->GetLastError(); std::cout << "ERROR: " << error.GetExceptionName() << ": " << error.GetMessage() << std::endl; return 0; @@ -287,7 +289,7 @@ namespace torchdata S3Handler::~S3Handler() {} - std::shared_ptr S3Handler::InitializeS3Client() + void S3Handler::InitializeS3Client() { std::lock_guard lock(this->initialization_lock_); Aws::SDKOptions options; @@ -300,21 +302,17 @@ namespace torchdata *S3Handler::s3_handler_cfg_, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, false)); - return this->s3_client_; } - std::shared_ptr - S3Handler::InitializeExecutor() + void S3Handler::InitializeExecutor() { std::lock_guard lock(this->initialization_lock_); this->executor_ = Aws::MakeShared( "executor", executorPoolSize); - return this->executor_; } - std::shared_ptr - S3Handler::InitializeTransferManager() + void S3Handler::InitializeTransferManager() { std::shared_ptr s3_client = GetS3Client(); std::lock_guard lock(this->initialization_lock_); @@ -328,7 +326,6 @@ namespace torchdata (executorPoolSize + 1) * s3MultiPartDownloadChunkSize; this->transfer_manager_ = Aws::Transfer::TransferManager::Create(transfer_config); - return this->transfer_manager_; } std::shared_ptr S3Handler::GetS3Client() @@ -405,7 +402,7 @@ namespace torchdata { Aws::S3::Model::HeadObjectRequest headObjectRequest; headObjectRequest.WithBucket(bucket.c_str()).WithKey(object.c_str()); - auto headObjectOutcome = + Aws::S3::Model::HeadObjectOutcome headObjectOutcome = this->GetS3Client()->HeadObject(headObjectRequest); if (headObjectOutcome.IsSuccess()) { @@ -417,13 +414,6 @@ namespace torchdata return 0; } - size_t S3Handler::GetFileSize(const std::string &file_url) - { - std::string bucket, object; - parseS3Path(file_url, &bucket, &object); - return this->GetFileSize(bucket, object); - } - void S3Handler::ListFiles(const std::string &file_url, std::vector *filenames) { @@ -443,7 +433,7 @@ namespace torchdata Aws::S3::Model::ListObjectsResult listObjectsResult; do { - auto listObjectsOutcome = + Aws::S3::Model::ListObjectsOutcome listObjectsOutcome = this->GetS3Client()->ListObjects(listObjectsRequest); if (!listObjectsOutcome.IsSuccess()) { @@ -457,7 +447,7 @@ namespace torchdata Aws::Vector objects = listObjectsResult.GetContents(); if (!objects.empty()) { - for (const auto &object : objects) + for (const Aws::S3::Model::Object &object : objects) { Aws::String key = default_key + object.GetKey(); if (key.back() == '/') diff --git a/torchdata/csrc/pybind/s3_io/s3_io.h b/torchdata/csrc/pybind/s3_io/s3_io.h index 4cfe2fbbe..808f23825 100644 --- a/torchdata/csrc/pybind/s3_io/s3_io.h +++ b/torchdata/csrc/pybind/s3_io/s3_io.h @@ -35,22 +35,20 @@ namespace torchdata size_t buffer_size_; bool multi_part_download_; - std::shared_ptr InitializeS3Client(); + void InitializeS3Client(); + void InitializeExecutor(); + void InitializeTransferManager(); + + std::shared_ptr GetS3Client(); std::shared_ptr - InitializeExecutor(); - std::shared_ptr InitializeTransferManager(); + GetExecutor(); + std::shared_ptr GetTransferManager(); size_t GetFileSize(const std::string &bucket, const std::string &object); - size_t GetFileSize(const std::string &file_url); public: S3Handler(); ~S3Handler(); - std::shared_ptr GetS3Client(); - std::shared_ptr - GetExecutor(); - std::shared_ptr GetTransferManager(); - void S3Read(const std::string &file_url, std::string *result); void ListFiles(const std::string &file_url, std::vector *filenames); From 1c479e2dc21c3b13eb2a1f940a1db328c236b85b Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Thu, 13 Jan 2022 22:54:54 +0800 Subject: [PATCH 016/153] expose request timeout and region python api's for temporary override --- torchdata/csrc/pybind/s3_io/pybind.cpp | 3 +-- torchdata/csrc/pybind/s3_io/s3_io.cpp | 36 +++++++++++++++++++------- torchdata/csrc/pybind/s3_io/s3_io.h | 3 ++- torchdata/datapipes/iter/load/s3io.py | 25 +++++++----------- 4 files changed, 40 insertions(+), 27 deletions(-) diff --git a/torchdata/csrc/pybind/s3_io/pybind.cpp b/torchdata/csrc/pybind/s3_io/pybind.cpp index 1ee733ffb..a48162752 100644 --- a/torchdata/csrc/pybind/s3_io/pybind.cpp +++ b/torchdata/csrc/pybind/s3_io/pybind.cpp @@ -12,8 +12,7 @@ using torchdata::S3Handler; PYBIND11_MODULE(_torchdata, m) { py::class_(m, "S3Handler") - // TODO: pass in timeout - .def(py::init<>()) + .def(py::init()) .def("s3_read", [](S3Handler *self, const std::string &file_url) { diff --git a/torchdata/csrc/pybind/s3_io/s3_io.cpp b/torchdata/csrc/pybind/s3_io/s3_io.cpp index 6389321e1..334f31258 100644 --- a/torchdata/csrc/pybind/s3_io/s3_io.cpp +++ b/torchdata/csrc/pybind/s3_io/s3_io.cpp @@ -34,7 +34,8 @@ namespace torchdata static const int executorPoolSize = 25; static const int S3GetFilesMaxKeys = 100; - std::shared_ptr setUpS3Config() + std::shared_ptr setUpS3Config(const long requestTimeoutMs, const std::string region) + // std::shared_ptr setUpS3Config() { std::shared_ptr cfg = std::shared_ptr(new Aws::Client::ClientConfiguration()); @@ -80,16 +81,26 @@ namespace torchdata cfg->verifySSL = true; } } - const char *region = getenv("AWS_REGION"); - if (region) - { - cfg->region = region; - } const char *endpoint_url = getenv("S3_ENDPOINT_URL"); if (endpoint_url) { cfg->endpointOverride = endpoint_url; } + if (region != "") + { + cfg->region = region; + } + else + { + const char *env_region = getenv("AWS_REGION"); + if (env_region) + { + cfg->region = env_region; + } + } + if (requestTimeoutMs > -1) { + cfg->requestTimeoutMs = requestTimeoutMs; + } return cfg; } @@ -260,7 +271,7 @@ namespace torchdata std::shared_ptr S3Handler::s3_handler_cfg_; - S3Handler::S3Handler() + S3Handler::S3Handler(const long requestTimeoutMs, const std::string region) : s3_client_(nullptr, ShutdownClient), transfer_manager_(nullptr, ShutdownTransferManager), executor_(nullptr, ShutdownExecutor), @@ -284,19 +295,26 @@ namespace torchdata multi_part_download_ = false; } } - InitializeS3Client(); + InitializeS3Client(requestTimeoutMs, region); } S3Handler::~S3Handler() {} void S3Handler::InitializeS3Client() + { + const long requestTimeoutMs = -1; + const std::string region = ""; + this->InitializeS3Client(requestTimeoutMs, region); + } + + void S3Handler::InitializeS3Client(const long requestTimeoutMs, const std::string region) { std::lock_guard lock(this->initialization_lock_); Aws::SDKOptions options; Aws::InitAPI(options); // Set up the request - S3Handler::s3_handler_cfg_ = setUpS3Config(); + S3Handler::s3_handler_cfg_ = setUpS3Config(requestTimeoutMs, region); this->s3_client_ = std::shared_ptr(new Aws::S3::S3Client( *S3Handler::s3_handler_cfg_, diff --git a/torchdata/csrc/pybind/s3_io/s3_io.h b/torchdata/csrc/pybind/s3_io/s3_io.h index 808f23825..e2f6d4c8a 100644 --- a/torchdata/csrc/pybind/s3_io/s3_io.h +++ b/torchdata/csrc/pybind/s3_io/s3_io.h @@ -36,6 +36,7 @@ namespace torchdata bool multi_part_download_; void InitializeS3Client(); + void InitializeS3Client(const long requestTimeoutMs, const std::string region); void InitializeExecutor(); void InitializeTransferManager(); @@ -46,7 +47,7 @@ namespace torchdata size_t GetFileSize(const std::string &bucket, const std::string &object); public: - S3Handler(); + S3Handler(const long requestTimeoutMs, const std::string region); ~S3Handler(); void S3Read(const std::string &file_url, std::string *result); diff --git a/torchdata/datapipes/iter/load/s3io.py b/torchdata/datapipes/iter/load/s3io.py index d4513dce6..fff272696 100644 --- a/torchdata/datapipes/iter/load/s3io.py +++ b/torchdata/datapipes/iter/load/s3io.py @@ -15,27 +15,21 @@ class S3FileListerIterDataPipe(IterDataPipe[str]): Args: source_datapipe: a DataPipe that contains URLs/URL prefixes to s3 files length: Nominal length of the datapipe + requestTimeoutMs: optional, overwrite the default timeout setting for this datapipe + region: optional, overwrite the default region inferred from credentials for this datapipe Note: AWS_CPP_SDK is necessary to use the S3 DataPipe(s). """ - def __init__(self, source_datapipe: IterDataPipe[str], length: int = -1) -> None: - # TODO: check whether AWS_CPP_SDK available + def __init__(self, source_datapipe: IterDataPipe[str], length: int = -1, requestTimeoutMs = -1, region = "") -> None: self.source_datapipe: IterDataPipe[str] = source_datapipe self.length: int = length + self.handler = torchdata._torchdata.S3Handler(requestTimeoutMs, region) def __iter__(self) -> Iterator[str]: - # TODO: timeout - handler = torchdata._torchdata.S3Handler() for prefix in self.source_datapipe: - # if handler.file_exists(prefix): - # yield prefix - # else: - # urls = handler.list_files(prefix) - # for url in urls: - # yield url - urls = handler.list_files(prefix) + urls = self.handler.list_files(prefix) for url in urls: yield url @@ -53,19 +47,20 @@ class S3FileLoaderIterDataPipe(IterDataPipe[Tuple[str, StreamWrapper]]): Args: source_datapipe: a DataPipe that contains URLs to s3 files + requestTimeoutMs: optional, overwrite the default timeout setting for this datapipe + region: optional, overwrite the default region inferred from credentials for this datapipe Note: AWS_CPP_SDK is necessary to use the S3 DataPipe(s). """ - def __init__(self, source_datapipe: IterDataPipe[str]) -> None: + def __init__(self, source_datapipe: IterDataPipe[str], requestTimeoutMs = -1, region = "") -> None: self.source_datapipe: IterDataPipe[str] = source_datapipe + self.handler = torchdata._torchdata.S3Handler(requestTimeoutMs, region) def __iter__(self) -> Iterator[Tuple[str, StreamWrapper]]: - # TODO: timeout - handler = torchdata._torchdata.S3Handler() for url in self.source_datapipe: - yield url, handler.s3_read(url) + yield url, self.handler.s3_read(url) def __len__(self) -> int: return len(self.source_datapipe) From 3e17da9e14f7459735acbc588333263c24532d42 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Sat, 15 Jan 2022 01:28:55 +0800 Subject: [PATCH 017/153] enable file loading while listing files, clean up code --- torchdata/csrc/pybind/s3_io/pybind.cpp | 11 ++- torchdata/csrc/pybind/s3_io/s3_io.cpp | 113 ++++++++++++------------- torchdata/csrc/pybind/s3_io/s3_io.h | 6 ++ torchdata/datapipes/iter/load/s3io.py | 18 ++-- 4 files changed, 83 insertions(+), 65 deletions(-) diff --git a/torchdata/csrc/pybind/s3_io/pybind.cpp b/torchdata/csrc/pybind/s3_io/pybind.cpp index a48162752..476d9eafe 100644 --- a/torchdata/csrc/pybind/s3_io/pybind.cpp +++ b/torchdata/csrc/pybind/s3_io/pybind.cpp @@ -7,7 +7,6 @@ #include "s3_io.h" namespace py = pybind11; -// TODO: change to S3Client using torchdata::S3Handler; PYBIND11_MODULE(_torchdata, m) { @@ -26,5 +25,15 @@ PYBIND11_MODULE(_torchdata, m) std::vector filenames; self->ListFiles(file_url, &filenames); return filenames; + }) + .def("set_max_keys", + [](S3Handler *self, const int max_keys) + { + self->SetMaxKeys(max_keys); + }) + .def("clear_marker", + [](S3Handler *self) + { + self->ClearMarker(); }); } diff --git a/torchdata/csrc/pybind/s3_io/s3_io.cpp b/torchdata/csrc/pybind/s3_io/s3_io.cpp index 334f31258..5de3525de 100644 --- a/torchdata/csrc/pybind/s3_io/s3_io.cpp +++ b/torchdata/csrc/pybind/s3_io/s3_io.cpp @@ -32,10 +32,10 @@ namespace torchdata static const size_t s3ReadBufferSize = 120 * 1024 * 1024; // 16 MB static const uint64_t s3MultiPartDownloadChunkSize = 50 * 1024 * 1024; // 50 MB static const int executorPoolSize = 25; - static const int S3GetFilesMaxKeys = 100; + static const int S3DefaultMaxKeys = 1000; + static const std::string S3DefaultMarker = ""; std::shared_ptr setUpS3Config(const long requestTimeoutMs, const std::string region) - // std::shared_ptr setUpS3Config() { std::shared_ptr cfg = std::shared_ptr(new Aws::Client::ClientConfiguration()); @@ -98,7 +98,8 @@ namespace torchdata cfg->region = env_region; } } - if (requestTimeoutMs > -1) { + if (requestTimeoutMs > -1) + { cfg->requestTimeoutMs = requestTimeoutMs; } return cfg; @@ -131,8 +132,8 @@ namespace torchdata } } - void parseS3Path(const std::string &fname, std::string *bucket, - std::string *object) + void parseS3Path(const Aws::String &fname, Aws::String *bucket, + Aws::String *object) { if (fname.empty()) { @@ -295,7 +296,11 @@ namespace torchdata multi_part_download_ = false; } } + InitializeS3Client(requestTimeoutMs, region); + + this->max_keys_ = S3DefaultMaxKeys; + this->last_marker_ = S3DefaultMarker; } S3Handler::~S3Handler() {} @@ -375,6 +380,25 @@ namespace torchdata return this->transfer_manager_; } + size_t S3Handler::GetFileSize(const std::string &bucket, + const std::string &object) + { + Aws::S3::Model::HeadObjectRequest headObjectRequest; + headObjectRequest.WithBucket(bucket.c_str()).WithKey(object.c_str()); + Aws::S3::Model::HeadObjectOutcome headObjectOutcome = + this->GetS3Client()->HeadObject(headObjectRequest); + if (headObjectOutcome.IsSuccess()) + { + return headObjectOutcome.GetResult().GetContentLength(); + } + Aws::String const &error_aws = headObjectOutcome.GetError().GetMessage(); + std::string error_str(error_aws.c_str(), error_aws.size()); + throw std::invalid_argument(error_str); + return 0; + } + + void S3Handler::ClearMarker() { this->last_marker_ = S3DefaultMarker; } + void S3Handler::S3Read(const std::string &file_url, std::string *result) { std::string bucket, object; @@ -415,70 +439,43 @@ namespace torchdata } } - size_t S3Handler::GetFileSize(const std::string &bucket, - const std::string &object) - { - Aws::S3::Model::HeadObjectRequest headObjectRequest; - headObjectRequest.WithBucket(bucket.c_str()).WithKey(object.c_str()); - Aws::S3::Model::HeadObjectOutcome headObjectOutcome = - this->GetS3Client()->HeadObject(headObjectRequest); - if (headObjectOutcome.IsSuccess()) - { - return headObjectOutcome.GetResult().GetContentLength(); - } - Aws::String const &error_aws = headObjectOutcome.GetError().GetMessage(); - std::string error_str(error_aws.c_str(), error_aws.size()); - throw std::invalid_argument(error_str); - return 0; - } - void S3Handler::ListFiles(const std::string &file_url, std::vector *filenames) { - std::string bucket, prefix; + Aws::String bucket, prefix; parseS3Path(file_url, &bucket, &prefix); - Aws::String default_key = ""; - if (prefix.empty()) - { - default_key = "/"; - } Aws::S3::Model::ListObjectsRequest listObjectsRequest; - listObjectsRequest.WithBucket(bucket.c_str()) - .WithPrefix(prefix.c_str()) - .WithMaxKeys(S3GetFilesMaxKeys); - - Aws::S3::Model::ListObjectsResult listObjectsResult; - do + listObjectsRequest.WithBucket(bucket) + .WithPrefix(prefix) + .WithMaxKeys(this->max_keys_) + .WithMarker(this->last_marker_); + std::cerr << "max_keys_: " << this->max_keys_ << std::endl; + std::cerr << "last_marker_: " << this->last_marker_ << std::endl; + + Aws::S3::Model::ListObjectsOutcome listObjectsOutcome = + this->GetS3Client()->ListObjects(listObjectsRequest); + if (!listObjectsOutcome.IsSuccess()) { - Aws::S3::Model::ListObjectsOutcome listObjectsOutcome = - this->GetS3Client()->ListObjects(listObjectsRequest); - if (!listObjectsOutcome.IsSuccess()) - { - Aws::String const &error_aws = - listObjectsOutcome.GetError().GetMessage(); - std::string error_str(error_aws.c_str(), error_aws.size()); - throw std::invalid_argument(error_str); - } + Aws::String const &error_aws = + listObjectsOutcome.GetError().GetMessage(); + throw std::invalid_argument(error_aws); + } - listObjectsResult = listObjectsOutcome.GetResult(); - Aws::Vector objects = listObjectsResult.GetContents(); - if (!objects.empty()) + Aws::S3::Model::ListObjectsResult listObjectsResult = listObjectsOutcome.GetResult(); + Aws::Vector objects = listObjectsResult.GetContents(); + if (!objects.empty()) + { + for (const Aws::S3::Model::Object &object : objects) { - for (const Aws::S3::Model::Object &object : objects) + if (object.GetKey().back() == '/') // ignore folders { - Aws::String key = default_key + object.GetKey(); - if (key.back() == '/') - { - continue; - } - Aws::String bucket_aws(bucket.c_str(), bucket.size()); - Aws::String entry = "s3://" + bucket_aws + "/" + object.GetKey(); - filenames->push_back(entry.c_str()); + continue; } - listObjectsRequest.SetMarker(listObjectsResult.GetContents().back().GetKey()); + Aws::String entry = "s3://" + bucket + "/" + object.GetKey(); + filenames->push_back(entry.c_str()); } - } while (listObjectsResult.GetIsTruncated()); + this->last_marker_ = listObjectsResult.GetContents().back().GetKey(); + } } - } // namespace torchdata diff --git a/torchdata/csrc/pybind/s3_io/s3_io.h b/torchdata/csrc/pybind/s3_io/s3_io.h index e2f6d4c8a..9fb469687 100644 --- a/torchdata/csrc/pybind/s3_io/s3_io.h +++ b/torchdata/csrc/pybind/s3_io/s3_io.h @@ -31,6 +31,9 @@ namespace torchdata std::shared_ptr s3_client_; std::shared_ptr executor_; std::shared_ptr transfer_manager_; + + Aws::String last_marker_; + int max_keys_; std::mutex initialization_lock_; size_t buffer_size_; bool multi_part_download_; @@ -50,6 +53,9 @@ namespace torchdata S3Handler(const long requestTimeoutMs, const std::string region); ~S3Handler(); + void SetMaxKeys(const int max_keys) { this->max_keys_ = max_keys; } + void ClearMarker(); + void S3Read(const std::string &file_url, std::string *result); void ListFiles(const std::string &file_url, std::vector *filenames); diff --git a/torchdata/datapipes/iter/load/s3io.py b/torchdata/datapipes/iter/load/s3io.py index fff272696..132456f62 100644 --- a/torchdata/datapipes/iter/load/s3io.py +++ b/torchdata/datapipes/iter/load/s3io.py @@ -22,16 +22,22 @@ class S3FileListerIterDataPipe(IterDataPipe[str]): AWS_CPP_SDK is necessary to use the S3 DataPipe(s). """ - def __init__(self, source_datapipe: IterDataPipe[str], length: int = -1, requestTimeoutMs = -1, region = "") -> None: + def __init__(self, source_datapipe: IterDataPipe[str], length: int = -1, request_timeout_ms=-1, region="", max_keys=None) -> None: self.source_datapipe: IterDataPipe[str] = source_datapipe self.length: int = length - self.handler = torchdata._torchdata.S3Handler(requestTimeoutMs, region) + self.handler = torchdata._torchdata.S3Handler(request_timeout_ms, region) + if max_keys: + self.handler.set_max_keys(max_keys) def __iter__(self) -> Iterator[str]: for prefix in self.source_datapipe: - urls = self.handler.list_files(prefix) - for url in urls: - yield url + while True: + urls = self.handler.list_files(prefix) + for url in urls: + yield url + if not urls: + break + self.handler.clear_marker() def __len__(self) -> int: if self.length == -1: @@ -54,7 +60,7 @@ class S3FileLoaderIterDataPipe(IterDataPipe[Tuple[str, StreamWrapper]]): AWS_CPP_SDK is necessary to use the S3 DataPipe(s). """ - def __init__(self, source_datapipe: IterDataPipe[str], requestTimeoutMs = -1, region = "") -> None: + def __init__(self, source_datapipe: IterDataPipe[str], requestTimeoutMs=-1, region="") -> None: self.source_datapipe: IterDataPipe[str] = source_datapipe self.handler = torchdata._torchdata.S3Handler(requestTimeoutMs, region) From eea6c787840e1acb599793b301533619daa87811 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Sat, 15 Jan 2022 01:33:36 +0800 Subject: [PATCH 018/153] rename s3_io to S3Handler --- torchdata/csrc/CMakeLists.txt | 4 ++-- .../csrc/pybind/{s3_io/s3_io.cpp => S3Handler/S3Handler.cpp} | 2 +- .../csrc/pybind/{s3_io/s3_io.h => S3Handler/S3Handler.h} | 0 torchdata/csrc/pybind/{s3_io => S3Handler}/pybind.cpp | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) rename torchdata/csrc/pybind/{s3_io/s3_io.cpp => S3Handler/S3Handler.cpp} (99%) rename torchdata/csrc/pybind/{s3_io/s3_io.h => S3Handler/S3Handler.h} (100%) rename torchdata/csrc/pybind/{s3_io => S3Handler}/pybind.cpp (97%) diff --git a/torchdata/csrc/CMakeLists.txt b/torchdata/csrc/CMakeLists.txt index ced337721..d5c423dbf 100644 --- a/torchdata/csrc/CMakeLists.txt +++ b/torchdata/csrc/CMakeLists.txt @@ -5,9 +5,9 @@ if(BUILD_S3) find_package(pybind11 REQUIRED) set(CMAKE_POSITION_INDEPENDENT_CODE ON) - set(SOURCES "pybind/s3_io/s3_io.cpp" ) + set(SOURCES "pybind/S3Handler/S3Handler.cpp" ) - pybind11_add_module(_torchdata ${SOURCES} "pybind/s3_io/pybind.cpp") + pybind11_add_module(_torchdata ${SOURCES} "pybind/S3Handler/pybind.cpp") message(STATUS "All linked libs: ${AWSSDK_LINK_LIBRARIES}") diff --git a/torchdata/csrc/pybind/s3_io/s3_io.cpp b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp similarity index 99% rename from torchdata/csrc/pybind/s3_io/s3_io.cpp rename to torchdata/csrc/pybind/S3Handler/S3Handler.cpp index 5de3525de..2e1f26a64 100644 --- a/torchdata/csrc/pybind/s3_io/s3_io.cpp +++ b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp @@ -1,4 +1,4 @@ -#include "s3_io.h" +#include "S3Handler.h" #include #include diff --git a/torchdata/csrc/pybind/s3_io/s3_io.h b/torchdata/csrc/pybind/S3Handler/S3Handler.h similarity index 100% rename from torchdata/csrc/pybind/s3_io/s3_io.h rename to torchdata/csrc/pybind/S3Handler/S3Handler.h diff --git a/torchdata/csrc/pybind/s3_io/pybind.cpp b/torchdata/csrc/pybind/S3Handler/pybind.cpp similarity index 97% rename from torchdata/csrc/pybind/s3_io/pybind.cpp rename to torchdata/csrc/pybind/S3Handler/pybind.cpp index 476d9eafe..b3fc7294c 100644 --- a/torchdata/csrc/pybind/s3_io/pybind.cpp +++ b/torchdata/csrc/pybind/S3Handler/pybind.cpp @@ -4,7 +4,7 @@ #include #include -#include "s3_io.h" +#include "S3Handler.h" namespace py = pybind11; using torchdata::S3Handler; From c4fddf53aee1831e083777f658f83265bc653c84 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Sat, 15 Jan 2022 01:34:01 +0800 Subject: [PATCH 019/153] clean up std::cerr lines --- torchdata/csrc/pybind/S3Handler/S3Handler.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp index 2e1f26a64..7cdc9694a 100644 --- a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp +++ b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp @@ -450,8 +450,6 @@ namespace torchdata .WithPrefix(prefix) .WithMaxKeys(this->max_keys_) .WithMarker(this->last_marker_); - std::cerr << "max_keys_: " << this->max_keys_ << std::endl; - std::cerr << "last_marker_: " << this->last_marker_ << std::endl; Aws::S3::Model::ListObjectsOutcome listObjectsOutcome = this->GetS3Client()->ListObjects(listObjectsRequest); From 54e49ccd77464154122e2c89776f185f403b8b8d Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Sat, 15 Jan 2022 18:09:11 +0800 Subject: [PATCH 020/153] deal with extreme cases when all results are folders --- torchdata/csrc/pybind/S3Handler/S3Handler.cpp | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp index 7cdc9694a..3f66b5cd0 100644 --- a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp +++ b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp @@ -460,20 +460,26 @@ namespace torchdata throw std::invalid_argument(error_aws); } - Aws::S3::Model::ListObjectsResult listObjectsResult = listObjectsOutcome.GetResult(); - Aws::Vector objects = listObjectsResult.GetContents(); - if (!objects.empty()) + Aws::Vector objects = listObjectsOutcome.GetResult().GetContents(); + if (objects.empty()) { - for (const Aws::S3::Model::Object &object : objects) + return; + } + for (const Aws::S3::Model::Object &object : objects) + { + if (object.GetKey().back() == '/') // ignore folders { - if (object.GetKey().back() == '/') // ignore folders - { - continue; - } - Aws::String entry = "s3://" + bucket + "/" + object.GetKey(); - filenames->push_back(entry.c_str()); + continue; } - this->last_marker_ = listObjectsResult.GetContents().back().GetKey(); + Aws::String entry = "s3://" + bucket + "/" + object.GetKey(); + filenames->push_back(entry.c_str()); + } + this->last_marker_ = objects.back().GetKey(); + + // extreme cases when all objects are folders + if (filenames->size() == 0) + { + this->ListFiles(file_url, filenames); } } } // namespace torchdata From 25ba1d7457afad91e3edd99fb090611db8bd122a Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Sun, 16 Jan 2022 13:17:43 +0800 Subject: [PATCH 021/153] change AWS init flow and clean up code --- torchdata/csrc/pybind/S3Handler/S3Handler.cpp | 31 +++++++------------ torchdata/csrc/pybind/S3Handler/S3Handler.h | 1 - 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp index 3f66b5cd0..97a6080e4 100644 --- a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp +++ b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp @@ -168,6 +168,13 @@ namespace torchdata class S3FS { + private: + std::string bucket_name_; + std::string object_name_; + bool multi_part_download_; + std::shared_ptr s3_client_; + std::shared_ptr transfer_manager_; + public: S3FS(const std::string &bucket, const std::string &object, const bool multi_part_download, @@ -260,13 +267,6 @@ namespace torchdata return downloadHandle->GetBytesTransferred(); } } - - private: - std::string bucket_name_; - std::string object_name_; - bool multi_part_download_; - std::shared_ptr s3_client_; - std::shared_ptr transfer_manager_; }; } // namespace @@ -297,7 +297,10 @@ namespace torchdata } } - InitializeS3Client(requestTimeoutMs, region); + Aws::SDKOptions options; + Aws::InitAPI(options); + S3Handler::s3_handler_cfg_ = setUpS3Config(requestTimeoutMs, region); + InitializeS3Client(); this->max_keys_ = S3DefaultMaxKeys; this->last_marker_ = S3DefaultMarker; @@ -306,20 +309,8 @@ namespace torchdata S3Handler::~S3Handler() {} void S3Handler::InitializeS3Client() - { - const long requestTimeoutMs = -1; - const std::string region = ""; - this->InitializeS3Client(requestTimeoutMs, region); - } - - void S3Handler::InitializeS3Client(const long requestTimeoutMs, const std::string region) { std::lock_guard lock(this->initialization_lock_); - Aws::SDKOptions options; - Aws::InitAPI(options); - - // Set up the request - S3Handler::s3_handler_cfg_ = setUpS3Config(requestTimeoutMs, region); this->s3_client_ = std::shared_ptr(new Aws::S3::S3Client( *S3Handler::s3_handler_cfg_, diff --git a/torchdata/csrc/pybind/S3Handler/S3Handler.h b/torchdata/csrc/pybind/S3Handler/S3Handler.h index 9fb469687..4f126a871 100644 --- a/torchdata/csrc/pybind/S3Handler/S3Handler.h +++ b/torchdata/csrc/pybind/S3Handler/S3Handler.h @@ -39,7 +39,6 @@ namespace torchdata bool multi_part_download_; void InitializeS3Client(); - void InitializeS3Client(const long requestTimeoutMs, const std::string region); void InitializeExecutor(); void InitializeTransferManager(); From c09be1e3bf79d59ba1acec754e064ab4c25c2d74 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Mon, 17 Jan 2022 14:55:30 +0800 Subject: [PATCH 022/153] rename S3FS api's --- torchdata/csrc/pybind/S3Handler/S3Handler.cpp | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp index 97a6080e4..08740d6fd 100644 --- a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp +++ b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp @@ -29,7 +29,7 @@ namespace torchdata { namespace { - static const size_t s3ReadBufferSize = 120 * 1024 * 1024; // 16 MB + static const size_t s3ReadBufferSize = 120 * 1024 * 1024; // 120 MB static const uint64_t s3MultiPartDownloadChunkSize = 50 * 1024 * 1024; // 50 MB static const int executorPoolSize = 25; static const int S3DefaultMaxKeys = 1000; @@ -186,19 +186,19 @@ namespace torchdata transfer_manager_(transfer_manager), s3_client_(s3_client) {} - size_t read(uint64_t offset, size_t n, char *buffer) + size_t Read(uint64_t offset, size_t n, char *buffer) { if (multi_part_download_) { - return readS3TransferManager(offset, n, buffer); + return ReadTransferManager(offset, n, buffer); } else { - return readS3Client(offset, n, buffer); + return ReadS3Client(offset, n, buffer); } } - size_t readS3Client(uint64_t offset, size_t n, char *buffer) + size_t ReadS3Client(uint64_t offset, size_t n, char *buffer) { Aws::S3::Model::GetObjectRequest getObjectRequest; @@ -235,7 +235,7 @@ namespace torchdata } } - size_t readS3TransferManager(uint64_t offset, size_t n, char *buffer) + size_t ReadTransferManager(uint64_t offset, size_t n, char *buffer) { auto create_stream_fn = [&]() { // create stream lambda fn return Aws::New( @@ -394,26 +394,25 @@ namespace torchdata { std::string bucket, object; parseS3Path(file_url, &bucket, &object); - S3FS s3handler(bucket, object, multi_part_download_, - GetTransferManager(), GetS3Client()); + S3FS s3fs(bucket, object, multi_part_download_, + GetTransferManager(), GetS3Client()); uint64_t offset = 0; uint64_t result_size = 0; uint64_t file_size = this->GetFileSize(bucket, object); - std::size_t part_count = (std::max)( + size_t part_count = (std::max)( static_cast((file_size + buffer_size_ - 1) / buffer_size_), - static_cast(1)); + static_cast(1)); result->resize(file_size); for (int i = 0; i < part_count; i++) { - offset = result_size; size_t buf_len = std::min(buffer_size_, file_size - result_size); size_t read_len = - s3handler.read(offset, buf_len, (char *)(result->data()) + offset); + s3fs.Read(offset, buf_len, (char *)(result->data()) + offset); result_size += read_len; From 9865675ea9685335e777d855a7586b213dfa142d Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Mon, 17 Jan 2022 18:37:48 +0800 Subject: [PATCH 023/153] thorough S3FileLister tests --- test/test_remote_io.py | 55 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/test/test_remote_io.py b/test/test_remote_io.py index ed8a434f2..3429f0095 100644 --- a/test/test_remote_io.py +++ b/test/test_remote_io.py @@ -8,7 +8,7 @@ from _utils._common_utils_for_test import check_hash_fn, create_temp_dir -from torchdata.datapipes.iter import EndOnDiskCacheHolder, FileOpener, HttpReader, IterableWrapper, OnDiskCacheHolder +from torchdata.datapipes.iter import EndOnDiskCacheHolder, FileOpener, HttpReader, IterableWrapper, OnDiskCacheHolder, S3FileLister, S3FileLoader class TestDataPipeRemoteIO(expecttest.TestCase): @@ -161,13 +161,52 @@ def _read_and_decode(x): self.assertTrue(os.path.exists(expected_csv_path)) self.assertEqual(expected_csv_path, csv_path) - def test_s3_lister_iterdatapipe(self): - file_urls = ["s3://ai2-public-datasets/charades"] - s3_lister_dp = S3FileLister(IterableWrapper(file_urls)) - count = 0 - for item in s3_lister_dp: - count = count + 1 - print("number of items:", count) + def test_s3_io_iterdatapipe(self): + # S3FileLister: different inputs + input_list = [ + [["s3://ai2-public-datasets"], 71], # bucket without '/' + [["s3://ai2-public-datasets/"], 71], # bucket with '/' + [["s3://ai2-public-datasets/charades"], 18], # folder without '/' + [["s3://ai2-public-datasets/charades/"], 18], # folder without '/' + [["s3://ai2-public-datasets/charad"], 18], # prefix + [["s3://ai2-public-datasets/charades/Charades_v1", + "s3://ai2-public-datasets/charades/Charades_vu17", ], 12], # prefixes + [["s3://ai2-public-datasets/charades/Charades_v1.zip"], 1], # single file + [["s3://ai2-public-datasets/charades/Charades_v1.zip", + "s3://ai2-public-datasets/charades/Charades_v1_flow.tar", + "s3://ai2-public-datasets/charades/Charades_v1_rgb.tar", + "s3://ai2-public-datasets/charades/Charades_v1_480.zip", ], 4], # multiple files + [["s3://ai2-public-datasets/charades/Charades_v1.zip", + "s3://ai2-public-datasets/charades/Charades_v1_flow.tar", + "s3://ai2-public-datasets/charades/Charades_v1_rgb.tar", + "s3://ai2-public-datasets/charades/Charades_v1_480.zip", + "s3://ai2-public-datasets/charades/Charades_vu17", ], 10], # files + prefixes + ] + for input in input_list: + s3_lister_dp = S3FileLister(IterableWrapper(input[0]), region="us-west-2") + self.assertEqual(sum(1 for _ in s3_lister_dp), input[1], f'{input[0]} failed') + + # S3FileLister: prefixes + different region + file_urls = ["s3://aft-vbi-pds/bin-images/111", "s3://aft-vbi-pds/bin-images/222", ] + s3_lister_dp = S3FileLister(IterableWrapper(file_urls), region="us-east-1") + self.assertEqual(sum(1 for _ in s3_lister_dp), 2212, f'{input[0]} failed') + + # S3FileLister: incorrect inputs + input_list = [ + [""], + ["ai2-public-datasets"], + ["s3://"], + ["s3:///bin-images"], + ] + for input in input_list: + with self.assertRaises(ValueError, msg=f"{input} should raise ValueError."): + s3_lister_dp = S3FileLister(IterableWrapper(file_urls)) + for _ in s3_lister_dp: + pass + + # S3FileLoader: loader + + # integration test if __name__ == "__main__": From c72a3fb0918bab3c7b06ff2d36058bc36dfd2552 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Mon, 17 Jan 2022 21:31:27 +0800 Subject: [PATCH 024/153] expose buffer size and multi part download api's --- torchdata/csrc/pybind/S3Handler/S3Handler.cpp | 20 +++++++++---------- torchdata/csrc/pybind/S3Handler/S3Handler.h | 2 ++ torchdata/csrc/pybind/S3Handler/pybind.cpp | 10 ++++++++++ torchdata/datapipes/iter/load/s3io.py | 8 ++++++-- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp index 08740d6fd..0e4b3615f 100644 --- a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp +++ b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp @@ -29,8 +29,8 @@ namespace torchdata { namespace { - static const size_t s3ReadBufferSize = 120 * 1024 * 1024; // 120 MB - static const uint64_t s3MultiPartDownloadChunkSize = 50 * 1024 * 1024; // 50 MB + static const size_t S3DefaultBufferSize = 120 * 1024 * 1024; // 120 MB + static const uint64_t S3DefaultMultiPartDownloadChunkSize = 50 * 1024 * 1024; // 50 MB static const int executorPoolSize = 25; static const int S3DefaultMaxKeys = 1000; static const std::string S3DefaultMarker = ""; @@ -279,19 +279,19 @@ namespace torchdata initialization_lock_() { // Load reading parameters - buffer_size_ = s3ReadBufferSize; + buffer_size_ = S3DefaultBufferSize; const char *bufferSizeStr = getenv("S3_BUFFER_SIZE"); if (bufferSizeStr) { buffer_size_ = std::stoull(bufferSizeStr); } multi_part_download_ = true; - const char *multi_download_disable_char = - getenv("S3_DISABLE_MULTI_PART_DOWNLOAD"); - if (multi_download_disable_char) + const char *multi_part_download_char = + getenv("S3_MULTI_PART_DOWNLOAD"); + if (multi_part_download_char) { - std::string multi_download_disable_str(multi_download_disable_char); - if (multi_download_disable_str == "ON") + std::string multi_download_disable_str(multi_part_download_char); + if (multi_download_disable_str == "OFF") { multi_part_download_ = false; } @@ -335,9 +335,9 @@ namespace torchdata this->GetExecutor().get()); transfer_config.s3Client = s3_client; // This buffer is what we used to initialize streambuf and is in memory - transfer_config.bufferSize = s3MultiPartDownloadChunkSize; + transfer_config.bufferSize = S3DefaultMultiPartDownloadChunkSize; transfer_config.transferBufferMaxHeapSize = - (executorPoolSize + 1) * s3MultiPartDownloadChunkSize; + (executorPoolSize + 1) * S3DefaultMultiPartDownloadChunkSize; this->transfer_manager_ = Aws::Transfer::TransferManager::Create(transfer_config); } diff --git a/torchdata/csrc/pybind/S3Handler/S3Handler.h b/torchdata/csrc/pybind/S3Handler/S3Handler.h index 4f126a871..e8d19c7f4 100644 --- a/torchdata/csrc/pybind/S3Handler/S3Handler.h +++ b/torchdata/csrc/pybind/S3Handler/S3Handler.h @@ -53,6 +53,8 @@ namespace torchdata ~S3Handler(); void SetMaxKeys(const int max_keys) { this->max_keys_ = max_keys; } + void SetBufferSize(const uint64_t buffer_size) { this->buffer_size_ = buffer_size; } + void SetMultiPartDownload(const bool multi_part_download) { this->multi_part_download_ = multi_part_download; } void ClearMarker(); void S3Read(const std::string &file_url, std::string *result); diff --git a/torchdata/csrc/pybind/S3Handler/pybind.cpp b/torchdata/csrc/pybind/S3Handler/pybind.cpp index b3fc7294c..3c0680898 100644 --- a/torchdata/csrc/pybind/S3Handler/pybind.cpp +++ b/torchdata/csrc/pybind/S3Handler/pybind.cpp @@ -31,6 +31,16 @@ PYBIND11_MODULE(_torchdata, m) { self->SetMaxKeys(max_keys); }) + .def("set_buffer_size", + [](S3Handler *self, const uint64_t buffer_size) + { + self->SetBufferSize(buffer_size); + }) + .def("set_multi_part_download", + [](S3Handler *self, const bool multi_part_download) + { + self->SetMultiPartDownload(multi_part_download); + }) .def("clear_marker", [](S3Handler *self) { diff --git a/torchdata/datapipes/iter/load/s3io.py b/torchdata/datapipes/iter/load/s3io.py index 132456f62..49a0ea72c 100644 --- a/torchdata/datapipes/iter/load/s3io.py +++ b/torchdata/datapipes/iter/load/s3io.py @@ -60,9 +60,13 @@ class S3FileLoaderIterDataPipe(IterDataPipe[Tuple[str, StreamWrapper]]): AWS_CPP_SDK is necessary to use the S3 DataPipe(s). """ - def __init__(self, source_datapipe: IterDataPipe[str], requestTimeoutMs=-1, region="") -> None: + def __init__(self, source_datapipe: IterDataPipe[str], request_timeout_ms=-1, region="", buffer_size=None, multi_part_download=None) -> None: self.source_datapipe: IterDataPipe[str] = source_datapipe - self.handler = torchdata._torchdata.S3Handler(requestTimeoutMs, region) + self.handler = torchdata._torchdata.S3Handler(request_timeout_ms, region) + if buffer_size: + self.handler.set_buffer_size(buffer_size) + if multi_part_download: + self.handler.set_multi_part_download(multi_part_download) def __iter__(self) -> Iterator[Tuple[str, StreamWrapper]]: for url in self.source_datapipe: From 1d645ac88dfc446be71e34678ba9f5ec4d9d9eb7 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Mon, 17 Jan 2022 23:07:52 +0800 Subject: [PATCH 025/153] fix s3 read, add BytesIO to pass file, improve read speed --- torchdata/csrc/pybind/S3Handler/S3Handler.cpp | 3 +-- torchdata/datapipes/iter/load/s3io.py | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp index 0e4b3615f..e957a3efd 100644 --- a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp +++ b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp @@ -30,7 +30,7 @@ namespace torchdata namespace { static const size_t S3DefaultBufferSize = 120 * 1024 * 1024; // 120 MB - static const uint64_t S3DefaultMultiPartDownloadChunkSize = 50 * 1024 * 1024; // 50 MB + static const uint64_t S3DefaultMultiPartDownloadChunkSize = 5 * 1024 * 1024; // 5 MB static const int executorPoolSize = 25; static const int S3DefaultMaxKeys = 1000; static const std::string S3DefaultMarker = ""; @@ -320,7 +320,6 @@ namespace torchdata void S3Handler::InitializeExecutor() { - std::lock_guard lock(this->initialization_lock_); this->executor_ = Aws::MakeShared( "executor", executorPoolSize); diff --git a/torchdata/datapipes/iter/load/s3io.py b/torchdata/datapipes/iter/load/s3io.py index 49a0ea72c..e755b766d 100644 --- a/torchdata/datapipes/iter/load/s3io.py +++ b/torchdata/datapipes/iter/load/s3io.py @@ -1,3 +1,4 @@ +from io import BytesIO from typing import Iterator, Tuple import torchdata @@ -70,7 +71,7 @@ def __init__(self, source_datapipe: IterDataPipe[str], request_timeout_ms=-1, re def __iter__(self) -> Iterator[Tuple[str, StreamWrapper]]: for url in self.source_datapipe: - yield url, self.handler.s3_read(url) + yield url, StreamWrapper(BytesIO(self.handler.s3_read(url))) def __len__(self) -> int: return len(self.source_datapipe) From ba58e6833972cc2dd695388a43ad8eb32fe4f00c Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Mon, 17 Jan 2022 23:37:17 +0800 Subject: [PATCH 026/153] improve reading efficiency, set default buffer size to 128MB --- torchdata/csrc/pybind/S3Handler/S3Handler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp index e957a3efd..cf6c6989c 100644 --- a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp +++ b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp @@ -29,7 +29,7 @@ namespace torchdata { namespace { - static const size_t S3DefaultBufferSize = 120 * 1024 * 1024; // 120 MB + static const size_t S3DefaultBufferSize = 128 * 1024 * 1024; // 128 MB static const uint64_t S3DefaultMultiPartDownloadChunkSize = 5 * 1024 * 1024; // 5 MB static const int executorPoolSize = 25; static const int S3DefaultMaxKeys = 1000; From 7d1b5f887575153b9aeb40d147ca79e69ed6ecfb Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Tue, 18 Jan 2022 17:26:59 +0800 Subject: [PATCH 027/153] rename multi_part_download to use_multi_part_download --- torchdata/csrc/pybind/S3Handler/S3Handler.cpp | 24 +++++++++---------- torchdata/csrc/pybind/S3Handler/S3Handler.h | 6 ++--- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp index cf6c6989c..74e60ce69 100644 --- a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp +++ b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp @@ -29,7 +29,7 @@ namespace torchdata { namespace { - static const size_t S3DefaultBufferSize = 128 * 1024 * 1024; // 128 MB + static const size_t S3DefaultBufferSize = 128 * 1024 * 1024; // 128 MB static const uint64_t S3DefaultMultiPartDownloadChunkSize = 5 * 1024 * 1024; // 5 MB static const int executorPoolSize = 25; static const int S3DefaultMaxKeys = 1000; @@ -171,24 +171,24 @@ namespace torchdata private: std::string bucket_name_; std::string object_name_; - bool multi_part_download_; + bool use_multi_part_download_; std::shared_ptr s3_client_; std::shared_ptr transfer_manager_; public: S3FS(const std::string &bucket, const std::string &object, - const bool multi_part_download, + const bool use_multi_part_download, std::shared_ptr transfer_manager, std::shared_ptr s3_client) : bucket_name_(bucket), object_name_(object), - multi_part_download_(multi_part_download), + use_multi_part_download_(use_multi_part_download), transfer_manager_(transfer_manager), s3_client_(s3_client) {} size_t Read(uint64_t offset, size_t n, char *buffer) { - if (multi_part_download_) + if (use_multi_part_download_) { return ReadTransferManager(offset, n, buffer); } @@ -285,15 +285,15 @@ namespace torchdata { buffer_size_ = std::stoull(bufferSizeStr); } - multi_part_download_ = true; - const char *multi_part_download_char = + use_multi_part_download_ = true; + const char *use_multi_part_download_char = getenv("S3_MULTI_PART_DOWNLOAD"); - if (multi_part_download_char) + if (use_multi_part_download_char) { - std::string multi_download_disable_str(multi_part_download_char); - if (multi_download_disable_str == "OFF") + std::string use_multi_part_download_str(use_multi_part_download_char); + if (use_multi_part_download_str == "OFF") { - multi_part_download_ = false; + use_multi_part_download_ = false; } } @@ -393,7 +393,7 @@ namespace torchdata { std::string bucket, object; parseS3Path(file_url, &bucket, &object); - S3FS s3fs(bucket, object, multi_part_download_, + S3FS s3fs(bucket, object, use_multi_part_download_, GetTransferManager(), GetS3Client()); uint64_t offset = 0; diff --git a/torchdata/csrc/pybind/S3Handler/S3Handler.h b/torchdata/csrc/pybind/S3Handler/S3Handler.h index e8d19c7f4..352e5c74b 100644 --- a/torchdata/csrc/pybind/S3Handler/S3Handler.h +++ b/torchdata/csrc/pybind/S3Handler/S3Handler.h @@ -34,9 +34,9 @@ namespace torchdata Aws::String last_marker_; int max_keys_; - std::mutex initialization_lock_; size_t buffer_size_; - bool multi_part_download_; + bool use_multi_part_download_; + std::mutex initialization_lock_; void InitializeS3Client(); void InitializeExecutor(); @@ -54,7 +54,7 @@ namespace torchdata void SetMaxKeys(const int max_keys) { this->max_keys_ = max_keys; } void SetBufferSize(const uint64_t buffer_size) { this->buffer_size_ = buffer_size; } - void SetMultiPartDownload(const bool multi_part_download) { this->multi_part_download_ = multi_part_download; } + void SetMultiPartDownload(const bool multi_part_download) { this->use_multi_part_download_ = multi_part_download; } void ClearMarker(); void S3Read(const std::string &file_url, std::string *result); From db5600df49d167eb547404536465b39f3cf4ef0e Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 19 Jan 2022 00:42:14 +0800 Subject: [PATCH 028/153] Make S3Handler Pickleable --- torchdata/csrc/pybind/S3Handler/S3Handler.cpp | 74 ++++++++++--------- torchdata/csrc/pybind/S3Handler/S3Handler.h | 9 ++- torchdata/csrc/pybind/S3Handler/pybind.cpp | 41 +++++++++- 3 files changed, 86 insertions(+), 38 deletions(-) diff --git a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp index 74e60ce69..23ed70728 100644 --- a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp +++ b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp @@ -202,8 +202,8 @@ namespace torchdata { Aws::S3::Model::GetObjectRequest getObjectRequest; - getObjectRequest.WithBucket(this->bucket_name_.c_str()) - .WithKey(this->object_name_.c_str()); + getObjectRequest.WithBucket(bucket_name_.c_str()) + .WithKey(object_name_.c_str()); std::string bytes = "bytes="; bytes += std::to_string(offset) + "-" + std::to_string(offset + n - 1); @@ -217,7 +217,7 @@ namespace torchdata []() { return Aws::New("S3IOAllocationTag"); }); // get the object - Aws::S3::Model::GetObjectOutcome getObjectOutcome = this->s3_client_->GetObject(getObjectRequest); + Aws::S3::Model::GetObjectOutcome getObjectOutcome = s3_client_->GetObject(getObjectRequest); if (!getObjectOutcome.IsSuccess()) { @@ -246,8 +246,8 @@ namespace torchdata }; // This buffer is what we used to initialize streambuf and is in memory std::shared_ptr downloadHandle = - this->transfer_manager_.get()->DownloadFile( - this->bucket_name_.c_str(), this->object_name_.c_str(), offset, + transfer_manager_.get()->DownloadFile( + bucket_name_.c_str(), object_name_.c_str(), offset, n, create_stream_fn); downloadHandle->WaitUntilFinished(); @@ -275,9 +275,10 @@ namespace torchdata S3Handler::S3Handler(const long requestTimeoutMs, const std::string region) : s3_client_(nullptr, ShutdownClient), transfer_manager_(nullptr, ShutdownTransferManager), - executor_(nullptr, ShutdownExecutor), - initialization_lock_() + executor_(nullptr, ShutdownExecutor) { + initialization_lock_ = std::shared_ptr(new std::mutex()); + // Load reading parameters buffer_size_ = S3DefaultBufferSize; const char *bufferSizeStr = getenv("S3_BUFFER_SIZE"); @@ -302,25 +303,26 @@ namespace torchdata S3Handler::s3_handler_cfg_ = setUpS3Config(requestTimeoutMs, region); InitializeS3Client(); - this->max_keys_ = S3DefaultMaxKeys; - this->last_marker_ = S3DefaultMarker; + max_keys_ = S3DefaultMaxKeys; + last_marker_ = S3DefaultMarker; } S3Handler::~S3Handler() {} void S3Handler::InitializeS3Client() { - std::lock_guard lock(this->initialization_lock_); - this->s3_client_ = - std::shared_ptr(new Aws::S3::S3Client( - *S3Handler::s3_handler_cfg_, - Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, - false)); + std::lock_guard lock(*initialization_lock_); + s3_client_ = + std::shared_ptr( + new Aws::S3::S3Client( + *S3Handler::s3_handler_cfg_, + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, + false)); } void S3Handler::InitializeExecutor() { - this->executor_ = + executor_ = Aws::MakeShared( "executor", executorPoolSize); } @@ -328,46 +330,46 @@ namespace torchdata void S3Handler::InitializeTransferManager() { std::shared_ptr s3_client = GetS3Client(); - std::lock_guard lock(this->initialization_lock_); + std::lock_guard lock(*initialization_lock_); Aws::Transfer::TransferManagerConfiguration transfer_config( - this->GetExecutor().get()); + GetExecutor().get()); transfer_config.s3Client = s3_client; // This buffer is what we used to initialize streambuf and is in memory transfer_config.bufferSize = S3DefaultMultiPartDownloadChunkSize; transfer_config.transferBufferMaxHeapSize = (executorPoolSize + 1) * S3DefaultMultiPartDownloadChunkSize; - this->transfer_manager_ = + transfer_manager_ = Aws::Transfer::TransferManager::Create(transfer_config); } std::shared_ptr S3Handler::GetS3Client() { - if (this->s3_client_.get() == nullptr) + if (s3_client_.get() == nullptr) { - this->InitializeS3Client(); + InitializeS3Client(); } - return this->s3_client_; + return s3_client_; } std::shared_ptr S3Handler::GetExecutor() { - if (this->executor_.get() == nullptr) + if (executor_.get() == nullptr) { - this->InitializeExecutor(); + InitializeExecutor(); } - return this->executor_; + return executor_; } std::shared_ptr S3Handler::GetTransferManager() { - if (this->transfer_manager_.get() == nullptr) + if (transfer_manager_.get() == nullptr) { - this->InitializeTransferManager(); + InitializeTransferManager(); } - return this->transfer_manager_; + return transfer_manager_; } size_t S3Handler::GetFileSize(const std::string &bucket, @@ -376,7 +378,7 @@ namespace torchdata Aws::S3::Model::HeadObjectRequest headObjectRequest; headObjectRequest.WithBucket(bucket.c_str()).WithKey(object.c_str()); Aws::S3::Model::HeadObjectOutcome headObjectOutcome = - this->GetS3Client()->HeadObject(headObjectRequest); + GetS3Client()->HeadObject(headObjectRequest); if (headObjectOutcome.IsSuccess()) { return headObjectOutcome.GetResult().GetContentLength(); @@ -387,7 +389,7 @@ namespace torchdata return 0; } - void S3Handler::ClearMarker() { this->last_marker_ = S3DefaultMarker; } + void S3Handler::ClearMarker() { last_marker_ = S3DefaultMarker; } void S3Handler::S3Read(const std::string &file_url, std::string *result) { @@ -398,7 +400,7 @@ namespace torchdata uint64_t offset = 0; uint64_t result_size = 0; - uint64_t file_size = this->GetFileSize(bucket, object); + uint64_t file_size = GetFileSize(bucket, object); size_t part_count = (std::max)( static_cast((file_size + buffer_size_ - 1) / buffer_size_), static_cast(1)); @@ -437,11 +439,11 @@ namespace torchdata Aws::S3::Model::ListObjectsRequest listObjectsRequest; listObjectsRequest.WithBucket(bucket) .WithPrefix(prefix) - .WithMaxKeys(this->max_keys_) - .WithMarker(this->last_marker_); + .WithMaxKeys(max_keys_) + .WithMarker(last_marker_); Aws::S3::Model::ListObjectsOutcome listObjectsOutcome = - this->GetS3Client()->ListObjects(listObjectsRequest); + GetS3Client()->ListObjects(listObjectsRequest); if (!listObjectsOutcome.IsSuccess()) { Aws::String const &error_aws = @@ -463,12 +465,12 @@ namespace torchdata Aws::String entry = "s3://" + bucket + "/" + object.GetKey(); filenames->push_back(entry.c_str()); } - this->last_marker_ = objects.back().GetKey(); + last_marker_ = objects.back().GetKey(); // extreme cases when all objects are folders if (filenames->size() == 0) { - this->ListFiles(file_url, filenames); + ListFiles(file_url, filenames); } } } // namespace torchdata diff --git a/torchdata/csrc/pybind/S3Handler/S3Handler.h b/torchdata/csrc/pybind/S3Handler/S3Handler.h index 352e5c74b..76dc5c538 100644 --- a/torchdata/csrc/pybind/S3Handler/S3Handler.h +++ b/torchdata/csrc/pybind/S3Handler/S3Handler.h @@ -28,6 +28,7 @@ namespace torchdata private: static std::shared_ptr s3_handler_cfg_; + std::shared_ptr initialization_lock_; std::shared_ptr s3_client_; std::shared_ptr executor_; std::shared_ptr transfer_manager_; @@ -36,7 +37,6 @@ namespace torchdata int max_keys_; size_t buffer_size_; bool use_multi_part_download_; - std::mutex initialization_lock_; void InitializeS3Client(); void InitializeExecutor(); @@ -57,6 +57,13 @@ namespace torchdata void SetMultiPartDownload(const bool multi_part_download) { this->use_multi_part_download_ = multi_part_download; } void ClearMarker(); + long GetRequestTimeoutMs() const { return s3_handler_cfg_->requestTimeoutMs; } + Aws::String GetRegion() const { return s3_handler_cfg_->region; } + Aws::String GetLastMarker() const { return last_marker_; } + int GetMaxKeys() const { return max_keys_; } + bool GetUseMultiPartDownload() const { return use_multi_part_download_; } + size_t GetBufferSize() const { return buffer_size_; } + void S3Read(const std::string &file_url, std::string *result); void ListFiles(const std::string &file_url, std::vector *filenames); diff --git a/torchdata/csrc/pybind/S3Handler/pybind.cpp b/torchdata/csrc/pybind/S3Handler/pybind.cpp index 3c0680898..242c44f60 100644 --- a/torchdata/csrc/pybind/S3Handler/pybind.cpp +++ b/torchdata/csrc/pybind/S3Handler/pybind.cpp @@ -45,5 +45,44 @@ PYBIND11_MODULE(_torchdata, m) [](S3Handler *self) { self->ClearMarker(); - }); + }) + .def(py::pickle( + [](const S3Handler &s3_handler) { // __getstate__ + /* Return a tuple that fully encodes the state of the object */ + return py::make_tuple(s3_handler.GetRequestTimeoutMs(), + s3_handler.GetRegion(), + s3_handler.GetLastMarker(), + s3_handler.GetMaxKeys(), + s3_handler.GetUseMultiPartDownload(), + s3_handler.GetBufferSize()); + }, + [](py::tuple t) { // __setstate__ + if (t.size() != 6) + throw std::runtime_error("Invalid state!"); + + /* Create a new C++ instance */ + S3Handler s3_handler(t[0].cast(), t[1].cast()); + + /* Assign any additional state */ + s3_handler.SetMaxKeys(t[4].cast()); + + return s3_handler; + })); + // .def(py::pickle( + // [](const Pickleable &p) { // __getstate__ + // /* Return a tuple that fully encodes the state of the object */ + // return py::make_tuple(p.value(), p.extra()); + // }, + // [](py::tuple t) { // __setstate__ + // if (t.size() != 2) + // throw std::runtime_error("Invalid state!"); + + // /* Create a new C++ instance */ + // Pickleable p(t[0].cast()); + + // /* Assign any additional state */ + // p.setExtra(t[1].cast()); + + // return p; + // })); } From 1a915b07249b83883101f4688f570e5f09f8ee13 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Thu, 3 Feb 2022 00:14:38 +0800 Subject: [PATCH 029/153] Remove max_keys api in S3FileLister and clean up code --- torchdata/csrc/pybind/S3Handler/S3Handler.cpp | 3 -- torchdata/csrc/pybind/S3Handler/S3Handler.h | 4 +-- torchdata/csrc/pybind/S3Handler/pybind.cpp | 29 +++---------------- torchdata/datapipes/iter/load/s3io.py | 4 +-- 4 files changed, 6 insertions(+), 34 deletions(-) diff --git a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp index 23ed70728..03b8f3ff2 100644 --- a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp +++ b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp @@ -32,7 +32,6 @@ namespace torchdata static const size_t S3DefaultBufferSize = 128 * 1024 * 1024; // 128 MB static const uint64_t S3DefaultMultiPartDownloadChunkSize = 5 * 1024 * 1024; // 5 MB static const int executorPoolSize = 25; - static const int S3DefaultMaxKeys = 1000; static const std::string S3DefaultMarker = ""; std::shared_ptr setUpS3Config(const long requestTimeoutMs, const std::string region) @@ -303,7 +302,6 @@ namespace torchdata S3Handler::s3_handler_cfg_ = setUpS3Config(requestTimeoutMs, region); InitializeS3Client(); - max_keys_ = S3DefaultMaxKeys; last_marker_ = S3DefaultMarker; } @@ -439,7 +437,6 @@ namespace torchdata Aws::S3::Model::ListObjectsRequest listObjectsRequest; listObjectsRequest.WithBucket(bucket) .WithPrefix(prefix) - .WithMaxKeys(max_keys_) .WithMarker(last_marker_); Aws::S3::Model::ListObjectsOutcome listObjectsOutcome = diff --git a/torchdata/csrc/pybind/S3Handler/S3Handler.h b/torchdata/csrc/pybind/S3Handler/S3Handler.h index 76dc5c538..4529aabae 100644 --- a/torchdata/csrc/pybind/S3Handler/S3Handler.h +++ b/torchdata/csrc/pybind/S3Handler/S3Handler.h @@ -34,7 +34,6 @@ namespace torchdata std::shared_ptr transfer_manager_; Aws::String last_marker_; - int max_keys_; size_t buffer_size_; bool use_multi_part_download_; @@ -52,7 +51,7 @@ namespace torchdata S3Handler(const long requestTimeoutMs, const std::string region); ~S3Handler(); - void SetMaxKeys(const int max_keys) { this->max_keys_ = max_keys; } + void SetLastMarker(const Aws::String last_marker) { this->last_marker_ = last_marker; } void SetBufferSize(const uint64_t buffer_size) { this->buffer_size_ = buffer_size; } void SetMultiPartDownload(const bool multi_part_download) { this->use_multi_part_download_ = multi_part_download; } void ClearMarker(); @@ -60,7 +59,6 @@ namespace torchdata long GetRequestTimeoutMs() const { return s3_handler_cfg_->requestTimeoutMs; } Aws::String GetRegion() const { return s3_handler_cfg_->region; } Aws::String GetLastMarker() const { return last_marker_; } - int GetMaxKeys() const { return max_keys_; } bool GetUseMultiPartDownload() const { return use_multi_part_download_; } size_t GetBufferSize() const { return buffer_size_; } diff --git a/torchdata/csrc/pybind/S3Handler/pybind.cpp b/torchdata/csrc/pybind/S3Handler/pybind.cpp index 242c44f60..d27baa6a0 100644 --- a/torchdata/csrc/pybind/S3Handler/pybind.cpp +++ b/torchdata/csrc/pybind/S3Handler/pybind.cpp @@ -26,11 +26,6 @@ PYBIND11_MODULE(_torchdata, m) self->ListFiles(file_url, &filenames); return filenames; }) - .def("set_max_keys", - [](S3Handler *self, const int max_keys) - { - self->SetMaxKeys(max_keys); - }) .def("set_buffer_size", [](S3Handler *self, const uint64_t buffer_size) { @@ -52,37 +47,21 @@ PYBIND11_MODULE(_torchdata, m) return py::make_tuple(s3_handler.GetRequestTimeoutMs(), s3_handler.GetRegion(), s3_handler.GetLastMarker(), - s3_handler.GetMaxKeys(), s3_handler.GetUseMultiPartDownload(), s3_handler.GetBufferSize()); }, [](py::tuple t) { // __setstate__ - if (t.size() != 6) + if (t.size() != 5) throw std::runtime_error("Invalid state!"); /* Create a new C++ instance */ S3Handler s3_handler(t[0].cast(), t[1].cast()); /* Assign any additional state */ - s3_handler.SetMaxKeys(t[4].cast()); + s3_handler.SetLastMarker(t[2].cast()); + s3_handler.SetMultiPartDownload(t[3].cast()); + s3_handler.SetBufferSize(t[4].cast()); return s3_handler; })); - // .def(py::pickle( - // [](const Pickleable &p) { // __getstate__ - // /* Return a tuple that fully encodes the state of the object */ - // return py::make_tuple(p.value(), p.extra()); - // }, - // [](py::tuple t) { // __setstate__ - // if (t.size() != 2) - // throw std::runtime_error("Invalid state!"); - - // /* Create a new C++ instance */ - // Pickleable p(t[0].cast()); - - // /* Assign any additional state */ - // p.setExtra(t[1].cast()); - - // return p; - // })); } diff --git a/torchdata/datapipes/iter/load/s3io.py b/torchdata/datapipes/iter/load/s3io.py index e755b766d..2a3d687ce 100644 --- a/torchdata/datapipes/iter/load/s3io.py +++ b/torchdata/datapipes/iter/load/s3io.py @@ -23,12 +23,10 @@ class S3FileListerIterDataPipe(IterDataPipe[str]): AWS_CPP_SDK is necessary to use the S3 DataPipe(s). """ - def __init__(self, source_datapipe: IterDataPipe[str], length: int = -1, request_timeout_ms=-1, region="", max_keys=None) -> None: + def __init__(self, source_datapipe: IterDataPipe[str], length: int = -1, request_timeout_ms=-1, region="") -> None: self.source_datapipe: IterDataPipe[str] = source_datapipe self.length: int = length self.handler = torchdata._torchdata.S3Handler(request_timeout_ms, region) - if max_keys: - self.handler.set_max_keys(max_keys) def __iter__(self) -> Iterator[str]: for prefix in self.source_datapipe: From 8cb884935603db3741c1915cd286a2a1b228e39c Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Thu, 3 Feb 2022 00:16:32 +0800 Subject: [PATCH 030/153] add S3 IO readme --- torchdata/datapipes/iter/load/README.md | 66 +++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 torchdata/datapipes/iter/load/README.md diff --git a/torchdata/datapipes/iter/load/README.md b/torchdata/datapipes/iter/load/README.md new file mode 100644 index 000000000..e53026b08 --- /dev/null +++ b/torchdata/datapipes/iter/load/README.md @@ -0,0 +1,66 @@ +# S3 IO Datapipe Documentation + +## Installation + +Torchdata S3 IO datapipes depends on [aws-sdk-cpp](https://github.com/aws/aws-sdk-cpp). The following is just a recommended way to installing aws-sdk-cpp, please refer to official documentation for detailed instructions. + +```bash +git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp +cd aws-sdk-cpp/ +mkdir sdk-build +cd sdk-build +cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" +make +make install # may need sudo +``` + +`ninja` and `pybind11` are also required to link PyThon implementation to C++ source code. + +```bash +conda install ninja pybind11 +``` + +S3 IO datapipes are't included when building by default. To build S3 IO in `torchdata`, at the `/data` root folder, run the following commands. + +```bash +export BUILD_S3=ON +pip uninstall torchdata -y +python setup.py clean +python setup.py install +``` + +## Using S3 IO datapies + +### S3FileLister + +`S3FileLister` accepts a list of S3 prefixes and iterates all matching s3 urls. The functional API is `list_file_by_s3`. Acceptable prefixes include `s3://bucket-name`, `s3://bucket-name/`, `s3://bucket-name/folder`, `s3://bucket-name/folder/`, and `s3://bucket-name/prefix`. You may also set `length`, `request_timeout_ms` (default 3000 ms in aws-sdk-cpp), and `region`. Note that: + +1. Input __must__ be a list and direct S3 URLs are skipped. +2. `length` is `-1` by default, and any call to `__len__()` is invalid, because the length is unknown until all files are iterated. +3. `request_timeout_ms` and `region` will overwrite settings in the configuration file or environment variables. + +### S3FileLoader + +`S3FileLoader` accepts a list of S3 URLs and iterates all files in `BytesIO` format with `(url, BytesIO)` tuples. The functional API is `load_file_by_s3`. You may also set `request_timeout_ms` (default 3000 ms in aws-sdk-cpp), `region`, `buffer_size` (default 120Mb), and `multi_part_download` (default to use multi-part downloading). Note that: + +1. Input __must__ be a list and S3 URLs must be valid. +2. `request_timeout_ms` and `region` will overwrite settings in the configuration file or environment variables. + +### Example + +```py +from torchdata.datapipes.iter import S3FileLister, S3FileLoader + +s3_prefixes = ['s3://bucket-name/folder/', ...] +dp_s3_urls = S3FileLister(s3_prefixes) +dp_s3_files = S3FileLoader(s3_urls) # outputs in (url, BytesIO) +# more datapipes to convert loaded bytes, e.g. +datapipe = StreamWrapper(dp_s3_files).parse_csv_files(delimiter=' ') + +for d in datapipe: # Start loading data + pass +``` + +### Note + +It's recommended to set up a detailed configuration file with the `AWS_CONFIG_FILE` environment variable. The following environment variables are also parsed: `HOME`, `S3_USE_HTTPS`, `S3_VERIFY_SSL`, `S3_ENDPOINT_URL`, `AWS_REGION` (would be overwritten by the `region` variable). \ No newline at end of file From d7414bcf8ea86d4002a451acd869d6d1dedf4351 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Thu, 3 Feb 2022 00:23:44 +0800 Subject: [PATCH 031/153] add S3FileLoader and integration tests --- test/test_remote_io.py | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/test/test_remote_io.py b/test/test_remote_io.py index 3429f0095..9969e7ac2 100644 --- a/test/test_remote_io.py +++ b/test/test_remote_io.py @@ -189,7 +189,7 @@ def test_s3_io_iterdatapipe(self): # S3FileLister: prefixes + different region file_urls = ["s3://aft-vbi-pds/bin-images/111", "s3://aft-vbi-pds/bin-images/222", ] s3_lister_dp = S3FileLister(IterableWrapper(file_urls), region="us-east-1") - self.assertEqual(sum(1 for _ in s3_lister_dp), 2212, f'{input[0]} failed') + self.assertEqual(sum(1 for _ in s3_lister_dp), 2212, f'{input} failed') # S3FileLister: incorrect inputs input_list = [ @@ -204,9 +204,35 @@ def test_s3_io_iterdatapipe(self): for _ in s3_lister_dp: pass - # S3FileLoader: loader + # S3FileLoader: loader + input = ["s3://charades-tar-shards/charades-video-0.tar", + "s3://charades-tar-shards/charades-video-1.tar", ] # multiple files + s3_loader_dp = S3FileLoader(input, region="us-west-2") + self.assertEqual(sum(1 for _ in s3_loader_dp), 2, f'{input} failed') + + input = [["s3://aft-vbi-pds/bin-images/100730.jpg"], 1] + s3_loader_dp = S3FileLoader(input[0], region="us-east-1") + self.assertEqual(sum(1 for _ in s3_loader_dp), input[1], f'{input[0]} failed') + + # S3FileLoader: incorrect inputs + input_list = [ + [""], + ["ai2-public-datasets"], + ["s3://"], + ["s3:///bin-images"], + ["s3://ai2-public-datasets/bin-image"], + ] + for input in input_list: + with self.assertRaises(ValueError, msg=f"{input} should raise ValueError."): + s3_loader_dp = S3FileLoader(file_urls) + for _ in s3_loader_dp: + pass # integration test + input = [["s3://charades-tar-shards/"], 10] + s3_lister_dp = S3FileLister(IterableWrapper(input[0]), region="us-west-2") + s3_loader_dp = S3FileLoader(s3_lister_dp, region="us-west-2") + self.assertEqual(sum(1 for _ in s3_loader_dp), input[1], f'{input[0]} failed') if __name__ == "__main__": From 2af94ec9aa0b92a7814c4fdb97d0da5af47ebb82 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Thu, 3 Feb 2022 00:35:36 +0800 Subject: [PATCH 032/153] prettier README.md --- torchdata/datapipes/iter/load/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/torchdata/datapipes/iter/load/README.md b/torchdata/datapipes/iter/load/README.md index e53026b08..1a4bdc3a6 100644 --- a/torchdata/datapipes/iter/load/README.md +++ b/torchdata/datapipes/iter/load/README.md @@ -14,7 +14,7 @@ make make install # may need sudo ``` -`ninja` and `pybind11` are also required to link PyThon implementation to C++ source code. +`ninja` and `pybind11` are also required to link PyThon implementation to C++ source code. ```bash conda install ninja pybind11 @@ -31,7 +31,7 @@ python setup.py install ## Using S3 IO datapies -### S3FileLister +### S3FileLister `S3FileLister` accepts a list of S3 prefixes and iterates all matching s3 urls. The functional API is `list_file_by_s3`. Acceptable prefixes include `s3://bucket-name`, `s3://bucket-name/`, `s3://bucket-name/folder`, `s3://bucket-name/folder/`, and `s3://bucket-name/prefix`. You may also set `length`, `request_timeout_ms` (default 3000 ms in aws-sdk-cpp), and `region`. Note that: @@ -63,4 +63,4 @@ for d in datapipe: # Start loading data ### Note -It's recommended to set up a detailed configuration file with the `AWS_CONFIG_FILE` environment variable. The following environment variables are also parsed: `HOME`, `S3_USE_HTTPS`, `S3_VERIFY_SSL`, `S3_ENDPOINT_URL`, `AWS_REGION` (would be overwritten by the `region` variable). \ No newline at end of file +It's recommended to set up a detailed configuration file with the `AWS_CONFIG_FILE` environment variable. The following environment variables are also parsed: `HOME`, `S3_USE_HTTPS`, `S3_VERIFY_SSL`, `S3_ENDPOINT_URL`, `AWS_REGION` (would be overwritten by the `region` variable). From 47ea59149fe454793968b106ac97cdf9cdf04ed2 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Sat, 5 Feb 2022 00:09:42 +0800 Subject: [PATCH 033/153] add pip3 install cmake ninja to ci --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b69739a6f..8729bb1bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,6 +46,7 @@ jobs: run: | pip3 install -r requirements.txt pip3 install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html + pip3 install cmake ninja - name: Install test requirements run: pip3 install expecttest fsspec iopath==0.1.9 numpy pytest rarfile - name: Build TorchData From aed2af10121dc531bfa68a973b53014b9c231176 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Sat, 5 Feb 2022 00:11:52 +0800 Subject: [PATCH 034/153] remove scipy requirement and update clean command --- setup.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index a43916cef..aa56a5e62 100644 --- a/setup.py +++ b/setup.py @@ -61,8 +61,8 @@ def run(self): # Run default behavior first distutils.command.clean.clean.run(self) - # Remove torchaudio extension - for path in (ROOT_DIR / "torchaudio").glob("**/*.so"): + # Remove torchdata extension + for path in (ROOT_DIR / "torchdata").glob("**/*.so"): print(f"removing '{path}'") path.unlink() # Remove build directory @@ -75,6 +75,7 @@ def run(self): shutil.rmtree(str(path), ignore_errors=True) +<<<<<<< HEAD if __name__ == "__main__": VERSION, SHA = _get_version() _export_version(VERSION, SHA) @@ -108,9 +109,6 @@ def run(self): # Package Info packages=find_packages(exclude=["test*", "examples*"]), zip_safe=False, - extras_require={ - "scipy": ["scipy"], - }, ext_modules=setup_helpers.get_ext_modules(), cmdclass={ "build_ext": setup_helpers.CMakeBuild, From 4436bd25f46fe4c50a2c37d85bbd1645452e994f Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Sat, 5 Feb 2022 00:41:45 +0800 Subject: [PATCH 035/153] add pip3 install pybind11 and BUILD_S3=1 to ci --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8729bb1bb..8370f7675 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,9 +47,11 @@ jobs: pip3 install -r requirements.txt pip3 install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html pip3 install cmake ninja + pip3 install pybind11 - name: Install test requirements run: pip3 install expecttest fsspec iopath==0.1.9 numpy pytest rarfile - name: Build TorchData + env: BUILD_S3=1 run: python setup.py develop - name: Run DataPipes tests with pytest if: ${{ ! contains(github.event.pull_request.labels.*.name, 'ciflow/slow') }} From 6d78f993530c4ca932b247002dc7c56c656f508c Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Sat, 5 Feb 2022 00:50:11 +0800 Subject: [PATCH 036/153] add step to install aws-sdk-cpp for S3 IO datapipes --- .github/workflows/ci.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8370f7675..a54260c11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,6 +50,15 @@ jobs: pip3 install pybind11 - name: Install test requirements run: pip3 install expecttest fsspec iopath==0.1.9 numpy pytest rarfile + - name: Install AWS-SDK-CPP for S3 IO datapipes + run: | + git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp + cd aws-sdk-cpp/ + mkdir sdk-build + cd sdk-build + cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" + make + make install - name: Build TorchData env: BUILD_S3=1 run: python setup.py develop From 739c4edba935276449a19860668fa983c70368e7 Mon Sep 17 00:00:00 2001 From: Daiming Yang <66369380+ydaiming@users.noreply.github.com> Date: Sat, 5 Feb 2022 11:24:26 +0800 Subject: [PATCH 037/153] Update .github/workflows/ci.yml Co-authored-by: Erjia Guan <68879799+ejguan@users.noreply.github.com> --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a54260c11..55f189b66 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,8 +60,9 @@ jobs: make make install - name: Build TorchData - env: BUILD_S3=1 run: python setup.py develop + env: + BUILD_S3=1 - name: Run DataPipes tests with pytest if: ${{ ! contains(github.event.pull_request.labels.*.name, 'ciflow/slow') }} run: From 9711d89d67098897811cb90bb1c6030934bf085d Mon Sep 17 00:00:00 2001 From: Daiming Yang <66369380+ydaiming@users.noreply.github.com> Date: Wed, 9 Feb 2022 09:18:57 +0800 Subject: [PATCH 038/153] Update .github/workflows/ci.yml Co-authored-by: Erjia Guan <68879799+ejguan@users.noreply.github.com> --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 55f189b66..21b5de14c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,8 @@ jobs: - name: Build TorchData run: python setup.py develop env: - BUILD_S3=1 + env: + BUILD_S3: 1 - name: Run DataPipes tests with pytest if: ${{ ! contains(github.event.pull_request.labels.*.name, 'ciflow/slow') }} run: From 9e27382ca334e3d31e329912bc80919fc52e21cc Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Thu, 10 Feb 2022 13:25:16 -0800 Subject: [PATCH 039/153] Add python installed packages path to GITHUB_PATH. --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21b5de14c..43da8ea69 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,6 +48,7 @@ jobs: pip3 install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html pip3 install cmake ninja pip3 install pybind11 + echo "/home/runner/.local/bin" >> $GITHUB_PATH - name: Install test requirements run: pip3 install expecttest fsspec iopath==0.1.9 numpy pytest rarfile - name: Install AWS-SDK-CPP for S3 IO datapipes From 6367a0496c5a42ae92cd0ae6e20e856fcc5132d0 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Thu, 3 Feb 2022 01:22:09 +0000 Subject: [PATCH 040/153] Apply requested changes from lint. --- setup.cfg | 1 - setup.py | 1 - test/test_remote_io.py | 68 +++++++++++++++++-------- tools/setup_helpers/extension.py | 45 ++++++++-------- torchdata/__init__.py | 1 + torchdata/_internal/__init__.py | 2 +- torchdata/datapipes/iter/load/README.md | 26 +++++++--- torchdata/datapipes/iter/load/s3io.py | 14 +++-- 8 files changed, 98 insertions(+), 60 deletions(-) diff --git a/setup.cfg b/setup.cfg index 8e87bf939..5c6311daf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,4 +3,3 @@ universal=1 [metadata] license_file = LICENSE - diff --git a/setup.py b/setup.py index aa56a5e62..87eeedefc 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,6 @@ from setuptools import find_packages, setup from torchdata.datapipes.gen_pyi import gen_pyi - from tools import setup_helpers ROOT_DIR = Path(__file__).parent.resolve() diff --git a/test/test_remote_io.py b/test/test_remote_io.py index 9969e7ac2..bc7cd8e48 100644 --- a/test/test_remote_io.py +++ b/test/test_remote_io.py @@ -8,7 +8,15 @@ from _utils._common_utils_for_test import check_hash_fn, create_temp_dir -from torchdata.datapipes.iter import EndOnDiskCacheHolder, FileOpener, HttpReader, IterableWrapper, OnDiskCacheHolder, S3FileLister, S3FileLoader +from torchdata.datapipes.iter import ( + EndOnDiskCacheHolder, + FileOpener, + HttpReader, + IterableWrapper, + OnDiskCacheHolder, + S3FileLister, + S3FileLoader, +) class TestDataPipeRemoteIO(expecttest.TestCase): @@ -169,27 +177,45 @@ def test_s3_io_iterdatapipe(self): [["s3://ai2-public-datasets/charades"], 18], # folder without '/' [["s3://ai2-public-datasets/charades/"], 18], # folder without '/' [["s3://ai2-public-datasets/charad"], 18], # prefix - [["s3://ai2-public-datasets/charades/Charades_v1", - "s3://ai2-public-datasets/charades/Charades_vu17", ], 12], # prefixes + [ + [ + "s3://ai2-public-datasets/charades/Charades_v1", + "s3://ai2-public-datasets/charades/Charades_vu17", + ], + 12, + ], # prefixes [["s3://ai2-public-datasets/charades/Charades_v1.zip"], 1], # single file - [["s3://ai2-public-datasets/charades/Charades_v1.zip", - "s3://ai2-public-datasets/charades/Charades_v1_flow.tar", - "s3://ai2-public-datasets/charades/Charades_v1_rgb.tar", - "s3://ai2-public-datasets/charades/Charades_v1_480.zip", ], 4], # multiple files - [["s3://ai2-public-datasets/charades/Charades_v1.zip", - "s3://ai2-public-datasets/charades/Charades_v1_flow.tar", - "s3://ai2-public-datasets/charades/Charades_v1_rgb.tar", - "s3://ai2-public-datasets/charades/Charades_v1_480.zip", - "s3://ai2-public-datasets/charades/Charades_vu17", ], 10], # files + prefixes + [ + [ + "s3://ai2-public-datasets/charades/Charades_v1.zip", + "s3://ai2-public-datasets/charades/Charades_v1_flow.tar", + "s3://ai2-public-datasets/charades/Charades_v1_rgb.tar", + "s3://ai2-public-datasets/charades/Charades_v1_480.zip", + ], + 4, + ], # multiple files + [ + [ + "s3://ai2-public-datasets/charades/Charades_v1.zip", + "s3://ai2-public-datasets/charades/Charades_v1_flow.tar", + "s3://ai2-public-datasets/charades/Charades_v1_rgb.tar", + "s3://ai2-public-datasets/charades/Charades_v1_480.zip", + "s3://ai2-public-datasets/charades/Charades_vu17", + ], + 10, + ], # files + prefixes ] for input in input_list: s3_lister_dp = S3FileLister(IterableWrapper(input[0]), region="us-west-2") - self.assertEqual(sum(1 for _ in s3_lister_dp), input[1], f'{input[0]} failed') + self.assertEqual(sum(1 for _ in s3_lister_dp), input[1], f"{input[0]} failed") # S3FileLister: prefixes + different region - file_urls = ["s3://aft-vbi-pds/bin-images/111", "s3://aft-vbi-pds/bin-images/222", ] + file_urls = [ + "s3://aft-vbi-pds/bin-images/111", + "s3://aft-vbi-pds/bin-images/222", + ] s3_lister_dp = S3FileLister(IterableWrapper(file_urls), region="us-east-1") - self.assertEqual(sum(1 for _ in s3_lister_dp), 2212, f'{input} failed') + self.assertEqual(sum(1 for _ in s3_lister_dp), 2212, f"{input} failed") # S3FileLister: incorrect inputs input_list = [ @@ -205,14 +231,16 @@ def test_s3_io_iterdatapipe(self): pass # S3FileLoader: loader - input = ["s3://charades-tar-shards/charades-video-0.tar", - "s3://charades-tar-shards/charades-video-1.tar", ] # multiple files + input = [ + "s3://charades-tar-shards/charades-video-0.tar", + "s3://charades-tar-shards/charades-video-1.tar", + ] # multiple files s3_loader_dp = S3FileLoader(input, region="us-west-2") - self.assertEqual(sum(1 for _ in s3_loader_dp), 2, f'{input} failed') + self.assertEqual(sum(1 for _ in s3_loader_dp), 2, f"{input} failed") input = [["s3://aft-vbi-pds/bin-images/100730.jpg"], 1] s3_loader_dp = S3FileLoader(input[0], region="us-east-1") - self.assertEqual(sum(1 for _ in s3_loader_dp), input[1], f'{input[0]} failed') + self.assertEqual(sum(1 for _ in s3_loader_dp), input[1], f"{input[0]} failed") # S3FileLoader: incorrect inputs input_list = [ @@ -232,7 +260,7 @@ def test_s3_io_iterdatapipe(self): input = [["s3://charades-tar-shards/"], 10] s3_lister_dp = S3FileLister(IterableWrapper(input[0]), region="us-west-2") s3_loader_dp = S3FileLoader(s3_lister_dp, region="us-west-2") - self.assertEqual(sum(1 for _ in s3_loader_dp), input[1], f'{input[0]} failed') + self.assertEqual(sum(1 for _ in s3_loader_dp), input[1], f"{input[0]} failed") if __name__ == "__main__": diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index 52dfa4e74..24325fa77 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -8,8 +8,8 @@ from setuptools.command.build_ext import build_ext __all__ = [ - 'get_ext_modules', - 'CMakeBuild', + "get_ext_modules", + "CMakeBuild", ] _THIS_DIR = Path(__file__).parent.resolve() @@ -20,15 +20,13 @@ def _get_build(var, default=False): if var not in os.environ: return default - val = os.environ.get(var, '0') - trues = ['1', 'true', 'TRUE', 'on', 'ON', 'yes', 'YES'] - falses = ['0', 'false', 'FALSE', 'off', 'OFF', 'no', 'NO'] + val = os.environ.get(var, "0") + trues = ["1", "true", "TRUE", "on", "ON", "yes", "YES"] + falses = ["0", "false", "FALSE", "off", "OFF", "no", "NO"] if val in trues: return True if val not in falses: - print( - f'WARNING: Unexpected environment variable value `{var}={val}`. ' - f'Expected one of {trues + falses}') + print(f"WARNING: Unexpected environment variable value `{var}={val}`. " f"Expected one of {trues + falses}") return False @@ -36,9 +34,7 @@ def _get_build(var, default=False): def get_ext_modules(): - return [ - Extension(name='torchdata._torchdata', sources=[]) - ] + return [Extension(name="torchdata._torchdata", sources=[])] # Based off of pybiind cmake_example @@ -51,7 +47,7 @@ def build_extension(self, ext): # we would like to prevent multiple calls to `cmake`. # Therefore, we call `cmake` only for `torchdata._torchdata`, # in case `ext_modules` contains more than one module. - if ext.name != 'torchdata._torchdata': + if ext.name != "torchdata._torchdata": return extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) @@ -72,26 +68,25 @@ def build_extension(self, ext): f"-DBUILD_S3:BOOL={'ON' if _BUILD_S3 else 'OFF'}", ] - build_args = [ - '--config', cfg - ] + build_args = ["--config", cfg] - if 'CMAKE_GENERATOR' not in os.environ or platform.system() == 'Windows': + if "CMAKE_GENERATOR" not in os.environ or platform.system() == "Windows": cmake_args += ["-GNinja"] - if platform.system() == 'Windows': + if platform.system() == "Windows": import sys + python_version = sys.version_info cmake_args += [ "-DCMAKE_C_COMPILER=cl", "-DCMAKE_CXX_COMPILER=cl", f"-DPYTHON_VERSION={python_version.major}.{python_version.minor}", - '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}'.format(cfg.upper(), extdir), + f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}", ] - if sys.maxsize > 2**32: - cmake_args += ['-A', 'x64'] - build_args += ['--', '/m'] + if sys.maxsize > 2 ** 32: + cmake_args += ["-A", "x64"] + build_args += ["--", "/m"] else: - build_args += ['--', '-j2'] + build_args += ["--", "-j2"] # Set CMAKE_BUILD_PARALLEL_LEVEL to control the parallel build level # across all generators. @@ -100,9 +95,9 @@ def build_extension(self, ext): # using -j in the build_ext call, not supported by pip or PyPA-build. if hasattr(self, "parallel") and self.parallel: # CMake 3.12+ only. - build_args += ["-j{}".format(self.parallel)] + build_args += [f"-j{self.parallel}"] if not os.path.exists(self.build_temp): os.makedirs(self.build_temp) - subprocess.check_call(['cmake', str(_ROOT_DIR)] + cmake_args, cwd=self.build_temp) - subprocess.check_call(['cmake', '--build', '.'] + build_args, cwd=self.build_temp) + subprocess.check_call(["cmake", str(_ROOT_DIR)] + cmake_args, cwd=self.build_temp) + subprocess.check_call(["cmake", "--build", "."] + build_args, cwd=self.build_temp) diff --git a/torchdata/__init__.py b/torchdata/__init__.py index 8b99d848f..d294f6886 100644 --- a/torchdata/__init__.py +++ b/torchdata/__init__.py @@ -1,5 +1,6 @@ # Copyright (c) Facebook, Inc. and its affiliates. from torchdata import _extension # noqa: F401 + from . import datapipes try: diff --git a/torchdata/_internal/__init__.py b/torchdata/_internal/__init__.py index 654425399..5d6cabb03 100644 --- a/torchdata/_internal/__init__.py +++ b/torchdata/_internal/__init__.py @@ -1,4 +1,4 @@ -from torch.hub import load_state_dict_from_url, download_url_to_file +from torch.hub import download_url_to_file, load_state_dict_from_url __all__ = [ diff --git a/torchdata/datapipes/iter/load/README.md b/torchdata/datapipes/iter/load/README.md index 1a4bdc3a6..171668d7c 100644 --- a/torchdata/datapipes/iter/load/README.md +++ b/torchdata/datapipes/iter/load/README.md @@ -2,7 +2,8 @@ ## Installation -Torchdata S3 IO datapipes depends on [aws-sdk-cpp](https://github.com/aws/aws-sdk-cpp). The following is just a recommended way to installing aws-sdk-cpp, please refer to official documentation for detailed instructions. +Torchdata S3 IO datapipes depends on [aws-sdk-cpp](https://github.com/aws/aws-sdk-cpp). The following is just a +recommended way to installing aws-sdk-cpp, please refer to official documentation for detailed instructions. ```bash git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp @@ -20,7 +21,8 @@ make install # may need sudo conda install ninja pybind11 ``` -S3 IO datapipes are't included when building by default. To build S3 IO in `torchdata`, at the `/data` root folder, run the following commands. +S3 IO datapipes are't included when building by default. To build S3 IO in `torchdata`, at the `/data` root folder, run +the following commands. ```bash export BUILD_S3=ON @@ -33,17 +35,23 @@ python setup.py install ### S3FileLister -`S3FileLister` accepts a list of S3 prefixes and iterates all matching s3 urls. The functional API is `list_file_by_s3`. Acceptable prefixes include `s3://bucket-name`, `s3://bucket-name/`, `s3://bucket-name/folder`, `s3://bucket-name/folder/`, and `s3://bucket-name/prefix`. You may also set `length`, `request_timeout_ms` (default 3000 ms in aws-sdk-cpp), and `region`. Note that: +`S3FileLister` accepts a list of S3 prefixes and iterates all matching s3 urls. The functional API is `list_file_by_s3`. +Acceptable prefixes include `s3://bucket-name`, `s3://bucket-name/`, `s3://bucket-name/folder`, +`s3://bucket-name/folder/`, and `s3://bucket-name/prefix`. You may also set `length`, `request_timeout_ms` (default 3000 +ms in aws-sdk-cpp), and `region`. Note that: -1. Input __must__ be a list and direct S3 URLs are skipped. -2. `length` is `-1` by default, and any call to `__len__()` is invalid, because the length is unknown until all files are iterated. +1. Input **must** be a list and direct S3 URLs are skipped. +2. `length` is `-1` by default, and any call to `__len__()` is invalid, because the length is unknown until all files + are iterated. 3. `request_timeout_ms` and `region` will overwrite settings in the configuration file or environment variables. ### S3FileLoader -`S3FileLoader` accepts a list of S3 URLs and iterates all files in `BytesIO` format with `(url, BytesIO)` tuples. The functional API is `load_file_by_s3`. You may also set `request_timeout_ms` (default 3000 ms in aws-sdk-cpp), `region`, `buffer_size` (default 120Mb), and `multi_part_download` (default to use multi-part downloading). Note that: +`S3FileLoader` accepts a list of S3 URLs and iterates all files in `BytesIO` format with `(url, BytesIO)` tuples. The +functional API is `load_file_by_s3`. You may also set `request_timeout_ms` (default 3000 ms in aws-sdk-cpp), `region`, +`buffer_size` (default 120Mb), and `multi_part_download` (default to use multi-part downloading). Note that: -1. Input __must__ be a list and S3 URLs must be valid. +1. Input **must** be a list and S3 URLs must be valid. 2. `request_timeout_ms` and `region` will overwrite settings in the configuration file or environment variables. ### Example @@ -63,4 +71,6 @@ for d in datapipe: # Start loading data ### Note -It's recommended to set up a detailed configuration file with the `AWS_CONFIG_FILE` environment variable. The following environment variables are also parsed: `HOME`, `S3_USE_HTTPS`, `S3_VERIFY_SSL`, `S3_ENDPOINT_URL`, `AWS_REGION` (would be overwritten by the `region` variable). +It's recommended to set up a detailed configuration file with the `AWS_CONFIG_FILE` environment variable. The following +environment variables are also parsed: `HOME`, `S3_USE_HTTPS`, `S3_VERIFY_SSL`, `S3_ENDPOINT_URL`, `AWS_REGION` (would +be overwritten by the `region` variable). diff --git a/torchdata/datapipes/iter/load/s3io.py b/torchdata/datapipes/iter/load/s3io.py index 2a3d687ce..0ae44c027 100644 --- a/torchdata/datapipes/iter/load/s3io.py +++ b/torchdata/datapipes/iter/load/s3io.py @@ -32,15 +32,14 @@ def __iter__(self) -> Iterator[str]: for prefix in self.source_datapipe: while True: urls = self.handler.list_files(prefix) - for url in urls: - yield url + yield from urls if not urls: break self.handler.clear_marker() def __len__(self) -> int: if self.length == -1: - raise TypeError("{} instance doesn't have valid length".format(type(self).__name__)) + raise TypeError(f"{type(self).__name__} instance doesn't have valid length") return self.length @@ -59,7 +58,14 @@ class S3FileLoaderIterDataPipe(IterDataPipe[Tuple[str, StreamWrapper]]): AWS_CPP_SDK is necessary to use the S3 DataPipe(s). """ - def __init__(self, source_datapipe: IterDataPipe[str], request_timeout_ms=-1, region="", buffer_size=None, multi_part_download=None) -> None: + def __init__( + self, + source_datapipe: IterDataPipe[str], + request_timeout_ms=-1, + region="", + buffer_size=None, + multi_part_download=None, + ) -> None: self.source_datapipe: IterDataPipe[str] = source_datapipe self.handler = torchdata._torchdata.S3Handler(request_timeout_ms, region) if buffer_size: From bed7090b51c1125ba3e9480a10bd8e8bf54d2e04 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Thu, 10 Feb 2022 14:28:55 -0800 Subject: [PATCH 041/153] Combine pip package install lines, remove extra env line. --- .github/workflows/ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43da8ea69..a79093cea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,8 +46,7 @@ jobs: run: | pip3 install -r requirements.txt pip3 install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - pip3 install cmake ninja - pip3 install pybind11 + pip3 install cmake ninja pybind11 echo "/home/runner/.local/bin" >> $GITHUB_PATH - name: Install test requirements run: pip3 install expecttest fsspec iopath==0.1.9 numpy pytest rarfile @@ -62,7 +61,6 @@ jobs: make install - name: Build TorchData run: python setup.py develop - env: env: BUILD_S3: 1 - name: Run DataPipes tests with pytest From 746ca2ffa55f51a19146f0fb255e9f83f189af01 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Thu, 10 Feb 2022 14:29:08 -0800 Subject: [PATCH 042/153] Change minimum cmake version to 3.13. --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index dec4f0fc7..8a2de8f1e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.18 FATAL_ERROR) +cmake_minimum_required(VERSION 3.13 FATAL_ERROR) # Most of the configurations are taken from PyTorch # https://github.com/pytorch/pytorch/blob/0c9fb4aff0d60eaadb04e4d5d099fb1e1d5701a9/CMakeLists.txt From 193a0ea976ae14a6c3c3287da08b15455be8e307 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Mon, 14 Feb 2022 10:22:53 -0800 Subject: [PATCH 043/153] Install curl as dependency for aws sdk for ubuntu builds. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a79093cea..af4323711 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,7 @@ jobs: run: | sudo add-apt-repository multiverse sudo apt update - sudo apt install rar unrar + sudo apt install rar unrar curl - name: Setup Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: @@ -46,7 +46,7 @@ jobs: run: | pip3 install -r requirements.txt pip3 install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - pip3 install cmake ninja pybind11 + pip3 install cmake ninja pybind11 pybind11-global echo "/home/runner/.local/bin" >> $GITHUB_PATH - name: Install test requirements run: pip3 install expecttest fsspec iopath==0.1.9 numpy pytest rarfile From 372c0e0e942528c373573f26664764d381c4c6a1 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Mon, 14 Feb 2022 10:24:13 -0800 Subject: [PATCH 044/153] Raise ModuleNotFoundError when attempting to use S3 datapipe and not compiled in. --- torchdata/datapipes/iter/load/s3io.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/torchdata/datapipes/iter/load/s3io.py b/torchdata/datapipes/iter/load/s3io.py index 0ae44c027..30878f3f1 100644 --- a/torchdata/datapipes/iter/load/s3io.py +++ b/torchdata/datapipes/iter/load/s3io.py @@ -24,6 +24,11 @@ class S3FileListerIterDataPipe(IterDataPipe[str]): """ def __init__(self, source_datapipe: IterDataPipe[str], length: int = -1, request_timeout_ms=-1, region="") -> None: + if not hasattr(torchdata, '_torchdata') or not hasattr(torchdata._torchdata, 'S3Handler'): + raise ModuleNotFoundError( + "Torchdata must be built with BUILD_S3=1 to use this datapipe." + ) + self.source_datapipe: IterDataPipe[str] = source_datapipe self.length: int = length self.handler = torchdata._torchdata.S3Handler(request_timeout_ms, region) From 17749d1d6957c328610c29bcecc55c87577059f7 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Mon, 14 Feb 2022 10:28:59 -0800 Subject: [PATCH 045/153] Fix lint. --- torchdata/datapipes/iter/load/s3io.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/torchdata/datapipes/iter/load/s3io.py b/torchdata/datapipes/iter/load/s3io.py index 30878f3f1..dc80044ae 100644 --- a/torchdata/datapipes/iter/load/s3io.py +++ b/torchdata/datapipes/iter/load/s3io.py @@ -24,10 +24,8 @@ class S3FileListerIterDataPipe(IterDataPipe[str]): """ def __init__(self, source_datapipe: IterDataPipe[str], length: int = -1, request_timeout_ms=-1, region="") -> None: - if not hasattr(torchdata, '_torchdata') or not hasattr(torchdata._torchdata, 'S3Handler'): - raise ModuleNotFoundError( - "Torchdata must be built with BUILD_S3=1 to use this datapipe." - ) + if not hasattr(torchdata, "_torchdata") or not hasattr(torchdata._torchdata, "S3Handler"): + raise ModuleNotFoundError("Torchdata must be built with BUILD_S3=1 to use this datapipe.") self.source_datapipe: IterDataPipe[str] = source_datapipe self.length: int = length From 48738d0c401f9d9f6484d3bd3d389e24bdb82718 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Mon, 14 Feb 2022 11:16:14 -0800 Subject: [PATCH 046/153] Install header dependencies for aws-sdk-cpp. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index af4323711..f1f5adfd2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,7 @@ jobs: run: | sudo add-apt-repository multiverse sudo apt update - sudo apt install rar unrar curl + sudo apt install rar unrar libssl-dev libcurl4-openssl-dev zlib1g-dev - name: Setup Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: From d7982f03dd52a664b58bad87d2fbd7f2ef8f896a Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Mon, 14 Feb 2022 21:37:50 +0000 Subject: [PATCH 047/153] Only add the _torchdata extension if BUILD_S3 is enabled (can be extended to support other optional extensions as well.) --- tools/setup_helpers/extension.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index 24325fa77..6a3038a0a 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -34,7 +34,10 @@ def _get_build(var, default=False): def get_ext_modules(): - return [Extension(name="torchdata._torchdata", sources=[])] + if _BUILD_S3: + return [Extension(name="torchdata._torchdata", sources=[])] + else: + return [] # Based off of pybiind cmake_example From 71e1cd027b1a8d04b932570491b693203c6ada52 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Mon, 14 Feb 2022 21:55:30 +0000 Subject: [PATCH 048/153] Set PYTHON_EXECUTABLE so we use the same version as what is running the setup.py script. --- tools/setup_helpers/extension.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index 6a3038a0a..0e04bf459 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -1,4 +1,5 @@ import os +import sys import platform import subprocess from pathlib import Path @@ -67,6 +68,7 @@ def build_extension(self, ext): f"-DCMAKE_PREFIX_PATH={torch.utils.cmake_prefix_path}", f"-DCMAKE_INSTALL_PREFIX={extdir}", f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}", + f"-DPYTHON_EXECUTABLE={sys.executable}", "-DCMAKE_CXX_FLAGS=-fPIC", f"-DBUILD_S3:BOOL={'ON' if _BUILD_S3 else 'OFF'}", ] @@ -76,8 +78,6 @@ def build_extension(self, ext): if "CMAKE_GENERATOR" not in os.environ or platform.system() == "Windows": cmake_args += ["-GNinja"] if platform.system() == "Windows": - import sys - python_version = sys.version_info cmake_args += [ "-DCMAKE_C_COMPILER=cl", From f1a27bbd3fcfc5e2d128719d974713af2281ac51 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Tue, 15 Feb 2022 23:29:10 +0000 Subject: [PATCH 049/153] Fix merge conflict. --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 87eeedefc..d8b95e2a3 100644 --- a/setup.py +++ b/setup.py @@ -74,7 +74,6 @@ def run(self): shutil.rmtree(str(path), ignore_errors=True) -<<<<<<< HEAD if __name__ == "__main__": VERSION, SHA = _get_version() _export_version(VERSION, SHA) From 8f16b7fb17011558f5c0a594cbd95404c8553686 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Tue, 15 Feb 2022 23:30:41 +0000 Subject: [PATCH 050/153] Add warning (but don't error out) when failing to import c++ extensions. --- torchdata/_extension.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/torchdata/_extension.py b/torchdata/_extension.py index daac5f59d..a2e2d1e3e 100644 --- a/torchdata/_extension.py +++ b/torchdata/_extension.py @@ -31,9 +31,12 @@ def _init_extension(): return _load_lib("libtorchdata") - # This import is for initializing the methods registered via PyBind11 - # This has to happen after the base library is loaded - from torchdata import _torchdata # noqa + try: + # This import is for initializing the methods registered via PyBind11 + # This has to happen after the base library is loaded + from torchdata import _torchdata # noqa + except ImportError as e: + warnings.warn(f"torchdata C++ extension unable to load: {e}") _init_extension() From 89e6fe84193e9cf26a6e0de7177cafb70941eecf Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Mon, 21 Feb 2022 19:49:49 -0800 Subject: [PATCH 051/153] add sudo for admin rights when make install aws-sdk-cpp --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f1f5adfd2..c97fed6c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,7 +58,7 @@ jobs: cd sdk-build cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" make - make install + sudo make install - name: Build TorchData run: python setup.py develop env: From 0cd94f606ef1e27b4c8c8abfe160fe1c008465c9 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Tue, 22 Feb 2022 20:43:57 -0800 Subject: [PATCH 052/153] Catch Exception in incorrect use case --- test/test_remote_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_remote_io.py b/test/test_remote_io.py index bc7cd8e48..9ae6916ee 100644 --- a/test/test_remote_io.py +++ b/test/test_remote_io.py @@ -225,7 +225,7 @@ def test_s3_io_iterdatapipe(self): ["s3:///bin-images"], ] for input in input_list: - with self.assertRaises(ValueError, msg=f"{input} should raise ValueError."): + with self.assertRaises(BaseException, msg=f"{input} should raise an exception."): s3_lister_dp = S3FileLister(IterableWrapper(file_urls)) for _ in s3_lister_dp: pass From 82f0d4e17ef95eb03f762a92a1932df5b3f2b94d Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Tue, 22 Feb 2022 21:12:25 -0800 Subject: [PATCH 053/153] ignore s3 io tests if _torchdata or aws-sdk-cpp doesn't exist --- test/test_remote_io.py | 9 +++++++++ torchdata/datapipes/iter/load/s3io.py | 3 +++ 2 files changed, 12 insertions(+) diff --git a/test/test_remote_io.py b/test/test_remote_io.py index 9ae6916ee..690f8faee 100644 --- a/test/test_remote_io.py +++ b/test/test_remote_io.py @@ -170,6 +170,15 @@ def _read_and_decode(x): self.assertEqual(expected_csv_path, csv_path) def test_s3_io_iterdatapipe(self): + # sanity test + file_urls = ["s3://ai2-public-datasets"] + try: + s3_lister_dp = S3FileLister(IterableWrapper(file_urls)) + s3_loader_dp = S3FileLoader(IterableWrapper(file_urls)) + except ModuleNotFoundError: + warnings.warn("S3 IO datapipes or C++ extension '_torchdata' isn't built in the current 'torchdata' package") + return + # S3FileLister: different inputs input_list = [ [["s3://ai2-public-datasets"], 71], # bucket without '/' diff --git a/torchdata/datapipes/iter/load/s3io.py b/torchdata/datapipes/iter/load/s3io.py index dc80044ae..9c59c0565 100644 --- a/torchdata/datapipes/iter/load/s3io.py +++ b/torchdata/datapipes/iter/load/s3io.py @@ -69,6 +69,9 @@ def __init__( buffer_size=None, multi_part_download=None, ) -> None: + if not hasattr(torchdata, "_torchdata") or not hasattr(torchdata._torchdata, "S3Handler"): + raise ModuleNotFoundError("Torchdata must be built with BUILD_S3=1 to use this datapipe.") + self.source_datapipe: IterDataPipe[str] = source_datapipe self.handler = torchdata._torchdata.S3Handler(request_timeout_ms, region) if buffer_size: From 5ba0b0a354ea7ed2d48800e2774ac775a8fd9d0f Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 07:57:01 -0800 Subject: [PATCH 054/153] ignore mypy error for torchdata._torchdata --- mypy.ini | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mypy.ini b/mypy.ini index 8bb1e7024..2617f0aad 100644 --- a/mypy.ini +++ b/mypy.ini @@ -12,6 +12,13 @@ exclude = examples, test, packaging python_version = 3.7 +# +# Extension modules without stubs. +# + +[mypy-torchdata._torchdata] +ignore_missing_imports = True + # # Third party dependencies that don't have types. # From 757ace5dbafa05c7490b143f332454e111ba93a6 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 08:04:05 -0800 Subject: [PATCH 055/153] find the last python version in PATH --- torchdata/csrc/CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/torchdata/csrc/CMakeLists.txt b/torchdata/csrc/CMakeLists.txt index d5c423dbf..87f713cb3 100644 --- a/torchdata/csrc/CMakeLists.txt +++ b/torchdata/csrc/CMakeLists.txt @@ -1,5 +1,11 @@ if(BUILD_S3) message(STATUS "Building S3 IO functionality") + if (NOT DEFINED Python_FIND_REGISTRY) + set(Python_FIND_REGISTRY "LAST") + endif () + if (NOT DEFINED Python_FIND_FRAMEWORK) + set(Python_FIND_FRAMEWORK "LAST") + endif () find_package(Python3 COMPONENTS Interpreter Development) find_package(AWSSDK REQUIRED COMPONENTS s3 transfer) find_package(pybind11 REQUIRED) From 6bfadd551ac95283209546ed191a2819130e6703 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 08:24:21 -0800 Subject: [PATCH 056/153] remove unnecessary sudo in ci --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c97fed6c0..f1f5adfd2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,7 +58,7 @@ jobs: cd sdk-build cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" make - sudo make install + make install - name: Build TorchData run: python setup.py develop env: From ddd2048cfb8110a6ce8741142911070a513708de Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 09:03:36 -0800 Subject: [PATCH 057/153] Change PyThon_FIND_FRAMEWORK to CMAKE_FIND_FRAMEWORK --- torchdata/csrc/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchdata/csrc/CMakeLists.txt b/torchdata/csrc/CMakeLists.txt index 87f713cb3..74ef26b0b 100644 --- a/torchdata/csrc/CMakeLists.txt +++ b/torchdata/csrc/CMakeLists.txt @@ -3,7 +3,7 @@ if(BUILD_S3) if (NOT DEFINED Python_FIND_REGISTRY) set(Python_FIND_REGISTRY "LAST") endif () - if (NOT DEFINED Python_FIND_FRAMEWORK) + if (NOT DEFINED CMAKE_FIND_FRAMEWORK) set(Python_FIND_FRAMEWORK "LAST") endif () find_package(Python3 COMPONENTS Interpreter Development) From 19e30d5994c75ff3b8c56b68494e218263a92d61 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 09:07:55 -0800 Subject: [PATCH 058/153] separate Windows installation for AWS-SDK-CPP --- .github/workflows/ci.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f1f5adfd2..f0812131f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,7 +50,17 @@ jobs: echo "/home/runner/.local/bin" >> $GITHUB_PATH - name: Install test requirements run: pip3 install expecttest fsspec iopath==0.1.9 numpy pytest rarfile - - name: Install AWS-SDK-CPP for S3 IO datapipes + - name: Install AWS-SDK-CPP on Windows for S3 IO datapipes + if: ${{ matrix.os }} == 'windows-latest' + run: | + git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp + mkdir sdk_build + cd sdk_build + cmake ..\aws-sdk-cpp -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3:transfer" + msbuild ALL_BUILD.vcxproj -p:Configuration=Release + msbuild INSTALL.vcxproj -p:Configuration=Release + - name: Install AWS-SDK-CPP on Non-Windows for S3 IO datapipes + if: ${{ matrix.os }} != 'windows-latest' run: | git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp cd aws-sdk-cpp/ From 65bc893faf08ce31a666753665f41d641d03a87a Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 09:14:29 -0800 Subject: [PATCH 059/153] still need administrative privilege to install aws-sdk-cpp --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f0812131f..9645dc6fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,7 +68,7 @@ jobs: cd sdk-build cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" make - make install + sudo make install - name: Build TorchData run: python setup.py develop env: From 140c0da5a8bc93c38877227a7d2e7b10f2dfc27e Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 09:22:39 -0800 Subject: [PATCH 060/153] remove redundant parentheses --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9645dc6fb..b69888eee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,7 +51,7 @@ jobs: - name: Install test requirements run: pip3 install expecttest fsspec iopath==0.1.9 numpy pytest rarfile - name: Install AWS-SDK-CPP on Windows for S3 IO datapipes - if: ${{ matrix.os }} == 'windows-latest' + if: matrix.os == 'windows-latest' run: | git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp mkdir sdk_build @@ -60,7 +60,7 @@ jobs: msbuild ALL_BUILD.vcxproj -p:Configuration=Release msbuild INSTALL.vcxproj -p:Configuration=Release - name: Install AWS-SDK-CPP on Non-Windows for S3 IO datapipes - if: ${{ matrix.os }} != 'windows-latest' + if: matrix.os != 'windows-latest' run: | git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp cd aws-sdk-cpp/ From e2fac1c7fb3d40935065bdd61752e1f975627d6e Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 09:56:22 -0800 Subject: [PATCH 061/153] edit test for incorrect filenames --- test/test_remote_io.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_remote_io.py b/test/test_remote_io.py index 690f8faee..4b25076ff 100644 --- a/test/test_remote_io.py +++ b/test/test_remote_io.py @@ -234,8 +234,8 @@ def test_s3_io_iterdatapipe(self): ["s3:///bin-images"], ] for input in input_list: - with self.assertRaises(BaseException, msg=f"{input} should raise an exception."): - s3_lister_dp = S3FileLister(IterableWrapper(file_urls)) + with self.assertRaises(ValueError, msg=f"{input} should raise ValueError."): + s3_lister_dp = S3FileLister(IterableWrapper(input)) for _ in s3_lister_dp: pass From 98e8e36090d9465e3f6b9a8bae4e932b62070759 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 09:58:34 -0800 Subject: [PATCH 062/153] update test cases for incorrect user input --- test/test_remote_io.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_remote_io.py b/test/test_remote_io.py index 4b25076ff..437a5b8dd 100644 --- a/test/test_remote_io.py +++ b/test/test_remote_io.py @@ -235,7 +235,7 @@ def test_s3_io_iterdatapipe(self): ] for input in input_list: with self.assertRaises(ValueError, msg=f"{input} should raise ValueError."): - s3_lister_dp = S3FileLister(IterableWrapper(input)) + s3_lister_dp = S3FileLister(IterableWrapper(input), region="us-east-1") for _ in s3_lister_dp: pass @@ -261,7 +261,7 @@ def test_s3_io_iterdatapipe(self): ] for input in input_list: with self.assertRaises(ValueError, msg=f"{input} should raise ValueError."): - s3_loader_dp = S3FileLoader(file_urls) + s3_loader_dp = S3FileLoader(input, region="us-east-1") for _ in s3_loader_dp: pass From 0e2d1f9bcf627f6a5d0a4334d0d2af431d020f3c Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 10:15:12 -0800 Subject: [PATCH 063/153] add a step to setup msbuild in ci --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b69888eee..6bdff6e3c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,6 +40,9 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + - name: Setup msbuild on Windows + if: matrix.os == 'windows-latest' + uses: microsoft/setup-msbuild@v1.1 - name: Check out source repository uses: actions/checkout@v2 - name: Install dependencies From 0ce24b9c292c02c3ef386c14dca88e3a786d1471 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 10:25:03 -0800 Subject: [PATCH 064/153] fix a typo in ci step to install AWS-SDK-CPP on Windows --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6bdff6e3c..a65d30983 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp mkdir sdk_build cd sdk_build - cmake ..\aws-sdk-cpp -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3:transfer" + cmake ..\aws-sdk-cpp -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" msbuild ALL_BUILD.vcxproj -p:Configuration=Release msbuild INSTALL.vcxproj -p:Configuration=Release - name: Install AWS-SDK-CPP on Non-Windows for S3 IO datapipes From ab3954ffa463dcdf3e19c8e22325795191cfdc81 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 10:54:21 -0800 Subject: [PATCH 065/153] enable long names for Windows to install AWS-SDK-CPP --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a65d30983..ab6fa18b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,6 +56,7 @@ jobs: - name: Install AWS-SDK-CPP on Windows for S3 IO datapipes if: matrix.os == 'windows-latest' run: | + reg add HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /d 1 /f git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp mkdir sdk_build cd sdk_build From f26236651fea58e4be77c7a79255b253b0352d0e Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 11:31:13 -0800 Subject: [PATCH 066/153] Display runtime Python version after setup-python@v2 in ci --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab6fa18b2..a53c18368 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,6 +40,8 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + - name: Display Python version + run: python --version - name: Setup msbuild on Windows if: matrix.os == 'windows-latest' uses: microsoft/setup-msbuild@v1.1 From 8e3eee4ff3219b69eda80c944fd74212a45bb1ad Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 11:41:51 -0800 Subject: [PATCH 067/153] try ping specific version at build --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a53c18368..4f1aa37b8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,8 +76,9 @@ jobs: make sudo make install - name: Build TorchData - run: python setup.py develop + run: python$PYTHON_VERSION setup.py develop env: + PYTHON_VERSION: ${{ matrix.python-version }} BUILD_S3: 1 - name: Run DataPipes tests with pytest if: ${{ ! contains(github.event.pull_request.labels.*.name, 'ciflow/slow') }} From 03d75765ffd75c68b29b28586540d153091201c6 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 13:12:18 -0800 Subject: [PATCH 068/153] use cmake flags to bypass find_package(Python3) --- .github/workflows/ci.yml | 3 +-- tools/setup_helpers/extension.py | 2 ++ torchdata/csrc/CMakeLists.txt | 7 ------- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f1aa37b8..a53c18368 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,9 +76,8 @@ jobs: make sudo make install - name: Build TorchData - run: python$PYTHON_VERSION setup.py develop + run: python setup.py develop env: - PYTHON_VERSION: ${{ matrix.python-version }} BUILD_S3: 1 - name: Run DataPipes tests with pytest if: ${{ ! contains(github.event.pull_request.labels.*.name, 'ciflow/slow') }} diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index 0e04bf459..26a477d8b 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -69,6 +69,8 @@ def build_extension(self, ext): f"-DCMAKE_INSTALL_PREFIX={extdir}", f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}", f"-DPYTHON_EXECUTABLE={sys.executable}", + f"-DPYTHON_INCLUDE_DIRS={sys.executable}", + f"-DPYTHON_LIBRARIES={sys.executable}", "-DCMAKE_CXX_FLAGS=-fPIC", f"-DBUILD_S3:BOOL={'ON' if _BUILD_S3 else 'OFF'}", ] diff --git a/torchdata/csrc/CMakeLists.txt b/torchdata/csrc/CMakeLists.txt index 74ef26b0b..814f07c16 100644 --- a/torchdata/csrc/CMakeLists.txt +++ b/torchdata/csrc/CMakeLists.txt @@ -1,12 +1,5 @@ if(BUILD_S3) message(STATUS "Building S3 IO functionality") - if (NOT DEFINED Python_FIND_REGISTRY) - set(Python_FIND_REGISTRY "LAST") - endif () - if (NOT DEFINED CMAKE_FIND_FRAMEWORK) - set(Python_FIND_FRAMEWORK "LAST") - endif () - find_package(Python3 COMPONENTS Interpreter Development) find_package(AWSSDK REQUIRED COMPONENTS s3 transfer) find_package(pybind11 REQUIRED) set(CMAKE_POSITION_INDEPENDENT_CODE ON) From f2b4a48112ca1381121792eaf3ee9fd50e6f2f20 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 13:30:21 -0800 Subject: [PATCH 069/153] add a separate step for reg update in Windows, and find python3 in CMake --- .github/workflows/ci.yml | 4 +++- torchdata/csrc/CMakeLists.txt | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a53c18368..568121bb9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,10 +55,12 @@ jobs: echo "/home/runner/.local/bin" >> $GITHUB_PATH - name: Install test requirements run: pip3 install expecttest fsspec iopath==0.1.9 numpy pytest rarfile + - name: Allow Long Names for Windows + if: matrix.os == 'windows-latest' + run: eg add HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /d 1 /f - name: Install AWS-SDK-CPP on Windows for S3 IO datapipes if: matrix.os == 'windows-latest' run: | - reg add HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /d 1 /f git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp mkdir sdk_build cd sdk_build diff --git a/torchdata/csrc/CMakeLists.txt b/torchdata/csrc/CMakeLists.txt index 814f07c16..d5c423dbf 100644 --- a/torchdata/csrc/CMakeLists.txt +++ b/torchdata/csrc/CMakeLists.txt @@ -1,5 +1,6 @@ if(BUILD_S3) message(STATUS "Building S3 IO functionality") + find_package(Python3 COMPONENTS Interpreter Development) find_package(AWSSDK REQUIRED COMPONENTS s3 transfer) find_package(pybind11 REQUIRED) set(CMAKE_POSITION_INDEPENDENT_CODE ON) From 66d138dd904e483265e86c577a38b27581b19b47 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 13:36:31 -0800 Subject: [PATCH 070/153] Fix a typo in Windows registry step --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 568121bb9..ac93c35cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,7 +57,7 @@ jobs: run: pip3 install expecttest fsspec iopath==0.1.9 numpy pytest rarfile - name: Allow Long Names for Windows if: matrix.os == 'windows-latest' - run: eg add HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /d 1 /f + run: reg add HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /d 1 /f - name: Install AWS-SDK-CPP on Windows for S3 IO datapipes if: matrix.os == 'windows-latest' run: | From 84e58589f050e34111e2e08df6f6d44ae25adbac Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 13:45:35 -0800 Subject: [PATCH 071/153] turn DPYBIND11_FINDPYTHON --- tools/setup_helpers/extension.py | 1 + torchdata/csrc/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index 26a477d8b..38fca642a 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -71,6 +71,7 @@ def build_extension(self, ext): f"-DPYTHON_EXECUTABLE={sys.executable}", f"-DPYTHON_INCLUDE_DIRS={sys.executable}", f"-DPYTHON_LIBRARIES={sys.executable}", + "-DPYBIND11_FINDPYTHON=ON", "-DCMAKE_CXX_FLAGS=-fPIC", f"-DBUILD_S3:BOOL={'ON' if _BUILD_S3 else 'OFF'}", ] diff --git a/torchdata/csrc/CMakeLists.txt b/torchdata/csrc/CMakeLists.txt index d5c423dbf..727ce05e9 100644 --- a/torchdata/csrc/CMakeLists.txt +++ b/torchdata/csrc/CMakeLists.txt @@ -1,8 +1,8 @@ if(BUILD_S3) message(STATUS "Building S3 IO functionality") - find_package(Python3 COMPONENTS Interpreter Development) find_package(AWSSDK REQUIRED COMPONENTS s3 transfer) find_package(pybind11 REQUIRED) + find_package(Python3 COMPONENTS Interpreter Development) set(CMAKE_POSITION_INDEPENDENT_CODE ON) set(SOURCES "pybind/S3Handler/S3Handler.cpp" ) From b6d7ae6ed2680c08a93220aa2aee8a30491c0d0f Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 14:11:41 -0800 Subject: [PATCH 072/153] ignore unit tests for aws-sdk-cpp --- .github/workflows/ci.yml | 4 ++-- tools/setup_helpers/extension.py | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ac93c35cd..0a025db02 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,7 +64,7 @@ jobs: git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp mkdir sdk_build cd sdk_build - cmake ..\aws-sdk-cpp -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" + cmake ..\aws-sdk-cpp -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" -DAUTORUN_UNIT_TESTS=OFF msbuild ALL_BUILD.vcxproj -p:Configuration=Release msbuild INSTALL.vcxproj -p:Configuration=Release - name: Install AWS-SDK-CPP on Non-Windows for S3 IO datapipes @@ -74,7 +74,7 @@ jobs: cd aws-sdk-cpp/ mkdir sdk-build cd sdk-build - cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" + cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" -DAUTORUN_UNIT_TESTS=OFF make sudo make install - name: Build TorchData diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index 38fca642a..2b09221e2 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -32,6 +32,7 @@ def _get_build(var, default=False): _BUILD_S3 = _get_build("BUILD_S3", False) +_PYTHON_SPECIFIC_VERSION = os.environ.get("PYTHON_SPECIFIC_VERSION", None) def get_ext_modules(): @@ -78,6 +79,11 @@ def build_extension(self, ext): build_args = ["--config", cfg] + if _PYTHON_SPECIFIC_VERSION: + cmake_args += [ + f"-DPYBIND11_PYTHON_VERSION={_PYTHON_SPECIFIC_VERSION}", + ] + if "CMAKE_GENERATOR" not in os.environ or platform.system() == "Windows": cmake_args += ["-GNinja"] if platform.system() == "Windows": From e87656284c7e180414d8b1f4143fd008f6b4b731 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 14:40:32 -0800 Subject: [PATCH 073/153] turn off force new findpython --- tools/setup_helpers/extension.py | 2 ++ torchdata/csrc/CMakeLists.txt | 3 +++ 2 files changed, 5 insertions(+) diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index 2b09221e2..f978b655c 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -77,6 +77,8 @@ def build_extension(self, ext): f"-DBUILD_S3:BOOL={'ON' if _BUILD_S3 else 'OFF'}", ] + print(sys.executable) + build_args = ["--config", cfg] if _PYTHON_SPECIFIC_VERSION: diff --git a/torchdata/csrc/CMakeLists.txt b/torchdata/csrc/CMakeLists.txt index 727ce05e9..ebdcb5f4c 100644 --- a/torchdata/csrc/CMakeLists.txt +++ b/torchdata/csrc/CMakeLists.txt @@ -1,5 +1,8 @@ if(BUILD_S3) message(STATUS "Building S3 IO functionality") + include(CMakeDependentOption) + cmake_dependent_option(PYBIND11_FINDPYTHON "Force new FindPython" OFF + "NOT CMAKE_VERSION VERSION_LESS 3.12" OFF) find_package(AWSSDK REQUIRED COMPONENTS s3 transfer) find_package(pybind11 REQUIRED) find_package(Python3 COMPONENTS Interpreter Development) From 201d0d1953c554d320f56fb15d34679518e92ef0 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 15:15:20 -0800 Subject: [PATCH 074/153] add -DPython3_ROOT_DIR for cmake --- .github/workflows/ci.yml | 5 ++++- tools/setup_helpers/extension.py | 8 +++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a025db02..dc689c7ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,9 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Display Python version - run: python --version + run: | + python --version + echo pythonlocation - name: Setup msbuild on Windows if: matrix.os == 'windows-latest' uses: microsoft/setup-msbuild@v1.1 @@ -81,6 +83,7 @@ jobs: run: python setup.py develop env: BUILD_S3: 1 + PYTHON_ROOT_DIR: pythonlocaltion - name: Run DataPipes tests with pytest if: ${{ ! contains(github.event.pull_request.labels.*.name, 'ciflow/slow') }} run: diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index f978b655c..97fd46fff 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -32,7 +32,7 @@ def _get_build(var, default=False): _BUILD_S3 = _get_build("BUILD_S3", False) -_PYTHON_SPECIFIC_VERSION = os.environ.get("PYTHON_SPECIFIC_VERSION", None) +_PYTHON_ROOT_DIR = os.environ.get("PYTHON_ROOT_DIR", None) def get_ext_modules(): @@ -77,13 +77,11 @@ def build_extension(self, ext): f"-DBUILD_S3:BOOL={'ON' if _BUILD_S3 else 'OFF'}", ] - print(sys.executable) - build_args = ["--config", cfg] - if _PYTHON_SPECIFIC_VERSION: + if _PYTHON_ROOT_DIR: cmake_args += [ - f"-DPYBIND11_PYTHON_VERSION={_PYTHON_SPECIFIC_VERSION}", + f"-DPython3_ROOT_DIR={_PYTHON_ROOT_DIR}", ] if "CMAKE_GENERATOR" not in os.environ or platform.system() == "Windows": From 57f95475c1fc244f45ff89c5e3d46168d840ff64 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 15:18:24 -0800 Subject: [PATCH 075/153] pass in environment var for PYTHON_ROOT_DIR --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc689c7ea..aa9ef819b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,7 +83,7 @@ jobs: run: python setup.py develop env: BUILD_S3: 1 - PYTHON_ROOT_DIR: pythonlocaltion + PYTHON_ROOT_DIR: $pythonlocaltion - name: Run DataPipes tests with pytest if: ${{ ! contains(github.event.pull_request.labels.*.name, 'ciflow/slow') }} run: From 68646098cb0becfd6dd17ca27d0c30d371d99763 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 15:41:12 -0800 Subject: [PATCH 076/153] find exact python version at building --- .github/workflows/ci.yml | 4 ++-- tools/setup_helpers/extension.py | 9 +++------ torchdata/csrc/CMakeLists.txt | 6 +++++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa9ef819b..8a4bb80b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,7 @@ jobs: - name: Display Python version run: | python --version - echo pythonlocation + echo $pythonlocation - name: Setup msbuild on Windows if: matrix.os == 'windows-latest' uses: microsoft/setup-msbuild@v1.1 @@ -83,7 +83,7 @@ jobs: run: python setup.py develop env: BUILD_S3: 1 - PYTHON_ROOT_DIR: $pythonlocaltion + BUILD_PYTHON_VERSION: ${{ matrix.python-version }} - name: Run DataPipes tests with pytest if: ${{ ! contains(github.event.pull_request.labels.*.name, 'ciflow/slow') }} run: diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index 97fd46fff..947bb9f8d 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -32,7 +32,7 @@ def _get_build(var, default=False): _BUILD_S3 = _get_build("BUILD_S3", False) -_PYTHON_ROOT_DIR = os.environ.get("PYTHON_ROOT_DIR", None) +_BUILD_PYTHON_VERSION = os.environ.get("BUILD_PYTHON_VERSION", None) def get_ext_modules(): @@ -70,18 +70,15 @@ def build_extension(self, ext): f"-DCMAKE_INSTALL_PREFIX={extdir}", f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}", f"-DPYTHON_EXECUTABLE={sys.executable}", - f"-DPYTHON_INCLUDE_DIRS={sys.executable}", - f"-DPYTHON_LIBRARIES={sys.executable}", - "-DPYBIND11_FINDPYTHON=ON", "-DCMAKE_CXX_FLAGS=-fPIC", f"-DBUILD_S3:BOOL={'ON' if _BUILD_S3 else 'OFF'}", ] build_args = ["--config", cfg] - if _PYTHON_ROOT_DIR: + if _BUILD_PYTHON_VERSION: cmake_args += [ - f"-DPython3_ROOT_DIR={_PYTHON_ROOT_DIR}", + f"-DBUILD_PYTHON_VERSION={_BUILD_PYTHON_VERSION}", ] if "CMAKE_GENERATOR" not in os.environ or platform.system() == "Windows": diff --git a/torchdata/csrc/CMakeLists.txt b/torchdata/csrc/CMakeLists.txt index ebdcb5f4c..25961ca0c 100644 --- a/torchdata/csrc/CMakeLists.txt +++ b/torchdata/csrc/CMakeLists.txt @@ -5,7 +5,11 @@ if(BUILD_S3) "NOT CMAKE_VERSION VERSION_LESS 3.12" OFF) find_package(AWSSDK REQUIRED COMPONENTS s3 transfer) find_package(pybind11 REQUIRED) - find_package(Python3 COMPONENTS Interpreter Development) + if(BUILD_PYTHON_VERSION) + find_package(Python3 ${BUILD_PYTHON_VERSION} EXACT COMPONENTS Interpreter Development) + else() + find_package(Python3 COMPONENTS Interpreter Development) + endif() set(CMAKE_POSITION_INDEPENDENT_CODE ON) set(SOURCES "pybind/S3Handler/S3Handler.cpp" ) From d6ea5aed63bdb1b3d64fe3261a345cb199aab728 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 16:10:21 -0800 Subject: [PATCH 077/153] rewrite cmake-generator logic for Windows --- tools/setup_helpers/extension.py | 51 +++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index 947bb9f8d..e53f7f532 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -81,22 +81,45 @@ def build_extension(self, ext): f"-DBUILD_PYTHON_VERSION={_BUILD_PYTHON_VERSION}", ] - if "CMAKE_GENERATOR" not in os.environ or platform.system() == "Windows": - cmake_args += ["-GNinja"] - if platform.system() == "Windows": - python_version = sys.version_info - cmake_args += [ - "-DCMAKE_C_COMPILER=cl", - "-DCMAKE_CXX_COMPILER=cl", - f"-DPYTHON_VERSION={python_version.major}.{python_version.minor}", - f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}", - ] - if sys.maxsize > 2 ** 32: - cmake_args += ["-A", "x64"] - build_args += ["--", "/m"] + # CMake lets you override the generator - we need to check this. + # Can be set with Conda-Build, for example. + cmake_generator = os.environ.get("CMAKE_GENERATOR", "") + + if self.compiler.compiler_type != "msvc": + # Using Ninja-build since it a) is available as a wheel and b) + # multithreads automatically. MSVC would require all variables be + # exported for Ninja to pick it up, which is a little tricky to do. + # Users can override the generator with CMAKE_GENERATOR in CMake + # 3.15+. + if not cmake_generator: + try: + import ninja # noqa: F401 + + cmake_args += ["-GNinja"] + except ImportError: + pass + else: - build_args += ["--", "-j2"] + # Single config generators are handled "normally" + single_config = any(x in cmake_generator for x in {"NMake", "Ninja"}) + + # CMake allows an arch-in-generator style for backward compatibility + contains_arch = any(x in cmake_generator for x in {"ARM", "Win64"}) + + # Specify the arch if using MSVC generator, but only if it doesn't + # contain a backward-compatibility arch spec already in the + # generator name. + if not single_config and not contains_arch: + cmake_args += ["-A", PLAT_TO_CMAKE[self.plat_name]] + + # Multi-config generators have a different way to specify configs + if not single_config: + cmake_args += [ + f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}" + ] + build_args += ["--config", cfg] + # Set CMAKE_BUILD_PARALLEL_LEVEL to control the parallel build level # across all generators. if "CMAKE_BUILD_PARALLEL_LEVEL" not in os.environ: From 10b10a104815b32e31bf44c9c5185d49f4696573 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 16:23:27 -0800 Subject: [PATCH 078/153] set visual studio sheel before building for Windows --- .github/workflows/ci.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a4bb80b3..66cba3625 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,11 @@ jobs: - name: Setup msbuild on Windows if: matrix.os == 'windows-latest' uses: microsoft/setup-msbuild@v1.1 + - name: Set up Visual Studio shell + if: matrix.os == 'windows-latest' + uses: egor-tensin/vs-shell@v2 + with: + arch: x64 - name: Check out source repository uses: actions/checkout@v2 - name: Install dependencies @@ -57,9 +62,6 @@ jobs: echo "/home/runner/.local/bin" >> $GITHUB_PATH - name: Install test requirements run: pip3 install expecttest fsspec iopath==0.1.9 numpy pytest rarfile - - name: Allow Long Names for Windows - if: matrix.os == 'windows-latest' - run: reg add HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /d 1 /f - name: Install AWS-SDK-CPP on Windows for S3 IO datapipes if: matrix.os == 'windows-latest' run: | From 0da968479beeeb9dcda13f1e13b8e12ccb167644 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 16:35:12 -0800 Subject: [PATCH 079/153] removed unnecessary arch type argument in cmake --- tools/setup_helpers/extension.py | 50 ++++++++------------------------ 1 file changed, 12 insertions(+), 38 deletions(-) diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index e53f7f532..9c7b9ec46 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -81,44 +81,18 @@ def build_extension(self, ext): f"-DBUILD_PYTHON_VERSION={_BUILD_PYTHON_VERSION}", ] - # CMake lets you override the generator - we need to check this. - # Can be set with Conda-Build, for example. - cmake_generator = os.environ.get("CMAKE_GENERATOR", "") - - if self.compiler.compiler_type != "msvc": - # Using Ninja-build since it a) is available as a wheel and b) - # multithreads automatically. MSVC would require all variables be - # exported for Ninja to pick it up, which is a little tricky to do. - # Users can override the generator with CMAKE_GENERATOR in CMake - # 3.15+. - if not cmake_generator: - try: - import ninja # noqa: F401 - - cmake_args += ["-GNinja"] - except ImportError: - pass - - else: - - # Single config generators are handled "normally" - single_config = any(x in cmake_generator for x in {"NMake", "Ninja"}) - - # CMake allows an arch-in-generator style for backward compatibility - contains_arch = any(x in cmake_generator for x in {"ARM", "Win64"}) - - # Specify the arch if using MSVC generator, but only if it doesn't - # contain a backward-compatibility arch spec already in the - # generator name. - if not single_config and not contains_arch: - cmake_args += ["-A", PLAT_TO_CMAKE[self.plat_name]] - - # Multi-config generators have a different way to specify configs - if not single_config: - cmake_args += [ - f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}" - ] - build_args += ["--config", cfg] + # Default to Ninja + if "CMAKE_GENERATOR" not in os.environ or platform.system() == "Windows": + cmake_args += ["-GNinja"] + if platform.system() == "Windows": + import sys + + python_version = sys.version_info + cmake_args += [ + "-DCMAKE_C_COMPILER=cl", + "-DCMAKE_CXX_COMPILER=cl", + f"-DPYTHON_VERSION={python_version.major}.{python_version.minor}", + ] # Set CMAKE_BUILD_PARALLEL_LEVEL to control the parallel build level # across all generators. From d8d3c097f41b1f26cdbe4bc572b806f88d052c37 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 16:43:25 -0800 Subject: [PATCH 080/153] remove redundant import sys --- tools/setup_helpers/extension.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index 9c7b9ec46..37d1ee163 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -85,8 +85,6 @@ def build_extension(self, ext): if "CMAKE_GENERATOR" not in os.environ or platform.system() == "Windows": cmake_args += ["-GNinja"] if platform.system() == "Windows": - import sys - python_version = sys.version_info cmake_args += [ "-DCMAKE_C_COMPILER=cl", From aef5e492b154e774261f41e4fc189c4a7e552786 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 17:01:57 -0800 Subject: [PATCH 081/153] try add aws sdk to a sub cmake folder for cmake reference --- .github/workflows/ci.yml | 2 ++ CMakeLists.txt | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 66cba3625..31a8d246c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,6 +65,8 @@ jobs: - name: Install AWS-SDK-CPP on Windows for S3 IO datapipes if: matrix.os == 'windows-latest' run: | + mkdir cmake + cd cmake git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp mkdir sdk_build cd sdk_build diff --git a/CMakeLists.txt b/CMakeLists.txt index 8a2de8f1e..f77df542d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -56,6 +56,8 @@ endif() # Options option(BUILD_S3 "Build s3 io functionality" OFF) +set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${CMAKE_CURRENT_SOURCE_DIR}/cmake") + find_package(Torch REQUIRED) add_subdirectory(torchdata/csrc) From 7643752b442655189e6e8f4c2fae7fb6a8e9d112 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 17:19:03 -0800 Subject: [PATCH 082/153] build shared lib for windows, and append cmake_prefix_path --- .github/workflows/ci.yml | 2 +- torchdata/csrc/CMakeLists.txt | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31a8d246c..8898705af 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,7 +70,7 @@ jobs: git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp mkdir sdk_build cd sdk_build - cmake ..\aws-sdk-cpp -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" -DAUTORUN_UNIT_TESTS=OFF + cmake ..\aws-sdk-cpp -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" -DAUTORUN_UNIT_TESTS=OFF -DBUILD_SHARED_LIBS=ON msbuild ALL_BUILD.vcxproj -p:Configuration=Release msbuild INSTALL.vcxproj -p:Configuration=Release - name: Install AWS-SDK-CPP on Non-Windows for S3 IO datapipes diff --git a/torchdata/csrc/CMakeLists.txt b/torchdata/csrc/CMakeLists.txt index 25961ca0c..0b0772054 100644 --- a/torchdata/csrc/CMakeLists.txt +++ b/torchdata/csrc/CMakeLists.txt @@ -1,8 +1,6 @@ if(BUILD_S3) message(STATUS "Building S3 IO functionality") - include(CMakeDependentOption) - cmake_dependent_option(PYBIND11_FINDPYTHON "Force new FindPython" OFF - "NOT CMAKE_VERSION VERSION_LESS 3.12" OFF) + list(APPEND CMAKE_PREFIX_PATH "C:\\Program Files (x86)\\aws-cpp-sdk-all\\lib\\cmake") find_package(AWSSDK REQUIRED COMPONENTS s3 transfer) find_package(pybind11 REQUIRED) if(BUILD_PYTHON_VERSION) From 33a3b956ef90f25e758fade7715e2a2a28dc882d Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 17:24:43 -0800 Subject: [PATCH 083/153] add another flag for windows aws sdk cpp build --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8898705af..1538ce00a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,7 +70,7 @@ jobs: git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp mkdir sdk_build cd sdk_build - cmake ..\aws-sdk-cpp -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" -DAUTORUN_UNIT_TESTS=OFF -DBUILD_SHARED_LIBS=ON + cmake ..\aws-sdk-cpp -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" -DAUTORUN_UNIT_TESTS=OFF -DBUILD_SHARED_LIBS=ON -DFORCE_SHARED_CRT=OFF msbuild ALL_BUILD.vcxproj -p:Configuration=Release msbuild INSTALL.vcxproj -p:Configuration=Release - name: Install AWS-SDK-CPP on Non-Windows for S3 IO datapipes From d48f3ab7a8a36f3036c79c0683fa2f8e01a131a3 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 17:41:21 -0800 Subject: [PATCH 084/153] ping 1.8.x branch of aws-sdk-cpp for Windows --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1538ce00a..2b192287a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,7 +67,7 @@ jobs: run: | mkdir cmake cd cmake - git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp + git clone --branch 1.8.x --recurse-submodules https://github.com/aws/aws-sdk-cpp mkdir sdk_build cd sdk_build cmake ..\aws-sdk-cpp -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" -DAUTORUN_UNIT_TESTS=OFF -DBUILD_SHARED_LIBS=ON -DFORCE_SHARED_CRT=OFF From e8fd66302f4434b86bc159645da716010657f494 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 18:04:08 -0800 Subject: [PATCH 085/153] update aws-sdk-cpp build in Windows --- .github/workflows/ci.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b192287a..463a549c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,14 +65,11 @@ jobs: - name: Install AWS-SDK-CPP on Windows for S3 IO datapipes if: matrix.os == 'windows-latest' run: | - mkdir cmake - cd cmake - git clone --branch 1.8.x --recurse-submodules https://github.com/aws/aws-sdk-cpp - mkdir sdk_build - cd sdk_build - cmake ..\aws-sdk-cpp -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" -DAUTORUN_UNIT_TESTS=OFF -DBUILD_SHARED_LIBS=ON -DFORCE_SHARED_CRT=OFF - msbuild ALL_BUILD.vcxproj -p:Configuration=Release - msbuild INSTALL.vcxproj -p:Configuration=Release + git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp + cd aws-sdk-cpp + cmake -S . -B build -DBUILD_ONLY="s3;transfer" -DENABLE_TESTING=OFF -DCMAKE_BUILD_TYPE=Release + cmake --build build --config Release + cmake --install build --config Release - name: Install AWS-SDK-CPP on Non-Windows for S3 IO datapipes if: matrix.os != 'windows-latest' run: | From fc6fb22c3750b9e932b014e6d1ead30b4377afd6 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 18:28:06 -0800 Subject: [PATCH 086/153] point to aws-cpp-sdk location --- torchdata/csrc/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchdata/csrc/CMakeLists.txt b/torchdata/csrc/CMakeLists.txt index 0b0772054..e23c15c51 100644 --- a/torchdata/csrc/CMakeLists.txt +++ b/torchdata/csrc/CMakeLists.txt @@ -1,6 +1,6 @@ if(BUILD_S3) message(STATUS "Building S3 IO functionality") - list(APPEND CMAKE_PREFIX_PATH "C:\\Program Files (x86)\\aws-cpp-sdk-all\\lib\\cmake") + list(APPEND CMAKE_PREFIX_PATH "C:\\Program Files (x86)\\aws-cpp-sdk-all") find_package(AWSSDK REQUIRED COMPONENTS s3 transfer) find_package(pybind11 REQUIRED) if(BUILD_PYTHON_VERSION) From 5a20b37ca891a973b868a2a8ec5557f997a4271f Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 18:32:38 -0800 Subject: [PATCH 087/153] append CMAKE_INSTALL_PREFIX with windows aws sdk cpp path --- tools/setup_helpers/extension.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index 37d1ee163..e11728bfd 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -61,6 +61,9 @@ def build_extension(self, ext): if not extdir.endswith(os.path.sep): extdir += os.path.sep + if platform.system() == "Windows": + extdir += ";C:\\Program Files (x86)\\aws-cpp-sdk-all" + debug = int(os.environ.get("DEBUG", 0)) if self.debug is None else self.debug cfg = "Debug" if debug else "Release" From ae9a2e10a0832cd0692865adc28abe07d5fe06c0 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 18:48:01 -0800 Subject: [PATCH 088/153] remove unnecessary CMAKE PATH in CMakeLists --- torchdata/csrc/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/torchdata/csrc/CMakeLists.txt b/torchdata/csrc/CMakeLists.txt index e23c15c51..bba2e7948 100644 --- a/torchdata/csrc/CMakeLists.txt +++ b/torchdata/csrc/CMakeLists.txt @@ -1,6 +1,5 @@ if(BUILD_S3) message(STATUS "Building S3 IO functionality") - list(APPEND CMAKE_PREFIX_PATH "C:\\Program Files (x86)\\aws-cpp-sdk-all") find_package(AWSSDK REQUIRED COMPONENTS s3 transfer) find_package(pybind11 REQUIRED) if(BUILD_PYTHON_VERSION) From beebaa26f4b993e0dbe3ab2daf7431576e6adbea Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 19:16:21 -0800 Subject: [PATCH 089/153] change CMAKE_INSTALL_PREFIX in CMakeLists --- tools/setup_helpers/extension.py | 3 --- torchdata/csrc/CMakeLists.txt | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index e11728bfd..37d1ee163 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -61,9 +61,6 @@ def build_extension(self, ext): if not extdir.endswith(os.path.sep): extdir += os.path.sep - if platform.system() == "Windows": - extdir += ";C:\\Program Files (x86)\\aws-cpp-sdk-all" - debug = int(os.environ.get("DEBUG", 0)) if self.debug is None else self.debug cfg = "Debug" if debug else "Release" diff --git a/torchdata/csrc/CMakeLists.txt b/torchdata/csrc/CMakeLists.txt index bba2e7948..210595ee4 100644 --- a/torchdata/csrc/CMakeLists.txt +++ b/torchdata/csrc/CMakeLists.txt @@ -1,5 +1,6 @@ if(BUILD_S3) message(STATUS "Building S3 IO functionality") + list(APPEND CMAKE_INSTALL_PREFIX "C:\\Program Files (x86)\\aws-cpp-sdk-all") find_package(AWSSDK REQUIRED COMPONENTS s3 transfer) find_package(pybind11 REQUIRED) if(BUILD_PYTHON_VERSION) From ef21b4b5dffcd62be38537d152569f206e85a6d5 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 19:35:10 -0800 Subject: [PATCH 090/153] add necessary ; for CMAKE_INSTALL_PREFIX --- torchdata/csrc/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchdata/csrc/CMakeLists.txt b/torchdata/csrc/CMakeLists.txt index 210595ee4..760c198de 100644 --- a/torchdata/csrc/CMakeLists.txt +++ b/torchdata/csrc/CMakeLists.txt @@ -1,6 +1,6 @@ if(BUILD_S3) message(STATUS "Building S3 IO functionality") - list(APPEND CMAKE_INSTALL_PREFIX "C:\\Program Files (x86)\\aws-cpp-sdk-all") + list(APPEND CMAKE_INSTALL_PREFIX ";C:\\Program Files (x86)\\aws-cpp-sdk-all") find_package(AWSSDK REQUIRED COMPONENTS s3 transfer) find_package(pybind11 REQUIRED) if(BUILD_PYTHON_VERSION) From 9713c9649bc30138159227c73414c69212869fe9 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 19:38:38 -0800 Subject: [PATCH 091/153] change CMAKE_INSTALL_PREFIX to only sdk_dir in extension --- tools/setup_helpers/extension.py | 4 +++- torchdata/csrc/CMakeLists.txt | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index 37d1ee163..1bf6f1809 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -64,10 +64,12 @@ def build_extension(self, ext): debug = int(os.environ.get("DEBUG", 0)) if self.debug is None else self.debug cfg = "Debug" if debug else "Release" + sdk_dir = "C:\\Program Files (x86)\\aws-cpp-sdk-all" + cmake_args = [ f"-DCMAKE_BUILD_TYPE={cfg}", # not used on MSVC, but no harm f"-DCMAKE_PREFIX_PATH={torch.utils.cmake_prefix_path}", - f"-DCMAKE_INSTALL_PREFIX={extdir}", + f"-DCMAKE_INSTALL_PREFIX={sdk_dir}", f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}", f"-DPYTHON_EXECUTABLE={sys.executable}", "-DCMAKE_CXX_FLAGS=-fPIC", diff --git a/torchdata/csrc/CMakeLists.txt b/torchdata/csrc/CMakeLists.txt index 760c198de..07f1112c9 100644 --- a/torchdata/csrc/CMakeLists.txt +++ b/torchdata/csrc/CMakeLists.txt @@ -1,6 +1,6 @@ if(BUILD_S3) message(STATUS "Building S3 IO functionality") - list(APPEND CMAKE_INSTALL_PREFIX ";C:\\Program Files (x86)\\aws-cpp-sdk-all") + # list(APPEND CMAKE_INSTALL_PREFIX ";C:\\Program Files (x86)\\aws-cpp-sdk-all") find_package(AWSSDK REQUIRED COMPONENTS s3 transfer) find_package(pybind11 REQUIRED) if(BUILD_PYTHON_VERSION) From 0308bf69f0e5199c5d7797bff710b2bc42a124c4 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 20:04:26 -0800 Subject: [PATCH 092/153] use ninja for aws sdk cpp --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 463a549c9..27f18b1ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,7 +67,7 @@ jobs: run: | git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp cd aws-sdk-cpp - cmake -S . -B build -DBUILD_ONLY="s3;transfer" -DENABLE_TESTING=OFF -DCMAKE_BUILD_TYPE=Release + cmake -S . -B build -GNinja -DBUILD_ONLY="s3;transfer" -DENABLE_TESTING=OFF -DCMAKE_BUILD_TYPE=Release cmake --build build --config Release cmake --install build --config Release - name: Install AWS-SDK-CPP on Non-Windows for S3 IO datapipes From aeeea10fcc071cc955e43385489c4545401e3eab Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 20:31:27 -0800 Subject: [PATCH 093/153] move all includes to a precompile.h --- torchdata/csrc/pybind/S3Handler/S3Handler.cpp | 34 +++---------------- torchdata/csrc/pybind/S3Handler/S3Handler.h | 12 +------ 2 files changed, 6 insertions(+), 40 deletions(-) diff --git a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp index 03b8f3ff2..9943474b4 100644 --- a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp +++ b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp @@ -1,30 +1,5 @@ #include "S3Handler.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - namespace torchdata { namespace @@ -380,11 +355,12 @@ namespace torchdata if (headObjectOutcome.IsSuccess()) { return headObjectOutcome.GetResult().GetContentLength(); + } else { + Aws::String const &error_aws = headObjectOutcome.GetError().GetMessage(); + std::string error_str(error_aws.c_str(), error_aws.size()); + throw std::invalid_argument(error_str); + return 0; } - Aws::String const &error_aws = headObjectOutcome.GetError().GetMessage(); - std::string error_str(error_aws.c_str(), error_aws.size()); - throw std::invalid_argument(error_str); - return 0; } void S3Handler::ClearMarker() { last_marker_ = S3DefaultMarker; } diff --git a/torchdata/csrc/pybind/S3Handler/S3Handler.h b/torchdata/csrc/pybind/S3Handler/S3Handler.h index 4529aabae..8aa73bae5 100644 --- a/torchdata/csrc/pybind/S3Handler/S3Handler.h +++ b/torchdata/csrc/pybind/S3Handler/S3Handler.h @@ -1,12 +1,4 @@ -#ifndef TORCHDATA_S3_IO_H -#define TORCHDATA_S3_IO_H - -#include -#include -#include -#include - -#include +#include "precompile.h" namespace torchdata { @@ -67,5 +59,3 @@ namespace torchdata std::vector *filenames); }; } // namespace torchdata - -#endif // TORCHDATA_S3_IO_H From a2bf507f3f9b718d2acced0490c7802c591af668 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 20:33:00 -0800 Subject: [PATCH 094/153] add precompile.h --- torchdata/csrc/pybind/S3Handler/S3Handler.cpp | 3 ++ torchdata/csrc/pybind/S3Handler/precompile.h | 30 +++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 torchdata/csrc/pybind/S3Handler/precompile.h diff --git a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp index 9943474b4..1b7b947ed 100644 --- a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp +++ b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp @@ -1,5 +1,8 @@ #include "S3Handler.h" +#undef GetMessage +#undef GetObject + namespace torchdata { namespace diff --git a/torchdata/csrc/pybind/S3Handler/precompile.h b/torchdata/csrc/pybind/S3Handler/precompile.h new file mode 100644 index 000000000..5f5080bc3 --- /dev/null +++ b/torchdata/csrc/pybind/S3Handler/precompile.h @@ -0,0 +1,30 @@ +#ifndef TORCHDATA_S3_IO_H +#define TORCHDATA_S3_IO_H + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#endif // TORCHDATA_S3_IO_H From 97f977433575d51c20b0d035b0dd425e763e518d Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 20:33:41 -0800 Subject: [PATCH 095/153] remove undef functions in cpp --- torchdata/csrc/pybind/S3Handler/S3Handler.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp index 1b7b947ed..9943474b4 100644 --- a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp +++ b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp @@ -1,8 +1,5 @@ #include "S3Handler.h" -#undef GetMessage -#undef GetObject - namespace torchdata { namespace From 17617e7c248ebd9c29d6ec986c3ac4657046a886 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 20:51:05 -0800 Subject: [PATCH 096/153] remove msbuild from ci, because using ninja for aws-sdk-cpp in Windows --- torchdata/csrc/pybind/S3Handler/S3Handler.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp index 9943474b4..1b7b947ed 100644 --- a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp +++ b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp @@ -1,5 +1,8 @@ #include "S3Handler.h" +#undef GetMessage +#undef GetObject + namespace torchdata { namespace From d667aefaaac2b54d64f496054c88a875975b9b5c Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 20:53:25 -0800 Subject: [PATCH 097/153] remove msbuild from ci, because using ninja for aws-sdk-cpp in Windows --- .github/workflows/ci.yml | 12 ------------ torchdata/csrc/pybind/S3Handler/S3Handler.cpp | 3 --- 2 files changed, 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 27f18b1ed..83e5e3ae0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,18 +40,6 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Display Python version - run: | - python --version - echo $pythonlocation - - name: Setup msbuild on Windows - if: matrix.os == 'windows-latest' - uses: microsoft/setup-msbuild@v1.1 - - name: Set up Visual Studio shell - if: matrix.os == 'windows-latest' - uses: egor-tensin/vs-shell@v2 - with: - arch: x64 - name: Check out source repository uses: actions/checkout@v2 - name: Install dependencies diff --git a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp index 1b7b947ed..9943474b4 100644 --- a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp +++ b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp @@ -1,8 +1,5 @@ #include "S3Handler.h" -#undef GetMessage -#undef GetObject - namespace torchdata { namespace From 70554d7dc708b8663c2b2ce6ce17f3c1cc1d7c86 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 20:57:33 -0800 Subject: [PATCH 098/153] remove unnecessary CMakeList lines --- CMakeLists.txt | 2 -- torchdata/csrc/CMakeLists.txt | 1 - 2 files changed, 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f77df542d..8a2de8f1e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -56,8 +56,6 @@ endif() # Options option(BUILD_S3 "Build s3 io functionality" OFF) -set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${CMAKE_CURRENT_SOURCE_DIR}/cmake") - find_package(Torch REQUIRED) add_subdirectory(torchdata/csrc) diff --git a/torchdata/csrc/CMakeLists.txt b/torchdata/csrc/CMakeLists.txt index 07f1112c9..bba2e7948 100644 --- a/torchdata/csrc/CMakeLists.txt +++ b/torchdata/csrc/CMakeLists.txt @@ -1,6 +1,5 @@ if(BUILD_S3) message(STATUS "Building S3 IO functionality") - # list(APPEND CMAKE_INSTALL_PREFIX ";C:\\Program Files (x86)\\aws-cpp-sdk-all") find_package(AWSSDK REQUIRED COMPONENTS s3 transfer) find_package(pybind11 REQUIRED) if(BUILD_PYTHON_VERSION) From 9f6c847cbbca59df6e3cad89f1c74f00deafb240 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Feb 2022 21:04:57 -0800 Subject: [PATCH 099/153] set up msbuild for Windows in CI --- .github/workflows/ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 83e5e3ae0..1e89d8b0c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,6 +40,14 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + - name: Setup msbuild on Windows + if: matrix.os == 'windows-latest' + uses: microsoft/setup-msbuild@v1.1 + - name: Set up Visual Studio shell + if: matrix.os == 'windows-latest' + uses: egor-tensin/vs-shell@v2 + with: + arch: x64 - name: Check out source repository uses: actions/checkout@v2 - name: Install dependencies From c997c66cb66cc77a7bdf9b65531ef59724b8942c Mon Sep 17 00:00:00 2001 From: erjia Date: Thu, 24 Feb 2022 16:02:17 +0000 Subject: [PATCH 100/153] Fix mypy --- mypy.ini | 7 ------- torchdata/_torchdata/__init__.pyi | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 7 deletions(-) create mode 100644 torchdata/_torchdata/__init__.pyi diff --git a/mypy.ini b/mypy.ini index 2617f0aad..8bb1e7024 100644 --- a/mypy.ini +++ b/mypy.ini @@ -12,13 +12,6 @@ exclude = examples, test, packaging python_version = 3.7 -# -# Extension modules without stubs. -# - -[mypy-torchdata._torchdata] -ignore_missing_imports = True - # # Third party dependencies that don't have types. # diff --git a/torchdata/_torchdata/__init__.pyi b/torchdata/_torchdata/__init__.pyi new file mode 100644 index 000000000..e97aaa9e4 --- /dev/null +++ b/torchdata/_torchdata/__init__.pyi @@ -0,0 +1,14 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + + +from typing import List, Tuple + + +# TODO: Add pyi generate script +class S3Handler(): + def __init__(self, request_timeout_ms: int, region: str) -> None: ... + def s3_read(self, url: str) -> bytes: ... + def list_files(self, prefix: str) -> List[str]: ... + def set_buffer_size(self, buffer_size: int) -> None: ... + def set_multi_part_download(self, multi_part_download: bool) -> None: ... + def clear_marker(self) -> None: ... From 2ae4f2d46fb90907363a868d1d443ccb7ae24ce9 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Thu, 3 Mar 2022 10:38:02 -0800 Subject: [PATCH 101/153] update docstring with functional name and update examples --- torchdata/datapipes/iter/load/README.md | 4 ++-- torchdata/datapipes/iter/load/s3io.py | 24 ++++++++++++++++++++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/torchdata/datapipes/iter/load/README.md b/torchdata/datapipes/iter/load/README.md index 171668d7c..ffed9464b 100644 --- a/torchdata/datapipes/iter/load/README.md +++ b/torchdata/datapipes/iter/load/README.md @@ -61,9 +61,9 @@ from torchdata.datapipes.iter import S3FileLister, S3FileLoader s3_prefixes = ['s3://bucket-name/folder/', ...] dp_s3_urls = S3FileLister(s3_prefixes) -dp_s3_files = S3FileLoader(s3_urls) # outputs in (url, BytesIO) +dp_s3_files = S3FileLoader(s3_urls) # outputs in (url, StreamWrapper(BytesIO)) # more datapipes to convert loaded bytes, e.g. -datapipe = StreamWrapper(dp_s3_files).parse_csv_files(delimiter=' ') +datapipe = StreamWrapper(dp_s3_files).parse_csv(delimiter=' ') for d in datapipe: # Start loading data pass diff --git a/torchdata/datapipes/iter/load/s3io.py b/torchdata/datapipes/iter/load/s3io.py index 9c59c0565..06d18635c 100644 --- a/torchdata/datapipes/iter/load/s3io.py +++ b/torchdata/datapipes/iter/load/s3io.py @@ -11,7 +11,7 @@ class S3FileListerIterDataPipe(IterDataPipe[str]): r""":class:`S3FileListerIterDataPipe`. - Iterable DataPipe that lists URLs with the given prefixes. + Iterable DataPipe that lists URLs with the given prefixes (functional name: ``list_file_by_s3``). Args: source_datapipe: a DataPipe that contains URLs/URL prefixes to s3 files @@ -21,6 +21,16 @@ class S3FileListerIterDataPipe(IterDataPipe[str]): Note: AWS_CPP_SDK is necessary to use the S3 DataPipe(s). + + Example: + >>> from torchdata.datapipes.iter import S3FileLister, S3FileLoader + >>> s3_prefixes = ['s3://bucket-name/folder/', ...] + >>> dp_s3_urls = S3FileLister(s3_prefixes) + >>> dp_s3_files = S3FileLoader(s3_urls) # outputs in (url, StreamWrapper(BytesIO)) + >>> # more datapipes to convert loaded bytes, e.g. + >>> datapipe = dp_s3_files.parse_csv(delimiter=' ') + >>> for d in datapipe: # Start loading data + ... pass """ def __init__(self, source_datapipe: IterDataPipe[str], length: int = -1, request_timeout_ms=-1, region="") -> None: @@ -50,7 +60,7 @@ def __len__(self) -> int: class S3FileLoaderIterDataPipe(IterDataPipe[Tuple[str, StreamWrapper]]): r""":class:`S3FileListerIterDataPipe`. - Iterable DataPipe that loads S3 files given S3 URLs. + Iterable DataPipe that loads S3 files given S3 URLs (functional name: ``load_file_by_s3``). Args: source_datapipe: a DataPipe that contains URLs to s3 files @@ -59,6 +69,16 @@ class S3FileLoaderIterDataPipe(IterDataPipe[Tuple[str, StreamWrapper]]): Note: AWS_CPP_SDK is necessary to use the S3 DataPipe(s). + + Example: + >>> from torchdata.datapipes.iter import S3FileLister, S3FileLoader + >>> s3_prefixes = ['s3://bucket-name/folder/', ...] + >>> dp_s3_urls = S3FileLister(s3_prefixes) + >>> dp_s3_files = S3FileLoader(s3_urls) # outputs in (url, StreamWrapper(BytesIO)) + >>> # more datapipes to convert loaded bytes, e.g. + >>> datapipe = dp_s3_files.parse_csv(delimiter=' ') + >>> for d in datapipe: # Start loading data + ... pass """ def __init__( From 13741877494cbc5bdd49cbcead35a8a6693c01ad Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Thu, 3 Mar 2022 10:45:17 -0800 Subject: [PATCH 102/153] fix lint style --- test/test_remote_io.py | 4 +++- tools/setup_helpers/extension.py | 4 ++-- torchdata/_torchdata/__init__.pyi | 7 +++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/test/test_remote_io.py b/test/test_remote_io.py index 437a5b8dd..04f23349c 100644 --- a/test/test_remote_io.py +++ b/test/test_remote_io.py @@ -176,7 +176,9 @@ def test_s3_io_iterdatapipe(self): s3_lister_dp = S3FileLister(IterableWrapper(file_urls)) s3_loader_dp = S3FileLoader(IterableWrapper(file_urls)) except ModuleNotFoundError: - warnings.warn("S3 IO datapipes or C++ extension '_torchdata' isn't built in the current 'torchdata' package") + warnings.warn( + "S3 IO datapipes or C++ extension '_torchdata' isn't built in the current 'torchdata' package" + ) return # S3FileLister: different inputs diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index 1bf6f1809..4e9646811 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -1,7 +1,7 @@ import os -import sys import platform import subprocess +import sys from pathlib import Path import torch @@ -93,7 +93,7 @@ def build_extension(self, ext): "-DCMAKE_CXX_COMPILER=cl", f"-DPYTHON_VERSION={python_version.major}.{python_version.minor}", ] - + # Set CMAKE_BUILD_PARALLEL_LEVEL to control the parallel build level # across all generators. if "CMAKE_BUILD_PARALLEL_LEVEL" not in os.environ: diff --git a/torchdata/_torchdata/__init__.pyi b/torchdata/_torchdata/__init__.pyi index e97aaa9e4..0d7a9cf05 100644 --- a/torchdata/_torchdata/__init__.pyi +++ b/torchdata/_torchdata/__init__.pyi @@ -1,14 +1,13 @@ # Copyright (c) Facebook, Inc. and its affiliates. - - + from typing import List, Tuple # TODO: Add pyi generate script -class S3Handler(): +class S3Handler: def __init__(self, request_timeout_ms: int, region: str) -> None: ... def s3_read(self, url: str) -> bytes: ... - def list_files(self, prefix: str) -> List[str]: ... + def list_files(self, prefix: str) -> List[str]: ... def set_buffer_size(self, buffer_size: int) -> None: ... def set_multi_part_download(self, multi_part_download: bool) -> None: ... def clear_marker(self) -> None: ... From ba5b085ae0a98bff69258b2971f61014b718d4a7 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Thu, 3 Mar 2022 10:47:13 -0800 Subject: [PATCH 103/153] fix lint style --- torchdata/_torchdata/__init__.pyi | 1 - 1 file changed, 1 deletion(-) diff --git a/torchdata/_torchdata/__init__.pyi b/torchdata/_torchdata/__init__.pyi index 0d7a9cf05..55981abd1 100644 --- a/torchdata/_torchdata/__init__.pyi +++ b/torchdata/_torchdata/__init__.pyi @@ -2,7 +2,6 @@ from typing import List, Tuple - # TODO: Add pyi generate script class S3Handler: def __init__(self, request_timeout_ms: int, region: str) -> None: ... From 94d8e952f4902210dddfae7d776dd0a3f79d1327 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Thu, 3 Mar 2022 13:30:03 -0800 Subject: [PATCH 104/153] update github ci to include build-s3 = 0 --- .github/workflows/ci.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e89d8b0c..fbf410bea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,9 @@ jobs: - 3.7 - 3.8 - 3.9 + with-s3: + - 1 + - 0 steps: - name: Setup additional system libraries if: startsWith( matrix.os, 'ubuntu' ) @@ -41,10 +44,10 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Setup msbuild on Windows - if: matrix.os == 'windows-latest' + if: matrix.with-s3 == 1 && matrix.os == 'windows-latest' uses: microsoft/setup-msbuild@v1.1 - name: Set up Visual Studio shell - if: matrix.os == 'windows-latest' + if: matrix.with-s3 == 1 && matrix.os == 'windows-latest' uses: egor-tensin/vs-shell@v2 with: arch: x64 @@ -59,7 +62,7 @@ jobs: - name: Install test requirements run: pip3 install expecttest fsspec iopath==0.1.9 numpy pytest rarfile - name: Install AWS-SDK-CPP on Windows for S3 IO datapipes - if: matrix.os == 'windows-latest' + if: matrix.with-s3 == 1 && matrix.os == 'windows-latest' run: | git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp cd aws-sdk-cpp @@ -67,7 +70,7 @@ jobs: cmake --build build --config Release cmake --install build --config Release - name: Install AWS-SDK-CPP on Non-Windows for S3 IO datapipes - if: matrix.os != 'windows-latest' + if: matrix.with-s3 == 1 && matrix.os != 'windows-latest' run: | git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp cd aws-sdk-cpp/ @@ -79,7 +82,7 @@ jobs: - name: Build TorchData run: python setup.py develop env: - BUILD_S3: 1 + BUILD_S3: ${{ matrix.with-s3 }} BUILD_PYTHON_VERSION: ${{ matrix.python-version }} - name: Run DataPipes tests with pytest if: ${{ ! contains(github.event.pull_request.labels.*.name, 'ciflow/slow') }} From c1c445a8999028c15398df2b5b701e809b43a5b9 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Mar 2022 15:20:58 -0700 Subject: [PATCH 105/153] remove redundant setup.cfg --- setup.cfg | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 5c6311daf..000000000 --- a/setup.cfg +++ /dev/null @@ -1,5 +0,0 @@ -[bdist_wheel] -universal=1 - -[metadata] -license_file = LICENSE From d254f83ec3724c7b7436e95e97b8c35b21bc83b0 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Mar 2022 15:28:16 -0700 Subject: [PATCH 106/153] fix lint and update comment --- setup.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index d8b95e2a3..236c1fd82 100644 --- a/setup.py +++ b/setup.py @@ -6,11 +6,9 @@ import subprocess from pathlib import Path - from setuptools import find_packages, setup -from torchdata.datapipes.gen_pyi import gen_pyi - from tools import setup_helpers +from torchdata.datapipes.gen_pyi import gen_pyi ROOT_DIR = Path(__file__).parent.resolve() @@ -107,6 +105,7 @@ def run(self): # Package Info packages=find_packages(exclude=["test*", "examples*"]), zip_safe=False, + # C++ Extension Modules ext_modules=setup_helpers.get_ext_modules(), cmdclass={ "build_ext": setup_helpers.CMakeBuild, From 030f8d51aa9748b27433da9c85326063221fd72e Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Mar 2022 15:29:19 -0700 Subject: [PATCH 107/153] disable test building for aws-sdk-cpp --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fbf410bea..1fe0e3212 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,7 +76,7 @@ jobs: cd aws-sdk-cpp/ mkdir sdk-build cd sdk-build - cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" -DAUTORUN_UNIT_TESTS=OFF + cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" -DENABLE_TESTING=OFF make sudo make install - name: Build TorchData From a5c3a86b16b02f8724c8b3815dc0e1dce37f479d Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Wed, 23 Mar 2022 15:40:35 -0700 Subject: [PATCH 108/153] fix lint --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 236c1fd82..e77654fe8 100644 --- a/setup.py +++ b/setup.py @@ -6,6 +6,7 @@ import subprocess from pathlib import Path + from setuptools import find_packages, setup from tools import setup_helpers from torchdata.datapipes.gen_pyi import gen_pyi From 3c59f1888f72956a9fc92766689bb401d97a49ca Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Thu, 24 Mar 2022 14:17:35 -0700 Subject: [PATCH 109/153] remove warning when _torchdata fails to load --- torchdata/_extension.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/torchdata/_extension.py b/torchdata/_extension.py index a2e2d1e3e..daac5f59d 100644 --- a/torchdata/_extension.py +++ b/torchdata/_extension.py @@ -31,12 +31,9 @@ def _init_extension(): return _load_lib("libtorchdata") - try: - # This import is for initializing the methods registered via PyBind11 - # This has to happen after the base library is loaded - from torchdata import _torchdata # noqa - except ImportError as e: - warnings.warn(f"torchdata C++ extension unable to load: {e}") + # This import is for initializing the methods registered via PyBind11 + # This has to happen after the base library is loaded + from torchdata import _torchdata # noqa _init_extension() From 12003eadc8068a883acf12d065e8f84543cdf5c8 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Thu, 24 Mar 2022 16:06:28 -0700 Subject: [PATCH 110/153] load _torchdata instead of libtorchdata --- torchdata/_extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchdata/_extension.py b/torchdata/_extension.py index daac5f59d..f70db65bd 100644 --- a/torchdata/_extension.py +++ b/torchdata/_extension.py @@ -30,7 +30,7 @@ def _init_extension(): warnings.warn("torchdata C++ extension is not available.") return - _load_lib("libtorchdata") + _load_lib("_torchdata") # This import is for initializing the methods registered via PyBind11 # This has to happen after the base library is loaded from torchdata import _torchdata # noqa From 98709cb0b9d766a9a3f08c23e0c37e397068a09f Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Thu, 24 Mar 2022 16:48:22 -0700 Subject: [PATCH 111/153] print serach path and existence of _torchdata --- torchdata/_extension.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/torchdata/_extension.py b/torchdata/_extension.py index f70db65bd..cc0104599 100644 --- a/torchdata/_extension.py +++ b/torchdata/_extension.py @@ -20,6 +20,8 @@ def _load_lib(lib: str): # In this case, we expect that `libtorchdata` is available somewhere # in the search path of dynamic loading mechanism, and importing `_torchdata`, # which depends on `libtorchdata` and dynamic loader will handle it for us. + print("[Extension] _torchdata path: ", path) + print("[Extension] _torchdata path exists: ", path.exists()) if path.exists(): torch.ops.load_library(path) torch.classes.load_library(path) From ee972e6d48525de338598b747f06e69bb457c426 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Thu, 24 Mar 2022 17:03:43 -0700 Subject: [PATCH 112/153] print torch and torchdata paths --- test/test_remote_io.py | 3 +++ torchdata/_extension.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/test/test_remote_io.py b/test/test_remote_io.py index 04f23349c..efb02b817 100644 --- a/test/test_remote_io.py +++ b/test/test_remote_io.py @@ -18,6 +18,7 @@ S3FileLoader, ) +import torchdata class TestDataPipeRemoteIO(expecttest.TestCase): def setUp(self): @@ -170,6 +171,8 @@ def _read_and_decode(x): self.assertEqual(expected_csv_path, csv_path) def test_s3_io_iterdatapipe(self): + print("[Extension] torchdata path: ", torchdata.__file__) + # sanity test file_urls = ["s3://ai2-public-datasets"] try: diff --git a/torchdata/_extension.py b/torchdata/_extension.py index cc0104599..450005cad 100644 --- a/torchdata/_extension.py +++ b/torchdata/_extension.py @@ -32,6 +32,8 @@ def _init_extension(): warnings.warn("torchdata C++ extension is not available.") return + print("[Extension] torch path: ", torch.__file__) + _load_lib("_torchdata") # This import is for initializing the methods registered via PyBind11 # This has to happen after the base library is loaded From 97ccbe1e2d4af3665c12a46060946a79454ed32e Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Thu, 24 Mar 2022 17:27:27 -0700 Subject: [PATCH 113/153] print torchdata path --- test/test_remote_io.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/test_remote_io.py b/test/test_remote_io.py index efb02b817..b1e02d085 100644 --- a/test/test_remote_io.py +++ b/test/test_remote_io.py @@ -171,8 +171,6 @@ def _read_and_decode(x): self.assertEqual(expected_csv_path, csv_path) def test_s3_io_iterdatapipe(self): - print("[Extension] torchdata path: ", torchdata.__file__) - # sanity test file_urls = ["s3://ai2-public-datasets"] try: @@ -278,4 +276,5 @@ def test_s3_io_iterdatapipe(self): if __name__ == "__main__": + print("[Extension] torchdata path: ", torchdata.__file__) unittest.main() From 90062e396e3aba1776e0a09ea95aa75eee937cfa Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Thu, 24 Mar 2022 18:07:50 -0700 Subject: [PATCH 114/153] print torchdata path --- torchdata/_extension.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/torchdata/_extension.py b/torchdata/_extension.py index 450005cad..6b5aa0976 100644 --- a/torchdata/_extension.py +++ b/torchdata/_extension.py @@ -32,7 +32,9 @@ def _init_extension(): warnings.warn("torchdata C++ extension is not available.") return + import torchdata print("[Extension] torch path: ", torch.__file__) + print("[Extension] torchdata path: ", torchdata.__file__) _load_lib("_torchdata") # This import is for initializing the methods registered via PyBind11 From 4233a56580329bdbefbeab48e7afad99784de8c0 Mon Sep 17 00:00:00 2001 From: Daiming Yang <66369380+ydaiming@users.noreply.github.com> Date: Fri, 25 Mar 2022 11:19:06 -0700 Subject: [PATCH 115/153] find path for extension _torchdata --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1fe0e3212..768f74df2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,6 +84,8 @@ jobs: env: BUILD_S3: ${{ matrix.with-s3 }} BUILD_PYTHON_VERSION: ${{ matrix.python-version }} + - name: Find extension + run: ls -al _torchdata* - name: Run DataPipes tests with pytest if: ${{ ! contains(github.event.pull_request.labels.*.name, 'ciflow/slow') }} run: From b60fc7ed63686d896b24b9193ecf2152a7335ba4 Mon Sep 17 00:00:00 2001 From: Daiming Yang <66369380+ydaiming@users.noreply.github.com> Date: Fri, 25 Mar 2022 11:50:23 -0700 Subject: [PATCH 116/153] Update _torchdata find path --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 768f74df2..81023f840 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,7 +85,7 @@ jobs: BUILD_S3: ${{ matrix.with-s3 }} BUILD_PYTHON_VERSION: ${{ matrix.python-version }} - name: Find extension - run: ls -al _torchdata* + run: ls -al torchdata/_torchdata* - name: Run DataPipes tests with pytest if: ${{ ! contains(github.event.pull_request.labels.*.name, 'ciflow/slow') }} run: From 54a993bb35e1c2f30b5cb3df333c461390cdf5fb Mon Sep 17 00:00:00 2001 From: Daiming Yang <66369380+ydaiming@users.noreply.github.com> Date: Fri, 25 Mar 2022 12:22:31 -0700 Subject: [PATCH 117/153] add sheel bash to find _torchdata path --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81023f840..b82dea7c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,6 +85,7 @@ jobs: BUILD_S3: ${{ matrix.with-s3 }} BUILD_PYTHON_VERSION: ${{ matrix.python-version }} - name: Find extension + shell: bash run: ls -al torchdata/_torchdata* - name: Run DataPipes tests with pytest if: ${{ ! contains(github.event.pull_request.labels.*.name, 'ciflow/slow') }} From af9d7c1e3e1d1bfbf4cf5544655b53b3bbdb8728 Mon Sep 17 00:00:00 2001 From: Daiming Yang <66369380+ydaiming@users.noreply.github.com> Date: Fri, 25 Mar 2022 12:45:32 -0700 Subject: [PATCH 118/153] Update lib find path in _extension.py --- torchdata/_extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchdata/_extension.py b/torchdata/_extension.py index 6b5aa0976..6cbc3d52a 100644 --- a/torchdata/_extension.py +++ b/torchdata/_extension.py @@ -5,7 +5,7 @@ import torch from torchdata._internal import module_utils as _mod_utils # noqa: F401 -_LIB_DIR = Path(__file__).parent / "lib" +_LIB_DIR = Path(__file__).parent def _get_lib_path(lib: str): From 543294ea70f61b1561ca2b9ee4806321808db3b4 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Fri, 25 Mar 2022 15:27:41 -0700 Subject: [PATCH 119/153] use the method from torchtext to find _torchdata .so --- torchdata/_extension.py | 42 +++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/torchdata/_extension.py b/torchdata/_extension.py index 6cbc3d52a..07c190900 100644 --- a/torchdata/_extension.py +++ b/torchdata/_extension.py @@ -27,19 +27,37 @@ def _load_lib(lib: str): torch.classes.load_library(path) +# def _init_extension(): +# if not _mod_utils.is_module_available("torchdata._torchdata"): +# warnings.warn("torchdata C++ extension is not available.") +# return + +# import torchdata +# print("[Extension] torch path: ", torch.__file__) +# print("[Extension] torchdata path: ", torchdata.__file__) + +# _load_lib("_torchdata") +# # This import is for initializing the methods registered via PyBind11 +# # This has to happen after the base library is loaded +# from torchdata import _torchdata # noqa + + def _init_extension(): - if not _mod_utils.is_module_available("torchdata._torchdata"): - warnings.warn("torchdata C++ extension is not available.") - return - - import torchdata - print("[Extension] torch path: ", torch.__file__) - print("[Extension] torchdata path: ", torchdata.__file__) - - _load_lib("_torchdata") - # This import is for initializing the methods registered via PyBind11 - # This has to happen after the base library is loaded - from torchdata import _torchdata # noqa + import importlib + import os + + import torch + + # load the custom_op_library and register the custom ops + lib_dir = os.path.dirname(__file__) + loader_details = (importlib.machinery.ExtensionFileLoader, importlib.machinery.EXTENSION_SUFFIXES) + + extfinder = importlib.machinery.FileFinder(lib_dir, loader_details) + ext_specs = extfinder.find_spec("_torchdata") + if ext_specs is None: + raise ImportError("torchdata C++ Extension is not found.") + torch.ops.load_library(ext_specs.origin) + torch.classes.load_library(ext_specs.origin) _init_extension() From bcec5fca26c41bed74a8226e75f791c4475d7590 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Fri, 25 Mar 2022 15:40:57 -0700 Subject: [PATCH 120/153] remove _torchdata and try not to confuse importlib --- torchdata/_torchdata/__init__.pyi | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 torchdata/_torchdata/__init__.pyi diff --git a/torchdata/_torchdata/__init__.pyi b/torchdata/_torchdata/__init__.pyi deleted file mode 100644 index 55981abd1..000000000 --- a/torchdata/_torchdata/__init__.pyi +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. - -from typing import List, Tuple - -# TODO: Add pyi generate script -class S3Handler: - def __init__(self, request_timeout_ms: int, region: str) -> None: ... - def s3_read(self, url: str) -> bytes: ... - def list_files(self, prefix: str) -> List[str]: ... - def set_buffer_size(self, buffer_size: int) -> None: ... - def set_multi_part_download(self, multi_part_download: bool) -> None: ... - def clear_marker(self) -> None: ... From 7f69ddaa74a9f6262e06451a0fb7b8648fd6ebd7 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Fri, 25 Mar 2022 15:49:52 -0700 Subject: [PATCH 121/153] remove _init_extension() all time --- torchdata/_extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchdata/_extension.py b/torchdata/_extension.py index 07c190900..c7a3ba7c7 100644 --- a/torchdata/_extension.py +++ b/torchdata/_extension.py @@ -60,4 +60,4 @@ def _init_extension(): torch.classes.load_library(ext_specs.origin) -_init_extension() +# _init_extension() From 0d494a4c1f5b0940f8e830ca75247704d25c7741 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Fri, 25 Mar 2022 16:09:42 -0700 Subject: [PATCH 122/153] comment out gen_pyi in setup.py --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index e77654fe8..d0b2ca115 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ from setuptools import find_packages, setup from tools import setup_helpers -from torchdata.datapipes.gen_pyi import gen_pyi +# from torchdata.datapipes.gen_pyi import gen_pyi ROOT_DIR = Path(__file__).parent.resolve() @@ -113,4 +113,4 @@ def run(self): "clean": clean, }, ) - gen_pyi() + # gen_pyi() From 0f0d15835b9342686e7376867326281d123be823 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Fri, 25 Mar 2022 16:10:10 -0700 Subject: [PATCH 123/153] should _init_extension() at all time --- torchdata/_extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchdata/_extension.py b/torchdata/_extension.py index c7a3ba7c7..07c190900 100644 --- a/torchdata/_extension.py +++ b/torchdata/_extension.py @@ -60,4 +60,4 @@ def _init_extension(): torch.classes.load_library(ext_specs.origin) -# _init_extension() +_init_extension() From bff38d94d3170cc669833107c2c1e89bddcaca87 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Fri, 25 Mar 2022 16:34:23 -0700 Subject: [PATCH 124/153] add extension loading logic for windows --- torchdata/_extension.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/torchdata/_extension.py b/torchdata/_extension.py index 07c190900..121478660 100644 --- a/torchdata/_extension.py +++ b/torchdata/_extension.py @@ -50,6 +50,30 @@ def _init_extension(): # load the custom_op_library and register the custom ops lib_dir = os.path.dirname(__file__) + + if os.name == "nt": + # Register the main torchvision library location on the default DLL path + import ctypes + import sys + + kernel32 = ctypes.WinDLL("kernel32.dll", use_last_error=True) + with_load_library_flags = hasattr(kernel32, "AddDllDirectory") + prev_error_mode = kernel32.SetErrorMode(0x0001) + + if with_load_library_flags: + kernel32.AddDllDirectory.restype = ctypes.c_void_p + + if sys.version_info >= (3, 8): + os.add_dll_directory(lib_dir) + elif with_load_library_flags: + res = kernel32.AddDllDirectory(lib_dir) + if res is None: + err = ctypes.WinError(ctypes.get_last_error()) + err.strerror += f' Error adding "{lib_dir}" to the DLL directories.' + raise err + + kernel32.SetErrorMode(prev_error_mode) + loader_details = (importlib.machinery.ExtensionFileLoader, importlib.machinery.EXTENSION_SUFFIXES) extfinder = importlib.machinery.FileFinder(lib_dir, loader_details) From c137e03e4cd4dbe22bf2824d101df1009418ab24 Mon Sep 17 00:00:00 2001 From: erjia Date: Mon, 28 Mar 2022 16:27:24 +0000 Subject: [PATCH 125/153] Optimize pybind build --- .github/workflows/ci.yml | 8 ++++---- setup.py | 3 ++- tools/setup_helpers/extension.py | 12 ++++++++---- torchdata/_extension.py | 8 ++------ .../_torchdata.cpython-39-x86_64-linux-gnu.so | Bin 0 -> 171960 bytes 5 files changed, 16 insertions(+), 15 deletions(-) create mode 100755 torchdata/_torchdata.cpython-39-x86_64-linux-gnu.so diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b82dea7c0..36ac86c14 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,10 +57,8 @@ jobs: run: | pip3 install -r requirements.txt pip3 install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - pip3 install cmake ninja pybind11 pybind11-global + pip3 install cmake ninja pybind11 echo "/home/runner/.local/bin" >> $GITHUB_PATH - - name: Install test requirements - run: pip3 install expecttest fsspec iopath==0.1.9 numpy pytest rarfile - name: Install AWS-SDK-CPP on Windows for S3 IO datapipes if: matrix.with-s3 == 1 && matrix.os == 'windows-latest' run: | @@ -80,13 +78,15 @@ jobs: make sudo make install - name: Build TorchData - run: python setup.py develop + run: pybind11_DIR=`pybind11-config --cmakedir` python setup.py develop env: BUILD_S3: ${{ matrix.with-s3 }} BUILD_PYTHON_VERSION: ${{ matrix.python-version }} - name: Find extension shell: bash run: ls -al torchdata/_torchdata* + - name: Install test requirements + run: pip3 install expecttest fsspec iopath==0.1.9 numpy pytest rarfile - name: Run DataPipes tests with pytest if: ${{ ! contains(github.event.pull_request.labels.*.name, 'ciflow/slow') }} run: diff --git a/setup.py b/setup.py index d0b2ca115..d6514afec 100644 --- a/setup.py +++ b/setup.py @@ -9,6 +9,7 @@ from setuptools import find_packages, setup from tools import setup_helpers + # from torchdata.datapipes.gen_pyi import gen_pyi ROOT_DIR = Path(__file__).parent.resolve() @@ -104,7 +105,7 @@ def run(self): "Topic :: Scientific/Engineering :: Artificial Intelligence", ], # Package Info - packages=find_packages(exclude=["test*", "examples*"]), + packages=find_packages(exclude=["test*", "examples*", "tools*"]), zip_safe=False, # C++ Extension Modules ext_modules=setup_helpers.get_ext_modules(), diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index 4e9646811..7a0b6b8a1 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -4,15 +4,20 @@ import sys from pathlib import Path -import torch -from setuptools import Extension from setuptools.command.build_ext import build_ext +try: + from pybind11.setup_helpers import Pybind11Extension +except ImportError: + from setuptools import Extension as Pybind11Extension + + __all__ = [ "get_ext_modules", "CMakeBuild", ] + _THIS_DIR = Path(__file__).parent.resolve() _ROOT_DIR = _THIS_DIR.parent.parent.resolve() @@ -37,7 +42,7 @@ def _get_build(var, default=False): def get_ext_modules(): if _BUILD_S3: - return [Extension(name="torchdata._torchdata", sources=[])] + return [Pybind11Extension(name="torchdata._torchdata", sources=[])] else: return [] @@ -68,7 +73,6 @@ def build_extension(self, ext): cmake_args = [ f"-DCMAKE_BUILD_TYPE={cfg}", # not used on MSVC, but no harm - f"-DCMAKE_PREFIX_PATH={torch.utils.cmake_prefix_path}", f"-DCMAKE_INSTALL_PREFIX={sdk_dir}", f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}", f"-DPYTHON_EXECUTABLE={sys.executable}", diff --git a/torchdata/_extension.py b/torchdata/_extension.py index 121478660..c3ee2f84d 100644 --- a/torchdata/_extension.py +++ b/torchdata/_extension.py @@ -1,8 +1,6 @@ import os -import warnings from pathlib import Path -import torch from torchdata._internal import module_utils as _mod_utils # noqa: F401 _LIB_DIR = Path(__file__).parent @@ -46,8 +44,6 @@ def _init_extension(): import importlib import os - import torch - # load the custom_op_library and register the custom ops lib_dir = os.path.dirname(__file__) @@ -80,8 +76,8 @@ def _init_extension(): ext_specs = extfinder.find_spec("_torchdata") if ext_specs is None: raise ImportError("torchdata C++ Extension is not found.") - torch.ops.load_library(ext_specs.origin) - torch.classes.load_library(ext_specs.origin) + + from torchdata import _torchdata # noqa _init_extension() diff --git a/torchdata/_torchdata.cpython-39-x86_64-linux-gnu.so b/torchdata/_torchdata.cpython-39-x86_64-linux-gnu.so new file mode 100755 index 0000000000000000000000000000000000000000..38fb0d495f9b767fd6e9a1339aec57cb0b7c3649 GIT binary patch literal 171960 zcmb?^2|!%c_5P3*HStYSqqxz7aL3HNjY$|M#8u-f)@BYt`R>S~BnZ-d)Z; z_uO;Oz3;y9+A{yVQCV3o`!m{gri-ZSdXw>5H#V#rWnH;kURRN8JpMh(b)=*no4T|A zMP7@U&;E9~Jd%iYjKklCHuL_5wqu32zx%!>&=r_se%tw&n)%kp&3x|k0{g^RQI`1Ejzpvx(82s@w72n5N*Tmnz^>O$+9)HvD zcLM%S#NSEyd-u6r2Osv+e+N6?C|Y~E`_g$E5+5GCWz^(PURmUR^Y7oTJnGRmZoPWW z_?_j66VCXcwCsiV_FnSwYmePu-~G|@s}8w$&nchIefse3cjhhGcj61XpWNL3^}ql6 z8`qD%`8t2qZ=blp>Rn1O^79-zdh$q0kQegUx>avC9c1c4<#;{&u*~V#ADOxQQD4u@ z{{{A&son<;%*?;7W#&&nhh}Qm8xG}M4L(!)OJSIq_&pBgFFQDM`nl+VOyx{JAhY~O zL8l%nHtpj(HgozX9Q5!B@=%&`Z1H7i%8a3C3?meLC2~tq$$YJ0^2EGaSk}0F`Ev|2PNxFFHE2{*QLBt2@AF zYS&qqWHa^eY6z1l{dfoYA9j%cB!_YL?TMM&TZ;6V>fP%g{|63sb&G@D9_OI9uY=E| z|HmEt`S}ih`xA%sk2?76DhK^{JJ{8!7}uHlYd1z>CcSNP7%%5L__1l2>@$_W2mO*s zKYw*-?^zD|+3w(%bX1zD{5bk0lRQ}t{$XWKW<6Z&Q12fd{AYnDvpkPDl(Q51%p~Wf z4*DPGptpRdafI=bsei{f^jC|6e>>Gd&#yVy=i?6Z_)UoDGRcGC=z{9@`>b=Yw}T!0 z+Y6{SlRexDIWyHe!C@RNL!6%}{c(^#Q~8HE_<>s;`u!JZaHjHqjCn0n`RxvN(&8`= z{>?%DqajZwd;6(_p1m`!t7f{R#@nq|eu2 zX_@5NG&OU*l^E}t(x2ckPPaS6MTa@q+iVBFU555%(u3wOPMaO%+=cdLD(4di`+3`8 z9DT>Zo-e?-$fTc-9NK%NgTD9mdx| z4&!L6L-_+(CuWlWL5DcxE{Ae{<*+^)@1VE)FeozV!SCQ7e&Qg{VGi@)+3>fS#+~6{ z=TE?YW|Al7unuZ>@F#N|=Cy8zd1t*toN%+lI9lLP{w@c5yUjuVJ00p>>o6{!apUD zoRb{lE8Ri=F;8as7dXgsjYGfx+hP7X4f@ICAGo)biT{N|`b`e$_h$lrzD>er|KnPpgC9{)NMMU+xf}_#Mh0?cn#?9nxO{ z`^n@dxpA6F&tuSEnfP+>P{n?q7aZcKPaXWg5f1&b&S8E1Fyzl9&xg=MCi}V5A$|^x z%AEdMhxKDMl$J?v_d3kihdacjFFWu@JNSoN9mdgj9qh9RJpvW&_c;yv&!o3rhw=MM zhjDip^pi=?=cDjU<=0NlthWw_et*`%Z|`%k=e-X5SI0Q)W54WRhZPR;9P1G0>~NT8 zTO9oAfJ478c8EWZa@fE644s=v&(}KmlYI_x!fJ>4;uHsY+EHmH`#;7ZF52$UU)MU= z;dX~{_nbpq@h^w|n(eSoS?-{pd*BB$>C@|=|E&)C{E>q^jSlnOy$+q_4tK5nfnBnhK4S6NKOD(F z3q4PEO>nsjU|{&H9%ZI?S?%huV*;?Qd?#4bQq2OYSJq!y>sndAw0d>($u*(S z$<50yKRMh~-O#+ew&|4S#`Igw(=(7YjZL){9SYYq)mAS9IwhZZo5Ra$PCSut7U7DT znqV_g0a1;~2i63d8bh^B;Wfd6ZH-uVTSKn+V5Gs}2Ot7GKH5ietq~_a^`ley=sMJ-sNv#B#Zh2EJ zwMa`?5vguk7OXaNAPu#59yD9u*f1hR6C|x}PRq5lu?cDnvTxIS<3gxpc8%GvU~NlH zZ77T)(YKN+$(_QfM$G zkFCt6TC+BoB?hNCLYu=)HLF4{e66g7fv|vrpw8-+t_jyR2hP0$#dBaWhrhA$@<<43 z@-;Mvs~c)+%TZ$EGP9)AaP>9Yx2a&LoOxA+&ueU2Rn5^s<<6t&rt->|I={C1@hfNQGY^EzlgB0Tk ztSPMyHAf&H24bMGz5%XXX~(>###JfnlktpXzHsfT^XtQPw$g;JpnEqhc43VS#AWRF zRgG7WmL#Q(p*8k3JQ0gYl`S3UYp#ea4bwAU(cBP%&4-tpjh#)|TcD1@rPa;#@a!@m zd^I$On!4(yASROfaI>$btfCtKloi7Wt3x3SE^5U#-7j0p3`|?g%FOH);V+we*02C* zv^7^Ostt)x7K^Y{X1U20nw*skYO(Ah>{|unSp|1py9_KX#5O2c9aTsTVO)M$@yy0i zJ72YgnN6afR1sfZ23tjNzB7WR9ACbW;!uR{026o_CtZ%{iZzH(R+;7rQ&}cXp>$rP zp@x%!?GJ3jH)k>%L93;W(6-tga5x;_(7)-1OD)GTGABmHf@+2f9G+jcJ7}@1A`A@R zF~w$wM<&MxTBb-iJWy?4HbNq_JZMk6Eej?}`f2gRY{rtNL%rBYs7a#&spX?%!aG37Ki)zmGk4p)P* zgeZka4c8Wepv)L%2rf*KIbkj+$QPeey|n%ccwt#D0S%F?}W)UAwZNh{`e5-n@v~Y?kQ+4Pm$R%6Lf1|{3pD+XkaF9=ux#S{lCLcvwl zm)8cH;mXnOaP$_=tKZ-EQ&SjvMYs)r&|E7)u~pD4)5=oQ z0SlkGBwW9;c@~CsZS|_cf&ji4AHgcgB<&eK1Xc>{>Eryf9)0oK|rZ_Wi=92fzVrga;Teq@S;yVsEL{<_~6$KY9 z#T1IbU|DTVlgu`hd6;fsOhc7Q`oiK=`uc`uET@9ih+>=IC{{A+TI!G&LGPjpiLt-J zbTzxCaa9P`!9ab66tZ6r7Rok%Fm2C$_OgGSaI*owexV2vazoJ$EXyggwhKuy+4 zWrPRFmuN}`IUB%rH9{MPidbHMHRDAjRgRe`v=VWv9Y|voP=&CjqKfIIE9>E)WW#4c zMR1l0F)^65#50kmYAzGX7E}}l3uYp=DT##Z&YMwDzoMbGsW3lKy@m!(2#W~Fu1DF@ z1(@BY&sb6sEGwy494rVfDw~fPq{JUATQE1U(1$r^VPG*BT1Z8hp`y79(kx3A)6-#( zH6^MQuSJd38heirS`G=7<$j z%?}o)5?Z&-=+M@*-EK1=D>jW3%7wBU8&}pYGl{a6+L{Ogc?hs-0ivgZ8TM3XbpnbJ z@A}^YO0$X?=U^+hnE%Kus74BAg6Wm3Ru&ZCpSg`SGNnlpSUq>0f+fx7tWnYcGmvc$ zdnHoR%srpsImI$O=kVLs=9iz!WRIh{8DFJ*R9Eh4xpFj&$IQ=Pg~6H&o_ zQ$6JWKh2T*6)1{cfn92D-f~fltZ z;rQRY#8>aXSy)f9^AuUW@Pg_qYkjFb)U2usxpvF2Vw=H*n}ivvvr^+jQ*7LwVY{;Q zC{Ssb!=RF8z{}b~WgMJJqKfd08pt4)S}-dZY&7R%EXcVE$uun^kVoifPT^K+AHx6q zU|GeiVBp+}=|QojipJ@oNErJ^=<$linafvVOUVkd>StgI#5Q7$ZFDnhIBj8y(;To%^yRiyJ(EEDA@OOX+Qh}l^ogiygN>YKum zYIZ#470%B1YIIpo9=TnI2EsR8o<>C)gbYI8<`clnYu5x*k<>gy4`F)?OqN%{Qa%Vo z1mtgQSb@ofUqI%bTI!|+t(#WTB@Of;wabM4N_NF*#FVXUk!4xfk+Dp8W#bC0IVD2l zXdd2iGv_&b{`*Q5S67rR<&zx7z!>(3yU9CAkEMAPdW>_9}x#^HIdSVVAJcsPaNSp_S+h3Fvmy%r0<4Dz8t|CXHNUhpIQ^9AIIdz| z<-6wleP@>j3r{ILWu|#uS&0-gPAMoF0gGLwm6auD`+^0?otCmdy*FJ2#UsinR1>Oq z)az3U^VJdx&3h1VM(PUUrLGHRxK3qLPpxZQReNe}(`EJ5r><^ny1Y4rrTMAVI0VE4 zKsX3=D!*|?=T!Xh7!pq;jmF;?7Y;AU2Z!D8>kMP@zigZ_Q$I|>^!(1om6>A{uI&$V znfGy2%=+N)mZY##W#a^yd3_L4kCk84u+!q8W~zp(JYp@Fp}v!$@h;Qee%9;UJ7VLH}irej$q5(=m0%+i^Y|1-~Z4cExr!+J3# z=G_Tk*%sa%DI?1@OBrXillr$yWDfpO2jg7SC_gO0luyIeB>auV`Mj}m-_D`7Q!?Sc zSwE3kKC$_+Yb1k1-PXsJBFpt<{^vsKQLY}8!Sjsy_^S?n69)A)*OhWE(B%qza}Ao} zc13~tVd?Af&Ga7pMUJRP`h#3Ir|uu*+F;%9#&_M&51_Kiu60&H`?U^<|hlI>;-(Yef%6!Uk`(y=sMX-A3mJzJa#&59XsY>H*dFs-5Tb@uF?`ksTX-VU4d0NwWTb_pnNrSq14Hl*>kJiTeWd#z~)ed)Z_Upvxyt6%!lc$bylNaMS$ z@imyncUtz7ZTSh?FXdVCxYKy|xu*Ve(s<7~mR+Usb~#09e84Klo5qK%aw^k!yPT>t zzQZb~E{*TD%4tdCy;i$g)A&}ODgW9uzTeVMM;hO2;q_^J|2Ia7f8UVCdvZ;_JB{zO za9$%8*_one~s~kOzAGG?h zGL5(GrzMT=w(NOr8t=92w#a%v$ktoH(r;xN-)Y(5+BClZ5mV0{X?&N(cc$?jR$O7E z@pd_bX}n#I%Zj5@a$4oM)A*3}{*pJ1x69Ylc)R?{G(KdN-}(JxW!&5K+PFK7e`%`e zA9~aH{_je5S6>?6@l%tx{XMmKA^z-jj@$AR>AdYf(s{!ny*k#owwh2OXowD z{iO4)4!pgNNl$OjSLB~i+H22C>AXGvr1L}8yp+yYDtb`M4>|B_9e6ulPA`9hL;5}k ze#n7$TjMOf{5%J~${H8x>FxLa(s}#6zjWSyFE5?9-@i-e^A!2{TcP%6N5UKz+^13b z9P7PECRKT_QqGBXA^2XWv`giO9Qc4WPFcR1zSV)RQ|irASxaS+At$Z&%o#L51heh4(7ya}+sMzE0tfQsfCKe6GUx zIq-Q(d)4%H4t$@|UNyZ}8AmGLs?^Kh^i6$~a#X%cNq@YOeuKill`6#L>Qv-Z`7Wh= zmG4!`IYKGFU*V5ac#l%f2wrLL2wstgziDiLiWHu|Wo&=?ZW*>8m0zc%&r#C%D}GGf zxi=J^`{VX!P~p}0@P`zBco!JC6nXFpcj}X^@SYTl`Cs9uDtwN@n=LeNdKCV6C4HX4 z=P7(Z(X-0uE9trKY=4Rre!5NK+pF+J3a=~tX$l`u_!5QBQ{++kN+o@%lDN{e~rQ$3V)r#4=Q|2;aipV zs{D|W{uU*Dw&G`gtnhAy|Cz$)DEvJNpQrG@Quus@|FyywDg1*9?^XEUDZH-m4=a2? z;h$9aN`*f||cPjkr z3cp_A-&FV(rQcP4gOdI|C4INTf28oe3O}UqI~0DbqR)PXKS1GiMIMzml=KHF=?4|w zt?)w%|22iLQ_5F)x5AHBOaexAaQQTTj? z#{<69r%2(sHn2Zlg=gLNM^|`zjm>)jg&%${?s8Qs{6Tgh_^wiT^O?7Kvrggpq}=|5 z6kfAQe77jPy%y)~R)tr8S7@!m@En zDEx5>KdA8ERQMrNjg}->dL5mGrv8&r7jU`~rpV zQ}~4nzeC}_rSSa2~3QTQDSf33pzD}1ZM8w!8D!VfBZo5Bw%d{p6Gw+-8WyTWHH{0$24 zR`?qgK1bo#D!fPGZ&LU?g^w$IzQW(E@I?xrPO~(O$xtO;WsOMhr(}F_;m`u zP2oEgzDMELEBw<6-=*+>R`?AH{}+YtR`|ave6PYkqwswS->2|96#iL-?^pQk3U4U< za|%DG@Xss!kix&9@UA!nZ2?2MWJd z;dd!~hr$mk{5pmIP~kfj{$qt-ukim;_%4P2MBz6m{BDKsR`@*%->dMSDtw>9?^XC6 z3jdkH_bdFr72Z(z&lP@9;s2xXLkho7;a%4Gf$`*&jhk$RSN^l0F1NyuQqt!r{Ah*u zDEt_O&r|qu3ZJj=;}yP0;jEb%zvR>4OQ z(`=Fn}`o54hX)U_y}UJ;46ubB+eJyM0^ynNAP9DM-#gRFD3pu zu}koU#K#a1e$MtUBK8pX3qFTEco}i0;1h^zi8}-z zOT3)8RqzqSD~LmaClhl5N>&Lzkhq>WAb1q1mA>w|)PZNKexKHpV;wIv5!H*I*6L$&z9dVerQ}BJn5#kQP zcN1Sh+$#7^;?=|%_MZy9B>X{9WR~-O~TW9mM^DpC*6`^2q+?KaY*p3#OsKw1m8sbL*jto>xu6q_6oj|_%7mn!A-_#Wa;!6y*^ zg1AHQvBbY5ZWVk4@x8<$!IO#C6ITg7koZ@`0l}k)?<4jK-g^k}uZi;oe?)vgu}AQ` z#19a=1;0-G8)BE>mx&)F9{iW|KXDgvzu>2de@omacoXsOh`R+pO8k4`F2TPe{sVER z;QNUGNZcX#ZsLcCTLs@q{4jAy@U6rfh^qwOMEnSGK=Ad%j}m(YUrGEJalYUt;>U?S zf-fWf6R}(HQsO6wU4kzpev)|bW9fh5ZsLBy=MZls?h|}A@h0MK!KV>#ChihkNW6u( zQ}7AITZuaaA4|NAxK;2G#683z!IO!5iK_%3NcSa=8+zj6Tz7fi*lT+1gYtMdH`|zjo4_UeUc|q@&oXXDY&9^d zU?&PP*5X=^FV4+brcXaaid?%R+-Pis`T1hoklt916#m#A;oGRdWL_T6%}rLL6ZP0;R;;mNB%F@oPPLqBLJG?g@`I{smvC$LHv6o85ZxYvFMhC0_?~(Pk(mM~_`iNoM34mtw_$ zWuzXTk*mi+h7U02JnM3GWetosu7@=hbfcP_%~a>-Pr4~EmY+s18Y&Ixfn3IENT)}!iQ1JL@C6i+^19S zw#}Y_H}&{6SmN#rAHY^7b16M8Hh0*4i~QHRnP8$GKh+q+e6d{vAL#K?++oA>^~40) zsjHy-yJYQ_@WFoRYvXA!z|_D(Xre6oX81?|J@%^c3=Ek3c<7wp zr;4UtGmhu?2m_nDwmP3*O8Re@f60nibzC z#Z3DT^6MM_Gg^-g>Kl!-^sH_Av;PU_puk5cCkmXv0__H+m)Cm68kFOToT|5-@y#-gF zcx}kH=R;q7%=2iR7Uj&U$KeiM^2bZE2Tl?x6n(`H(A%HVq73+$pkTW{){Xpn@k^1g zzj$-xRewCItYCXN6y@{9Lr4HkN`k#w`wkXf{Be0~Q#dzCpWaa(zfj89VJG2C=P%x= zwQ~wCk54Xd-yV6{Kd0UuaSt53yraCh&RrhQ2_Ih`+fiQJ7uiL-DCoA$yrVp}*%#{_ zme&{eqI?uyo;c)rXvr5JrN=f79Ax*O9{W^}7yFBQBA@8Rz2QK4`_AwbAAE*n(u-dV zkN3qUJdY~Mx&s4fAOluYhDD;?tb$YCK;0=H5$Gkc4ws@V`r~A2^_T6;Y;W4xis`wo} z_BYNR*_=yE*Hf@6%N6YoAI*uzJ<@$dCP4Z;*dBc0z?;TlyY}s~>`=zr>^-!B0Q{zi zcceaTlis#hTYV@AG<)-~ev$uujG?mcX1R>~?Jk#TZ=b_uVqzH~HF`;p-w=StIK#xo zQ1~{ek@X}hb9P@(BOk>URbsSx-Tp-GX3httdi+E>!@GAYLT$o`w1k>RpBtFaOpeukXhiERXvF2>dojJfV{|bG z{Q8a$F=#)2pE`1Jws}F1eJpzVDgAG}h?CA?)QdR(9T%zofi%t29_Ny8#c;lfIzuDW z({^fBc(Tl|*_%6t^=n-8t~Kpui|4Xt_5kY~Cz;OB<5%g!@aupc!m);Hkm6mHO?3Q4 z$q&)X9mYW@Z(wK2e;J=8fgexdi=>v}`qk*EXheVJQP^wU@W5ja`)n!{HC~JU4J|S1 z-=?BB^rELZGo8c6vd6b-(Nn+~2MmBHHcGTRwrcH{&>j+3=epb4JCQldpIF8j_Ix*| zT;nGY78AG|Ga4ppF>5h$V+sOH+`ypZjU42J0CN$i{TnkpZg|Wap5YsB$c@Fh9v$%< z3=CBoPhvQ?Jfen(f7lXjEg}nSM;wZuJRc7GH>Fl9OEbEh15Z#~)z`ie;h8V~ zcy1q*R`MtO<1+sJo=H*LJ*{jCKRUzCt;Um7A1XW^1t-6c%*M&M>QLLK#Kt3f7Xh)| zyTzYcz1xS9jrf}>k=eT|F*fYp^+E>YMA(zryIZvOMle>#(nZg&S%Sm;sa&Mt|&x3m&X{K^-y4+h#PltG2kDs1v^uzwG?$x3nBLictgN-$y zQ}%9;7IQ3p8!%^hEG-ftQHgnXI(z&gXz+m16mzbU&Ax=za;tH+L9!y^T)Y zma5N)6W=?GuOePMjN`)!ZGTYdZ(}klX6NtM(9*P_O9k+31606qyk}0an6B zjOj3J<8QC0IuG{NiR8)8XcmVe14KalQDHm{>*V^v^r{+t^hLw*qp=f(Nr#V)ym`^) z4%G}Nl}Cme1@mDzpq7iLN;|g#mfQ+Ia*Lecr1j&^D4yk#tYB_F0zfk zm=|tivs_$~BTSWYmaGn{Kt}fe1y0{Xg_))O zZiO;S_MP8SvU^*W_E1YmYhMpWkGu6)nXbq6Y=3N3c2AkSq^QT1P*SfR59xZW94p&$ ztRr6xACVegy7u6KdfTRKz4#;EjO;?OV`A5MwDyl!Putr$C2jx7iC_^LD2gq~#aAT? zEkZ!zMSc0cwlY^10~yRA81pkEj?mhdP$Q|;R>`F$L3_UUM$z7dj78!tFnX*`TQ{&p zF0K7j{0p{{tY*$$3)tJTFA#6>D*P;k&(RZsJcQeJo+DECVm(hVGBtL*&Qiu!?}gt^ z%slo6F5HTYO>geow*i9elT}nI{*4Eqr6qaEEZjkSQ&zAqc{)rmb)R#y@k{I{VKj#k zU6y52dK5p(EdH)`v-p}+@voT{T!KmS`@g^jKfP`y|`CL2lFI zi#*tBn35G4(A$z(Pf+jr#^h+Ib7Gcp2*qMlRq75e)DvTj)^qc-fO6=Nu&k4aQ~c3n zz3ocRsMQJWA?yWNK~!7ssJ8x(*Oul>E+vPn;QfK`J;|z!kKSNYr#vOzEM%RA_we6fA@v>?vOIR76m*OLs=sGBSMS?+CX z3FFNzdg3vL?N%7hn2_ux!Jz>Iq(fH4xFgkj~*XKmoJWk>!v*tD)q7$tR@S5 z#ZN{4;ls+xA3I)eYom}{=;bL3JsI68e^MSh1WuwfbWVH>e3)n8>Gbt-+ukf6R}3#` z(N}08G6XT!j?s%ZX*a%2Dlz5-)FbMZ7`E~?YG`q8HcTtlgDphk`!u^mTc@Ow)tuP8 z-qY3!iXUk`)?X-1xYU+nS;0;$$IIZePBY~6$M|BWV`Jm0kUj@K z1OxYE7^Odk{*N_>lAq!Ez8;t6`mmn#$9~?5da#p$9yvF}(hi2-=JI$?GjilUYw6~k z*qDvUuZxp|RJSpcNP`_*J!UEvCf;_ID?C-4F~rtmG^p6=+mbIKpAyg5_mWS7_a}be z$-;W1r*P%+$HKYpWI65`$G!q-V?D;Z|6mhLvwjucjbq{p`F_p#o7{WHmkaJB;Nb8uF|j|91Ak zk%$jvKPP-VF_RB6dGWK6L$OU0d0mfx3uCF{t0^VC9VkE*){M>SLb?+Pq$zT7F#k88`M?G_+jr7r}*MG_EU55+&+X_ zTJ(ENh0)+CmtE;AF|N-;Y|{iE`5&!)AqJ2yKFc5bP;cL@MSp{6M2~+5CHZr6%Ht2R z3S%uQ)Zr0wSnDTxyg-unGHEet)!P4r%kmf^$Tjdev#fa+9d0}!z6w`*>>1;?0KT>d zhfrkJ?shsYWZfbIIGl?G=RV`7_~igS9&z~+*A|s0?%lx3@5;-9fKT!o1xJod{t7vi z_zE6~j_Wp-6EA~A4BTc%-YJiNw~Kk=^C5P5{HN;yWE<=&YhxHta)nf&IY>vMJ2ZNS$qsS}-UdbzE?r#Oe ziv4e`XPpq9Xf(qF8L%>b_leWgqT`T`BdR=pFBR*L*I{p?r1cut3GfNNtgYO4*$Fd$ zbbP&rX93%YFMk#L7ts5^EM0HHSfJr}eDQFs@M4}28yp# z6|Y5aLLpSX4}I$|?hl{o>nTGvcI@orD%|u#7j9oxHyo)i@qFKa9_$OcNVOr9_Be-DNftQR6cT!F?8s9(_wOVb!rdtHAc8iZ zxA|hvTNN4a!XSKDoGyBv(f|A5u^82nJk+=bHTq+(rz)j7&81YyrL4A<<@w`XT`*}_ zl0UZFI2dh(58H%mKIGWT~ z*>Ay@!T55F$q3-^7^L@le8pa6zj~izH^F6bCCP_FV~q#kpdtDgVU2NnnOdG2Is`vi!g#}N}IW>&)&n@wLW(}-#!wXj^h?Zm8b zkg>%MJ$|ulAs%J!OP>ejWBU+;%H{3;V3Y)(ZEt4{9LV{40)p`FEW}J&lztmw%>4yE z`OLVvFiwx5lo+S%cxynW9O(QDZ!&~ouXBuNU$wgh?qpMVerijBhbi=U9v7U2dVC2y zYIzl-!Q*`KNVYzwP*0rAr$LL^Ug!d&E;JaPrx$P3W1Au$*t-uuGIt*a9%Nza7F}cw z)N4H$HG_F2o}+s9Ey7eZCKrJ>b|qyuG1gPE?)zwkif7r`%Ak(s z?f{8F^y63P3zy|)f2@56!%;@3b||j2Y0=zH$q01>vcseO@e@k4hr+q}{@6Xa5=rz# zb2)eLls004-nI$j*et|9ZKEvK5lLNND1(b84#lx@e!?31D0#~8 z_!22cMGo}AW4n!m(Y?O5t8?>QVN6Kx7^l3*SpOa43XU;MTzD>BME^TUdFrD@^A56F z>(<&YK7ipCP8lHEzT3E?PB#IwhXn=tIDAc8P&S;WOhlsVwipo4o<4{7pAfF)Xb&Cb z?i(;w8~p{$*4T#gGw@n1Fmd*xQ64`pr0+q{FlM_i_8cU&y(iSXFni$m;rKz1pC$2X zJ8I)SX2(sze8vo zp8s0UknJP`V39;?e;pU)@dL16tUM3)#Xj}Li}m7XBA@t+d$s6a z#xiQ@<}+b)CCQeC$Ct+@U}>CH9?LTBgm3k={e~51m6+>-QCj<%h}anO;pvX)AK_yk z&t)R+!;Z%kGC$j&Se;`&+s9*1e|&WgNH>UiWJN`=|l&uDu7irtuFtYO-I!N0|`>G9jGfqi!VdW`F%(Bs_( zcAK&|v(7-G8HeFG7?%@r zYyqyE{yY!87{7u?7j&+%^*DAMEYnS}C2@EI3`9M?Aba2&YX4(>w>!Lwyx1-ZpzU6) z7k?Ug-5(!UfDQic@X6)ye%xT{!+n@;WC9-S`(r&?w3zmZ3;R4x@qfY||D5IS$du$i z;i<~!RJnceY%Tf*0F2LvmDvD{Or0mq?zH#g;eir?oZZ*S<8Q6~Tyb|^#7<*2VOpdb zinQo8Y!c>MIB58?BIAyi*hG6v9y@^ccKk&65A)FtbLNL|@$*%^ zY3)1Ft#T+r=1cJ&k(ZNCgOzv=yPW7wb7s;W8bd=#ZbykJ{jw9qBGBb*l2?m<4oQvC zy%2ZUCUgqx9bpqFci1MLV(Uzs@QO_wZ$D3>J!sKi$^8*Fp|$grFYV&xlwF*cW*1uf zVmJuXHo^tT`2Zu=OZ(~LL^HkCek?9bZ%Y3L1IkVQa1680Rrd!{_SA-Zwmq?}FmGfZ z4!)u?CucM&klf|RnF__IbS-)_n+K!9u9p~<7u%4EQC*las(Z)rh8bAT%?*UTs!>h1 zq`~^#vaG|C=V6do`c2u_+2BS=F#TvgOJ9#W%I01p0CvD!KZUNNPM3u z8SqEhS-zg}%;=AgPfkQC%sal=U~2z2b-qXji$8HxpX|_2+^OzV8n zpcXv@Rda5{ekUg)@jlymK8!;Rn-uUJdhB4M>~B;DU0vIHdf%+%hR-bjdUdYX8V_EM zhdOIKe7pq(508fy=B^tt9zqV|q3-|dc*ueON{xqu0RG2#7?6+vho)Y~McR0{5R}t+ zn2UQD57g`cgY*Ecr%q2yJP@%KHUo@89GpnrCU&Ik_ZWZP%xczpV2u*}_+$O$@$YlT zK<6Q`qYxDImB$Z4*r7%DLGz3T;z7nhmcKINpEq%fJm(N>v0@JC)7qb5dYm_czi4#p z@kSg^`J3J|-whg@!t-;iZx6o*vd!{h*cly%r&1B@oPX$#pM@C)E#u=oUiBF$}RM_zK2day-c89mcs^as4-*v}F&iXTPZ<={M%6kz-_zfv-YJb{nQ%kB%m9Rv#N!oKX zu9Nfe58~NaZ}KLzOj^TJ>}qSY=pb(m`vz$de1g>?o|PPKQF1d9%ky;xK&tU*(QT-V z2IPy6hJVABH2qsRVr7O|CGSBBcGmon0-P#-d2X)NI1vDi5&^ov5`d5}C zuJ6TUjU666VMeA#b|*2;%Twt%U(7W3AdPVrtvzwZdJ?yRfUzSVmu9<+H+m`7YsP0! z@mmUGkZnAMO`d^!ST@6C+%bMd<~3_Sa>RNaXTy0K@VfU961he!*JXX&t@D)U#kpQS zjH^TvN=cusbNj?$Q%YL%bIX{=GT}tBc>`c?lWt}iWxaA!QDofNjY%ain6p@Za@)iAp)?g zbNM2Gu?$yya8ZU$3p^IT2K_IEOlkf5R19Zyd}8KhD^c@sx6=sP zMq&G6F}Jd51b#$MQqsqGMU34&fbZltF_z5vDCz7UPZ{ z%m6EfaXaw9?Q9k$o=u6Nt5fmKhMkWBtc*v2N`8*+gj`JOXF)I1RdU2bn$Pg+AoG4h z^59rF+Q&4@kOmo&a)uP&d=7{Hw*48O3w;bvHbmzE=aR#-2e7IUUd_NwY(g4TUHn|c ztDG0wpW@@UyhvoX_kt`Q<>G!BH?H{MwQgN=i;{aGTJ)u_;Z9;jVP7wE*%!cu3YxPECIx|Xzf8r z;irqBko0-{3=epWvKBI!NHGP?&8^ZCShz!e0(fw2H3~3S^GR&^B35sjkPTJ)PzW$C^sW_0*; zt^Jx)SPM5tvEQ55JPAtj_)eUkqi-A_g#0;xCa+PuvryXRjp zN{@}n)uK_kAI+^qg?d|&3-4rMCJYck)ih^Y)w1_2n@~TJ7U*++luPMpVS4OkR?lj+ zXbE=Xcv9^k+6qh1W25EhAvT-YbUpruT((J{^9N}Jw#eBoE!ukkPECJ`mB6*=^+ZV) zO5_c_ryFB-pPc*FpX=7AV%cy~u6Dz}*l1Ki75ifE`8K|ZgSA{vf|EW6@cQDx9CngE zZHL)MhIS)=Z$poxDIA1z4M=YpI0O>`AsAbR{vU%X5QE@xh&={yBzhZ`s#Z#x`Lh)M zp^6;XjP}qb$JTK_Ut0HDq#thG22=!c5ft4c6L&P%KxQ+tfC10H)Q1Pz#c1IZp#5|35JbDal)Zsbz?8CseKvaB z?o;EFM`bW-(UZW+Xjt#nq8*S4=HK2k5qa9oywFD8bgWE;cO=ReM!?2vdq>q?U2Ey_02| zs}}7>KIzHTxJd6wbYuKrJdo4(Y}cngtsl^fld$MYc$RTy7qxAZ{`40QYByAJa)O0Y zEmS%`;z_S^yR`NqbT+ocwYCBz<8Y+rjLF!YOa2QhjS=lNz32BiLopUUF>gP8$k0`+ zeRpXN_Up4b7Uqnd(l-?qe#UKTzNWgoL(Gx~mtDydkfcLT%*%$5wgY0{aSSKVz)h~F zqd4u>Fb(=llp-2>THK}hNbQEdpg`N}w%;Uzd&Q|&!KRkc#Nbd{n79r*;z|$e;0@zY#F5Dh zp*2a5y;y8gnLTmyH*x0`EKKe59I-v-jkZ5R$J0JFwKi<8c)IpYq)zRx=^NjGS-xh> zeTYr6CNV^rGP51?7)fgiI5BDthO7ZdH3X^jnCX~hO4DQaTh8!))A~2|kLIbi;?3F( zZ$b)eikziy92jTU_dLYYW*yguInANr?u z>+Lu{rj>2P!1Be$MV^7*%jUj2ibK9gFJC{vn&BN*cg&+woj+@UPh05MV+5J!D${9v4Kd(fty>%AE zSZ7rYeHP);k2#d}>WQOX<1=h>ERxB%D@|ub&E^Y#yL=lhh{a%)Ef&W{>WMKteW~4Y z8AD$j)?p7}PtB6LjKl6`5tL(U5IIo55Hq6t1l}x)gS(;raMSG5+f#Y0u9iF~N-Bll zH@|NdO8`xoP=+a*IdEtvG>nv;M02Ts21~l?k9pmR8~DF;?Rs8V*Im{%l5CL6*0bOM ziHvWTdtGmz1!Kli%ceyG;|C`?vQ=vlH2 zz_W8W5zD99F`NoA#xWH~Yx^PFwD#{Jsjs*<($D2s9Zm=832PdPW3vr{88ujZChx_* zWZHhWB|Ee`L}k%8;TxkFY9GXv#0xLtKaC}EU2+B~S$sfT4$8c%hh7r8Lp zf}MGp*zxE!wJ`kb5iSf*%FU9*TJ$aqU7CI+Vuba$ZpGA#R->R)vKl^P?j92 zi*{3~Qu#!E&Xk9@A-a<18}e2Fmq}cLd8mYbstKNzU5t0u5jdV`M!sQ+p7<%1Q*Mn#W-_#?ZjB9MAo^w21zKujs;E7WqaaghgL?M$CPDy2ocR zTD!*hwT&@tl>b?D>PnBB@i}!QDogev(f^)*7b7M%?HemS7;&DI_cWEkOPa5T--#9v z_oyN~&|GRQn%qpa7R|<8ybG|^cnAN*!Q`J=FXFc;leS_vhP#S9g<`K9CNe&Sx)F?x zP_r12aRW+A@5JO16o~Qij%g*g{hDnTJ8hR|A#rYVy#Fv*Iftig|F zF$q7mLXE{Y0$>i|4P!n5vWmO2#)4_~h*0VBi#KX*mr*3#q=x+VjA+FVdTh+oGLalD zZy-c-b71bNvCT=<&N)KIXgL!aJvVSFc*l4W-xAOz3o#Ap@HFzl&+`05yOA#=S!g7Fo<$CLD2>S^sqz^;?W;2(Rvz+_BuubpBD?}sNp=S$)+ zm_f7}CNYFB`e>R4Twa~Pi}CcL*gUC8tK&WNw|Vb&(=D1&q#q@On9~do8P>8~cn~Sq z3L=rWMasS13J>k*(OdwhqQ~3K4LKxlMRG0LgW)4<)OSSD_GEdLC)S+jkCo=f&Mneo z6<+R*K@J%K7%vonADSyUesD>F_#sWlF4r;sz?13pLDSI-jq83TIYV6G@x~;R8@R_} z_-*lU#@5II&w*j_h##!29PplDK0rPX81G$H@|qJGuB$jZ)YZxF>DeRI`1u3qGSl1i zqg-qcOHhQ(ZlKgq6+p>B@{{rxxorR#9Q`1T!vK<3T0cDz(jM|&upX}>?Hh-|0E?V? z;UdPTW9Q?G)we<_Kenm5DV71CNX6yv&QfQV3#rH zZZbd0tzs#$cg?pojN`Dg=7%_5GyU7=qDH=bvClY&>1m6+lRD`iq4x)CmS2WbWqb`c zXtCW$>r2eDSf9nV3TvmG4%UZ9fShPUWd(1-<>kWPQEdmy`#h?WHugaK~pEFRe!#5EVq{OoCu?+}@t0 zjn6hy@Y_XK>@4b>4SgeHAAHV|(CKeoYc@CA*eO@1=NjAP!ju$-nq~it26Tnp9HW#` zGIocqKLQS)ih&b{M82_>s&JG8K2E0Z4fj~$9)p`uh}M1+=W0A8#M_CKt>;(l<9p2A z0`x3%(f6#geGjfXmFxA^wH(R!CGeQVmyqY>=7xiD7+S@kX*R3r!9z1P(s~|+q1uP+ z=0Z1dF51G)VeB4DkMprkaw6`F>w_5lQ2RUhjR8#uWIX&G02qhccz{o*B{{DRsL_7Dl+%@-7yj@1Mlx4ZF#Zm>jVc?gR@y0`7knfV;eCgZ^Ncw!;l)-2D>{Eqf~wAb0y&Ts z?`6QCjj;zKBQ#7vnb>*M3=D-7KR)aX#BeC#KsFaUq`$)fOAZKgWEUBeP1&?)l8R*& z6ioW}rK`!9oy}lm5>KO>C(-c=!auQB0lEv1~Ysy zj-lKM%Yr5YB)4HNz&u`_EKIVjquVw>7Dju&$NOe>w2tjq#+{WW}-b8}_0tUlqj z@Dwj`k}+`n2>&bt0R_;*8LwRlvnhU6i+)$~EPe%gH={;*71Ckeia1shy z!77$(LKql5m9ZG30L#Hn(zQg43(NAZY?q!o5ipG2aR-2JgnJ$UBpFzJ@vyGavRz%^Xb_S$<4!ZsV3txpfw9J%Zq0OgN8<&VibtkX7fddPz=t$1LcahLcu9H^je!Q}9w>puL3h%uUO2TkJex1s_q zx(8cQ#^GpI@_pIT@}^dXIG0wQs4ePXZ*o51hdXHEn85XeBR#@3fgiNe;&MnXv6G8y z7u94X&$g}&?*r5_c5hew0%`FyO5 z1rAr*c;*#=s%3qayqu%R4p|d3-+^mG9~g~z)Qwjd5CiQon$c?XV)uN-e)brDWreo= zY$!3a=}2m{iT%roE?+4@L~UmqLoz* zJAnOV@$fp%%NO<^)7$-s<%kE#*_WvT$fei3dREBu>wlZuy6F^MQJgm#YHU8w&05cFA>*x6|Y{Vb!m%)8qe- zv^Rl|vbz5NgAj-dPEe6psSY;P;8swhM2&_`CK@!ZMR7@6MOw8|O%z3Ia3;byPK|Y= ztqX0fOSOu{Di)MzfZ)=v5v|~wxN;u{1hoPR!vFm__nFBgXqW$QUXbT`&b{~CbI&>V z+;jJ1XY-sLYa3wT)`AByFP%VcRH=p>34m z1%Ianr;hC;RVS(ZcUzU(b>-xMJDzI^^RQ%WE#Dh}Gl;(-(zMb#2z@S#rP!rRXNskU zeXgtS^-e6EcviW|e6jd(TCJN?&sTa{%-!c-qVDHhoy35RMld3YAyTz?>tr|UPHHR1 zg)sUk3DPVS$%CE_Skj2~vdvsm<=o67gJ9lA1ZZxd;swU6S7Q8%1iqd(59S<;cXQxY z-qeJAwk>(n_^VWIAQZ`#tfq?=9Kg_@^?eTErH13=g8g`TYO;`C!;~gwA{90zmDq>i z$JTuNRxWsfkG%C3{#0w!S=K>6_wjEH!|q3tz4v-%sgX(-4u1$!k}CiEBRtXrTi+1t z=4iDHf|!q$$Cp}{2hwe zO6!Aks*`WUk{Rdw^B1aaW9I9}GX%z}u&&*NSKLaP)YY+k*m(xyFu$xa9r8Pi6iyNC ztuGpm%tPH(ZbyS}>{b6`8klYBPmtbR2`YE%y;9|=Bit(ZnrLa*&9!o_Oe}4iGu?6n zMiLjG(e7i4fR|9e+)Co!v=Vi=S3oUv`w89N(s(}hFfwbSb_zmjYB3cwDqiv|37^G! zx>N6Bedj%J#-&1hagoaRwD9{<|J|dBXUO4j>EBkqOsSx_fk*menYR_CPNt4A7lvj1 zsS%rzTTSwPh0V6G#kfK`qT)jdw0mm@^raBmNZfGUbi=KgS&*4o5S&9(=CxtP?aJG{ zz1366F;)KbgQg8Hsh^vt4M#&H-=RW;j_f@(cYIl z5|Ug$m=?Uw7^*K%kuQ%rG~e~*`Q*=5o-N_%!R4m@t?nf;9U5dN-LEeu13s4Oqlu5; zG}W;zQg`b#~K=U2^Cg89m}z>5`xqUgF#Z(I}_~4F4&QF!a}hdngda=`t6Ws zW(E?@JRuFd}P@=71G5=ujx`PyRC*>gfvc~Q7bI<=y@xPD{vvI!2hpJwB zV}0CZR^-7Cm_!e_PXTM(pG@Q|9ft94=|ys>SxCN+m&}7EpH}x7T*$l=JZy^U{cdv3 zx!zY$W_OqL_2RJf?-7&PG)ulqsrxJGYm&6ZyOah${bnRh<(o*aKVhD~v^49p+V2j8 zpzMVYCX9c&3=7MH&n-!7hNGYuka4}_G#wNvy#VAYD&|U zdmqYR$Z&BQEx2c6t~?J*=cywrB%>dZULV$gpCV*4yfiZb$ui5)BvJhGmr5?Wg302n z=Z4|+kmDjZ@&fUPS)&pDCF-`&MN^fqPMch=zioipp|?QogS^=VWWmcw7!ka} zV5)oA$Tfyx6kFY0e-dv#4vNG`a;uZe=9}lNHGbh|u> zY@f=2Q$E|Bne@?|^v+uJpy6dDJF}GT<@3Q!ufdjuXi#?iN&6e{)kYuQD?FNh`Kc${ z+H^K;!Hk;Jf!a4S!nQE5*I|cn%fSsc<}v)aYF#?Qc8CcYAF$<6vr#wwH0jvP6R#x} z8MDFt@dlBh(f)K8z_{%cJHb)lTK8T*9R$bhAG$e(RN5L+U+FGTIz}^Q2^R~lXX{CV zsce%d7W#Garu!-t`|g(T?&Ff2lkm=BQTIjBSVbn1_@!h^-i;s-x7A1q5re;pa2Z%1 z8_;hwCBUW3oWy?0OzNBR_sdl{cR(Y|w(wgdcxXNqbT3)|(+UNw$=jqH{kGFl?&fd7 zEAk?Vu|4n#+8r;`fxe&7bh=rAX~psnZ+7#gj_PiI$|`J~)?=$L=}Uvby+a?N24+r? zXc;8f@mq+PV?;Ff_l4UT{$^9DkP6KwMr#h>`{*L=99-`nfCAZ0itWFQx` zAWT2jJYoP?rqa22+odCCj%A0&<9kY)YXQcRUYd%LU`?clMiNsagoP4kAx;q|`mVF- zvQQJU(k}^~B4u;q#;aNFz{pkO#_va(*4q%)7{9nE8cQ*_b!=*Ir_J9!58^JpVwM3S z)GU1GeKM3lh7|FVDxtR3(;TQEa*gy8buD^Amob+C*LenK6q!2g*6l|EEYdJYvN?t| z?(;%{3J(R9m`w^bBEqWnLqB2dE|-$N!H6muVuQwY;*_Q@Xa7n;3w&0%-QMTE zK7>vYCXOvFA?9Vks?)r?GtU$L1-_)3&NNJ?S(>Y`VQO$k5C>78)2k+RF2`)_S(Ue* zzdGKg6Os}$D0t(>X_2NQXvi#bi319?q-yNx(S{vaKr;4Bt*%duV6uY)BQX_p zsD^4qNujN}pF(8Tv)NL%)FIp0QoS>QsaCn2X&Ci_?;L@Bt~cM7k0>z3(m_pEp!Pt~ zTSjmgvT1RBTRtvi{e{+Aaq=-M;D=(*bz3-3n#s~ba^Brtu$~JVpql{Q^TKULM)}xGC z%gfprr^UQZ=M+|RaP#|}vGge=kasoYoyuCQu_fb^=atEpG5?M2k32!cf^=vSmQ5O& z9D7E6O)PaKBgU1}XKS_G1yo6z!WurbcifJoaC<{0D+Ec!k`YPlh=CLD9#X(XGMqlz zc+@$OrpsZkG-qK{{OwD8wR&V<$257rrF3!EMFw!(dX+d<=Z_QRBpMcXaRlNIHFj0Z zAe>2Tf-2oiKJ=rP7M@+@rmN-hQ$z}7ORbrn^5AaZ^t^6DW3UTN69fiDBa!rj7n5aX zB@MNnRAV)Mkwm8^#M0%X4?^AUNor9c`<<_5U}F#pOu)F|1f+aGB=K7Tsj-cuC0d>I zM|2E-&K+jzR70~a^4JjZBTa$fwaTuS;VQj~ z&SAXBh}H7c@(0y%t=-0m6$xIaaJ9KM>A*{gmg`SNesA5WMG7n&O!t2cEMt>v!$k_* zspc+!on#m=a13lAv?jUPwKfWNi{Fq&(UhfnE*q^$3yn^AD)OM~Red};S(DQnn6yD@ z##x$6)$?>m=sjIN9A4^ke+!}!XQ1WaM)tiONZ>AEV8o|GE!Mc+KHZm)r_Xkbm1qb_4LAGMvD>vVa081V%n2J0|Ve*ZDcQ;Vqmf1)(sWRh0n4%e^|^_lkzS@86o zqJnzHDEF5UDhcsTvvO&90_QH)mlV050g$abyE^^zUXv)swioJ|f`rg6P=A=D>k zu!4p7nl=m7wO%c2&6wGGC|sevL+RhqSmZobE1+ST=Kj&wwE}WS6doMwrz8%x>O>* zT01bVm&upYH=9nz3AY>2ySHCzoVKawR5vum=15YPa)z+Rp^F=j+LwZGk}!9!xr7a; zcDTXMiZ{1}YtOuAXYJGH}#}RB>$-36JV-{*-X}@0A-L+BN@YdL_ z7bQQ&Vq`mN-SjZ9k<5enbj*GIys`u%8%gvwi4`V(GX%XI$8xdHtOp$S)mU{CYz6?n zlW?>-1c#<2p^lU_MvV7bI?iHglw9x4)n_&7Ob;<8q-Awnqq03LM4;@JGy@uG(tRS5 zI>B1%JgS8Klw#giPSA->$_=lU5UXoBN9!;Pxx=>@gsQoSRcx&1Ua+rw)MxcS;;lt4 z(Fn4Nx49pV_|1xME$-@s7sf0o$UEYqcM%ad;sR*5?d4TDk5``V?1`oUQ{?M}O^e@i z%3AZ&M;>(Q>GAhiKCu8j;Y{q)3nIyLtejJ6`OjingqF^8+plsG=O*Y#vNotgiB6)B zYA$DzUNRQZ zq-j@EvJQ9a)g;|kAEDUYrGk85)gj-%HC2|r&k;`^#+df(nFSXyIh?B>uqU%)9z6`C zRKr-<-O}*61Ol9fx?gBkv%HyG-^fYBK~k8y>^czTGjlTY{A9_qWQ!@?N=bTdDr>|LgQ;0WT{ zlFSZw|5Uib?0B)%OV(eDrgYOll}q0O2yv`X1fx`ube)#i$`}^TyL_2LZUV6e4V;nw=`YYmj*Yto-L2-HcHST z@fovEh1t@CMzF>(C$xFY5GFbwa~H_&3?zh1W3B!RFUl$_xLfnJ0_US=4O+hQ{KH zi`;J07%1oP=|+GQrt(uJhMkxvyzsUWusaue^f6N(Kjb5GgOnX#YC-7Qwz98=lX}+E zFB(U_3Xt}VPhT9K-!nXf4wFv$u5%~?w-5jLB@l%CAsAgVf53Q!FrLx_<1I?R1k;hjcifKkK4qFkf$ zPjTf>S0ye#4OF&>y2G_H9GEj0>46s#V&f5O+;bXaCGBl;9JFL+rQ2J$FCjNSQu@QI z6oTbh>@LLhz-_E-9s;(3$7>vcq)Swcu!3q>JvY#>8mpUQd9JoE2O3Tjity_6MT|?V zVNWkP23ihitx2ovpivDne1kJH585&>c0Px7vqpMC9e|nm#kHkPs zkRNYZN}S*;O>FFVCGd3R#Ze^wV>s*5$~;~96x*lViDGrS@_7Z0}G8fp~)6Yc0v=SR%LXOrZ`B+6CvK!Z{h7 zuqD_6!-TI`+RF2L;qwSR)0Jl^&%c2nnCl~U6>QB(K!=DlO1`lA+3}KFt{S}b z+ePpM;aiK~_dvf*1UI?3a9&@d&KvBPT8W6b9mW+&>?5Ak5hc)Bc!Xr;k$vX3cAsE% zlC#itO@(gUPMuA!xcp8E$?$xiaCn}MwfaLH?&>RSn4n&lk38;6^dfk4Iy?*YxGr}S$&2zQcQY_RP@mDIUW->e6Dl(Ntc@>cQ6M5%F zaaa_T=Sc-02Y>`a&2l?s*{hY5Z12?pN~bcsq_fP|xJC8gO(#lci(@p6_w&>bpLUL( z5(Sa7%!eqW*mn;Ir>=n-Zu;U~ z47y*&1fRQHf<~x=zgki}^yY7)BJzz?4223c3Ijw#9!q~A6GL}UBZ-m3Q*^sLLCM$L zu9tQNacxk;4#8anTmO&&&eTY{d@T*!#p4S!hyvB5eo^8g6v}$)7YG@`C@|jHV3#^C z$f!!-7AuT8f_$`xAF<31=(8o?pX73BZ#0X+xn*sx55&stz-i@JBDz#o(Ln0HS}oD~ zMcSzHAFr1x_y0Xk(a7Wq8#Ei zwun!cpDFh$$vI7Qkl5hq4yRC}yU~INULns=_uyzEQYDPK5f^y`Rm7n6Hdk_u+bbzL z?+Lfr4h#&fEi*47zDm$Z?h**IMjy!;Sc`d3L1ELtM~SW#vTreKZ7OJSQ9*l31$R=d zmI?}j3m&6gJCN2xuufoI2OE}z&?xj(#~0x)tt9R?Ayic3r>R8;y2`AYjc6_KMb0&} zFNZSq!g> zB}W#ymyMw}n32l06R{NYPYplO*>X_Xt@Gaq3UCw-ogob@KWR4;DShq{zitYM#*$d+;;a??*}K4fgGqfWrH457E-2HzKZNtnzPa2_B1r;d8_#!nMZ_^Xn6w5 zqIos65P1UgK{MTx1ZAeHJ&-nwIe2 zva!d~1bzoDL1Bi<*BbQqA9hG0-`?!*p=h7u=4d&~R`sA5eB*H&2kD|l~div19iVI_rc%*OH_7Nt`5@ufIXZ_do-RCqN*99n}h zg)o_V6&u&<+(s%ApDH=`(H9F?zQ-V{E=TX;giLnFVriIJV!nD{9_e@iO5#W68I(`l zrljG%X`k>!?Jz;YbA(4^aw&joT8v(dSadpF<=LBjM6VAGGeMIlI+FWVI>b}%XlTe?ylG{^_yHo0AW zU?h2*n8K;m?iBCR15bwr zJB_BpJ$IgOSPLZ4q#xc|>E;4`ins-C+ew_Rh7!1>5u~SodsHf&^Ow5*4s-AC*XubJk3&o8JMO+8S z@R=^$**ZS)c_K1hIM#o)_^&7A5hUZP1o8L!)b9%cBgCm>e728p+O?E9>4E7&MvET0tZAAcg0|dJI?D>idKI(@!Oc8gwT_gk`Bd8#wxRPeU-?iI zGYH8yTy2eyCb|l^whj%~D>Z!WPG$hzfr^2`niynl%nOl^yJ(aTu>^8W=zh6JLZFG? z1G)bt1WHZ9O%)A%PNd58^Vow@s@OJKMZq;@lOwx& zgB^>zUUt+5Cu^u{996VBqK%9U=u>JZeyGJz(1>Jv`^SJGlP#t16~#OI#FzFq$E%ho z+qRFSsc0s9i&N%a{gJ4}`w4TRcFVH18-d{srnXUKjoa;~T{oD@8GC?lP37U(#n#WY zVuzR3R?E&f(>pV1a?Eu2|RbMl7xA zLe9MzUCepM{l&tQVcoNl#2q^*uV`3N!5msiBl`!kzBFQ&?--VhR(@kKI8z$kZZu2d zofn*kT#HI$k)BYp?%D?_EEjRbQWIz#bXSfFoj#H!>>KOH)uawq5B9rSbl!FE(%{?` zIosn*_%^=2U*qg%P;Cwm(D+#WFs3A{D>@pUREwv))c~A$G}_sCk7?=GSy4oq&X5$) z9GMA4x7v+z>=X+P#NV+aAlSU6-8N@!2O9Q0Pn?ELq91@^EML4M)m>M8vMiSKLQyWH zL49OBr%VTWUk44ELYfIGoq=t03RC!ix$9lqUxhL|s?vYNV-PxDxsIHtzZ5L;cB~Lig$e%94ijh@*H(a~h5tZg1rq`V0SiQt@l<@*^LE6r%El<$Ye|#hl6j zxx>BxWW_zcM{#G1dS6^`9ryVy?tqcoe!e zEc8=9?_TI)U+CA^8t#7aiZAqR5c)!Q+^<|I^f8sZr2WLUtFVvqFS^P-<4#}hJr$By z?l*{{a?b&y8->T074xC0GI?U2r!r*AO{Vr!x!)$K%B={_0w-|q*WasLDeu`w@IrZw@DuH$JZgEF ze&78&U*5wMl2_jTVR<{j=+3~-XUn^zY(-Yi$hV{>rAD0+uLw)5v6T%ED|_$QA>=PV zqOH!C@#ege|6%*Ck_*SAJ6XXe=jwA<`38@+$US)3l0ZFkvj=; zXWK`4rDbA8e(1Np$YT_eSLB0;A`x#;vh!)`vPIsgMS^)nj^>KImZRXWz5R>iGD<+)V{PVKuO_LN~_g$>L(HJ^^kJ+O_`Z$O7(Q6 zt3Gd@gjLQKovQAl2w8^{^3hpmn-psfArkG?J!?p zqdaLJ<)<5djVrt5x08?-{$hpjeZHQMdOx);Z`IKC)p`jrp@08IN0ej!$k<-DmF( z(-4{6!}6zgtxjrVJ310&cQY3v)L(5=aMh{(OKb4NhqsK>oZ2ld)xFxJNBf@DPisrj8MQ8<|YqqC)6T1uudV|mkaEm$|2rp%It-3(iV4r<)k9jPLb zL}OoN1{3Km zVm&+GQH(VG9`*7g*KbdhX4_QbXKIvF<*gSFLHYwJWtG_o|thV zF$)o85B;Yv{{RkymVsj7sxdM%mCN6VX{GipCp+y_H?h^2i}GkMS&nDjS~3ysD@Z|N z*L*O1I}yI+tv{dE_Pw!L1ceGB{ZZe;4e;ViN z^fmNA%G@yUEA3SA3b{8O4T})C?0Rr8@7RNd=$)^9ZpaU1W}{jGNv*bqX|r)?KP}UV zB`?tb;^4W@^4GgrkGYr(n5QTSR<07#?)JxzE=zB!$X9ev$kAA*#C7s08IGlGzSLzh5O46s)4PMVXCFoQm+Km2R6Ko7Q!;C$?{T z2AG3Ko$9ZfD8|CoD&O*N3d1kStM9viL-;o9yV7qw-Q&+s z|F18)!r%Oc@ORC&`TXa9L-@;7)GU3!`-bo*sWRE{Y2Oh3`!4!x7~YnSSksm+ynKsT zxWMXmcO^$3CpP_+6XpL2P%)XJLq^h4BIfc1#2k;S!-5t({iE_B0*fb3Q1yzve*K6Qg*a7xg zf5+SBSZZ*o45J*dd|Iir%35s;B`e&-L#vQN`+B@l|LuC0#ipmHIMlma_xJXIA>KxIDJQu>)BcTD)Gs)e^6s;sZLxsCagM z`)NV7%FxgCbS=5~lLD;z&{@lG|5d)F5UiaajSE6cEkJPMS3U=k$Se$cFf7Ye-bZBL zF&1Uw&&?+iUC!QMIeT%P5_yZiek4>B2~nwpsV$ExQofnO!e;x1amEM6V*Uj5LKXRsrM=e^gbfZe~-PM!u`mo-S zrYErN;_TezZT!UR4MhG4?E|_j3AP}J9e}JS)T~{4 z;Ne}_Zi)G9UsRixr*$lV@>+K`CXQ~5!n!NoSlK?H%DZb_^bbO4qbS0~_e*+q3!yC* zH+$P@Z$V*f0Z{i*84Uf^CP6~?C0EW4x0^;U(gU}Xr=qO{E8Vz|?p=D&ovSVPqMUSY zdeGf)eJ0JYDrdpdiPgGZ{hOv|KS4FYg;Qv1O#OIqA2?I5FDYNvchhu1 z=ukaK1UWfgGw;ks4`No1*3A2OdT!U_P^qNv{cd-%)GO^AsK!HfJAtAGGxn@o@M{(q%~#bqNdiU9{(LM9OI^*9J&l_WrbIUW!_(bp zGOwKd8QA|g%xAU2V{`BCO+o)I8xG1!D#~?AHB20V{&`*W7rIS1`6_U3Hr>Q9-5jMmUg88vT)VQHP(S0Os@5CLmtY zn#=s$_d)gxd=CP}=r1LQY{?!{;2My_dJ?#|2n-G*a&VY{d1$Ux%zcF>4|tJ)dja&c z-9>E(X`5@UdwHVHXuI`n1*~!Xt-WHJva2Fc3)sL+W^UN=Wcx#-#R#p1mdvXj^O!Eo zr+du%ddx?JnCFt%L?5;VuarVrMOUeN3)#EG2nr;}bmg_H5ztxUpcKYeG*fTeW zVz>m2&RiF)u)1mGjf&)-cza#CZ!*3>Jzdp}k6Jo#=Lg zJO)I_%whB8`)seUe1TplpWHI{79YWNN{+N>t5sA4IjyLi|$WO7oJ2qs^AHD zSxj|sYWo&_2dXaTAsS);?Dl8Z|iWkM9DvOW@!=Q*@MGf8SD#Bimdpe zl8I-vhW!(*FE4ypCLlfVa$G`8M9U`{@MAOtMxBYl%RJuI|8#H40+RWm-mYCZj!!?} zIK50+mY+1yaQ))M1YWkmD%F2ass6bqxX|^l zNexh-ZOnqMaUTP6W7$_0OR;jOZ43LMlPoqZ{P;|nG1mW%C44cBFk_b5L+y66xC5WJ z@F|Sk**TCUUP57iHMK9R2BO)%syjw`rOPi{>#10dJl!Q8<=vo^@%#y&)&TPnK#~EE z7w||va{4#CrM0=+cx!50@$fBMI__5SA=2`TMrTT5DD{9^< zeR4xAO%xVLY>AR{&+wk#aKC=POLl|z4gFZa(>F4pA0&6l8)YAQDvV7EZ1Wq3bqnXL zmL0&+&^wk?Kr``!Q}W|uM7H;|+DLED{E5EcRe{j z4L8k%;8-{Ch%C6Q8@NQ^?|`Jpz!SA@q%&{4dSVa#*qVPoME^I$+Qt<@U%`@btXxeF zYHiN(Q(^rSl#Rl(Sl^0M#peNq7kib*+`9l+RANYA73?WYs$FchRG|VjGQ@! zpS-G*AF$QUCNOEtSkuEnO&pm{0aHe3(l0?w#YI>wd9xlA9K+ni9g=K|pa8aNQUxdx z<&gol%CS4qw6NWX-UFhCY1Xla{B3(c=33W>(x{Cl751Q|zP}aUHnNCF?B%nx5}L31 z|1FPkQ)9^!rp8h?HmYPMi4&NE_F@n$PYmNGcP|cX>Q6DvHyy)$Bqui@BSa?VW#th5ztc=(o>eDrz)SGvj2*n(vY72Nnfu) zDBrHH$01LQRw^#pp{aWG)YRilz@eIs?jh8kI{KZQj^^11ZC}a!+MRpUf6~vrM*WZV zvHsG3Q#N!#cYKYmV`i^zt5TU}WJ}dUA$pc-mt3iAJlD!|m%F9%Kb7aC|JAXqLAPr)q1O-Zh%$YB~gWNpses%pGpv`D;nTk`he-dt$dS zV&}Jec&8p#h>xY2srhp}yd?);=HbmAK062hyN6Hl@Wvec4<26Y;dMFq^&URV!>e-e zOFX>T!=pL)86J*PY4qpdqddID!`n?AB!qoEe71+r$-#H>@J0`B&cVNuCZ?vAIjJX6g}5dAm3R&PQzROg?z)kZKmeqQ1v}y z6)XI0?y>ZP8$|TqJEpC9YW+LYBGt*SIm7YWuij)O)HClD>U_y`=_{G<`2MEb9%4A7 z1aj*yh)rK?C{y#7dakULG*Zj4A3wM?-T> z#fN0oa4Tm};={bjwhpVR)YFk?jyx;4n&{p3_bJ$idH-1a@RiS4a=XbnHN)twc$Ui& z(i6(MuE=gL%k0^`Kj(HAK^0sDwxXW9hg-|&qK0Q@aKCm12S=@FaMetl8AcpA>`0sY zr~)R116)V%L4?L=(griVL~C`a6GR)nT@!P0lONTZQ6UCO6g z0OfaEn&wzKA2_>bU~^tzW6!`Td4Y9318egFt9k|w%L|P53@pwIEa(}yHm@k1ymu># zRg_%E#B)l$r>6V!Qpp48mZ~ufq+5OUl>89RMFp9_wcWtOvS2R)zwJWZGh%iRWw*|*g1(OuuMh%R<0I;*uw9Sv)_ z7=^sH)DOz7=h)Mgbqm+NqN_Ed9AC^bgk2U>!UA{R!BYFH+hc8God7&L?a3hkT;wJ_ z4c;xsGw|B~@$?k9H!&DzIIVtr{vHY-tnc38+rkc1 z8)aN@3`k_c7Zn8>mmwlIA=;nFvaMijZdL}{`^8$cJk@9Tp+ zXbiYCMNVb2r^lkFeyL^a0|Loqef^Hwqc+I>7_H-!5UR-NVvQR4@wDMB6I=8B(Y*9G zp?P`f_xfMc59pr$J%su{Kd*vPctIhjt^Yr@zMP z&~Eg-DJ2tq6Cn^I7HmhRUH$E?{PQe*+30Wfs4wZlmyZtXOE_l8cyUQAIgWAT!bt@2 z>ZnyGkLA_ZD7Ribq6VshPYib3-zrCLymopQz758r^fidY@a@`#uXi54`-Cq&ZEc8f z>3(9b#|M`X<^%^ZLzTD}y)ML+E_~rAzF5h`zU3mzMHiA6{$)&-%F!jRwi}t3fK~n1 z!grN2TUOR!hO}KNurKTRIV}Bx?Lm^?KS~$=-lz9vi??OVc6e9%ee%-3fV}e4Q=HyY zRh%QzdS9YiwFD z8ugKC?y|9#w#_{zwMkcQdcB~)xa>Zc%?)>yKMX!plJ%zt)I*9i|6epn)#;#5hu}xs$cQ!_~b9kB8e? z!WM2BIxXf0CAo?`p?0}C1wqL^rI|rl`#8Gz?-HOKsFPwEVZMM1RDXig#C>tA$48D? zGoJ3|&YFewXDzt^-eCB5tzoKx{vh4@E!SMLL#z@cjaNO=)H0RbeMf*Vii}FM(g{#f1o8+I+)Vxi>wnv!eU1ftX(0Z_Lp$X)XExdByj;%t*NDkfx@y#6Vstvy2=#! zRT;BqMCUcF!80Cuxqw=Fd$ZD3j{mb$H7A5HAG|6vi{c)ap^80mtBZR4YgdZ{TY zSgW_Z`8MxwEF7ju^0YrW#_{>O!~l3y4}-7TSH+*NJy*b&=0=NN_3&99et!zIrwl7Z}#xnIrwfKKE=ZubMRgsUhCm?Ie1`ZWBxD?ugbw+_Hfx<=#-W9^gQX| zPLfHR&Gzu$d$@@&o#$mIJ^X$jf1V!3`d@hXOb?%xgP(5jX|Yx2Tbtsx6lfNTzvukf ze0wYBZ)&$$yvaIdvaik@j&5mIgB^=sUB)8!1O9OY%O{#%0uml#vuK*QrJ>UD-QGqy z9@Dp-|LBH(rHrrA(RtWhE6izBV}Xe7eaAvK+Cb&E96rP>NPQQ+TbF_F+%9~tf4LRD zzBzokrOco2qgbbuPG;hAm#zgH>~o|Li>BsP{%+${VR62=QpFj|c@#QM1)C_&MLFEh ztbHW$CzV_SBsJ;tiZVNQ@mnXo5v~3-OkY_|@~fJPFJqBoUY4Ia7{g0*Qm<)OuGF^n z8wUbgRn#!3X6To(=@bl8AK?Y`jaSW0F2&maWqk5qU@Zn~!=Gy^&c#0OTb-_lcNp4# z43Vvzx|^6OWsxHuNj#>$xGrFoza|)Sj2xf?z~Z^(7^+}2pRRvtRuC|bML+&L+b0N* zS;^P97o4p+a;@-_2pN`xb?|ga8urW^nP)R=y75n^c~vLBz(;44Pc9ZpeIX|TLNI+s z^r)MDY5rQ4U+alARUD<dhx zklRD4Q8F9^i4|DXonckC+iWe1iu~mB$g@VCEvw$8FYW>e3eG_AZaT>9JXmv0w_0Nr ziyvLkz@b9a<*vDSIIP7-8hhp14`$Zc%)L?mXFbUDyunJu?X>LuTkQT%^(1{N?fF;R zMP@upq3A0llwq+_Ym^uf&loOUI zJTAOs9!GZpK75Gh!fnFZf&b)I4K@y(<*{C;@NQVI>VfqN!+L5I^FtkDEHJy_y3Oyz z=G#ny`y3etS1F66&*Cd9QF-fDS?ty$i(S8g@GlPbguhMY4fYn+gN0T7yTy!qqz2yM zu|L&~E$(S4QXWs{n|TO%QWCtq2}B?7ZXB5C5xlC5x^dvj9yl(~#}R7RX(4$>iM$zL z)wu~QwC6?I_3%D=7V|}oGif%@U)gZ-*vRANRXAPVxgp7p`cq0f)hKnU|Gah# zPKL9ixZcA5sY#D3BEcIY*S}e|Tkbk6d2;@~Yv> zy0v$T$}WWB$k1_Xt1Di;oQ3WkvG~f(TyGOgpYnA<{b8~6l_35O)N(~ z#h>Zb^jt$>HE@P@zKXr_-dL`wd@i=FrFl1-W@?P_nBmW5+6isAAiw{X{A~DhmRGFe zwT1yBE1s+0H($Q;$6x7;t#6MF*#;`=_asBbTuID^_ta8%Z~wJZ;S9e552wfg(Jm|s4_BVxH$Y)K?>thL}>In}>IGL`yWU4c=h@ipZ;D@%V3t9L)dexgXhK~xgP zAJ1Wdc;gW|fg;wcRSJDSm5XQgFt>@pVd#B0{(~3r<$?ccKIO1?RQtA`PuXJiXY2Wt zl^!0I@1H-P@{)(IHT}<2sE{|`IL#lhF81nrRYB(NB*5S7J5f691TpRQ-_zG?hY_Q=EK`2NUl2EX6K=jGtHd$OoiPzx$Jbb2yn|KYrro`y4^KcWd z!IyjZR&rhF;nvYl+kmv!r~3rg*u%r$^<3@hznJ`P59OyrK_>>)J9n4gO!V8^MCe6_<#NzM&~er8Pw9l==_1Fr_=f=@wIvwe2|CF@$i-$e9IuiGtwJ*_;+~tRw|P4@OE!XJN2;m7aF{<|IqGLfALGEBULuh?3PmrRsUxp zdFuaw?)vj2b=wbnKG7`EO>b+|!|P8YzN*`2g%!Hi%h$wgnXkc;UftMLLb%VvC0O*+#B6wO^l+16IzA;A|8fsEDWX?7_&FY4?D0f%@Nph) z6&=0G3E>AGZWW#5e{=9b9=?^3x9no&-Ac$SJ=~;7{mS8a$-}p@<4<__R(AYugBMoq z(_NSw{?XO`nLQX6+Jkv;c4ANw=`PRdBoF0TTWIpE(!PZsy7Cb zl?0}bGeY|^6-}BLJg`G|ef&{rsE?+BJ=VKCBz&pDhVP^DUGtX{gHr68%_nnz0rSGU zarA{vEvrP0}{WGJ1-bn~lxL_Y{!Ux!3z6hxj52 z63%zq^rfHkehT!tG{#7GhH20mx2%|CS|Fj}f2V6&s#8B?S%K}>*?(tlbM|~nj@g+z z#ZQ5c8k#k3_hCAhL=a|NCacWNVH6Eka|VlKa>&lv2*D-vvcmpayfO++W+3Z#S{uK( zi2Z~tIm-IOTz`Y< z1y1Q1SeF-A+cU5#FK}4*!1R3AAL91^m!vYkc3a+Oe_`*lsMYS(^(L*T``wqKo?ZY` zVmnO?9-=AA@1OVF|3_V&zDnjSZ%iv;$Ez_lp^fePQ;o!&6pL^`S?JCP8@FPq9^1&z zC{oUaL+9z!W-%0kR*oq({pVR_0Q4v(Xq|m=icO^6cc5t7H>)flwCr+Pt_M!gsfR zJ6J%+KJuz~@7dQiN3k7eeu~44qq#e`RZJVpdhZpza=s1Kv#UQL9`EP9gDk@XTs*Kp zT?`EqgG^u9^{)APhEwwOIV-1r!Cy(X^?u!OJw5^rJ8rfJXrZ-_)^N?=US+TuZZGej zDf`s?5o+>G0Z#XTt-QGDKBYJW@&d&-j6d~ovC|_c5dzjv?ur>p##;nWdGC(uHrB7W zn`}$lhG}UVD~mdrL@TzMtF^DO>}8-oP}2D7sOw7>tjuqCSL4DJ;xSA8JslFRr{i5o z=^I8(p7ZcJAAd?N{-YjV<>9qC_-{Ns>fyt3@aZ02;NitN_)k5&Q`(ufL)1JyCwjP4 z9QaZ_j9(QVt{MS+b`Czo!)FR$f@sHAq^g%;>BBYZo2}^C)$R2M`QcmJneaYAj6{lE zy3(~tmq&_Q+f#jmf5=MMcxqP)zkXfgSFD75vwl}8_|u^yFQ z61rzZZ(4VN(;TqGj9IRVm+KFVCE33n19}|IYmC<5IR4Y3S6N%Jgbnm+s6g9DLAfyE z;;Q1vfYHSn#!<)MWaQtMw1EiS?3M$dDf_~-JPWj`N>`Ob{fJnKi@+>;cwr=&M4B$d zjOr>d0`exr^X;X3uu!F_T<{o6h{lxmal7kVUJVUgB$1LGcH7w0ut1a1>jzj4Cp$M4 ztdvEYC#*HpD*w0p0b-=hkDzj*R||1mKRk;NXEc&{3r6VhF9qp5Dn0GmW7#5o=%+?o zXHaf>hong6o_mx?v*OGj_VYAG5|0Q~TI&oTqsqOc$k0+{XH5>dP6LD^PTID=gs1)F+RCG|g!* z|2$G$xB;h7cYmZu4f{MYU{o~L##$aC=wkvOs#$PqlsRp&Sx=vbzK3SJ|LzP z$!kmWNM9>|mahpss*|U3UJN?Vab9+D2RtouZ(6Nn4TA}A`cfiB62FGE79}-L05^$L zh`;H|5UxYV!=UYZn`4CwHCBuIQM`{(C8v1qGg2zgvy?_+eEBVtXR%wV7X;kG)23p^r_L*I%L)|Se_E#= zmGy9)3m=NAQN&tEp^ED8T)lRcJ%ZG0QkAnyL@~|1&e+{Da?9<*lps8!f233pof`&# zGj7lz&e;&EnEG__XE~$8J||dDuW-{NWPqMaO1i?wChp}+rV39~Hq;#@mN2PmpWvhF zv)F$w3_m41x0f&f%$u$Jl?y=Z&hACwJU8pO*wsSHTOS0MvC(b*K%D7t_i8|tS~1Ms zVMC(`8lI#;QIniK2kc!u*j^|p{D^NV{0Ng5fmZlA>Nx+q!tblXbBES=4h2*RzmXB? zLdU$nO5F&HV8Rv5@9J}w|J*M8^zBFCd?x*>B^);Z&jA_C+$b5OD<8y6x$zQujv@0b zr2Mac&M{>5`<@}UYJ?JojBqI%tArtT%S^g50UDSx4|2z+{-lT7zzxP`aO}+Gkz8<; zuMlfY1u#O!lLZp{zvD?M_^Yv|(Ge6cpCNj02Uf=aJS}-};8t1^8|)c00^;D;>r6O{ zUuFLDXZvOuwAR4X~W_6YG9I~sb_H0E{$y9+cLh!VeRqk@W(qyJ%%MHM1gN29x^6@|m@+~535v3rP@ znj$%`sU552QuVf}p8J4Q-&vt5P96ZR3>>Y#OU4R$*#L8;g&Z zWob+Ge`y!kRw3U&Y61qy7D+|nYHS0 zUsx7ne2tPN)-R|>-aYuCT9ykxk;#gG7e02=R^d7M`}KBrZW1#pcZC`5(+-hp&dFkT z2o!4Jb9ivxd0!LHuH>bTuT97!yMA~%JQ;BTI`KFV^77z5*f?_n|88J97b338a|+)s2KUKVfy;d5tG?W1ycx!f81S3Iev%X5L&zyOH2OZVQi{^k<>|v=g7Fdx#`K2@yiZpP5Ynr=*iNDEt@%Ap!jq zd*W+P!KtAs;B_f~qjy~^>9W3v)xc16uvatj5TiriU z>CCNp<1cyrQN~o_t5|NGcQoJxPP{wz(=a0y|DrG8Z0b}RH5e`j--f0qY<nGHpr^Atf?uA|4;r__~GcYUjFBOUkBuT)}4>2P1lJbIUgF3~g* z&Z0K=8B?mURJ;~6cB&$7L#+|%qYjGS*#T@2dCksnD)%U;37BuJdi3` z#oG-GUL-^e^;S2DvnDO-MnbULe~qYSEtW4WsOb%TUOjYU^)nlLBWzTf0&z-zI399* zYM1rZsZmAi8ZoHsZAxfwlkNA>g);3msp_g~E|f`)WW=Kn3iApDh$KF;NUY{ZX&5@p zTYnu3?`K>@3-o)9TcCdaGxZc>Sn& zT^H@t?;Sg0Xgym^$za#~@e||O5tBD?PzI{_X?4X}(MVz(MbfAYjODHpu1ZXv#r5!u zipE#0YdAXxJxnS~wm?M;kKPTq5}|@$!c@ z$zM`C!sJg3G`WGZd3FhfpZH0a@c)@-FI2jyJBAtKU}rT?Ts|3GD|4xrD+P&v!U+?f z;G%;*I2K4Zd2t1)9hjWcrz8symn!?R*fSe?tISPSRMfwu$Vw##@QbywaU9L5c)gaY zNNi5i-j26pL%)bsyu;?JV9VJ-zam> zC?*Rl%c{{BmMT}L2bXDJlD0>|mtg8FSeceoCs7*2Oy;zG(V)M%07v5HZ%}P0xs1sm zqT$k#h;M%c=F@!Yhw{)1x%{o+&&A>d7DN&U0gXM=iK)@x0Nc%W1K^J0fKJS`w=2m$ z$O6Og`U^zGZuD9wf~==tXVJdrPB@ISXgPO*Equg#G>rGU2=~>r)!iYWB{D;F=f9Kz z^M#7T)%71>=8}KFZ77vbCfNdY=aVS&oIYV+tDB(DJG?&SwHNVK7~#CeA~w7w@_6CF z__>J|mVrvWFw%$ky*uf`Ww?`(rrCX@SBoZev(x$Y8NZQ;?w(IK)7h_ovL;>mocz$X z!rORIGxT~huln)Y9KN;{{>t7OkFH5pp3f^`iLPkVyIz0U^*Swlb(21(rKC$5#WRZI zh0@l_e|Y2=@}L}L#u;1h=#E1~ueKt>$|I-*BV7I{9)<`jCGbd8)u% zMTcQxFC<;)8V+e;6B5kB8l%GiG{$!e)bX!q!Lk~$71Mjv8$P? zxj;Z7Qyx*tt1Z5nF!<_qX(UttQ2u zvo{-F8K1htkk%x3t4?}6H6)8yU@%6L*|EtTxTCE-|NLQ&hNHMlFq8!EVv_Ku7MLnl zS9~QyR!=+rfY~|TWFeJ@z&51yNr6 znrsj1VN`XR%Y=y{IqIojb@H=dFk$YARnRM|T6wBGX)NbivD6P--5P<{WLWM`7`8Hd zzshTFEJ2x<(D-xjM7@{e89i-{mxgfIuM`maC|2Ix$(vNzhK!|xE`^~ZqeXPAl+oeJ zXg#b6Gg|DnQv{6Bk1Z(^<81vdqBFJ3B6YaudAhNNrSLLgNmZWDgR%JR;W|YsPO9=m z1-2EQ%|kz4j|*RY?`*mdrYkQ2=mx5SQt{JP&4h77#+4(GX$cXz)qe-c$Rc<%c z{EEeqCc2d5eir2qgJ^19=3`|}!->Ci5c?T5wXcn?le{Hr_=Fiz0=Lu;^!^y<32jj3 zx$}g^GR<~%k)0QWuN|HjR4xVkmh*xJ`#HeiQP4EI7gj2!jBGhU#(>)`;^qSdy}&c;u526|dsc@#$TP*|Ly20rU7gS1DbFpD`f} zuE7x@D5jvx#<(rU(VjDiNB-9{h-1{m=m4BHcL*}f%w;}2H=nqOHP(LpWh}LDmW(n> z=k%~i*vCxf^QU_F%pCj<51-)SUF*gY2A?~^;PcCPxSJl9falvpICo#>P@L{jw4=Rb z*{O$RInu-Dcz8<=KGegTJ$!Z!KET7Lc(^Kq$MdDhcI#aL?|S$!AHOOW|2YpY^6-)z z{80~g5*w>BO?oZ~CB~mVS+U8fBKv zyb&T6PNk$Zsq(M-QD6Vbh9+8{;m?H!GfQXGU;hglxUhuSZZ0vXd?j>W8~+vM*g{el z=s(l2sQF}CUmmO*QzJFH1U?Oalc@l59?a1>Yix_^$g_hxK7#tZ{y^4`Yeh%fxd{bx zr!;sCYups#%810o1j1*3AQJE#v}Lr&-Guy~Rw4M7m3*1t47bI$x8nQUq(Wa)oacEaO3MJ!^r~OAN@|+A0E>dFax@=ssQK$C^S$dLO>2}{s&{;B5#19T zRbqF(oU~&s;ss?IW{4Q;c9vPY_wMA8g&&~^P z$$Re%--)sz4_p9K3|%r&TEx5V-zwIVcqL(&ms9L@@B5e4(36SAD!#7YM@MMxY>e1l zj4+PT;#vQw&<#=o?zSyc2|lL9Q+ERe{U}SW`x9km6oWp*v=Qc_NUtJ8Io1PLWmx?# z?$vMt#7uvoTg<11sty1%+177ooey`QGzL=8Oy=_~iI;Y6byB1$DxpqaB*n-oBd42W z`M`72C__P8lDUmnXl6* z?us)381Pl>d!fj2$GnG>>3GiL!z!74uCa{S6dw6$6ot9E`03pU(I~3>vlNF~L5*S1 z{hqI)1zWgzLCaz|y@r*X;&~+TW3qI2YyKoPIvUii^be%C~mv}shQ`MITi1hlqB;WZy833T65}rSi2l1Y5CV0 zmh4i59Szs&`bSjVd-0L^Q&#PTZXv@2nJ0zr1^rnNuIFWqS5-_0^;)yB9eGMk}8c@}-5hvY2!juV3zpsc?2q^IkAXk>a+p4Fn;Zz~l4 zKi=L2EY9Nk<6jbpm_!z>vHZNQ7At6sphh7V1p{$cmxP+6ru71ZU`Qo^uq4D=v4Cb> zx5d_YskKej*7T>gX=`k%M#+`fi$vK) z=fCfRRZN&~j1aq>?Hb)sEWo|@n2Orv90Mwy64P(f@A@_^yk)bggJjO35OVx^BNfud zEB;ggoHjTX?5SFsB9Zw{Ap9bxY|JV;rsFZwns(4Xhi+`dcGl?&$d=yd6M9ZLO{i-x z!S<1nVw|65kNFEs7TcN9~!hbj5 zIMb*-7;QDR(g-dE1&|K@=3&Qr+j)2rnKTrloLwEB0Tdnl&GYNqt$dTT+;5+?CPJ4% z5?qh0J@%X`ata>ZOep?L%^hL-63f8g2v(SGUx5PPrZ2NJOHH=SD2*R7C zvxKvX??OagET2vdFzkKmF62X&GuL$PPLa#ge=@wqDJRN#LP9*I=wA@6Es;qgYK7%dthkXj8&xF3$%592u10<^@>PV1{wKUL5QRjdk4w%FU#oayYjvK}N7+QB zzl3n?vt|?wvzmJy#^F>qqWEEhTYV66YX&+HdWc6JAutUZhQf#i1um56+b)!m2};@J z6vFenP+ve0)A!tQ(y1QGH=DHxjT=<*4D8*>m5WxjI2exm3w5HivFv((woywMwMOBZ zC-5FSFZ=kOTzu+e>bdFtu|!eb*jKc~v|j~41}re43{D|?i@oYRg1=VnRX3%2d+voL zLKj9V4$}jz@b=sVu7c5a+KA=s42BI&o|mmLcIqj#>CsA`#@~t=K4w>y1!yP*#FeZ- ziPz9fAC?A$x1IM-HrZ;6pI$MemJJ=P!BZL<8K{kFHlb-gVW&mIAP5*myV(Q(YetfO z=lV;xd2i|jW~n_#yewMY5Xkr)?xc)bw#)S zQ{9kx(f{P3=QnKH%NxDRp>bkNk*54w-KRRJH1u_2RTSDTjeC2KqdQ~k#!wFHTiD5@ zVOf{lTdXG?Zn=T(&AHYW_ZIh(Ib@^jXfj}3b?qW@c=D}nTxZckQ*>y+3Rt;HwAGl!=WTgMyjD=r2x@m{CJK3>v?cm z_o{UmhA^Waqgu)3jNq#ZFUq!1y()GZQ5Dv=&TNx03Y7S75{aOvM~6>A-~3f(cU?Ow zaa!oFs&zHdH0QHw^haB>oUywoZk?sl#Mk|&Sio2q&Gg=QJxG^{{3aGgVG%0NInv12 z+f4bou6>moKgvkH`}6yGN9gBtE4jH5{!T5&YXCA3Mg2R4a2T}dM@+4`>d&yQG`h`= z!}S^uY?;KzLmJiwQrKe>#&U>B8}E?{JV_B%et&e03|7u#2#rAGpg`&Q+1{ReU}9p6 zjlsx5$g6Y^Ct%s8(TOj_ed+UI zXDaNA6;lq-2(|lNnx@OPnOd0lsTR|1hmla$+EwnCHK{Lq8Qqe$jaCVcxvynLJ+YZz zG;!Ps3YkO49V={=9Z}ORcLvV+z2Xy?XHk3}7kN2G)wXXvv#185s^5SH=M% zsXwE-1Wq}~OtfH{SrQh##pa4cKJ!5BWYE@`5CM;X8A{h zEH=P#l3N)qk)ErzN&62(gP5|TVorseRq_&t(;Z%94w9i^)AyIB|9Sg{LMME(ecSeH zpC9-HmVq$?u&N>_0a>WXolgf?v#_v1k{8KGL9E^9yzr7JTW_T=bi7PgsVZ>&lJl7P zVo1ZRZClnJU>77PK(SxOa{c+au^)eE*B2;EMcfg8+}fWR4(4}|mTb$sH{An!(V}gc zL3Eya(c5#Q=Gq$u$A|>W(41md5@GXeF+?Nmj5pOI?YXg~?~mY5yvg1b-5-T>>cts<1l35d7MZx-sE zl>@R4!2IUx6J6X23{K>EEUdYJ;9Rvs(cn{BH1?F}8VNGzYE#gJcV72_waMch0;J{( zri@LoOCUDqSI;Wvrr080(pyAa&>V9053xJrn~@a$8$s1e-d^_JC1U0u3pJ*UQ&k3s z_?*dm>|7IROglRm7BN;tOd>>&!Z;@pCp{uJ!-I5#xwmIEADs|2#=ftoH-u@e#%!_E zfg`slZ#?#xEyMYm={^Wr)Bm1`vYf5VH|V>2V+}iI&E4q7ns!_xQeeeYL^xham-9sV z@z=PIaCm=|mR?@OgV%*<@u#4%?WK=N6Go!qsZwzzmfsWxl|KVSldEVhgIJAd}54g6y0VZg)$k z6?5nkJBNDGkYC$C~3;FEX2H#bsUNX~l`CcFfoYN#*=o5jdOiScoza-#Y|>~02ry-U7!44prbqEzFy%A8IA1%HvVcizfaM37ZmVem(>k;wG1 zZ&Io(kmlsRZ0`}mGJLf#%Zd9u4B@Mu!f4kGUngsk<-gxoRwUQ$uWEEp{s9SP`EYL6 zuw>Na+^(7|<4mdb9WB~oO2?086$vP-aRd!wr&~8ABO$P2P}6hlWT-lE`Md; z-6e&^PpVPLC521uo4ce?W#6#kYVqM*y#dJ`LQ{15&pWCA@ohpGQXTvcYZ>!Q4VpiV zGbMmwcPd6PC5AS!uil0TUWhz4ll?8u&@WgHc~lD3?E`2Ix2oTH3G<>_kKcgThz6Km zHY-JoNwFp7qYN!4aSMwXfDiC77=9~$lZ56s^skS9&LCjNCK>rZ;GCfXCb?C7kr#6d zl4yCcz8r3gb{Gw}c74^y^cwCEHMTQ*hcZ^OC-}0WTmK`y z{rnGi>p$KM+Z2ceXXZQ0=_r46?RN9!aRtuOr-_0CX{{&IIAC^oSr{lt8^O-=kb(6G z+u|%WQL?L%bbKw%YfZ4UN_%}*;jo;efFR^b00kaHHW#3XxNz?TX`Oeb_hA#hC z{~pmlwzq7F<7B=i{%rbu$J*=7`i+)u-jm1o4R79wd03lOBvQ>PP8um;S?6nUzW}|S z>Z&5AA?fBbZaD3f^8hqnar!GGLTi!26b`<@aIE-!=bwD6inRF1Aj7xG@Up?fZAMvC z2GyC$u$~MOJg8ffkM0nG?s4uy5LyuznYDwOXyQT|d7?fvz%6RijgNJKXY^AG0tuj$hScvRd^Kh%1-Gph}ZJAold@mO~ z6(sTW3y~R!)sa@hlk}Pto>mHp(L?ri3Fxc`9!(^SBZ>i>GgVNUz=D3V>$&E87R`>} zudpY-ix195<)!>S^*R21QlTV^^wG?E4I(iK`VB(y$m9dAmzYa#vEHDVg3U=}=xWOB zExg4xl{_lcduWeI7dp&63g<5g#-^=6nxwrl+HyQahc~LV7 zAybWH*t^r}rd?&yK9Vo?qvZ36XV1PoF(YM8RpgzUrE!?ph?usmsLcW zkkq|bhzc!I`6jhloa<93c7KovoyLgcMB~DZD7@t1#%I6x{O6o3!$c@M{?q;FOKvap z?bggZTX{UXpFPTSp0Q$s@sDYloUpRh%+{yWOt3I4#lD0T^II~)hja3e;HD^Q4>02V z2-@&Q@b%$(+aZtyy05WmQ1Jcf1%F-zL(4w{XU=a}1}9nkujZd!IR41Wq66%3kdsFE z)kL30AVa?Q4?X~pyG15dW#f#N^t(&Q^9CTR9nj5neklX=HD@Z#>N*yQZB%|q5Y z_aPht>S9xVk4D69XPqG*w@}ndX#EwZn{wkXSoxgthSK?a-ET63;Gk~KAcIMI&0eC- z+^A^%kNgn~>cpRf>q-Icz+gg*1Y_)py6RicxS?qOk;&7?`DXDK()%~!cGShQBeZQ3eno=@GnDIN!*#}AqwUE%KKjbThvz?zF!ID-GWKHUZb{DG{)+G$ zp;w(}ARizI;WM?%x#3Au>BD;7qF#r_oQFu^kDQsWiSX+VTeFFN9dC$7?tr)VI3Ej} zusFWmON_Gqh8)A=UGxoc;~kTxQZZGPFVg<_DR^NRKMNoKEd0C=j~&KOp6JWP&uLZR ziEe{Ym&-f{z|Y0$_;Dfn@2?s*P4@4a)WN=W3i;M@D z#>Ve(hUill87nohCF^?8$O!%OzraV{j$wSv#r*l{^zsiRz%V{`3e5i;AD526$2An? zER~RUsBjBOzWZE%-UV~95NxSsD*B|^&0hHEA%0qN6U-C8@Io~Jp*?COTWY7+ckOJ zgBd-{^lvd-uLWlCR_J)~9S;ti=7w3lyt>XU<4Z-R1kUMi*SPbKF3;=oal8=IL~A;f zx(f?3`U%_I9y23g#N4I~j4`*t_X9S26|73r%WLwyk7kasu=Iw623=K+w2idpIt|nw zznk=3bw&RMD_iJJGalXX5jJ}q*v|F{I-MG`S&flT@!dS6@h@asH4Y%1>YXgwr7NA; z>=425RJtj2YA}jpHM~0||5;v*t}{CIS74!glP}1PrmvB!R^_mxr8CcI1H`WB zr%=BqRGQo*k>$KXpFi0pXh7ok$*#rTgS8|4D%#iwCbC`m=&naySNkMIa9#pw?wIb?OxSaSV)eB>>LMcW%SD!3rn+Qi7kl2CZ4W^Ot@Q8Xb%rGxk32a4<$KCrj((G^cI$VF`=pp0uwoyGE85N97Md^WjgiwzB zO(N9WWAGjRL)@D#51F;7e-C!PU<7`s#rADznff(g)_^0QAiX;t9qw1*`XpG5`jx(T zv1585as-8dC4b~W7AN96)85fuil3hoR6D{$ z?3H7yEasNz!WW_8qw~#y0~-8+$Wd&>tqQ-)KG*Dc;c$PsYHX)?k(~0G_@FJhJ8nDb z{H4!u-bCpR`YyCk9d`tqHI>lp$knuD?u(*Arf*0Xe7?q7%WT$*66<)`wdPebH%Y&dG9D;j?Pz8FWCIt)GQ2rs@cZZ)@&$0?_tQHsVxEedv;rT0a;Z>*su=Vg|+15Zn zDZ`WfD0wGI&{>b@0&I%+zBx3@=PwY7<&+=haLWw%(lD89ya>gdko7}*RUd4d^ycy7n z=+dD334h!NTCP}Z({RGwT1b_*W5S6eQ;jfwUL~>;3ud+&QKBtAMw6CKma)+(HVZvi zmUqc?<(vmWO8UTZc4IIReOC6o=I!~efy_2V29=tK!L}3PjMSTiCG1Tet*o`Ht=0!s`ap z2)MU|w4&dc#|EQOA1{IMHhbOn0BSp()jc>g(5nwVsvGFlc90^T{DsI6`kWEMDDTx= z!XND|GdW%z5@CgVi+FRsNGF0eV}`=m6@VT1>$m$eUUnY(w`gI#AfT(Jr3_jJ`=bp* zmy}=1nd61fPpbb@@~->M*0d(zRHs=5150{n)+k`ZVR>1(^~`!n6d0Xca)n5hne!&+ zM6k-Oke6@tK|GU}DS$zrYZyVr2>p(`ZIMm1u#8pQgYLmoE%GE2zY0{AoF5SMajI#CiYFa&Si!- z8mMI8{-${assJlZb{ocSRYY@rW{wuU&@>j4+io3fD1KpcjQ=`$yX9kK;27A-+d5fNloy}8{KZEI$<{?@@ zMLDR5-$}$!gFv4CLu49O;NpJcn(n;)K+-X9r#d@2`A}q?+x=hs3B`US|Fhryule?{ zE#h)pgf%4poC2gB8uM$iA(t9v#{8ECS>n3dmOjpwt|7Z^H5nrlGWvZYt|JLFu>VF* zo5OE&+)!`eQI6Yfj%V>6BZotv9BaryD0-a$Wtv+iboj&;(b;HmE3gnJ5wn3Le&+b) zS&Z0$<%)8BWGyDhbQCaW~C1A}sM(b9DXnmYd#lv_b(%t#46Teqm%P>aM z%5Qy3b2tk0?`cP>nPQGoF*5ASri>fLVG7q@BD=FR1G|3we$s5CE#eZTX{>?8KqAN= zmTZe=z5N(Sj&g#WWb%AxoUp`eEtN1(oKtr5&OE2qAAwx83fk#?^s*u|cYc4 zUKjhG(5aO+QL5?Rii>%TF%c60HaY(D6`1< z7_!${Idk)UZ1q%gN!MA0)3d1*iOqP)#5%1>ePr4WPOCA~2^~-ujC{Td>p~#>N+5Cp zGpRo?FYA*nLoE9>Dz|eov;7Wc!<-Dez;?H&dK8Ua&!X5F+;hGW*<8GjqO+37^Y&f{cS*yq+Wmf65k{!X`JmqP z_DthR%}VRbo-++o@^o*@Tv1tMuFCWthvewqmd$&zbaRy^v#R0ptHPfb%33){2dQ0R z+;!6RYojBBUK}KUFKWS|{U?lofOccMyAbJ!k^n2UT{K+B*C^D_Aey_tS^Be)l27Kw z63J=~Cwksk-kM&h&+PQXPD zD}E(agFpOvafNV6$I7xE1T1idj+V)T!VXUsrpo}ha3M1ghK`JeH`a|T~Mg92y>r`^{$YvncXj)a7*H#wTV^-#;nUjN* zhLeLf#J-Jy&?r%afq1Tz09%Nh7`f1$V*#nYh*b^k3kA|m=D^tD{&v5_egL^t;SI8i zvean`^;2U?VyE)eDZgJ-JK!w;q1ZE}25@B1&~p|cuEZY`5)-i7i%Fp+{%3#8?BxRY z&jZs`w;XEZ?0y+-5*|gC%X*%(Um?T)A(lPHlq?E zOxBht!9Z^|2?X=j-dsJAvq&Gpy{0&Az8UB(NTn&VX?j(xQ^~(7x;fVvC>Bv~fX>1# z%5F@3>(@RRbf3+(8&j)bNkxXEZYM_LrG(C+qHWY?cCtQUGd!aVW!lLa?V<*jS`8?X zl>(J3lC_7*?C5>gACC43SWQoQmDV}^U&>%(BWndEOSA%?99Y7aYju{itCV((()PYm zv0{Bcufg!c+mtSR&vpXNay%F`c26&`FkJvl{)hopdv}82wL6vAz(2@S2v%jk3UK=S z>6NB%jFxom6TAeY>xU4=^l&h|ehpANW9}76+~Xnp*4~|HU}W6o`0i8e%6{J2;;@aL z!6CpO{v(b~S)!%LJwB763mwp8*lmhg*~mK?Tt_FH;RVVv3&m&)-ku*Dw7VVjIocLn zrw-~lmUy!x3APQC+l}(RQXm3sqAxd#rYE-zzs+iP*LCa2b+f_9J=M&Zv=u|jdH9~y zYS*kMO-|>mUx9*%ZR^!5H0px+wbBIsf(z9DB>a2+#f!n<3bJ{7m*QSSdJ(R3vXN}P z`T)a3wD<1h(H}jg_)>xagKFZLj`#NS!8}Hyq9l5Eok`leodB<-d(3MxHmk3lx{h!o zZ4kX@2$Yi4=1eB-ttA#Gr9p4uJ+-!Ed%0C_86Uhy{740R3x8B?zjY~y_72)NzkM6B zZ)F7WmehPF7J01zMRI$0f*C{Zuz-|J!qVt!5_)?!Wx`xa*P4oZ zm!gXJyG{F1I!x#GglqxRm0p4*qeML6SU*}^erl~{f^tpEK}SctOnbw zA>c$!g0-u_BI$%vupF&5Rjl1^UsqNWdJCiK8E@fA&mQu<_pUOm?Xvm0NZ|a9sH*V4 zg2bBahA_aI3Z-nnBE$botS#fA{NlvgHH345;kTXT_h>AwS5M-!4cp(0T+%^O)rL{` zx&_p#u9W-Tnf9B2k(^2Tmf_mBOG|{kDxE^r$l*+hRi-APM)S)_sro%sIkad$&=Cp?-z@xT>)CX%i=C!@Z7RjAVWF zmp%`6gYA4F7$gWr8qovGPyw%V;CaF?$vXu|#x8UQ=iKj`stTwIgBDZUeq4KdO}a?d zVT$e6#ek*;=Wk*@QS^EXt}{xR$@Q4D6V5)c@-MQ|X204|9nuWHibf1a+hm%x-n4od zMRo7ZqQ!c}J|?BPd-?;k0OjY#y?PDzw(>1R0|%U&zNh7QZ|`aFr$F@4z3>T%&$1n~ zqr@}ZrK8KU`?x2bw>&y$SDu1moH+3FiV(4|@-U)oy zbWE!=WC%a2)Rtam=rr8BS84uE51`~aj2}Y}E(s=yb-VpmWb^q(e7o9yE3w}^Bfed3zEy5Zo3M=)u|*qpVyh~@OgB?rHo{(Rq-9E< z=`CWcX=;DsFQpHm<;0F9Ce5BXBy}b>kx089;YwS>tUjSiU>(xf?e5PW(xoWu$pq-w zgEn&ybj7?h@~8+bn6cJ;dlr>;#C#;O%-8))rqe)w#_lmD%>k#A5#Wvn z@-kmryV_#rP6Fp)5E2WEWw`6rX1nrVfWV#Kvr(n=XJ#ZFIfwC42rSMDI9lurcK+n* zLI-Ht`@+1{l+;27>Hq~XEY|ZOrfIwAL&)5Q@2SyUVt!L``g)tbujac^K4UitbSuMY z$RHpf`5XnV^~|VzW)yr5Cx2_Mx$7>CJ2({G%7NAI)QjV790jc4p=jU zpR($nDrORX)m3|(%h#&=6VA+^s%ES2-28R(JgDXxD*tuFL=02;Y7>#9b z25Thwt;>FMseFz7c0ek>&3-!|m3QhJ`eh%$!*4#gQ*D#O5o@^4-Lw?YQzN*Xj&Q~R zp>^54_70{b5Ae_EC)mFqk<{H2-yrNHRl}EV78+iS)sWX<&B|ir z6Z<-v6f1YD`jQDb=XV~=%?P_dcuz|D^-NIzn!{-&yQsqX4Y|1wO-{n}{{ec!+jA6k zIwwFz#?yn^Nte_NxSrL}O#1b#2i6Y$N&^YWwKWL0ap_MGHIaAJ2vdJklu;jV1GgW<9e12?X^e{m)Q5&e3oP z`ln96^FcI|o^QiCc=^Krvbk+J4@a6<>trT-RavbRE$j+qem&%%X4P-Un@ARwrC)Q} zX^&IqMo?Q)CCYp|B^j*||HhS-i0&5q*J13FXl$TBAwS(fcw3zf?sEgv1m+KOKE_AJuNI=V(b&YIuoy{S<;PITIfAkFAHkK8IM z{m`sEGc=udzr%ZU4%QPa`~x#`l}`Q+E!c>HSpAddSNkJ)ryOdGHLR_GkPqk*SyRs1_2{)K=nO+Gkd3im>4f!srXVckhBI^GKg56;%LJ>b@}liU$-UlHoEQBB zqr=(rE$9bU@Dw3i)C!zOVSo%5XO;=MkB<*BVPOxG2LE{|P=d)z=-lk4J;6#|rz>)@T_Fy{<|k zqd3L{vAhA|FW<|38y&CuqYT9GAN_?J`!A#Ws$f$9mCcEAkI~_w}un)4Z8T<-}dF-h-F z@4v0yn@&_;ztg`qmqbm&4LvsAM$dhX`{!p~CQT~(HdE}kw&p;0CcAjHo z3@>W!cIx`px2Q>Uv-`5z9#O% z*^prElTMLeO{-R_bHwU|;X5ToT3{)C^Z^uf6Ew`*uIVZr3din!l{1naoHs%(DJ0w|HgE;XOX=+@b zG^0q12L(EDHc8gz+~r49m2rS!b;j0Nh?!XJ6OZiiST$4a1jVaXbItJD5E~x`4)a|rimtte>5k3Hqlgb2X$`z zE{z1!nixJJNbf6Ck|@YZPeE*vm>E2$B&?x!Wr1@VLSJHN07{lZe;gk~>34ui=P>u< z|GFRVrTOlDZhL+v%treFA}zW8sV%;+&SMKP6ok&m`P>~{Y*dW041@=qqP%pj- zF$)I)IsAsd@NG`(-(C^ktIcA)&tK26obk%=-vVO(`gN5X-prVBGRZfeWI4&vyU8 z&v8^+mktf^R&FTo{^(qGAcYo|cW=(92rrUbH@gLr^L1%)Y4kwuS<-MR=8Xvhy8pv5vNDEn3eU}{wVSHZ)oX7cq7uhkM%J7Hg zYML=tpF}tW!!H#-Uv9WW{3ur*MlaxY-`_^RIx4UOalJc()xt_{ywSaFo9MT|!K{S+ z(Z6Sh|M;DO-t2(4a#Qz*!hQ!PubB)@P2{&q(;{!V*$8F+bv#b8K$ue#U!k!q#Rm_5 zIk0C(@c5lwj9GQ##`4IQ8Ovh=k(s%{!a-!*#$e$e;QiMx(z3fJ{&~~h^6r6*?th;f zn%TVx#r&VRiUuONKm-{=Kj9CcmB0?Pn&2?fQl9NqF16kav31K~;K12fe{p7QYJd4Y`GRpFVrRpIN=Oh=HJwdVw* zGbu23r20_(Lz~mkV0DY=^fP9_#IJ&hqfabj_$~8AX6J=wbboMe=-6m^M)-X8#`H7V z6aF3j{^Q4_`v*{divBPmk4OfkLv}+_`G$(0j^$$S`TttK2=+ow|L^sSQ%&~&Q@>c+ z^9lW8lVN)Q|LYfa|JV(0u7Ci7kuO@0OT00S5ad`JU%D>}OVe6Ztz;SWH3>hwIR8dI zt-GK&;~<~z-?@P=#=n!P!mxkmI7umS#|4J{E@S~bymi>OZmt%@WOsbU&b(*es46Fy!7X7u$PMRX6h&sq|EvM<`h?f`6K9rM%q?F z9FfWYzLOw2A#<~OgqcYt_GGF}&d-PGBYwN|CFdGG#_vydo*=X2{~d7}I`2Wgr`rcm zk5&r=Lr;i8)APHX*G$v!eje!z?2^Ut2{B!T!x3+7OktmJ!c_J>*>dl-P^mIWh|zj606Q3 zmoWB9o8V8B-LVl%Qfn~c(?Vla>6_l3>mh?6w`0T)P4*`p#;3oBF2=-aDmpx$+XZ9g zPXM1M_8k94&JW+8!G1n$THanUR(}M)hu<2~-k}DCD|6kPceptPM>>Cb|As!kWOJA- zMlRcwz||U@hkxU={f1=p8Geq^-9m>j=|jh7*>N=p zFbfE~o!_&a67yZ}G?CTRW4?U5ESvhCY%k^qF@gChR$CDs^!D~;QbPBBx7&6BD6yLu zYTIIY6lm!EN@@V@@n5_827Q``M}^uMWCs}7_C9*6+Q`N$=~3aW{tf#^YZpTI-fUdm z1Z(%c4DXE>fiUh%#l*n_yK|Saj|UrD{K%2|GNc(O#nAo*Qn3LkFm;}%;|*~o@Wa71 z=3s-Dbf6-qcboH6cH16s?tKbQn`^*M<7et`!OB&tc0XZNWFkhoV%DHN&aP|CwEC6! z4`^JEnokd(#}wsigXo?z*5WYxhDXN^GaMXAB%iP%XJ%9w;I6VI8uhYBZ}Ijl*2*0x z7Rcnd)4ykr<(9g^aeFX2(O6E^eLZ1O4tq7&6&XK1dEacvL|!FLmVId4qbE9rYJV&T?HZ&;q% zBgmTanSWhIhN?&dupyOnGF5?d4ieu;f8iZ*v`|+PFdQWt%7iZGcdVV=u?Ev>VVhhmrJ!20SI~cyu5%pV4s#-$23wuzmZ*YYxQKl- zJ_hpWjM&xcB22ujqv0-0D(r5Y|4ia4?UCRn0oUNIQF-Qb=%ji0JQl{+J2x+vkQyC( zR$gLQzmH&u8vG1YntsD^H?{+^c5b~!3AxfF;hZ&(+7PCfBRw>!e6&;T0Qrzpk+Zrh zx%4uN0gQ0})?}gH2=|X95~Eowqdw?(^0BuPL?@i|go?Am{XwAwfjOE&RK+C}EpyNg z7Vw-|CejGhcX*7o0;>}L0XWRQJ*_82PfgrHv05#}_Q%Kqu}Gj1va*1KV=|soW&zC= zOG*dy3^dWHk3IuFeSy-7;Fb>OIY3pRbJa{zPoDE#fewqgpc_CV$6V9LyS68uD(*se z7Nl5u2m6~gm)P|qyPt4ZioVpuyk)2?PdmjBxrP7S6#V{S_)+?DN6syDTF^A3i5P|5 zHi9{G69CePf=@Q$N!H7;2$)u3p@c5#{Zq3F0NqYZc=_cw_3|Ur{yDz)M~wPp5BXUQ7pLOB65*m4-@EMPJl@*!Fg}Ik20J$p#W>F0VO)v$4)4`y$`E~ zZXND3!hgb)9i4p6nfRuUH$YsS_s);KcWmr>8KH!a3_TwI=`jD{Yz%{#(07a90?zk< zUOiNxt|(j4*cdv8?udUQg>N@~M#uB928!w{?k8pisP*ZlHO%U7ljwxNS| zJJWA45;c&*haw9!O3isC1FQ%3aDo(LLEPq;ADdz7qB9uotcCtTr*|0t2kcw3bZh!H z&FJRJZ}3?V0-wJE1mv|7VH`U*YFtB&-*BE_5NHkI0Rh&+z##T+J^#{hAEc;rmy*CV z;lbJ#bn;`G-sw!7Rn<5FOMjIT-@`pHHH0z(=f;Y7a<(reqZruhP6tWVwf9ig2>nL$ zw_xDEQT)x-d)eyVPGZuBV)~9yKuT^Nl{}_z@&0#K{!aDv*xi#l zECjQBFnqt=X_4k0Zg`G1ej1+R#f5QrZ>a72aYhl@e{bkyVl`8Ni2`3XtxY)3iIYm9 z*=8E>89PHiZzfmT+vrR*oLs!n+`h60JNZ|LGCM=1MY=me-TNU>8>nu+(7`CIC1h^A;SGYJXU}X#Szn=$z{01UtvsO@9X?`E2WdEtD1L&SZ-k zef9J#hCG`Wx>SvGw?^SG{aLL+<+K{r*z8JsfO0;uAymg^wPL$accyF?W^b*#AM}SJ zkF@=ue-$b|bwB8<8`Yz!{h;qEGf-$|?Csr#yo~9eb1Afr55$Di0DaBH%7UF^v-4h+ zaFf>fEaz>vgtT_-B1+V{hxU-mxSVihR;saZD%0LyDJw(aS<>xik1_kZUSM_dJuxce z=qmsuEra7R&2e(z7mi`q96l$tH_AceY{S5GJdO=IbSDl2I>;q_A%7Sc%d*uE_O2bX z4}S#zws*B)u8IF9%|GM-hr{~2zn#b$U*h3><&vsr_ zZ@>&D`;F@xt6NsG>)0Td;cBTO+xaf%sTgVQ^zjt`zVpX6)Da0o=c#t#2|Z+!^TJ$p zz~mp@Q+sre>Jlo6Al+fHISnMkP0J=mc2t z7_$P+k*ChH7g)Fk6h=GHsb6=ldBeh3fNM+oW+)wfRfxx{c&+(evcBfsySm}(XYPQHQdR?Bg8*c)OKW5>exhKDiF!?;PIj?GzsyXjw zjcks=mfIp4Q%wnAKy{omwX-xU1btF&wmWv+kOk(8w!C=WA-@b%|9|NeY7I~IDxRDCqED3b_SHh&>0v{ zL6UqP3CZJPxWdy9Q`Ba={nPQ(=t5F!s(Hi`2i0Yp9(V5gneFZ$$bL{SngKKk^IXAp z3YqM=2jxxFETge|aEeUoJ?7L9;E@I0>|}bo$A|T&w}-rDClg8f)hJ_l?=i<0+vLpn zzfK~&8=taS{zbB6mhCnR?9;ts%G6L0%1kC0veP%M` z?RiMWgUd3_isS))CN_rHiTtzrpB-*Td`*7`Lmo$EB{G$cuPE$}%zG=Bu(ouKA zy5aLYX8Diu(Yi!)uCtqPP@zf4n4ZH#PzmMBqlB>!jG?BCT=Kt$RixnA<*ZoDY?6i; z`v_?GG-S8aH(#Ko9&Zow1D@EQ9a~B^D{L-WB62XR9*fXm5H=+@fR*Tk0Fza5vJEXr z7NOZd8phLNWM6rE<|a#JBqrhetn~RN{q%2r!@wk)SiB$QV}1^Fdd6(qM|8v{XK5ub zF#PWWg7fPulhgGdh=QY&`%VON*8922u&S(oTnFTl$t|`4oezmQa$3b0=i_E+Cisb5 z!xmlEoQ_e!E=#d?_oGO`GQZ(>i3$rtf%BgGnIF))O zNnL2SgGlv^bJGl(Q3zzQGl$#qi{#hE`Z|se(g`4ryZ8{BCsN`RCe5?CO-;-rJX&Z_HBk_L|#tEUHnZX#$8i zEz0#Y5byjAq?qfH)U%r{qV_Q|UCR8N(N!kDrY?<6c#qkkieF-Ra4bpkHB)uxZ%+!yivQ3&wD2ERNW3YQ4JY#%8RV15VCu-w zZY?uYCzd{HE4+`kHd>8)m(@8U)qWk`@vTRNkE}cP0x$nJB8^sFGF&o}JEO$yO%fI$?x2n&&4>al9^bUyY3R_Q*@w zR4mK+Q(08$Jgx#IoZ$Z6o_YY%dUmVewKKz{cIXdRV=&ff3H}%oIzqZIVw{5xOU)`lay~UYkbdbr1T;mg2AB$~P z8t3f*1Rq-mIYGD_$Ml#{ehN2MMJN8Ds`Nd39BZKaExeO!gOrK3ji12nIHH8jFHnnf4%BPm%(rk(^%^+EhMqUfKkIYL44f4f&X0ej zLCL5Y?R3hVqhB=!|?vQk$>$<`bVjG>6KNoabgwiNlO`Yne7zWu{I>KoG}uxGVNe{GJNKB0mM>aZV=v? z7sO+I8Y{6{50lSqWNPvA&c-?y_)AP$ObdVpv$;qMCnuoounzi{j%RfND@GHoG5|)l zz;Hf44I1L~1NaqH1WNxL;ueM;eWLuSeyxgW33W*ClaI-%GFtJ50LRd_AhVm8>(`@L(xXqKaGEe&O_Vf+*H$horqA#M}Fh;>JTI zb`0k)AcNIwO3_Cc!(*X;#_&M|&xMJ>u0mXl)m>m7G48;hYPPE8gAH=#*6ggr-<6m-^A`y*yH4bdF!|UFO`gAKdH`sq^Z#P=Il)( z)&u-nmoHe{ys)@<%9Mo-p}NIQQ>H9x2-Sp^w>H!)ZoZ;ricgQe#=3Ujl8&a(;?|}H zUoy+7Z4HYSw}%?q8W#E%)U`LXtN7%{8j4xaam5u4Z3Tr>e1T??HPS&;_g}y6V+IXv5*wWmz+}9pzX{%e*;PEVOZV%Np*Ejg;nwnbb z>q3iLntfLQZ^ObVzUGEyzSMVt+T8M~K$`XdX-&<-#r2_@8k*G}@-1lawKs%(p%!1A zPw|vq*S35j1%-$?>BLE=7ZtUiB)IM4q#CfZnCu0GsRHVO4DgpVHZ=QMmeSD03l}#p z@+DI$GrtsmS{8h*0RR>@m<9<+bK4e^9~j)Yx=@3!qd5)9$u=HOIrEwg_DEgd(oAh3 zpX&G3wJqvc($H+0QoTIX2$pO`zJmIeC9U8KOi{stjzzvDEejhAwr$V17Ye=^Egek@ zZM!X^8+@S-^|~!aRhwV3Sq1G4O;=3xO`0^R&_EyzEDkkN%hI~0#S49O=90zj?ZQN@ zlGdg`wB;P2K6OBWM^Nxr#_f)}usskq^$LZR)v(uWw z%3|8u&>3p0^EEP18rs?gEi~+cyKt-_LfheLD$Qz3jR^I7M^nQji=R)@Q4(ru3U#zK z`{>6Gh(JTbr6{q4<4te0FJ9DKN2&(jvby%M2UBEaXMKYK*VoWqU)QSf+}5%Lio2o? z{sGj<&p_YY-c-l8Pd||dwq9M2>*KrfI`xcAAhK2{*R9Mibv6dw*(1y!jOx5WS|3~oeSADV>H8sg0 zS!3zBMq;6^_`t{3lJk>xMR8vLIlAYtVo6H{y7T@5FY)8TK1r2n`p_beaLLhVjwREMVLY zC@7>6vy41~!*hoN4-ERNS8PlgJJvIHY3m6?oiVA@ zQv(UqH!iFT)p=4QBwce8Lzq>P9LHx_Hc@vBu~i}w3GvM}C;(Xwy zeqYei(ljE0%1zIZdOy+4I5K~F;(4v2m1+pbGsCcsDl?Pd-G1>@cZ6meZP+7d?PI&g z4^6t!=Qku1%sE5!J10arOFSs$XPsJf#;HZco*8u=iyG0@o6(ppgHLH1$1`#Qv&&P{ zhPic%Jax<3YuXoHS>v%4_*#&~V%#SU>*rPi*0i>@w4z-vAO5hQV=)t<=Jw$jWgUKN zu}wh)UC~hEYNQ_KM+go20eL%RN?mJfL-Rrg-jvf$1G<{?=FOa0c~Q-rz*j3_PH5mw zimjR#oExaAF287QO~vesX9Z`MS9oU6oXM3kb82SHo;5RYe$C84u+rn7T~+BhZ4zxd z4ZeBB;zi0lZ%$>6f9~AsIePhW7MQIPa?kAPUF(>uas%>+VDUogYEyPuqNM0 zNp*v+O{oHO*GaBwWF}>#gZ03>txSjG*Qcg2?H%>?4ejk$fYjxbFf6DMh?ZOV=FXYp zyRu=qQS6q~T}iJQ?I+duM%YXGnAvM;28yz$p11noXXX>Pm6D7UP3OV|Fkh3;)sAaW z!|Ljr>e|~6SXt_;YW(zkpHW#dpC!>6P1^sjwxz@CPPo+!Q~d#oNfUf#UMoFjSwr2G zOeQb|G%$g;=<--??Z56Zagv6%5o60YH#ppVl>3PU(i+m0J+P2Ie@^Q8c5p&DuA1*F zNX}u^%Vu&+z7ruXXt|}`SHK*lo{4U|@5HM+3Vr9GVl8ez^&H7dtO6synkoC>>#MpSk-)H5?~`+o_^XV-tA_Lt1n%oJ)Zo}7HDs%ypw z1qlB=HmG58I{t-b&^~!dN2sCmKS1jN^Bc6CWxJ@j)P^$*>+fhY3f6z#VV_?9>FIRv zA3%Cw0-*fjnkqHcu$#btTM+rrO2?mg)_+sToY3iKq~{0!$cfgby7~r7?*Ae6F$^R+ z|6A}qCx3NTX-W-d=$s#%eO`I6re;=6@tJeZm^r8TbfeHD$J>%6)%BJ0nwC~-uBu;p zZieN&X4qfTaz=^Ofkx<#Hpj^S|7ZQBs(*h%{{$E#3r%}V8^M#y+uG`uqjQJOW>yEI zC@wxdU2h1~ix}!*H#Hc47Khpc^%_t0469G)Aug#K;y(R!RG;d$7R^yCxd#omPe;PA zzFk}-zFlKv_HYVRFW8nDB&T&QLCv&1~e^Se`D*^jYR@?;v*9KJQ zOXfE#07j2SuUfQtDMs*QL7!!MRu?7hLq^1=6U>w;HB~YXAek{I_?l7SjOEE#mfhAE zn=tgCNqEv6gC!Jgv0+JTNDBy3xhHu>q%$^ulsv5F##k-m$ec5L?e&cfOI*X~i1Z6v zFfhyJCFKBH1*Xsi9ragYpi{NRUUE?bwnU6#vH{6V>n>}U_k~!NmD^ZsT)5M$J$0Cu zTx(4VWtdxMoN|htl>A4Q`-`>g;4VS5bcEU$W3>iKr2eo$VIsYS`G)#XP+Zg1!j@#+ z%raFaO>}KT?Xtq6;b>__7H46}*nVVy(u@fVvpUnZ+?t5Z^3Q?vKQpgL8PMGJkL>66 z4$O6Jo~5jaw6smrD7NN$c`S`c#AQ*N_06i+<@Uly15E2gCf} z^rDmwS|c3n&lzmaG=5u}vbYShkF@!GanXTF8LqGRYftgk3mV zttK2I+)P-6^Oy&&k^5!tE+(ud3=#Gb-X?BxF|HAWgY)+#_7V<#m2!d8cgem)D`DTI zz^AZ=dRUq9u?X5kSi}L6oh8+Zam3?F(n;7)_z2-DTrXZD>|@pc@bQ$- zDscs2Elz&75$3WJ#UUJIFJj3dYSBEmkx3c@11WET?pMkf;ABkU)9g0MO>k@yqgAmK-f$5-ae z!^k%VI0$PGqJF|w!gYGj#@Uu|4dKzyUq4|H;da6b(Ia6eVe7$(#A?E7oDg>s*5VeB ze>m?siA1YHyh$D+93mVf^c+e%2y+RuCjckm(S${WMTDNi@D3oXCTt|ECA^lfi|{tW zRfG=__7iR<93p&^u;_D%#P}nChj1d{8p0sq5MdKxAI_Z*5f)9Le!?!o+|SbJ0+3mCv0Ud_$R_X!o7ss2@gLC zc$ha95mplh30sT67hxY^4`JEqiNq?xTEZs?x8ps!lW>UeUBaTtwEt-8J3Eo6B^)B` zQdmO$ipR-fCt*L~yM%*;*~ie1>Ci7>8DS^kD#AMni-PzJ5!POmNR%8)I6skCL0I+` z+DkY@_!?pVC6sd<@Ze*5DdDQxL}Cr0r!JA$PS{5{L^!k{ktpyHk3VTAVSgLrfw1*z zF1AAqthtu{AzXC>ud ziANNM85e}DIL1%Q2fhgSBpiwY7vbPHsrN+QzXjZcUAKZ?!hXUh2y4F$z6i^{10CY? zJh&1(DZCxL5_-N1{DftMZxRj>7Muh=SJ6JguAc%2VL#z(glq80nOFcEKZkAzyMCES zY|}g8YlOLL>Hk8?TgNyj-2OP@=w#aU1o4Ee>&Zvh_Z09E_HUqlghPb6r;z_?_ypm0 z!rKU2e+#|{tA8*119;jFyo9R=KO$^>4tP!_{uRn4+`bEV2y0)39w$-nYw!g^{WQ$= z%eeX?Pex~M#!=%B%I?eXctE2EQu1soMb^Q?Jvq5EbMh|mj$4-9dqt468yJd7m zj_=0I@|=S1tn+h$s_ocBpdFSWkmJ61O zoV*$4(ZIBdJTH((^$Xrw;0=zuFh{ud@wbk@clo}6Fmne)O}X8pmXm7cxH?iP-=Fv^ zLnqNuEt$V0%`n_RnZj}H=t&v-0X}=&ndCj1e1qsLV@HvHnazKTV83xR`MWdE&nX){ zo2=>_s;6_#8+SDME+zdsbehw6ws6e@t_#M^0xo^mFAd*+o__#*uO`3nmO1)U;Vv4r zXFsrKj`NgJw$9?ouGp7&pXbb%=snu~P0?0#^m#eHTSQwoW|ij@b&oker+U<1a*Bwc zK_svwCfdxS4Y#3xy{oqW%$D&jmGRBdl{vm>=6N{LL(RHLjfhv&VUa^E*d-JP(mho|sDG(hB50 zDhjbc2@uQ7af?#V(6`{-M)^JHe>LRK{KBWe3vk2mj@kscGd}_D1-86A%DL?;`x0km zQeFb43u@f<(>S;N>E*SK`pyBHf8IFsC&oYJ^jwBM%X3=07y}Dv3Js-M)#K(RA3Y@` zyPrI3$P-bX1N9;J?;g`(i6DjVz_^Jd7rrwu-zzx36kUQ4BS`=^n%+>f4Juk;_DkdDs-(NjfE ztvKHS$f;_au^h<9c{BR@Pta#GpBz!%aQ_U`=LI>dMvXpu)B#KM!v+lvjn9I&T(vLp zd&Xf}AyDMt%Q)l%ldr@ZRE(D z`YF6!Y~sBAcnw$+Ol9PM^T~aQIR}wH-{$|!ak3_J^zSnd2$Sgtm3bRw?%lF4@c=qT z=IsBV%%QB&|2Xu3Wx`d!8GjCC&V6fNV%aFld>rK@eLNJ6J`~>Nj>7Ym#mH=VTd&%^ zFELsX{0`?&^V9DUCpl!wAmrVN|>g1>R5b-OL@n5&`DnoPE zF5;Qen4jWS6L$-7hor?lL7eg@>ry&hY}d12+-q)mXQk!UeE1>Kp$40u@{T_kc|}}$ zy*}bL6G!q?x-*IE-!INj+!Oo7UFw#Xo>%kkNA^qCL)@DE;_f6)@X=l7=i~qV6ttSS zU+|nc2cqyx){nXqsZl)+zG>P|zKWrdIC+h@GU6C6=BIRfi3<|P^dc2EeyqncpSWZ{ zE1i$Hxx}Tn;Y{MHiDMY1^7@HWdFgSN66a5glgw{);rl+%ncsn?r7zt^3*f(J61Rfy zo5}M}eP3p2ex$BZJ?h?cRRhXax&7q3_FwxF-&VdVx7-!BTzwzp`$zvzdtU-wMU}0+ ztLpB1Z>CEa5;BPiD3cI_BBG)Ofe@yEh=?O-+emlolczS%9_R*w0W@F`2-2WN83YH2 zOh!=%A`uWIGDH+aKn(~8h!8-L`@UVZE5l8qfB)n9*P_~l`lErjPv zcn1w1R~1f3>%%8CJf>#2q)Wo3g6uBXg8;@u3KM}dDb<5>e zm&1r8a(R$TI;C>+|HtI)`bV!_4mr2JMAshZDg@0g(}(KT6I3NxYfh%>{4uwsQ|?6G3B<$0d9tK^tF8X{_9FMlg+;a{KeqkCbm_e0mKJDSI4ukA8P8V zTFXF}wGN|f4cXqcl!@IVduY4x0`(c0lir7Ifta<;L0`7V`bb9WzIJlDN8IM-%O2-< zAsusmz7U)VJ=N;2C+x?6Ru1tQvem(Y?y^Muekf<~YT$Ys|* zJm(RgD;o#;J)oThZM`r#V^xtXTqNvCguMYBy(+{0E-P?D3%wZhZ|b1$0{v(R{W9>M0R0r`ZA4Gi+=p@*>^5n8Jc)xnbsl`uf1EZs9Br~! zL^s4MZC3UnlYXf`d+r93JxLvQLD&d{NxOII_ivS9vTr&EVObJp_h~az0;R1yft&2*UL#?+ zgf*ogbBG#47? z!8*;utcY~zi-h4b#1{iPe(X}%2lj3CjevQ^G zMyZiVtJK>t$shKiCwh?9K2BQe7-!S}7&0=-o{qHV0?6k^27}+j?OEl>*SD4*W@6$G zO`fjro<#k3j0y$^iJqsSr)ms9YWr0VjS_D%cou>`9{K4l{C}*$-!}jT;j#Drq0{CW zSevIEct&8u>tXN|XxpkWjxUsP90|d+BJ*hy`14SXdm04oamo|!=*%{MCtb*H6GhP1 zrPGDrpW&A|&isAAFYTxl^c|2Z61p?yzAi!%b`)VxMq|%W>emjlS{MkFFh43Uw_(t+ z5$*POs~wkw$r(W&!t8!%%RO2dmX5IH2$O%cRh>Oto%bN@Y@@n$K8djN2uqamaMBWx z_Ac>FM;KwR-?mpn(o%>p55g8!#W&lHZ!^N4#~DCfa)%H$2Vsv?#W&22&x13G6F4iV zOD-8Egr9`l6OZ`pvN899=~gk8(mkNp!`38=NwuR zcy#ZmX8>!?2eheyU7EOE-X5H>6kHMvUIjm8NW6EglL|)D$5{*Vi7;fT^MX zVXmoDUxu^)0e#6h%OTMh@tgX(2l9-+)~ODILuX`F&c;gd-s*O^J!eAdIN5xuauCai z7*y#-ZA)36KqG(R`e5*NxV8OELRn(R)EUd7(E&X-;8{oMUv~Sl3Q>#LmPCsIX#jq5 z23Oc6Xxq!4zd7YyeJswb+*dVooLRVXQx5IaX%dwbq)T*0BToab(H|wx> z;KXHCyjf={^^&|uUiTm`hte@N!L51iW#<)Hk?`{fZ{0QMtSg+p^ccdi4;%1zM0gW~ zKacQRr7y`r9NKx!HD9{yQzch!dv|m8O#4Ev81D|tT<-bWk<<2AwZ~UAF7(E((mCX$ z_OO8Zma>@xePg-@gR5cF*Q*cGm3z9y;nrimI?D#ThE5%xfX<`027_;65608#H_N*p z_T;RmxK5qeDR%Y^cc5Kl;_R@aex5zYVY$+$JuM=yMw`lVl(uv zp|7eOtNLNTKxt!ZyH%$`yG?XKI_2DV$570}Qm1_!ojP{mbV=>UUa;=24jsF?+BQ-V zudqzC$IM6ncbUFdvQ{YVz zUVGjhWcOm!jJKhAPtR@#@8R~+~~pv6jl?|`SSv0nCYri5e{`~4z61R~!+-V*sM z{(br9A+r2;IENPbAP&|4jrhgx+6DRX4+euj!|mA)`Remwy7^Y+_=x>^Os^D;;dy#{ z-2=orvlmK7@=*xUE;!$AC+V10Pda2h_I5})U}Tj#8UelSh6jUpK;Ee%S&AciIq<+y z9UR@|^iS3yxu}O>4+Vqq(vPlp;;J>DRqyWutXrfX)jq`OBNIU9Znc&o1MGQ8o)w7u zLooO);`EfEv!QHj#Z%S4q?VUKUF~rSb=?^{`r?^^BIt{fJXW<|^ee147gjc4OeJvR zj1%UrJ$9sneGJMc_m9EgSSjBuCoR?I6f8FMtj;3snboHdyKQeq{JZ{y@dx^(U%_6+ zaNadS+WjGfok!So33JB!=cM7M!Rvu}J?3FN6Chz#>9+fwR4KG1SN1uq;xfUO27T=w z!Fj&a(d&o-a<$Tnl9IE{!(?0J&wbNO|+3xXR@FtBj+Z>1^IJ*(r zbs3HDz6dWwxTlj7SM@cEd0%CHW=1?2i0ffjr!x=qg={foW3ln)@j0@!=D(_T)jLpX zm>8i|IbjIeYCM z0UNZ^4Rcc6P$V|;fMo37;JKApX&;U2DTAs$d>f9?TS*_T-MM;2NZl_?LD^0U1{Z7J z`jvhPd&06r7m7G!3!=Z>7fwM}A)cMN1$v!z#(!cAT4%Sh=;j!&5!R_@JXP~;Rhs2A ztfbCpFx!rR>OqKS7oN>YmG*iq>J5FQR=TU}8)}8NUrk z_Z&R;^DN@37Ee{WtJ<5KseV>xy0aooOq+F-Tek@7c-R{y^6Po| zA<_R5)=t9ztpkw0YL^rj#P`SpfT(c+3T375na-;FNK(JM!{-$ZUe!a}?G~Rh?q)Qd=|= zrM9$Yq*88Y5ntcA!QkKE7P%T@`<+gVn4j(Sg7C}pbaTMJ9&XPR#8p)L$l$|TA{ zvM+cBfWON;%pYOk|4%1vztT4P22Pmf*gE@Oe|!9s=lABkiFK<7`i?pJYR$KqGWW@J zVm*STVd&fkE5+%TN1=eGcvyR{BovQ1~Vzs_=0i0Av(!tbFP?XOn3 zU>2Z}tuVfVKN)T*!)#Nmf9DDYjp#c&t~aRIVBEJGEKedE)qf4rABR0`GPbA>nh2)g z?0BO=f9s#d27}8C`bK2XwCxMBiI4L#rV2wIQ+b{dwwQ&mPMo3|@@c%?!2^DKkjGSD7DZqO1uiwxdk zP`PFy`jRn-Yq`PbA}h=`He2)`d80|=%^tXaE_s3I;MAqqBU$F>qGmKLJQnwrk^I!9 zG(QlB`!fxB=B2cv$s!YQLad8par}KV*yeC9YbHV;H-q!DX2N%`m-A9?tkpI&1A(ob7h+}U1!GmK_= zMYdQnzthW;!sv)sL}&Xzc-6;m_-MP2@%+JeK8BX({9Np}v*M>yey}VI!wuzq%43vz zuD*}xoY`Xg03S*l_nh!SKMnKkjAFQFMx$M>kM0cTs}28r>OSA@7=HC`-!E~kw%;4J zB;K=aXxLMUeGz*6l8GjyK1lw^0vx(55%-@iXWWZg^3%h7pR|(usn%D({7n^d#g#}x zP8zSgmkQE&>piqGjpq%b(lp+98-3M*KfH~8=)jY2qwK4A(w#K>D*ki`{czRepl|KS z8;8)(9eM4LsOi`6#350I*YMkSL~Xc+O9w|i-$_E(bsF{2U9`M2FT0aAcjozb(U|MF z;I61S*YTR$qn2OCQ|^p9cHJQG70MBuQZ*1zS~!ByXSUoix{)JwOurbR&{Un&D~+&b+^{s9ri{K zWU;8n?T7E8@(d}qWBnKrF3aKxchin6iFR>+No-Mn22bGt-ap9q$p9($@>^THFp#DW zl)94Jih=z7Kq?=|zYL_aLYR0PX42Jzc;@ZCJ%gk?C*RJHIDR`X8ce5dmw2WO=B;-{ z%^NJ){^SnM8$#Rfkl4PzgI^dDl|4iXZr`0e?Jlaglacv%?%`+d^=-O`SKmwH@00Yt zcptwv)c4+flA=%V zM$>2y`YI6Xi7S)sUONK#Fn+<=-Uaw?0T}<*1_S0hM%?L087N-}&`h(* zw~@5X0j1n1EeX;-$&9?=G77ODTbs7`jL5NBu(=2>Hy94^11-6^76+4 z+Un&AQS_acE28K#KSOqGIIP93;rvzveH+e)!|9C(q$D`|JvIvYMS~0b(SCEj@qRxU z_-6K}v!3P)yU`)$pR#D2m%qrO=e>L|i(c_I`lTxdw`7FA>tEM?xD=y)m z`_S=AcvfGU-i#OZrIKd6z8}4EDevn?yD#N4{b*b=Ps*aVlX+ENnw7#+v!v-P$)d9< z{8E2f)SSQTOV706S^en47Q7~l3NL$JGIQW^UfRw0u{FazrVS&{i`ww4p0uV7e|a69 zY{RF!QeInrJA+oX<(0i@b6cL@n=0B0|LiMxOLq!h!Q=YVyX}Nh(w^V!MvGJV+%2>| zm9sNx+?7177j3;V2B+sQr}2#5bT~~o&!=%-Z_4k$i+a+F9r)|(sknnAbYBN4)-f2D zv#8)I$!hUcQV{z<=|N+zmO?DJT8jAe)qF66zU+8ZvT?AJROymyqD2p0xLR z)c(geNQ@J^@Rymi$G(4ZBct+;-^izKq?d1!l&rdmzwb&rZ{lye(w8^!ft%>$O^ix^ z`ey#JE6ux^w{)dsm2$iC)ULF)8|Qa7e(Hv3 z-|Nl?y3>a4e5yNb&fxPIbRdIK0Ec?={7kygQwrvTOg@=OyD}wdQ+n|az3Ag!oZXv_ z_L2gt>Swci7+1y0Da8?kU&e4|qLm*ud7VXLtR!?g>0>`wywgJ~JWURJXrqS@z;<-T zN$kh9O{PKkp&Qoq3%}XiY|wXgSYQajH&Q>>knkJKHw@ZXx369uG0E+THW@s_q(cV3 zZPHS;f705({OQSWdgu<$_Rs^o)Zhm4dTOj>28Y_wz?A)0HYd~DG#R>~KP5X%#{Mt-v{THMZ{-$A5u?LLhmH1+_Ko(AW=u2IXlKHaN#@};$rOj* zApFws>yBS%{QBaDH(G6aH1M0&c6m3x92(QO)S!Rxe1kqVnj;%QBT@{jmfV%PIHRLK ziLa%rYP*FuQ7pa^gWs{}u)&Kw^oqrME&AL7S!*=nLz90rX>2vpdUGVg#~8dZk)|2E zp$Xm);Ab0Q*lm>Cg!Y*HZW85Ndrbr^!oV6spL;vQ{fe)^*x8i6^z-aE$_o>@m%^4C zpTyF!a9)&1b0dVbG~xqeZBu$Sii_juR20ulq8SbNd?MvVH#!wV-$o15)Ybun|ri={y&{TMSvE7s6cnWyp zczPl|7thZo(!6+{m`F?Fd1eAVm%vXa(*8sqpG0R8B@_}<5*BKpl z8cE-U(R$Bw##7;Rf_YWAG2hQ1E%%=`7KKx3I3Ei)@+0`i2s$Rh&JbYN^6blvgic&+ zP!=CI@F>hK!&qftlN4=YtXXZW2pZQcG^ntbEOt+n`#cv|e?$qB}{9)2ppK$4fl8oRw{ImR>}Pf0Yk`S~?) zhW9(#R9fcQcv=?0OA=^mB)^hCuKXQJ`iQh!4frK>^3;3fw9u$sR3yR+XBWi>uCd?Gu~FqV3_!L zaex;4cvgUle0)60nBi|aBZ`*zr^1+oIT8b=w<8$tBN0sykQd1_B58aSl93lB&cjh$ z6rkJyPisJ10(>+;2LpUAKvNoUK?65osHQ7=LF9g3;-S&}nulf?Jl{hawo^8=zQSDS_~+w*Ahc}KgM)3k`Ed5 zH{M{-*D_e!veDPhL-l`Uz-*?eIvGS?I~BCeM&lC(UCiXpxlMdJoU=(qRvSFhN-7GM`|2hlQ|{d5o8i*zP>tA@IxESO#765$rRh z?}s^jfO%_}@l6=|xHUGMbHZtAI5-fM)B5f5SY_lG{6Rd;GWq3rT5s}^1X^$LyYW;V55Pn!eJ80X?BzcEjWrv*(I!ONO71!ZrO>k+}%F^o8~ zVXTBfxf&pg&^Ia?8!q(aNB z>BzLhmaI4?d*~}O$ABkW+kA@b*32`moM6!h=&(}r&EYPP_PNZu7S7=8+l}7+Zy+%_ zf{}FGkQcA!djoJ!_U}RkoesMS?zQ3kT$J&t&D-_sVGf+z>R5;MLyV!F+Mqtt{UG|p;LY|$s$Pg)G-xlFmnBQG*-GMmhD+9o`^ zJ-p9D(-{(w5C1Plk9k+SMT1PF1F2qW(3^&1GHfs;zvE1v)j-Uvxs72L%6*Z=(*h`z z%W*&J;iC=leH6KW%=~F1%J=fdM)Z|^FYxj126WQC|I5!mHX=0I9~#o~NS+=|n<7P9 zQ53%)MO&m8jz#hKXqtkdDnJJUAY2GA-r}F$KvJ-x0iS6=Wer5*@@SslkTylzF-7z0 zhIBew=ubD~bq(>LqoihHV}89cR#c)4xqaxmC$WaZ-W3KQgQtekX@g<7O*dr}+G_HI zFsaA4y|je+q@UI^Z(^G2J>W1D=amAywy*ii}RR|S75)(&y)Q0f}daUi+N)g z>_j)Czc1^&z z!a3C>xbz{L`{SOWj=N?_OoC+?9s4koKVdIEM~ zt6lzQ{?GW&=G4^dM(M_1U%1ORMAKP#HI|RKPV{iFK>dX`I{tm~f?&04WIb}lA#%x? zHRUGOBUc(CH!MW1K;@Dp!o}3(zSNw`#9-K?(LvoGjt;8TrS@L*gO;54~PJApz;Bb?c=o>!ufk($i}4 zWoo+)iKdo!-!)Eq7@+Mn&3@<-=S+&x8t(dsby0rBF*i8=!s{I@z0SdO#lp^xKUy*Q zTE{;|^^a5>rZ_+`UD1u##b~9U&~&*Oa`WN(L$NrdeABi3Li=T?oa=Y{nTswx6y5$C zN)Pp))&A>ZXn3f9fcD=|y6Z2{`g8rih5O!d>Y-54<!P+2FdQ#v>s(NNd__YnWo5dXpue^H44qY(d2_2cx-zOwL4tXw$ev#-2A zI~b+@ba}Z27vA)+ucGYcHsW<1`;upl<-*%9_O(I7akOt=Zu`XFeAur&;f%z-uovlE zci#o>yh6mt~w6&EP3R4h>}Q!H1kQ1t$n z#;@2?v6Etk;$X$$ilY>B6!R4qD6Uj2Q7ls|SFBLj#A7~%vW5XxKgo1u}raCu|mi#o>yh6mt~w6&EP3 zR4h>}Q!H1kQ1p(|_!V0!c2dkx9IQB8ag<_?V!ooIt@gbjzjVOYFhiN|7`m@{G&tX)U=Lm94b-j<0J4b4}9F6 zQt{D~->2fUVX5*Zj0fBQ;X$%}@{w0ceef^;@%UdJaS-4ButW6ABQ-oj5%$>cAHH7% z)HU+qCn#0E)oY3c)yKTfImhOT^g z{eM@L%6dU;e5du`y;H-{?4m9!Dx-7lc$d%&jtbcDuDrW$aj{q#-E~Oi`a$WffszJS z-d)$YI3USklz1g+u6&$Bw%>Ob-L5=cssW1bx=5*25Ai6WGf>tjuDrW0aZ&#BTP`>K zE}x6C%#k&Z>vz{ZE)LW3l=REbO+TFWH3)a1yYlWj%SCsc<;L&IyZOIQ8@*C6q)Lmq_z6>J9w(g7ph(CqWxd{)dbviu#5i@B404h zVRq3?udBlKyC`O8jrdi*P%$rGD?k;v@w)nC8m}(zuH#*F*S9WzX#TTR-c*KRx)0zY zo7BGqshhGRw}6AmYYkR8zMjGcaFS^ zJKPNd6$p)g1?U0%qJNmrr!d<@#OR0-BVc zD_@%3T-c~o<#)Lo7&g^;w_Tk!e1W?Zu`BPc0~bEyq~G1Xb@T1U>*nh*$X1Vk*JLOD zU6UR8)(+Tl$<38_@#_$I(tR4ra|9=b)W0j|;?WR!*G?$NcjOOAq}8tEYHo3!#7}g& z`7fB+T*x$WrXxQ>GFR Date: Mon, 28 Mar 2022 17:04:43 +0000 Subject: [PATCH 126/153] Ignore compiled c code --- .gitignore | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.gitignore b/.gitignore index 65607dd22..82a1fe2a1 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,24 @@ torchdata/datapipes/iter/__init__.pyi # macOS dir files .DS_Store +## General + +*/*.so* +*/**/*.so* +torchdata/*.so* + +# Compiled Object files +*.slo +*.lo +*.o +*.cuo +*.obj + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + # Compiled python *.pyc *.pyd From 5de5f4c5331a6ff2cf7cc33002819dbc5a03a2f3 Mon Sep 17 00:00:00 2001 From: erjia Date: Mon, 28 Mar 2022 19:21:11 +0000 Subject: [PATCH 127/153] Revamp cmake --- tools/setup_helpers/extension.py | 7 +++---- torchdata/csrc/CMakeLists.txt | 25 +++++++++++++++++++++---- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index 7a0b6b8a1..4981813b1 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -1,3 +1,4 @@ +import distutils.sysconfig import os import platform import subprocess @@ -47,10 +48,6 @@ def get_ext_modules(): return [] -# Based off of pybiind cmake_example -# https://github.com/pybind/cmake_example/blob/2440893c60ed2578fb127dc16e6b348fa0be92c1/setup.py -# and torchaudio CMakeBuild() -# https://github.com/pytorch/audio/blob/ece03edc3fc28a1ce2c28ef438d2898ed0a78d3f/tools/setup_helpers/extension.py#L65 class CMakeBuild(build_ext): def build_extension(self, ext): # Because the following `cmake` command will build all of `ext_modules`` at the same time, @@ -76,6 +73,7 @@ def build_extension(self, ext): f"-DCMAKE_INSTALL_PREFIX={sdk_dir}", f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}", f"-DPYTHON_EXECUTABLE={sys.executable}", + f"-DPython_INCLUDE_DIR={distutils.sysconfig.get_python_inc()}", "-DCMAKE_CXX_FLAGS=-fPIC", f"-DBUILD_S3:BOOL={'ON' if _BUILD_S3 else 'OFF'}", ] @@ -109,5 +107,6 @@ def build_extension(self, ext): if not os.path.exists(self.build_temp): os.makedirs(self.build_temp) + subprocess.check_call(["cmake", str(_ROOT_DIR)] + cmake_args, cwd=self.build_temp) subprocess.check_call(["cmake", "--build", "."] + build_args, cwd=self.build_temp) diff --git a/torchdata/csrc/CMakeLists.txt b/torchdata/csrc/CMakeLists.txt index bba2e7948..df21f5046 100644 --- a/torchdata/csrc/CMakeLists.txt +++ b/torchdata/csrc/CMakeLists.txt @@ -1,19 +1,36 @@ if(BUILD_S3) + message(STATUS "Building S3 IO functionality") + find_package(AWSSDK REQUIRED COMPONENTS s3 transfer) find_package(pybind11 REQUIRED) + if(BUILD_PYTHON_VERSION) find_package(Python3 ${BUILD_PYTHON_VERSION} EXACT COMPONENTS Interpreter Development) else() find_package(Python3 COMPONENTS Interpreter Development) endif() - set(CMAKE_POSITION_INDEPENDENT_CODE ON) - set(SOURCES "pybind/S3Handler/S3Handler.cpp" ) + set(CMAKE_POSITION_INDEPENDENT_CODE ON) - pybind11_add_module(_torchdata ${SOURCES} "pybind/S3Handler/pybind.cpp") + add_library(_torchdata SHARED "pybind/S3Handler/S3Handler.cpp") + target_include_directories(_torchdata PRIVATE ${PROJECT_SOURCE_DIR} ${Python_INCLUDE_DIR}) message(STATUS "All linked libs: ${AWSSDK_LINK_LIBRARIES}") + target_link_libraries(_torchdata PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} Python3::Python) + + set_target_properties(_torchdata PROPERTIES PREFIX "") + if (MSVC) + set_target_properties(_torchdata PROPERTIES SUFFIX ".pyd") + endif(MSVC) + if (APPLE) + set_target_properties(_torchdata PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") + endif(APPLE) + + install( + TARGETS _torchdata + LIBRARY DESTINATION . + RUNTIME DESTINATION . # For Windows + ) - target_link_libraries(_torchdata PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS}) endif() From f374fc889dabcc40ee4cbe394b525e66ae73e221 Mon Sep 17 00:00:00 2001 From: erjia Date: Mon, 28 Mar 2022 19:30:54 +0000 Subject: [PATCH 128/153] Remove libtorch --- CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8a2de8f1e..531dca41c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -56,6 +56,4 @@ endif() # Options option(BUILD_S3 "Build s3 io functionality" OFF) -find_package(Torch REQUIRED) - add_subdirectory(torchdata/csrc) From b616a2c7fbbf40bd6243d74796c4abef4d581b50 Mon Sep 17 00:00:00 2001 From: erjia Date: Mon, 28 Mar 2022 19:33:45 +0000 Subject: [PATCH 129/153] Temporary disable a few tests --- .github/workflows/domain_ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/domain_ci.yml b/.github/workflows/domain_ci.yml index 7729848d9..5e10ce596 100644 --- a/.github/workflows/domain_ci.yml +++ b/.github/workflows/domain_ci.yml @@ -4,12 +4,12 @@ on: branches: - main - release/* - pull_request: - branches: - - main - # For PR created by ghstack - - gh/*/*/base - - release/* + # pull_request: + # branches: + # - main + # # For PR created by ghstack + # - gh/*/*/base + # - release/* jobs: torchvision: From 32b42b259efd53dcfe1f3a90c97ffdc6e4c1368b Mon Sep 17 00:00:00 2001 From: erjia Date: Mon, 28 Mar 2022 19:39:39 +0000 Subject: [PATCH 130/153] Fix import _torchdata --- .flake8 | 1 + torchdata/_extension.py | 6 ++---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.flake8 b/.flake8 index 14f53be8e..0e17765c9 100644 --- a/.flake8 +++ b/.flake8 @@ -4,6 +4,7 @@ ignore = E203,E402,E501,F821,W503,W504, per-file-ignores = __init__.py: F401, F403, F405 test/*: F401 + _extension.py: F401 exclude = ./.git, ./third_party, diff --git a/torchdata/_extension.py b/torchdata/_extension.py index c3ee2f84d..091b3923c 100644 --- a/torchdata/_extension.py +++ b/torchdata/_extension.py @@ -74,10 +74,8 @@ def _init_extension(): extfinder = importlib.machinery.FileFinder(lib_dir, loader_details) ext_specs = extfinder.find_spec("_torchdata") - if ext_specs is None: - raise ImportError("torchdata C++ Extension is not found.") - - from torchdata import _torchdata # noqa + if ext_specs is not None: + from torchdata import _torchdata _init_extension() From 3e4ca67ae13e37b75068f56f131ee9dfdc938a80 Mon Sep 17 00:00:00 2001 From: erjia Date: Mon, 28 Mar 2022 19:57:07 +0000 Subject: [PATCH 131/153] Fix pybind11 path --- .github/workflows/ci.yml | 12 ++++++++++-- setup.py | 2 +- tools/setup_helpers/extension.py | 10 +++++++--- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 36ac86c14..aaf6eef5d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,8 +57,15 @@ jobs: run: | pip3 install -r requirements.txt pip3 install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - pip3 install cmake ninja pybind11 + pip3 install cmake ninja echo "/home/runner/.local/bin" >> $GITHUB_PATH + - name: Install pybind11 + shell: bash + run: | + pip3 install pybind11 + PYBIND11_PATH=`pybind11-config --cmakedir` + echo "::set-output name=path::$PYBIND11_PATH" + id: install_pybind11 - name: Install AWS-SDK-CPP on Windows for S3 IO datapipes if: matrix.with-s3 == 1 && matrix.os == 'windows-latest' run: | @@ -78,10 +85,11 @@ jobs: make sudo make install - name: Build TorchData - run: pybind11_DIR=`pybind11-config --cmakedir` python setup.py develop + run: python setup.py develop env: BUILD_S3: ${{ matrix.with-s3 }} BUILD_PYTHON_VERSION: ${{ matrix.python-version }} + pybind11_DIR: ${{ steps.install_pybind11.outputs.path }} - name: Find extension shell: bash run: ls -al torchdata/_torchdata* diff --git a/setup.py b/setup.py index d6514afec..03c9543e8 100644 --- a/setup.py +++ b/setup.py @@ -105,7 +105,7 @@ def run(self): "Topic :: Scientific/Engineering :: Artificial Intelligence", ], # Package Info - packages=find_packages(exclude=["test*", "examples*", "tools*"]), + packages=find_packages(exclude=["test*", "examples*", "torchdata.csrc*", "tools*"]), zip_safe=False, # C++ Extension Modules ext_modules=setup_helpers.get_ext_modules(), diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index 4981813b1..3349bd823 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -49,6 +49,13 @@ def get_ext_modules(): class CMakeBuild(build_ext): + def run(self): + try: + subprocess.check_output(["cmake", "--version"]) + except OSError: + raise RuntimeError("CMake is not available.") from None + super().run() + def build_extension(self, ext): # Because the following `cmake` command will build all of `ext_modules`` at the same time, # we would like to prevent multiple calls to `cmake`. @@ -66,11 +73,8 @@ def build_extension(self, ext): debug = int(os.environ.get("DEBUG", 0)) if self.debug is None else self.debug cfg = "Debug" if debug else "Release" - sdk_dir = "C:\\Program Files (x86)\\aws-cpp-sdk-all" - cmake_args = [ f"-DCMAKE_BUILD_TYPE={cfg}", # not used on MSVC, but no harm - f"-DCMAKE_INSTALL_PREFIX={sdk_dir}", f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}", f"-DPYTHON_EXECUTABLE={sys.executable}", f"-DPython_INCLUDE_DIR={distutils.sysconfig.get_python_inc()}", From 62b0ea0456691670a033e05902f4043b492251cf Mon Sep 17 00:00:00 2001 From: erjia Date: Mon, 28 Mar 2022 20:11:39 +0000 Subject: [PATCH 132/153] Remove binary --- .../_torchdata.cpython-39-x86_64-linux-gnu.so | Bin 171960 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100755 torchdata/_torchdata.cpython-39-x86_64-linux-gnu.so diff --git a/torchdata/_torchdata.cpython-39-x86_64-linux-gnu.so b/torchdata/_torchdata.cpython-39-x86_64-linux-gnu.so deleted file mode 100755 index 38fb0d495f9b767fd6e9a1339aec57cb0b7c3649..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 171960 zcmb?^2|!%c_5P3*HStYSqqxz7aL3HNjY$|M#8u-f)@BYt`R>S~BnZ-d)Z; z_uO;Oz3;y9+A{yVQCV3o`!m{gri-ZSdXw>5H#V#rWnH;kURRN8JpMh(b)=*no4T|A zMP7@U&;E9~Jd%iYjKklCHuL_5wqu32zx%!>&=r_se%tw&n)%kp&3x|k0{g^RQI`1Ejzpvx(82s@w72n5N*Tmnz^>O$+9)HvD zcLM%S#NSEyd-u6r2Osv+e+N6?C|Y~E`_g$E5+5GCWz^(PURmUR^Y7oTJnGRmZoPWW z_?_j66VCXcwCsiV_FnSwYmePu-~G|@s}8w$&nchIefse3cjhhGcj61XpWNL3^}ql6 z8`qD%`8t2qZ=blp>Rn1O^79-zdh$q0kQegUx>avC9c1c4<#;{&u*~V#ADOxQQD4u@ z{{{A&son<;%*?;7W#&&nhh}Qm8xG}M4L(!)OJSIq_&pBgFFQDM`nl+VOyx{JAhY~O zL8l%nHtpj(HgozX9Q5!B@=%&`Z1H7i%8a3C3?meLC2~tq$$YJ0^2EGaSk}0F`Ev|2PNxFFHE2{*QLBt2@AF zYS&qqWHa^eY6z1l{dfoYA9j%cB!_YL?TMM&TZ;6V>fP%g{|63sb&G@D9_OI9uY=E| z|HmEt`S}ih`xA%sk2?76DhK^{JJ{8!7}uHlYd1z>CcSNP7%%5L__1l2>@$_W2mO*s zKYw*-?^zD|+3w(%bX1zD{5bk0lRQ}t{$XWKW<6Z&Q12fd{AYnDvpkPDl(Q51%p~Wf z4*DPGptpRdafI=bsei{f^jC|6e>>Gd&#yVy=i?6Z_)UoDGRcGC=z{9@`>b=Yw}T!0 z+Y6{SlRexDIWyHe!C@RNL!6%}{c(^#Q~8HE_<>s;`u!JZaHjHqjCn0n`RxvN(&8`= z{>?%DqajZwd;6(_p1m`!t7f{R#@nq|eu2 zX_@5NG&OU*l^E}t(x2ckPPaS6MTa@q+iVBFU555%(u3wOPMaO%+=cdLD(4di`+3`8 z9DT>Zo-e?-$fTc-9NK%NgTD9mdx| z4&!L6L-_+(CuWlWL5DcxE{Ae{<*+^)@1VE)FeozV!SCQ7e&Qg{VGi@)+3>fS#+~6{ z=TE?YW|Al7unuZ>@F#N|=Cy8zd1t*toN%+lI9lLP{w@c5yUjuVJ00p>>o6{!apUD zoRb{lE8Ri=F;8as7dXgsjYGfx+hP7X4f@ICAGo)biT{N|`b`e$_h$lrzD>er|KnPpgC9{)NMMU+xf}_#Mh0?cn#?9nxO{ z`^n@dxpA6F&tuSEnfP+>P{n?q7aZcKPaXWg5f1&b&S8E1Fyzl9&xg=MCi}V5A$|^x z%AEdMhxKDMl$J?v_d3kihdacjFFWu@JNSoN9mdgj9qh9RJpvW&_c;yv&!o3rhw=MM zhjDip^pi=?=cDjU<=0NlthWw_et*`%Z|`%k=e-X5SI0Q)W54WRhZPR;9P1G0>~NT8 zTO9oAfJ478c8EWZa@fE644s=v&(}KmlYI_x!fJ>4;uHsY+EHmH`#;7ZF52$UU)MU= z;dX~{_nbpq@h^w|n(eSoS?-{pd*BB$>C@|=|E&)C{E>q^jSlnOy$+q_4tK5nfnBnhK4S6NKOD(F z3q4PEO>nsjU|{&H9%ZI?S?%huV*;?Qd?#4bQq2OYSJq!y>sndAw0d>($u*(S z$<50yKRMh~-O#+ew&|4S#`Igw(=(7YjZL){9SYYq)mAS9IwhZZo5Ra$PCSut7U7DT znqV_g0a1;~2i63d8bh^B;Wfd6ZH-uVTSKn+V5Gs}2Ot7GKH5ietq~_a^`ley=sMJ-sNv#B#Zh2EJ zwMa`?5vguk7OXaNAPu#59yD9u*f1hR6C|x}PRq5lu?cDnvTxIS<3gxpc8%GvU~NlH zZ77T)(YKN+$(_QfM$G zkFCt6TC+BoB?hNCLYu=)HLF4{e66g7fv|vrpw8-+t_jyR2hP0$#dBaWhrhA$@<<43 z@-;Mvs~c)+%TZ$EGP9)AaP>9Yx2a&LoOxA+&ueU2Rn5^s<<6t&rt->|I={C1@hfNQGY^EzlgB0Tk ztSPMyHAf&H24bMGz5%XXX~(>###JfnlktpXzHsfT^XtQPw$g;JpnEqhc43VS#AWRF zRgG7WmL#Q(p*8k3JQ0gYl`S3UYp#ea4bwAU(cBP%&4-tpjh#)|TcD1@rPa;#@a!@m zd^I$On!4(yASROfaI>$btfCtKloi7Wt3x3SE^5U#-7j0p3`|?g%FOH);V+we*02C* zv^7^Ostt)x7K^Y{X1U20nw*skYO(Ah>{|unSp|1py9_KX#5O2c9aTsTVO)M$@yy0i zJ72YgnN6afR1sfZ23tjNzB7WR9ACbW;!uR{026o_CtZ%{iZzH(R+;7rQ&}cXp>$rP zp@x%!?GJ3jH)k>%L93;W(6-tga5x;_(7)-1OD)GTGABmHf@+2f9G+jcJ7}@1A`A@R zF~w$wM<&MxTBb-iJWy?4HbNq_JZMk6Eej?}`f2gRY{rtNL%rBYs7a#&spX?%!aG37Ki)zmGk4p)P* zgeZka4c8Wepv)L%2rf*KIbkj+$QPeey|n%ccwt#D0S%F?}W)UAwZNh{`e5-n@v~Y?kQ+4Pm$R%6Lf1|{3pD+XkaF9=ux#S{lCLcvwl zm)8cH;mXnOaP$_=tKZ-EQ&SjvMYs)r&|E7)u~pD4)5=oQ z0SlkGBwW9;c@~CsZS|_cf&ji4AHgcgB<&eK1Xc>{>Eryf9)0oK|rZ_Wi=92fzVrga;Teq@S;yVsEL{<_~6$KY9 z#T1IbU|DTVlgu`hd6;fsOhc7Q`oiK=`uc`uET@9ih+>=IC{{A+TI!G&LGPjpiLt-J zbTzxCaa9P`!9ab66tZ6r7Rok%Fm2C$_OgGSaI*owexV2vazoJ$EXyggwhKuy+4 zWrPRFmuN}`IUB%rH9{MPidbHMHRDAjRgRe`v=VWv9Y|voP=&CjqKfIIE9>E)WW#4c zMR1l0F)^65#50kmYAzGX7E}}l3uYp=DT##Z&YMwDzoMbGsW3lKy@m!(2#W~Fu1DF@ z1(@BY&sb6sEGwy494rVfDw~fPq{JUATQE1U(1$r^VPG*BT1Z8hp`y79(kx3A)6-#( zH6^MQuSJd38heirS`G=7<$j z%?}o)5?Z&-=+M@*-EK1=D>jW3%7wBU8&}pYGl{a6+L{Ogc?hs-0ivgZ8TM3XbpnbJ z@A}^YO0$X?=U^+hnE%Kus74BAg6Wm3Ru&ZCpSg`SGNnlpSUq>0f+fx7tWnYcGmvc$ zdnHoR%srpsImI$O=kVLs=9iz!WRIh{8DFJ*R9Eh4xpFj&$IQ=Pg~6H&o_ zQ$6JWKh2T*6)1{cfn92D-f~fltZ z;rQRY#8>aXSy)f9^AuUW@Pg_qYkjFb)U2usxpvF2Vw=H*n}ivvvr^+jQ*7LwVY{;Q zC{Ssb!=RF8z{}b~WgMJJqKfd08pt4)S}-dZY&7R%EXcVE$uun^kVoifPT^K+AHx6q zU|GeiVBp+}=|QojipJ@oNErJ^=<$linafvVOUVkd>StgI#5Q7$ZFDnhIBj8y(;To%^yRiyJ(EEDA@OOX+Qh}l^ogiygN>YKum zYIZ#470%B1YIIpo9=TnI2EsR8o<>C)gbYI8<`clnYu5x*k<>gy4`F)?OqN%{Qa%Vo z1mtgQSb@ofUqI%bTI!|+t(#WTB@Of;wabM4N_NF*#FVXUk!4xfk+Dp8W#bC0IVD2l zXdd2iGv_&b{`*Q5S67rR<&zx7z!>(3yU9CAkEMAPdW>_9}x#^HIdSVVAJcsPaNSp_S+h3Fvmy%r0<4Dz8t|CXHNUhpIQ^9AIIdz| z<-6wleP@>j3r{ILWu|#uS&0-gPAMoF0gGLwm6auD`+^0?otCmdy*FJ2#UsinR1>Oq z)az3U^VJdx&3h1VM(PUUrLGHRxK3qLPpxZQReNe}(`EJ5r><^ny1Y4rrTMAVI0VE4 zKsX3=D!*|?=T!Xh7!pq;jmF;?7Y;AU2Z!D8>kMP@zigZ_Q$I|>^!(1om6>A{uI&$V znfGy2%=+N)mZY##W#a^yd3_L4kCk84u+!q8W~zp(JYp@Fp}v!$@h;Qee%9;UJ7VLH}irej$q5(=m0%+i^Y|1-~Z4cExr!+J3# z=G_Tk*%sa%DI?1@OBrXillr$yWDfpO2jg7SC_gO0luyIeB>auV`Mj}m-_D`7Q!?Sc zSwE3kKC$_+Yb1k1-PXsJBFpt<{^vsKQLY}8!Sjsy_^S?n69)A)*OhWE(B%qza}Ao} zc13~tVd?Af&Ga7pMUJRP`h#3Ir|uu*+F;%9#&_M&51_Kiu60&H`?U^<|hlI>;-(Yef%6!Uk`(y=sMX-A3mJzJa#&59XsY>H*dFs-5Tb@uF?`ksTX-VU4d0NwWTb_pnNrSq14Hl*>kJiTeWd#z~)ed)Z_Upvxyt6%!lc$bylNaMS$ z@imyncUtz7ZTSh?FXdVCxYKy|xu*Ve(s<7~mR+Usb~#09e84Klo5qK%aw^k!yPT>t zzQZb~E{*TD%4tdCy;i$g)A&}ODgW9uzTeVMM;hO2;q_^J|2Ia7f8UVCdvZ;_JB{zO za9$%8*_one~s~kOzAGG?h zGL5(GrzMT=w(NOr8t=92w#a%v$ktoH(r;xN-)Y(5+BClZ5mV0{X?&N(cc$?jR$O7E z@pd_bX}n#I%Zj5@a$4oM)A*3}{*pJ1x69Ylc)R?{G(KdN-}(JxW!&5K+PFK7e`%`e zA9~aH{_je5S6>?6@l%tx{XMmKA^z-jj@$AR>AdYf(s{!ny*k#owwh2OXowD z{iO4)4!pgNNl$OjSLB~i+H22C>AXGvr1L}8yp+yYDtb`M4>|B_9e6ulPA`9hL;5}k ze#n7$TjMOf{5%J~${H8x>FxLa(s}#6zjWSyFE5?9-@i-e^A!2{TcP%6N5UKz+^13b z9P7PECRKT_QqGBXA^2XWv`giO9Qc4WPFcR1zSV)RQ|irASxaS+At$Z&%o#L51heh4(7ya}+sMzE0tfQsfCKe6GUx zIq-Q(d)4%H4t$@|UNyZ}8AmGLs?^Kh^i6$~a#X%cNq@YOeuKill`6#L>Qv-Z`7Wh= zmG4!`IYKGFU*V5ac#l%f2wrLL2wstgziDiLiWHu|Wo&=?ZW*>8m0zc%&r#C%D}GGf zxi=J^`{VX!P~p}0@P`zBco!JC6nXFpcj}X^@SYTl`Cs9uDtwN@n=LeNdKCV6C4HX4 z=P7(Z(X-0uE9trKY=4Rre!5NK+pF+J3a=~tX$l`u_!5QBQ{++kN+o@%lDN{e~rQ$3V)r#4=Q|2;aipV zs{D|W{uU*Dw&G`gtnhAy|Cz$)DEvJNpQrG@Quus@|FyywDg1*9?^XEUDZH-m4=a2? z;h$9aN`*f||cPjkr z3cp_A-&FV(rQcP4gOdI|C4INTf28oe3O}UqI~0DbqR)PXKS1GiMIMzml=KHF=?4|w zt?)w%|22iLQ_5F)x5AHBOaexAaQQTTj? z#{<69r%2(sHn2Zlg=gLNM^|`zjm>)jg&%${?s8Qs{6Tgh_^wiT^O?7Kvrggpq}=|5 z6kfAQe77jPy%y)~R)tr8S7@!m@En zDEx5>KdA8ERQMrNjg}->dL5mGrv8&r7jU`~rpV zQ}~4nzeC}_rSSa2~3QTQDSf33pzD}1ZM8w!8D!VfBZo5Bw%d{p6Gw+-8WyTWHH{0$24 zR`?qgK1bo#D!fPGZ&LU?g^w$IzQW(E@I?xrPO~(O$xtO;WsOMhr(}F_;m`u zP2oEgzDMELEBw<6-=*+>R`?AH{}+YtR`|ave6PYkqwswS->2|96#iL-?^pQk3U4U< za|%DG@Xss!kix&9@UA!nZ2?2MWJd z;dd!~hr$mk{5pmIP~kfj{$qt-ukim;_%4P2MBz6m{BDKsR`@*%->dMSDtw>9?^XC6 z3jdkH_bdFr72Z(z&lP@9;s2xXLkho7;a%4Gf$`*&jhk$RSN^l0F1NyuQqt!r{Ah*u zDEt_O&r|qu3ZJj=;}yP0;jEb%zvR>4OQ z(`=Fn}`o54hX)U_y}UJ;46ubB+eJyM0^ynNAP9DM-#gRFD3pu zu}koU#K#a1e$MtUBK8pX3qFTEco}i0;1h^zi8}-z zOT3)8RqzqSD~LmaClhl5N>&Lzkhq>WAb1q1mA>w|)PZNKexKHpV;wIv5!H*I*6L$&z9dVerQ}BJn5#kQP zcN1Sh+$#7^;?=|%_MZy9B>X{9WR~-O~TW9mM^DpC*6`^2q+?KaY*p3#OsKw1m8sbL*jto>xu6q_6oj|_%7mn!A-_#Wa;!6y*^ zg1AHQvBbY5ZWVk4@x8<$!IO#C6ITg7koZ@`0l}k)?<4jK-g^k}uZi;oe?)vgu}AQ` z#19a=1;0-G8)BE>mx&)F9{iW|KXDgvzu>2de@omacoXsOh`R+pO8k4`F2TPe{sVER z;QNUGNZcX#ZsLcCTLs@q{4jAy@U6rfh^qwOMEnSGK=Ad%j}m(YUrGEJalYUt;>U?S zf-fWf6R}(HQsO6wU4kzpev)|bW9fh5ZsLBy=MZls?h|}A@h0MK!KV>#ChihkNW6u( zQ}7AITZuaaA4|NAxK;2G#683z!IO!5iK_%3NcSa=8+zj6Tz7fi*lT+1gYtMdH`|zjo4_UeUc|q@&oXXDY&9^d zU?&PP*5X=^FV4+brcXaaid?%R+-Pis`T1hoklt916#m#A;oGRdWL_T6%}rLL6ZP0;R;;mNB%F@oPPLqBLJG?g@`I{smvC$LHv6o85ZxYvFMhC0_?~(Pk(mM~_`iNoM34mtw_$ zWuzXTk*mi+h7U02JnM3GWetosu7@=hbfcP_%~a>-Pr4~EmY+s18Y&Ixfn3IENT)}!iQ1JL@C6i+^19S zw#}Y_H}&{6SmN#rAHY^7b16M8Hh0*4i~QHRnP8$GKh+q+e6d{vAL#K?++oA>^~40) zsjHy-yJYQ_@WFoRYvXA!z|_D(Xre6oX81?|J@%^c3=Ek3c<7wp zr;4UtGmhu?2m_nDwmP3*O8Re@f60nibzC z#Z3DT^6MM_Gg^-g>Kl!-^sH_Av;PU_puk5cCkmXv0__H+m)Cm68kFOToT|5-@y#-gF zcx}kH=R;q7%=2iR7Uj&U$KeiM^2bZE2Tl?x6n(`H(A%HVq73+$pkTW{){Xpn@k^1g zzj$-xRewCItYCXN6y@{9Lr4HkN`k#w`wkXf{Be0~Q#dzCpWaa(zfj89VJG2C=P%x= zwQ~wCk54Xd-yV6{Kd0UuaSt53yraCh&RrhQ2_Ih`+fiQJ7uiL-DCoA$yrVp}*%#{_ zme&{eqI?uyo;c)rXvr5JrN=f79Ax*O9{W^}7yFBQBA@8Rz2QK4`_AwbAAE*n(u-dV zkN3qUJdY~Mx&s4fAOluYhDD;?tb$YCK;0=H5$Gkc4ws@V`r~A2^_T6;Y;W4xis`wo} z_BYNR*_=yE*Hf@6%N6YoAI*uzJ<@$dCP4Z;*dBc0z?;TlyY}s~>`=zr>^-!B0Q{zi zcceaTlis#hTYV@AG<)-~ev$uujG?mcX1R>~?Jk#TZ=b_uVqzH~HF`;p-w=StIK#xo zQ1~{ek@X}hb9P@(BOk>URbsSx-Tp-GX3httdi+E>!@GAYLT$o`w1k>RpBtFaOpeukXhiERXvF2>dojJfV{|bG z{Q8a$F=#)2pE`1Jws}F1eJpzVDgAG}h?CA?)QdR(9T%zofi%t29_Ny8#c;lfIzuDW z({^fBc(Tl|*_%6t^=n-8t~Kpui|4Xt_5kY~Cz;OB<5%g!@aupc!m);Hkm6mHO?3Q4 z$q&)X9mYW@Z(wK2e;J=8fgexdi=>v}`qk*EXheVJQP^wU@W5ja`)n!{HC~JU4J|S1 z-=?BB^rELZGo8c6vd6b-(Nn+~2MmBHHcGTRwrcH{&>j+3=epb4JCQldpIF8j_Ix*| zT;nGY78AG|Ga4ppF>5h$V+sOH+`ypZjU42J0CN$i{TnkpZg|Wap5YsB$c@Fh9v$%< z3=CBoPhvQ?Jfen(f7lXjEg}nSM;wZuJRc7GH>Fl9OEbEh15Z#~)z`ie;h8V~ zcy1q*R`MtO<1+sJo=H*LJ*{jCKRUzCt;Um7A1XW^1t-6c%*M&M>QLLK#Kt3f7Xh)| zyTzYcz1xS9jrf}>k=eT|F*fYp^+E>YMA(zryIZvOMle>#(nZg&S%Sm;sa&Mt|&x3m&X{K^-y4+h#PltG2kDs1v^uzwG?$x3nBLictgN-$y zQ}%9;7IQ3p8!%^hEG-ftQHgnXI(z&gXz+m16mzbU&Ax=za;tH+L9!y^T)Y zma5N)6W=?GuOePMjN`)!ZGTYdZ(}klX6NtM(9*P_O9k+31606qyk}0an6B zjOj3J<8QC0IuG{NiR8)8XcmVe14KalQDHm{>*V^v^r{+t^hLw*qp=f(Nr#V)ym`^) z4%G}Nl}Cme1@mDzpq7iLN;|g#mfQ+Ia*Lecr1j&^D4yk#tYB_F0zfk zm=|tivs_$~BTSWYmaGn{Kt}fe1y0{Xg_))O zZiO;S_MP8SvU^*W_E1YmYhMpWkGu6)nXbq6Y=3N3c2AkSq^QT1P*SfR59xZW94p&$ ztRr6xACVegy7u6KdfTRKz4#;EjO;?OV`A5MwDyl!Putr$C2jx7iC_^LD2gq~#aAT? zEkZ!zMSc0cwlY^10~yRA81pkEj?mhdP$Q|;R>`F$L3_UUM$z7dj78!tFnX*`TQ{&p zF0K7j{0p{{tY*$$3)tJTFA#6>D*P;k&(RZsJcQeJo+DECVm(hVGBtL*&Qiu!?}gt^ z%slo6F5HTYO>geow*i9elT}nI{*4Eqr6qaEEZjkSQ&zAqc{)rmb)R#y@k{I{VKj#k zU6y52dK5p(EdH)`v-p}+@voT{T!KmS`@g^jKfP`y|`CL2lFI zi#*tBn35G4(A$z(Pf+jr#^h+Ib7Gcp2*qMlRq75e)DvTj)^qc-fO6=Nu&k4aQ~c3n zz3ocRsMQJWA?yWNK~!7ssJ8x(*Oul>E+vPn;QfK`J;|z!kKSNYr#vOzEM%RA_we6fA@v>?vOIR76m*OLs=sGBSMS?+CX z3FFNzdg3vL?N%7hn2_ux!Jz>Iq(fH4xFgkj~*XKmoJWk>!v*tD)q7$tR@S5 z#ZN{4;ls+xA3I)eYom}{=;bL3JsI68e^MSh1WuwfbWVH>e3)n8>Gbt-+ukf6R}3#` z(N}08G6XT!j?s%ZX*a%2Dlz5-)FbMZ7`E~?YG`q8HcTtlgDphk`!u^mTc@Ow)tuP8 z-qY3!iXUk`)?X-1xYU+nS;0;$$IIZePBY~6$M|BWV`Jm0kUj@K z1OxYE7^Odk{*N_>lAq!Ez8;t6`mmn#$9~?5da#p$9yvF}(hi2-=JI$?GjilUYw6~k z*qDvUuZxp|RJSpcNP`_*J!UEvCf;_ID?C-4F~rtmG^p6=+mbIKpAyg5_mWS7_a}be z$-;W1r*P%+$HKYpWI65`$G!q-V?D;Z|6mhLvwjucjbq{p`F_p#o7{WHmkaJB;Nb8uF|j|91Ak zk%$jvKPP-VF_RB6dGWK6L$OU0d0mfx3uCF{t0^VC9VkE*){M>SLb?+Pq$zT7F#k88`M?G_+jr7r}*MG_EU55+&+X_ zTJ(ENh0)+CmtE;AF|N-;Y|{iE`5&!)AqJ2yKFc5bP;cL@MSp{6M2~+5CHZr6%Ht2R z3S%uQ)Zr0wSnDTxyg-unGHEet)!P4r%kmf^$Tjdev#fa+9d0}!z6w`*>>1;?0KT>d zhfrkJ?shsYWZfbIIGl?G=RV`7_~igS9&z~+*A|s0?%lx3@5;-9fKT!o1xJod{t7vi z_zE6~j_Wp-6EA~A4BTc%-YJiNw~Kk=^C5P5{HN;yWE<=&YhxHta)nf&IY>vMJ2ZNS$qsS}-UdbzE?r#Oe ziv4e`XPpq9Xf(qF8L%>b_leWgqT`T`BdR=pFBR*L*I{p?r1cut3GfNNtgYO4*$Fd$ zbbP&rX93%YFMk#L7ts5^EM0HHSfJr}eDQFs@M4}28yp# z6|Y5aLLpSX4}I$|?hl{o>nTGvcI@orD%|u#7j9oxHyo)i@qFKa9_$OcNVOr9_Be-DNftQR6cT!F?8s9(_wOVb!rdtHAc8iZ zxA|hvTNN4a!XSKDoGyBv(f|A5u^82nJk+=bHTq+(rz)j7&81YyrL4A<<@w`XT`*}_ zl0UZFI2dh(58H%mKIGWT~ z*>Ay@!T55F$q3-^7^L@le8pa6zj~izH^F6bCCP_FV~q#kpdtDgVU2NnnOdG2Is`vi!g#}N}IW>&)&n@wLW(}-#!wXj^h?Zm8b zkg>%MJ$|ulAs%J!OP>ejWBU+;%H{3;V3Y)(ZEt4{9LV{40)p`FEW}J&lztmw%>4yE z`OLVvFiwx5lo+S%cxynW9O(QDZ!&~ouXBuNU$wgh?qpMVerijBhbi=U9v7U2dVC2y zYIzl-!Q*`KNVYzwP*0rAr$LL^Ug!d&E;JaPrx$P3W1Au$*t-uuGIt*a9%Nza7F}cw z)N4H$HG_F2o}+s9Ey7eZCKrJ>b|qyuG1gPE?)zwkif7r`%Ak(s z?f{8F^y63P3zy|)f2@56!%;@3b||j2Y0=zH$q01>vcseO@e@k4hr+q}{@6Xa5=rz# zb2)eLls004-nI$j*et|9ZKEvK5lLNND1(b84#lx@e!?31D0#~8 z_!22cMGo}AW4n!m(Y?O5t8?>QVN6Kx7^l3*SpOa43XU;MTzD>BME^TUdFrD@^A56F z>(<&YK7ipCP8lHEzT3E?PB#IwhXn=tIDAc8P&S;WOhlsVwipo4o<4{7pAfF)Xb&Cb z?i(;w8~p{$*4T#gGw@n1Fmd*xQ64`pr0+q{FlM_i_8cU&y(iSXFni$m;rKz1pC$2X zJ8I)SX2(sze8vo zp8s0UknJP`V39;?e;pU)@dL16tUM3)#Xj}Li}m7XBA@t+d$s6a z#xiQ@<}+b)CCQeC$Ct+@U}>CH9?LTBgm3k={e~51m6+>-QCj<%h}anO;pvX)AK_yk z&t)R+!;Z%kGC$j&Se;`&+s9*1e|&WgNH>UiWJN`=|l&uDu7irtuFtYO-I!N0|`>G9jGfqi!VdW`F%(Bs_( zcAK&|v(7-G8HeFG7?%@r zYyqyE{yY!87{7u?7j&+%^*DAMEYnS}C2@EI3`9M?Aba2&YX4(>w>!Lwyx1-ZpzU6) z7k?Ug-5(!UfDQic@X6)ye%xT{!+n@;WC9-S`(r&?w3zmZ3;R4x@qfY||D5IS$du$i z;i<~!RJnceY%Tf*0F2LvmDvD{Or0mq?zH#g;eir?oZZ*S<8Q6~Tyb|^#7<*2VOpdb zinQo8Y!c>MIB58?BIAyi*hG6v9y@^ccKk&65A)FtbLNL|@$*%^ zY3)1Ft#T+r=1cJ&k(ZNCgOzv=yPW7wb7s;W8bd=#ZbykJ{jw9qBGBb*l2?m<4oQvC zy%2ZUCUgqx9bpqFci1MLV(Uzs@QO_wZ$D3>J!sKi$^8*Fp|$grFYV&xlwF*cW*1uf zVmJuXHo^tT`2Zu=OZ(~LL^HkCek?9bZ%Y3L1IkVQa1680Rrd!{_SA-Zwmq?}FmGfZ z4!)u?CucM&klf|RnF__IbS-)_n+K!9u9p~<7u%4EQC*las(Z)rh8bAT%?*UTs!>h1 zq`~^#vaG|C=V6do`c2u_+2BS=F#TvgOJ9#W%I01p0CvD!KZUNNPM3u z8SqEhS-zg}%;=AgPfkQC%sal=U~2z2b-qXji$8HxpX|_2+^OzV8n zpcXv@Rda5{ekUg)@jlymK8!;Rn-uUJdhB4M>~B;DU0vIHdf%+%hR-bjdUdYX8V_EM zhdOIKe7pq(508fy=B^tt9zqV|q3-|dc*ueON{xqu0RG2#7?6+vho)Y~McR0{5R}t+ zn2UQD57g`cgY*Ecr%q2yJP@%KHUo@89GpnrCU&Ik_ZWZP%xczpV2u*}_+$O$@$YlT zK<6Q`qYxDImB$Z4*r7%DLGz3T;z7nhmcKINpEq%fJm(N>v0@JC)7qb5dYm_czi4#p z@kSg^`J3J|-whg@!t-;iZx6o*vd!{h*cly%r&1B@oPX$#pM@C)E#u=oUiBF$}RM_zK2day-c89mcs^as4-*v}F&iXTPZ<={M%6kz-_zfv-YJb{nQ%kB%m9Rv#N!oKX zu9Nfe58~NaZ}KLzOj^TJ>}qSY=pb(m`vz$de1g>?o|PPKQF1d9%ky;xK&tU*(QT-V z2IPy6hJVABH2qsRVr7O|CGSBBcGmon0-P#-d2X)NI1vDi5&^ov5`d5}C zuJ6TUjU666VMeA#b|*2;%Twt%U(7W3AdPVrtvzwZdJ?yRfUzSVmu9<+H+m`7YsP0! z@mmUGkZnAMO`d^!ST@6C+%bMd<~3_Sa>RNaXTy0K@VfU961he!*JXX&t@D)U#kpQS zjH^TvN=cusbNj?$Q%YL%bIX{=GT}tBc>`c?lWt}iWxaA!QDofNjY%ain6p@Za@)iAp)?g zbNM2Gu?$yya8ZU$3p^IT2K_IEOlkf5R19Zyd}8KhD^c@sx6=sP zMq&G6F}Jd51b#$MQqsqGMU34&fbZltF_z5vDCz7UPZ{ z%m6EfaXaw9?Q9k$o=u6Nt5fmKhMkWBtc*v2N`8*+gj`JOXF)I1RdU2bn$Pg+AoG4h z^59rF+Q&4@kOmo&a)uP&d=7{Hw*48O3w;bvHbmzE=aR#-2e7IUUd_NwY(g4TUHn|c ztDG0wpW@@UyhvoX_kt`Q<>G!BH?H{MwQgN=i;{aGTJ)u_;Z9;jVP7wE*%!cu3YxPECIx|Xzf8r z;irqBko0-{3=epWvKBI!NHGP?&8^ZCShz!e0(fw2H3~3S^GR&^B35sjkPTJ)PzW$C^sW_0*; zt^Jx)SPM5tvEQ55JPAtj_)eUkqi-A_g#0;xCa+PuvryXRjp zN{@}n)uK_kAI+^qg?d|&3-4rMCJYck)ih^Y)w1_2n@~TJ7U*++luPMpVS4OkR?lj+ zXbE=Xcv9^k+6qh1W25EhAvT-YbUpruT((J{^9N}Jw#eBoE!ukkPECJ`mB6*=^+ZV) zO5_c_ryFB-pPc*FpX=7AV%cy~u6Dz}*l1Ki75ifE`8K|ZgSA{vf|EW6@cQDx9CngE zZHL)MhIS)=Z$poxDIA1z4M=YpI0O>`AsAbR{vU%X5QE@xh&={yBzhZ`s#Z#x`Lh)M zp^6;XjP}qb$JTK_Ut0HDq#thG22=!c5ft4c6L&P%KxQ+tfC10H)Q1Pz#c1IZp#5|35JbDal)Zsbz?8CseKvaB z?o;EFM`bW-(UZW+Xjt#nq8*S4=HK2k5qa9oywFD8bgWE;cO=ReM!?2vdq>q?U2Ey_02| zs}}7>KIzHTxJd6wbYuKrJdo4(Y}cngtsl^fld$MYc$RTy7qxAZ{`40QYByAJa)O0Y zEmS%`;z_S^yR`NqbT+ocwYCBz<8Y+rjLF!YOa2QhjS=lNz32BiLopUUF>gP8$k0`+ zeRpXN_Up4b7Uqnd(l-?qe#UKTzNWgoL(Gx~mtDydkfcLT%*%$5wgY0{aSSKVz)h~F zqd4u>Fb(=llp-2>THK}hNbQEdpg`N}w%;Uzd&Q|&!KRkc#Nbd{n79r*;z|$e;0@zY#F5Dh zp*2a5y;y8gnLTmyH*x0`EKKe59I-v-jkZ5R$J0JFwKi<8c)IpYq)zRx=^NjGS-xh> zeTYr6CNV^rGP51?7)fgiI5BDthO7ZdH3X^jnCX~hO4DQaTh8!))A~2|kLIbi;?3F( zZ$b)eikziy92jTU_dLYYW*yguInANr?u z>+Lu{rj>2P!1Be$MV^7*%jUj2ibK9gFJC{vn&BN*cg&+woj+@UPh05MV+5J!D${9v4Kd(fty>%AE zSZ7rYeHP);k2#d}>WQOX<1=h>ERxB%D@|ub&E^Y#yL=lhh{a%)Ef&W{>WMKteW~4Y z8AD$j)?p7}PtB6LjKl6`5tL(U5IIo55Hq6t1l}x)gS(;raMSG5+f#Y0u9iF~N-Bll zH@|NdO8`xoP=+a*IdEtvG>nv;M02Ts21~l?k9pmR8~DF;?Rs8V*Im{%l5CL6*0bOM ziHvWTdtGmz1!Kli%ceyG;|C`?vQ=vlH2 zz_W8W5zD99F`NoA#xWH~Yx^PFwD#{Jsjs*<($D2s9Zm=832PdPW3vr{88ujZChx_* zWZHhWB|Ee`L}k%8;TxkFY9GXv#0xLtKaC}EU2+B~S$sfT4$8c%hh7r8Lp zf}MGp*zxE!wJ`kb5iSf*%FU9*TJ$aqU7CI+Vuba$ZpGA#R->R)vKl^P?j92 zi*{3~Qu#!E&Xk9@A-a<18}e2Fmq}cLd8mYbstKNzU5t0u5jdV`M!sQ+p7<%1Q*Mn#W-_#?ZjB9MAo^w21zKujs;E7WqaaghgL?M$CPDy2ocR zTD!*hwT&@tl>b?D>PnBB@i}!QDogev(f^)*7b7M%?HemS7;&DI_cWEkOPa5T--#9v z_oyN~&|GRQn%qpa7R|<8ybG|^cnAN*!Q`J=FXFc;leS_vhP#S9g<`K9CNe&Sx)F?x zP_r12aRW+A@5JO16o~Qij%g*g{hDnTJ8hR|A#rYVy#Fv*Iftig|F zF$q7mLXE{Y0$>i|4P!n5vWmO2#)4_~h*0VBi#KX*mr*3#q=x+VjA+FVdTh+oGLalD zZy-c-b71bNvCT=<&N)KIXgL!aJvVSFc*l4W-xAOz3o#Ap@HFzl&+`05yOA#=S!g7Fo<$CLD2>S^sqz^;?W;2(Rvz+_BuubpBD?}sNp=S$)+ zm_f7}CNYFB`e>R4Twa~Pi}CcL*gUC8tK&WNw|Vb&(=D1&q#q@On9~do8P>8~cn~Sq z3L=rWMasS13J>k*(OdwhqQ~3K4LKxlMRG0LgW)4<)OSSD_GEdLC)S+jkCo=f&Mneo z6<+R*K@J%K7%vonADSyUesD>F_#sWlF4r;sz?13pLDSI-jq83TIYV6G@x~;R8@R_} z_-*lU#@5II&w*j_h##!29PplDK0rPX81G$H@|qJGuB$jZ)YZxF>DeRI`1u3qGSl1i zqg-qcOHhQ(ZlKgq6+p>B@{{rxxorR#9Q`1T!vK<3T0cDz(jM|&upX}>?Hh-|0E?V? z;UdPTW9Q?G)we<_Kenm5DV71CNX6yv&QfQV3#rH zZZbd0tzs#$cg?pojN`Dg=7%_5GyU7=qDH=bvClY&>1m6+lRD`iq4x)CmS2WbWqb`c zXtCW$>r2eDSf9nV3TvmG4%UZ9fShPUWd(1-<>kWPQEdmy`#h?WHugaK~pEFRe!#5EVq{OoCu?+}@t0 zjn6hy@Y_XK>@4b>4SgeHAAHV|(CKeoYc@CA*eO@1=NjAP!ju$-nq~it26Tnp9HW#` zGIocqKLQS)ih&b{M82_>s&JG8K2E0Z4fj~$9)p`uh}M1+=W0A8#M_CKt>;(l<9p2A z0`x3%(f6#geGjfXmFxA^wH(R!CGeQVmyqY>=7xiD7+S@kX*R3r!9z1P(s~|+q1uP+ z=0Z1dF51G)VeB4DkMprkaw6`F>w_5lQ2RUhjR8#uWIX&G02qhccz{o*B{{DRsL_7Dl+%@-7yj@1Mlx4ZF#Zm>jVc?gR@y0`7knfV;eCgZ^Ncw!;l)-2D>{Eqf~wAb0y&Ts z?`6QCjj;zKBQ#7vnb>*M3=D-7KR)aX#BeC#KsFaUq`$)fOAZKgWEUBeP1&?)l8R*& z6ioW}rK`!9oy}lm5>KO>C(-c=!auQB0lEv1~Ysy zj-lKM%Yr5YB)4HNz&u`_EKIVjquVw>7Dju&$NOe>w2tjq#+{WW}-b8}_0tUlqj z@Dwj`k}+`n2>&bt0R_;*8LwRlvnhU6i+)$~EPe%gH={;*71Ckeia1shy z!77$(LKql5m9ZG30L#Hn(zQg43(NAZY?q!o5ipG2aR-2JgnJ$UBpFzJ@vyGavRz%^Xb_S$<4!ZsV3txpfw9J%Zq0OgN8<&VibtkX7fddPz=t$1LcahLcu9H^je!Q}9w>puL3h%uUO2TkJex1s_q zx(8cQ#^GpI@_pIT@}^dXIG0wQs4ePXZ*o51hdXHEn85XeBR#@3fgiNe;&MnXv6G8y z7u94X&$g}&?*r5_c5hew0%`FyO5 z1rAr*c;*#=s%3qayqu%R4p|d3-+^mG9~g~z)Qwjd5CiQon$c?XV)uN-e)brDWreo= zY$!3a=}2m{iT%roE?+4@L~UmqLoz* zJAnOV@$fp%%NO<^)7$-s<%kE#*_WvT$fei3dREBu>wlZuy6F^MQJgm#YHU8w&05cFA>*x6|Y{Vb!m%)8qe- zv^Rl|vbz5NgAj-dPEe6psSY;P;8swhM2&_`CK@!ZMR7@6MOw8|O%z3Ia3;byPK|Y= ztqX0fOSOu{Di)MzfZ)=v5v|~wxN;u{1hoPR!vFm__nFBgXqW$QUXbT`&b{~CbI&>V z+;jJ1XY-sLYa3wT)`AByFP%VcRH=p>34m z1%Ianr;hC;RVS(ZcUzU(b>-xMJDzI^^RQ%WE#Dh}Gl;(-(zMb#2z@S#rP!rRXNskU zeXgtS^-e6EcviW|e6jd(TCJN?&sTa{%-!c-qVDHhoy35RMld3YAyTz?>tr|UPHHR1 zg)sUk3DPVS$%CE_Skj2~vdvsm<=o67gJ9lA1ZZxd;swU6S7Q8%1iqd(59S<;cXQxY z-qeJAwk>(n_^VWIAQZ`#tfq?=9Kg_@^?eTErH13=g8g`TYO;`C!;~gwA{90zmDq>i z$JTuNRxWsfkG%C3{#0w!S=K>6_wjEH!|q3tz4v-%sgX(-4u1$!k}CiEBRtXrTi+1t z=4iDHf|!q$$Cp}{2hwe zO6!Aks*`WUk{Rdw^B1aaW9I9}GX%z}u&&*NSKLaP)YY+k*m(xyFu$xa9r8Pi6iyNC ztuGpm%tPH(ZbyS}>{b6`8klYBPmtbR2`YE%y;9|=Bit(ZnrLa*&9!o_Oe}4iGu?6n zMiLjG(e7i4fR|9e+)Co!v=Vi=S3oUv`w89N(s(}hFfwbSb_zmjYB3cwDqiv|37^G! zx>N6Bedj%J#-&1hagoaRwD9{<|J|dBXUO4j>EBkqOsSx_fk*menYR_CPNt4A7lvj1 zsS%rzTTSwPh0V6G#kfK`qT)jdw0mm@^raBmNZfGUbi=KgS&*4o5S&9(=CxtP?aJG{ zz1366F;)KbgQg8Hsh^vt4M#&H-=RW;j_f@(cYIl z5|Ug$m=?Uw7^*K%kuQ%rG~e~*`Q*=5o-N_%!R4m@t?nf;9U5dN-LEeu13s4Oqlu5; zG}W;zQg`b#~K=U2^Cg89m}z>5`xqUgF#Z(I}_~4F4&QF!a}hdngda=`t6Ws zW(E?@JRuFd}P@=71G5=ujx`PyRC*>gfvc~Q7bI<=y@xPD{vvI!2hpJwB zV}0CZR^-7Cm_!e_PXTM(pG@Q|9ft94=|ys>SxCN+m&}7EpH}x7T*$l=JZy^U{cdv3 zx!zY$W_OqL_2RJf?-7&PG)ulqsrxJGYm&6ZyOah${bnRh<(o*aKVhD~v^49p+V2j8 zpzMVYCX9c&3=7MH&n-!7hNGYuka4}_G#wNvy#VAYD&|U zdmqYR$Z&BQEx2c6t~?J*=cywrB%>dZULV$gpCV*4yfiZb$ui5)BvJhGmr5?Wg302n z=Z4|+kmDjZ@&fUPS)&pDCF-`&MN^fqPMch=zioipp|?QogS^=VWWmcw7!ka} zV5)oA$Tfyx6kFY0e-dv#4vNG`a;uZe=9}lNHGbh|u> zY@f=2Q$E|Bne@?|^v+uJpy6dDJF}GT<@3Q!ufdjuXi#?iN&6e{)kYuQD?FNh`Kc${ z+H^K;!Hk;Jf!a4S!nQE5*I|cn%fSsc<}v)aYF#?Qc8CcYAF$<6vr#wwH0jvP6R#x} z8MDFt@dlBh(f)K8z_{%cJHb)lTK8T*9R$bhAG$e(RN5L+U+FGTIz}^Q2^R~lXX{CV zsce%d7W#Garu!-t`|g(T?&Ff2lkm=BQTIjBSVbn1_@!h^-i;s-x7A1q5re;pa2Z%1 z8_;hwCBUW3oWy?0OzNBR_sdl{cR(Y|w(wgdcxXNqbT3)|(+UNw$=jqH{kGFl?&fd7 zEAk?Vu|4n#+8r;`fxe&7bh=rAX~psnZ+7#gj_PiI$|`J~)?=$L=}Uvby+a?N24+r? zXc;8f@mq+PV?;Ff_l4UT{$^9DkP6KwMr#h>`{*L=99-`nfCAZ0itWFQx` zAWT2jJYoP?rqa22+odCCj%A0&<9kY)YXQcRUYd%LU`?clMiNsagoP4kAx;q|`mVF- zvQQJU(k}^~B4u;q#;aNFz{pkO#_va(*4q%)7{9nE8cQ*_b!=*Ir_J9!58^JpVwM3S z)GU1GeKM3lh7|FVDxtR3(;TQEa*gy8buD^Amob+C*LenK6q!2g*6l|EEYdJYvN?t| z?(;%{3J(R9m`w^bBEqWnLqB2dE|-$N!H6muVuQwY;*_Q@Xa7n;3w&0%-QMTE zK7>vYCXOvFA?9Vks?)r?GtU$L1-_)3&NNJ?S(>Y`VQO$k5C>78)2k+RF2`)_S(Ue* zzdGKg6Os}$D0t(>X_2NQXvi#bi319?q-yNx(S{vaKr;4Bt*%duV6uY)BQX_p zsD^4qNujN}pF(8Tv)NL%)FIp0QoS>QsaCn2X&Ci_?;L@Bt~cM7k0>z3(m_pEp!Pt~ zTSjmgvT1RBTRtvi{e{+Aaq=-M;D=(*bz3-3n#s~ba^Brtu$~JVpql{Q^TKULM)}xGC z%gfprr^UQZ=M+|RaP#|}vGge=kasoYoyuCQu_fb^=atEpG5?M2k32!cf^=vSmQ5O& z9D7E6O)PaKBgU1}XKS_G1yo6z!WurbcifJoaC<{0D+Ec!k`YPlh=CLD9#X(XGMqlz zc+@$OrpsZkG-qK{{OwD8wR&V<$257rrF3!EMFw!(dX+d<=Z_QRBpMcXaRlNIHFj0Z zAe>2Tf-2oiKJ=rP7M@+@rmN-hQ$z}7ORbrn^5AaZ^t^6DW3UTN69fiDBa!rj7n5aX zB@MNnRAV)Mkwm8^#M0%X4?^AUNor9c`<<_5U}F#pOu)F|1f+aGB=K7Tsj-cuC0d>I zM|2E-&K+jzR70~a^4JjZBTa$fwaTuS;VQj~ z&SAXBh}H7c@(0y%t=-0m6$xIaaJ9KM>A*{gmg`SNesA5WMG7n&O!t2cEMt>v!$k_* zspc+!on#m=a13lAv?jUPwKfWNi{Fq&(UhfnE*q^$3yn^AD)OM~Red};S(DQnn6yD@ z##x$6)$?>m=sjIN9A4^ke+!}!XQ1WaM)tiONZ>AEV8o|GE!Mc+KHZm)r_Xkbm1qb_4LAGMvD>vVa081V%n2J0|Ve*ZDcQ;Vqmf1)(sWRh0n4%e^|^_lkzS@86o zqJnzHDEF5UDhcsTvvO&90_QH)mlV050g$abyE^^zUXv)swioJ|f`rg6P=A=D>k zu!4p7nl=m7wO%c2&6wGGC|sevL+RhqSmZobE1+ST=Kj&wwE}WS6doMwrz8%x>O>* zT01bVm&upYH=9nz3AY>2ySHCzoVKawR5vum=15YPa)z+Rp^F=j+LwZGk}!9!xr7a; zcDTXMiZ{1}YtOuAXYJGH}#}RB>$-36JV-{*-X}@0A-L+BN@YdL_ z7bQQ&Vq`mN-SjZ9k<5enbj*GIys`u%8%gvwi4`V(GX%XI$8xdHtOp$S)mU{CYz6?n zlW?>-1c#<2p^lU_MvV7bI?iHglw9x4)n_&7Ob;<8q-Awnqq03LM4;@JGy@uG(tRS5 zI>B1%JgS8Klw#giPSA->$_=lU5UXoBN9!;Pxx=>@gsQoSRcx&1Ua+rw)MxcS;;lt4 z(Fn4Nx49pV_|1xME$-@s7sf0o$UEYqcM%ad;sR*5?d4TDk5``V?1`oUQ{?M}O^e@i z%3AZ&M;>(Q>GAhiKCu8j;Y{q)3nIyLtejJ6`OjingqF^8+plsG=O*Y#vNotgiB6)B zYA$DzUNRQZ zq-j@EvJQ9a)g;|kAEDUYrGk85)gj-%HC2|r&k;`^#+df(nFSXyIh?B>uqU%)9z6`C zRKr-<-O}*61Ol9fx?gBkv%HyG-^fYBK~k8y>^czTGjlTY{A9_qWQ!@?N=bTdDr>|LgQ;0WT{ zlFSZw|5Uib?0B)%OV(eDrgYOll}q0O2yv`X1fx`ube)#i$`}^TyL_2LZUV6e4V;nw=`YYmj*Yto-L2-HcHST z@fovEh1t@CMzF>(C$xFY5GFbwa~H_&3?zh1W3B!RFUl$_xLfnJ0_US=4O+hQ{KH zi`;J07%1oP=|+GQrt(uJhMkxvyzsUWusaue^f6N(Kjb5GgOnX#YC-7Qwz98=lX}+E zFB(U_3Xt}VPhT9K-!nXf4wFv$u5%~?w-5jLB@l%CAsAgVf53Q!FrLx_<1I?R1k;hjcifKkK4qFkf$ zPjTf>S0ye#4OF&>y2G_H9GEj0>46s#V&f5O+;bXaCGBl;9JFL+rQ2J$FCjNSQu@QI z6oTbh>@LLhz-_E-9s;(3$7>vcq)Swcu!3q>JvY#>8mpUQd9JoE2O3Tjity_6MT|?V zVNWkP23ihitx2ovpivDne1kJH585&>c0Px7vqpMC9e|nm#kHkPs zkRNYZN}S*;O>FFVCGd3R#Ze^wV>s*5$~;~96x*lViDGrS@_7Z0}G8fp~)6Yc0v=SR%LXOrZ`B+6CvK!Z{h7 zuqD_6!-TI`+RF2L;qwSR)0Jl^&%c2nnCl~U6>QB(K!=DlO1`lA+3}KFt{S}b z+ePpM;aiK~_dvf*1UI?3a9&@d&KvBPT8W6b9mW+&>?5Ak5hc)Bc!Xr;k$vX3cAsE% zlC#itO@(gUPMuA!xcp8E$?$xiaCn}MwfaLH?&>RSn4n&lk38;6^dfk4Iy?*YxGr}S$&2zQcQY_RP@mDIUW->e6Dl(Ntc@>cQ6M5%F zaaa_T=Sc-02Y>`a&2l?s*{hY5Z12?pN~bcsq_fP|xJC8gO(#lci(@p6_w&>bpLUL( z5(Sa7%!eqW*mn;Ir>=n-Zu;U~ z47y*&1fRQHf<~x=zgki}^yY7)BJzz?4223c3Ijw#9!q~A6GL}UBZ-m3Q*^sLLCM$L zu9tQNacxk;4#8anTmO&&&eTY{d@T*!#p4S!hyvB5eo^8g6v}$)7YG@`C@|jHV3#^C z$f!!-7AuT8f_$`xAF<31=(8o?pX73BZ#0X+xn*sx55&stz-i@JBDz#o(Ln0HS}oD~ zMcSzHAFr1x_y0Xk(a7Wq8#Ei zwun!cpDFh$$vI7Qkl5hq4yRC}yU~INULns=_uyzEQYDPK5f^y`Rm7n6Hdk_u+bbzL z?+Lfr4h#&fEi*47zDm$Z?h**IMjy!;Sc`d3L1ELtM~SW#vTreKZ7OJSQ9*l31$R=d zmI?}j3m&6gJCN2xuufoI2OE}z&?xj(#~0x)tt9R?Ayic3r>R8;y2`AYjc6_KMb0&} zFNZSq!g> zB}W#ymyMw}n32l06R{NYPYplO*>X_Xt@Gaq3UCw-ogob@KWR4;DShq{zitYM#*$d+;;a??*}K4fgGqfWrH457E-2HzKZNtnzPa2_B1r;d8_#!nMZ_^Xn6w5 zqIos65P1UgK{MTx1ZAeHJ&-nwIe2 zva!d~1bzoDL1Bi<*BbQqA9hG0-`?!*p=h7u=4d&~R`sA5eB*H&2kD|l~div19iVI_rc%*OH_7Nt`5@ufIXZ_do-RCqN*99n}h zg)o_V6&u&<+(s%ApDH=`(H9F?zQ-V{E=TX;giLnFVriIJV!nD{9_e@iO5#W68I(`l zrljG%X`k>!?Jz;YbA(4^aw&joT8v(dSadpF<=LBjM6VAGGeMIlI+FWVI>b}%XlTe?ylG{^_yHo0AW zU?h2*n8K;m?iBCR15bwr zJB_BpJ$IgOSPLZ4q#xc|>E;4`ins-C+ew_Rh7!1>5u~SodsHf&^Ow5*4s-AC*XubJk3&o8JMO+8S z@R=^$**ZS)c_K1hIM#o)_^&7A5hUZP1o8L!)b9%cBgCm>e728p+O?E9>4E7&MvET0tZAAcg0|dJI?D>idKI(@!Oc8gwT_gk`Bd8#wxRPeU-?iI zGYH8yTy2eyCb|l^whj%~D>Z!WPG$hzfr^2`niynl%nOl^yJ(aTu>^8W=zh6JLZFG? z1G)bt1WHZ9O%)A%PNd58^Vow@s@OJKMZq;@lOwx& zgB^>zUUt+5Cu^u{996VBqK%9U=u>JZeyGJz(1>Jv`^SJGlP#t16~#OI#FzFq$E%ho z+qRFSsc0s9i&N%a{gJ4}`w4TRcFVH18-d{srnXUKjoa;~T{oD@8GC?lP37U(#n#WY zVuzR3R?E&f(>pV1a?Eu2|RbMl7xA zLe9MzUCepM{l&tQVcoNl#2q^*uV`3N!5msiBl`!kzBFQ&?--VhR(@kKI8z$kZZu2d zofn*kT#HI$k)BYp?%D?_EEjRbQWIz#bXSfFoj#H!>>KOH)uawq5B9rSbl!FE(%{?` zIosn*_%^=2U*qg%P;Cwm(D+#WFs3A{D>@pUREwv))c~A$G}_sCk7?=GSy4oq&X5$) z9GMA4x7v+z>=X+P#NV+aAlSU6-8N@!2O9Q0Pn?ELq91@^EML4M)m>M8vMiSKLQyWH zL49OBr%VTWUk44ELYfIGoq=t03RC!ix$9lqUxhL|s?vYNV-PxDxsIHtzZ5L;cB~Lig$e%94ijh@*H(a~h5tZg1rq`V0SiQt@l<@*^LE6r%El<$Ye|#hl6j zxx>BxWW_zcM{#G1dS6^`9ryVy?tqcoe!e zEc8=9?_TI)U+CA^8t#7aiZAqR5c)!Q+^<|I^f8sZr2WLUtFVvqFS^P-<4#}hJr$By z?l*{{a?b&y8->T074xC0GI?U2r!r*AO{Vr!x!)$K%B={_0w-|q*WasLDeu`w@IrZw@DuH$JZgEF ze&78&U*5wMl2_jTVR<{j=+3~-XUn^zY(-Yi$hV{>rAD0+uLw)5v6T%ED|_$QA>=PV zqOH!C@#ege|6%*Ck_*SAJ6XXe=jwA<`38@+$US)3l0ZFkvj=; zXWK`4rDbA8e(1Np$YT_eSLB0;A`x#;vh!)`vPIsgMS^)nj^>KImZRXWz5R>iGD<+)V{PVKuO_LN~_g$>L(HJ^^kJ+O_`Z$O7(Q6 zt3Gd@gjLQKovQAl2w8^{^3hpmn-psfArkG?J!?p zqdaLJ<)<5djVrt5x08?-{$hpjeZHQMdOx);Z`IKC)p`jrp@08IN0ej!$k<-DmF( z(-4{6!}6zgtxjrVJ310&cQY3v)L(5=aMh{(OKb4NhqsK>oZ2ld)xFxJNBf@DPisrj8MQ8<|YqqC)6T1uudV|mkaEm$|2rp%It-3(iV4r<)k9jPLb zL}OoN1{3Km zVm&+GQH(VG9`*7g*KbdhX4_QbXKIvF<*gSFLHYwJWtG_o|thV zF$)o85B;Yv{{RkymVsj7sxdM%mCN6VX{GipCp+y_H?h^2i}GkMS&nDjS~3ysD@Z|N z*L*O1I}yI+tv{dE_Pw!L1ceGB{ZZe;4e;ViN z^fmNA%G@yUEA3SA3b{8O4T})C?0Rr8@7RNd=$)^9ZpaU1W}{jGNv*bqX|r)?KP}UV zB`?tb;^4W@^4GgrkGYr(n5QTSR<07#?)JxzE=zB!$X9ev$kAA*#C7s08IGlGzSLzh5O46s)4PMVXCFoQm+Km2R6Ko7Q!;C$?{T z2AG3Ko$9ZfD8|CoD&O*N3d1kStM9viL-;o9yV7qw-Q&+s z|F18)!r%Oc@ORC&`TXa9L-@;7)GU3!`-bo*sWRE{Y2Oh3`!4!x7~YnSSksm+ynKsT zxWMXmcO^$3CpP_+6XpL2P%)XJLq^h4BIfc1#2k;S!-5t({iE_B0*fb3Q1yzve*K6Qg*a7xg zf5+SBSZZ*o45J*dd|Iir%35s;B`e&-L#vQN`+B@l|LuC0#ipmHIMlma_xJXIA>KxIDJQu>)BcTD)Gs)e^6s;sZLxsCagM z`)NV7%FxgCbS=5~lLD;z&{@lG|5d)F5UiaajSE6cEkJPMS3U=k$Se$cFf7Ye-bZBL zF&1Uw&&?+iUC!QMIeT%P5_yZiek4>B2~nwpsV$ExQofnO!e;x1amEM6V*Uj5LKXRsrM=e^gbfZe~-PM!u`mo-S zrYErN;_TezZT!UR4MhG4?E|_j3AP}J9e}JS)T~{4 z;Ne}_Zi)G9UsRixr*$lV@>+K`CXQ~5!n!NoSlK?H%DZb_^bbO4qbS0~_e*+q3!yC* zH+$P@Z$V*f0Z{i*84Uf^CP6~?C0EW4x0^;U(gU}Xr=qO{E8Vz|?p=D&ovSVPqMUSY zdeGf)eJ0JYDrdpdiPgGZ{hOv|KS4FYg;Qv1O#OIqA2?I5FDYNvchhu1 z=ukaK1UWfgGw;ks4`No1*3A2OdT!U_P^qNv{cd-%)GO^AsK!HfJAtAGGxn@o@M{(q%~#bqNdiU9{(LM9OI^*9J&l_WrbIUW!_(bp zGOwKd8QA|g%xAU2V{`BCO+o)I8xG1!D#~?AHB20V{&`*W7rIS1`6_U3Hr>Q9-5jMmUg88vT)VQHP(S0Os@5CLmtY zn#=s$_d)gxd=CP}=r1LQY{?!{;2My_dJ?#|2n-G*a&VY{d1$Ux%zcF>4|tJ)dja&c z-9>E(X`5@UdwHVHXuI`n1*~!Xt-WHJva2Fc3)sL+W^UN=Wcx#-#R#p1mdvXj^O!Eo zr+du%ddx?JnCFt%L?5;VuarVrMOUeN3)#EG2nr;}bmg_H5ztxUpcKYeG*fTeW zVz>m2&RiF)u)1mGjf&)-cza#CZ!*3>Jzdp}k6Jo#=Lg zJO)I_%whB8`)seUe1TplpWHI{79YWNN{+N>t5sA4IjyLi|$WO7oJ2qs^AHD zSxj|sYWo&_2dXaTAsS);?Dl8Z|iWkM9DvOW@!=Q*@MGf8SD#Bimdpe zl8I-vhW!(*FE4ypCLlfVa$G`8M9U`{@MAOtMxBYl%RJuI|8#H40+RWm-mYCZj!!?} zIK50+mY+1yaQ))M1YWkmD%F2ass6bqxX|^l zNexh-ZOnqMaUTP6W7$_0OR;jOZ43LMlPoqZ{P;|nG1mW%C44cBFk_b5L+y66xC5WJ z@F|Sk**TCUUP57iHMK9R2BO)%syjw`rOPi{>#10dJl!Q8<=vo^@%#y&)&TPnK#~EE z7w||va{4#CrM0=+cx!50@$fBMI__5SA=2`TMrTT5DD{9^< zeR4xAO%xVLY>AR{&+wk#aKC=POLl|z4gFZa(>F4pA0&6l8)YAQDvV7EZ1Wq3bqnXL zmL0&+&^wk?Kr``!Q}W|uM7H;|+DLED{E5EcRe{j z4L8k%;8-{Ch%C6Q8@NQ^?|`Jpz!SA@q%&{4dSVa#*qVPoME^I$+Qt<@U%`@btXxeF zYHiN(Q(^rSl#Rl(Sl^0M#peNq7kib*+`9l+RANYA73?WYs$FchRG|VjGQ@! zpS-G*AF$QUCNOEtSkuEnO&pm{0aHe3(l0?w#YI>wd9xlA9K+ni9g=K|pa8aNQUxdx z<&gol%CS4qw6NWX-UFhCY1Xla{B3(c=33W>(x{Cl751Q|zP}aUHnNCF?B%nx5}L31 z|1FPkQ)9^!rp8h?HmYPMi4&NE_F@n$PYmNGcP|cX>Q6DvHyy)$Bqui@BSa?VW#th5ztc=(o>eDrz)SGvj2*n(vY72Nnfu) zDBrHH$01LQRw^#pp{aWG)YRilz@eIs?jh8kI{KZQj^^11ZC}a!+MRpUf6~vrM*WZV zvHsG3Q#N!#cYKYmV`i^zt5TU}WJ}dUA$pc-mt3iAJlD!|m%F9%Kb7aC|JAXqLAPr)q1O-Zh%$YB~gWNpses%pGpv`D;nTk`he-dt$dS zV&}Jec&8p#h>xY2srhp}yd?);=HbmAK062hyN6Hl@Wvec4<26Y;dMFq^&URV!>e-e zOFX>T!=pL)86J*PY4qpdqddID!`n?AB!qoEe71+r$-#H>@J0`B&cVNuCZ?vAIjJX6g}5dAm3R&PQzROg?z)kZKmeqQ1v}y z6)XI0?y>ZP8$|TqJEpC9YW+LYBGt*SIm7YWuij)O)HClD>U_y`=_{G<`2MEb9%4A7 z1aj*yh)rK?C{y#7dakULG*Zj4A3wM?-T> z#fN0oa4Tm};={bjwhpVR)YFk?jyx;4n&{p3_bJ$idH-1a@RiS4a=XbnHN)twc$Ui& z(i6(MuE=gL%k0^`Kj(HAK^0sDwxXW9hg-|&qK0Q@aKCm12S=@FaMetl8AcpA>`0sY zr~)R116)V%L4?L=(griVL~C`a6GR)nT@!P0lONTZQ6UCO6g z0OfaEn&wzKA2_>bU~^tzW6!`Td4Y9318egFt9k|w%L|P53@pwIEa(}yHm@k1ymu># zRg_%E#B)l$r>6V!Qpp48mZ~ufq+5OUl>89RMFp9_wcWtOvS2R)zwJWZGh%iRWw*|*g1(OuuMh%R<0I;*uw9Sv)_ z7=^sH)DOz7=h)Mgbqm+NqN_Ed9AC^bgk2U>!UA{R!BYFH+hc8God7&L?a3hkT;wJ_ z4c;xsGw|B~@$?k9H!&DzIIVtr{vHY-tnc38+rkc1 z8)aN@3`k_c7Zn8>mmwlIA=;nFvaMijZdL}{`^8$cJk@9Tp+ zXbiYCMNVb2r^lkFeyL^a0|Loqef^Hwqc+I>7_H-!5UR-NVvQR4@wDMB6I=8B(Y*9G zp?P`f_xfMc59pr$J%su{Kd*vPctIhjt^Yr@zMP z&~Eg-DJ2tq6Cn^I7HmhRUH$E?{PQe*+30Wfs4wZlmyZtXOE_l8cyUQAIgWAT!bt@2 z>ZnyGkLA_ZD7Ribq6VshPYib3-zrCLymopQz758r^fidY@a@`#uXi54`-Cq&ZEc8f z>3(9b#|M`X<^%^ZLzTD}y)ML+E_~rAzF5h`zU3mzMHiA6{$)&-%F!jRwi}t3fK~n1 z!grN2TUOR!hO}KNurKTRIV}Bx?Lm^?KS~$=-lz9vi??OVc6e9%ee%-3fV}e4Q=HyY zRh%QzdS9YiwFD z8ugKC?y|9#w#_{zwMkcQdcB~)xa>Zc%?)>yKMX!plJ%zt)I*9i|6epn)#;#5hu}xs$cQ!_~b9kB8e? z!WM2BIxXf0CAo?`p?0}C1wqL^rI|rl`#8Gz?-HOKsFPwEVZMM1RDXig#C>tA$48D? zGoJ3|&YFewXDzt^-eCB5tzoKx{vh4@E!SMLL#z@cjaNO=)H0RbeMf*Vii}FM(g{#f1o8+I+)Vxi>wnv!eU1ftX(0Z_Lp$X)XExdByj;%t*NDkfx@y#6Vstvy2=#! zRT;BqMCUcF!80Cuxqw=Fd$ZD3j{mb$H7A5HAG|6vi{c)ap^80mtBZR4YgdZ{TY zSgW_Z`8MxwEF7ju^0YrW#_{>O!~l3y4}-7TSH+*NJy*b&=0=NN_3&99et!zIrwl7Z}#xnIrwfKKE=ZubMRgsUhCm?Ie1`ZWBxD?ugbw+_Hfx<=#-W9^gQX| zPLfHR&Gzu$d$@@&o#$mIJ^X$jf1V!3`d@hXOb?%xgP(5jX|Yx2Tbtsx6lfNTzvukf ze0wYBZ)&$$yvaIdvaik@j&5mIgB^=sUB)8!1O9OY%O{#%0uml#vuK*QrJ>UD-QGqy z9@Dp-|LBH(rHrrA(RtWhE6izBV}Xe7eaAvK+Cb&E96rP>NPQQ+TbF_F+%9~tf4LRD zzBzokrOco2qgbbuPG;hAm#zgH>~o|Li>BsP{%+${VR62=QpFj|c@#QM1)C_&MLFEh ztbHW$CzV_SBsJ;tiZVNQ@mnXo5v~3-OkY_|@~fJPFJqBoUY4Ia7{g0*Qm<)OuGF^n z8wUbgRn#!3X6To(=@bl8AK?Y`jaSW0F2&maWqk5qU@Zn~!=Gy^&c#0OTb-_lcNp4# z43Vvzx|^6OWsxHuNj#>$xGrFoza|)Sj2xf?z~Z^(7^+}2pRRvtRuC|bML+&L+b0N* zS;^P97o4p+a;@-_2pN`xb?|ga8urW^nP)R=y75n^c~vLBz(;44Pc9ZpeIX|TLNI+s z^r)MDY5rQ4U+alARUD<dhx zklRD4Q8F9^i4|DXonckC+iWe1iu~mB$g@VCEvw$8FYW>e3eG_AZaT>9JXmv0w_0Nr ziyvLkz@b9a<*vDSIIP7-8hhp14`$Zc%)L?mXFbUDyunJu?X>LuTkQT%^(1{N?fF;R zMP@upq3A0llwq+_Ym^uf&loOUI zJTAOs9!GZpK75Gh!fnFZf&b)I4K@y(<*{C;@NQVI>VfqN!+L5I^FtkDEHJy_y3Oyz z=G#ny`y3etS1F66&*Cd9QF-fDS?ty$i(S8g@GlPbguhMY4fYn+gN0T7yTy!qqz2yM zu|L&~E$(S4QXWs{n|TO%QWCtq2}B?7ZXB5C5xlC5x^dvj9yl(~#}R7RX(4$>iM$zL z)wu~QwC6?I_3%D=7V|}oGif%@U)gZ-*vRANRXAPVxgp7p`cq0f)hKnU|Gah# zPKL9ixZcA5sY#D3BEcIY*S}e|Tkbk6d2;@~Yv> zy0v$T$}WWB$k1_Xt1Di;oQ3WkvG~f(TyGOgpYnA<{b8~6l_35O)N(~ z#h>Zb^jt$>HE@P@zKXr_-dL`wd@i=FrFl1-W@?P_nBmW5+6isAAiw{X{A~DhmRGFe zwT1yBE1s+0H($Q;$6x7;t#6MF*#;`=_asBbTuID^_ta8%Z~wJZ;S9e552wfg(Jm|s4_BVxH$Y)K?>thL}>In}>IGL`yWU4c=h@ipZ;D@%V3t9L)dexgXhK~xgP zAJ1Wdc;gW|fg;wcRSJDSm5XQgFt>@pVd#B0{(~3r<$?ccKIO1?RQtA`PuXJiXY2Wt zl^!0I@1H-P@{)(IHT}<2sE{|`IL#lhF81nrRYB(NB*5S7J5f691TpRQ-_zG?hY_Q=EK`2NUl2EX6K=jGtHd$OoiPzx$Jbb2yn|KYrro`y4^KcWd z!IyjZR&rhF;nvYl+kmv!r~3rg*u%r$^<3@hznJ`P59OyrK_>>)J9n4gO!V8^MCe6_<#NzM&~er8Pw9l==_1Fr_=f=@wIvwe2|CF@$i-$e9IuiGtwJ*_;+~tRw|P4@OE!XJN2;m7aF{<|IqGLfALGEBULuh?3PmrRsUxp zdFuaw?)vj2b=wbnKG7`EO>b+|!|P8YzN*`2g%!Hi%h$wgnXkc;UftMLLb%VvC0O*+#B6wO^l+16IzA;A|8fsEDWX?7_&FY4?D0f%@Nph) z6&=0G3E>AGZWW#5e{=9b9=?^3x9no&-Ac$SJ=~;7{mS8a$-}p@<4<__R(AYugBMoq z(_NSw{?XO`nLQX6+Jkv;c4ANw=`PRdBoF0TTWIpE(!PZsy7Cb zl?0}bGeY|^6-}BLJg`G|ef&{rsE?+BJ=VKCBz&pDhVP^DUGtX{gHr68%_nnz0rSGU zarA{vEvrP0}{WGJ1-bn~lxL_Y{!Ux!3z6hxj52 z63%zq^rfHkehT!tG{#7GhH20mx2%|CS|Fj}f2V6&s#8B?S%K}>*?(tlbM|~nj@g+z z#ZQ5c8k#k3_hCAhL=a|NCacWNVH6Eka|VlKa>&lv2*D-vvcmpayfO++W+3Z#S{uK( zi2Z~tIm-IOTz`Y< z1y1Q1SeF-A+cU5#FK}4*!1R3AAL91^m!vYkc3a+Oe_`*lsMYS(^(L*T``wqKo?ZY` zVmnO?9-=AA@1OVF|3_V&zDnjSZ%iv;$Ez_lp^fePQ;o!&6pL^`S?JCP8@FPq9^1&z zC{oUaL+9z!W-%0kR*oq({pVR_0Q4v(Xq|m=icO^6cc5t7H>)flwCr+Pt_M!gsfR zJ6J%+KJuz~@7dQiN3k7eeu~44qq#e`RZJVpdhZpza=s1Kv#UQL9`EP9gDk@XTs*Kp zT?`EqgG^u9^{)APhEwwOIV-1r!Cy(X^?u!OJw5^rJ8rfJXrZ-_)^N?=US+TuZZGej zDf`s?5o+>G0Z#XTt-QGDKBYJW@&d&-j6d~ovC|_c5dzjv?ur>p##;nWdGC(uHrB7W zn`}$lhG}UVD~mdrL@TzMtF^DO>}8-oP}2D7sOw7>tjuqCSL4DJ;xSA8JslFRr{i5o z=^I8(p7ZcJAAd?N{-YjV<>9qC_-{Ns>fyt3@aZ02;NitN_)k5&Q`(ufL)1JyCwjP4 z9QaZ_j9(QVt{MS+b`Czo!)FR$f@sHAq^g%;>BBYZo2}^C)$R2M`QcmJneaYAj6{lE zy3(~tmq&_Q+f#jmf5=MMcxqP)zkXfgSFD75vwl}8_|u^yFQ z61rzZZ(4VN(;TqGj9IRVm+KFVCE33n19}|IYmC<5IR4Y3S6N%Jgbnm+s6g9DLAfyE z;;Q1vfYHSn#!<)MWaQtMw1EiS?3M$dDf_~-JPWj`N>`Ob{fJnKi@+>;cwr=&M4B$d zjOr>d0`exr^X;X3uu!F_T<{o6h{lxmal7kVUJVUgB$1LGcH7w0ut1a1>jzj4Cp$M4 ztdvEYC#*HpD*w0p0b-=hkDzj*R||1mKRk;NXEc&{3r6VhF9qp5Dn0GmW7#5o=%+?o zXHaf>hong6o_mx?v*OGj_VYAG5|0Q~TI&oTqsqOc$k0+{XH5>dP6LD^PTID=gs1)F+RCG|g!* z|2$G$xB;h7cYmZu4f{MYU{o~L##$aC=wkvOs#$PqlsRp&Sx=vbzK3SJ|LzP z$!kmWNM9>|mahpss*|U3UJN?Vab9+D2RtouZ(6Nn4TA}A`cfiB62FGE79}-L05^$L zh`;H|5UxYV!=UYZn`4CwHCBuIQM`{(C8v1qGg2zgvy?_+eEBVtXR%wV7X;kG)23p^r_L*I%L)|Se_E#= zmGy9)3m=NAQN&tEp^ED8T)lRcJ%ZG0QkAnyL@~|1&e+{Da?9<*lps8!f233pof`&# zGj7lz&e;&EnEG__XE~$8J||dDuW-{NWPqMaO1i?wChp}+rV39~Hq;#@mN2PmpWvhF zv)F$w3_m41x0f&f%$u$Jl?y=Z&hACwJU8pO*wsSHTOS0MvC(b*K%D7t_i8|tS~1Ms zVMC(`8lI#;QIniK2kc!u*j^|p{D^NV{0Ng5fmZlA>Nx+q!tblXbBES=4h2*RzmXB? zLdU$nO5F&HV8Rv5@9J}w|J*M8^zBFCd?x*>B^);Z&jA_C+$b5OD<8y6x$zQujv@0b zr2Mac&M{>5`<@}UYJ?JojBqI%tArtT%S^g50UDSx4|2z+{-lT7zzxP`aO}+Gkz8<; zuMlfY1u#O!lLZp{zvD?M_^Yv|(Ge6cpCNj02Uf=aJS}-};8t1^8|)c00^;D;>r6O{ zUuFLDXZvOuwAR4X~W_6YG9I~sb_H0E{$y9+cLh!VeRqk@W(qyJ%%MHM1gN29x^6@|m@+~535v3rP@ znj$%`sU552QuVf}p8J4Q-&vt5P96ZR3>>Y#OU4R$*#L8;g&Z zWob+Ge`y!kRw3U&Y61qy7D+|nYHS0 zUsx7ne2tPN)-R|>-aYuCT9ykxk;#gG7e02=R^d7M`}KBrZW1#pcZC`5(+-hp&dFkT z2o!4Jb9ivxd0!LHuH>bTuT97!yMA~%JQ;BTI`KFV^77z5*f?_n|88J97b338a|+)s2KUKVfy;d5tG?W1ycx!f81S3Iev%X5L&zyOH2OZVQi{^k<>|v=g7Fdx#`K2@yiZpP5Ynr=*iNDEt@%Ap!jq zd*W+P!KtAs;B_f~qjy~^>9W3v)xc16uvatj5TiriU z>CCNp<1cyrQN~o_t5|NGcQoJxPP{wz(=a0y|DrG8Z0b}RH5e`j--f0qY<nGHpr^Atf?uA|4;r__~GcYUjFBOUkBuT)}4>2P1lJbIUgF3~g* z&Z0K=8B?mURJ;~6cB&$7L#+|%qYjGS*#T@2dCksnD)%U;37BuJdi3` z#oG-GUL-^e^;S2DvnDO-MnbULe~qYSEtW4WsOb%TUOjYU^)nlLBWzTf0&z-zI399* zYM1rZsZmAi8ZoHsZAxfwlkNA>g);3msp_g~E|f`)WW=Kn3iApDh$KF;NUY{ZX&5@p zTYnu3?`K>@3-o)9TcCdaGxZc>Sn& zT^H@t?;Sg0Xgym^$za#~@e||O5tBD?PzI{_X?4X}(MVz(MbfAYjODHpu1ZXv#r5!u zipE#0YdAXxJxnS~wm?M;kKPTq5}|@$!c@ z$zM`C!sJg3G`WGZd3FhfpZH0a@c)@-FI2jyJBAtKU}rT?Ts|3GD|4xrD+P&v!U+?f z;G%;*I2K4Zd2t1)9hjWcrz8symn!?R*fSe?tISPSRMfwu$Vw##@QbywaU9L5c)gaY zNNi5i-j26pL%)bsyu;?JV9VJ-zam> zC?*Rl%c{{BmMT}L2bXDJlD0>|mtg8FSeceoCs7*2Oy;zG(V)M%07v5HZ%}P0xs1sm zqT$k#h;M%c=F@!Yhw{)1x%{o+&&A>d7DN&U0gXM=iK)@x0Nc%W1K^J0fKJS`w=2m$ z$O6Og`U^zGZuD9wf~==tXVJdrPB@ISXgPO*Equg#G>rGU2=~>r)!iYWB{D;F=f9Kz z^M#7T)%71>=8}KFZ77vbCfNdY=aVS&oIYV+tDB(DJG?&SwHNVK7~#CeA~w7w@_6CF z__>J|mVrvWFw%$ky*uf`Ww?`(rrCX@SBoZev(x$Y8NZQ;?w(IK)7h_ovL;>mocz$X z!rORIGxT~huln)Y9KN;{{>t7OkFH5pp3f^`iLPkVyIz0U^*Swlb(21(rKC$5#WRZI zh0@l_e|Y2=@}L}L#u;1h=#E1~ueKt>$|I-*BV7I{9)<`jCGbd8)u% zMTcQxFC<;)8V+e;6B5kB8l%GiG{$!e)bX!q!Lk~$71Mjv8$P? zxj;Z7Qyx*tt1Z5nF!<_qX(UttQ2u zvo{-F8K1htkk%x3t4?}6H6)8yU@%6L*|EtTxTCE-|NLQ&hNHMlFq8!EVv_Ku7MLnl zS9~QyR!=+rfY~|TWFeJ@z&51yNr6 znrsj1VN`XR%Y=y{IqIojb@H=dFk$YARnRM|T6wBGX)NbivD6P--5P<{WLWM`7`8Hd zzshTFEJ2x<(D-xjM7@{e89i-{mxgfIuM`maC|2Ix$(vNzhK!|xE`^~ZqeXPAl+oeJ zXg#b6Gg|DnQv{6Bk1Z(^<81vdqBFJ3B6YaudAhNNrSLLgNmZWDgR%JR;W|YsPO9=m z1-2EQ%|kz4j|*RY?`*mdrYkQ2=mx5SQt{JP&4h77#+4(GX$cXz)qe-c$Rc<%c z{EEeqCc2d5eir2qgJ^19=3`|}!->Ci5c?T5wXcn?le{Hr_=Fiz0=Lu;^!^y<32jj3 zx$}g^GR<~%k)0QWuN|HjR4xVkmh*xJ`#HeiQP4EI7gj2!jBGhU#(>)`;^qSdy}&c;u526|dsc@#$TP*|Ly20rU7gS1DbFpD`f} zuE7x@D5jvx#<(rU(VjDiNB-9{h-1{m=m4BHcL*}f%w;}2H=nqOHP(LpWh}LDmW(n> z=k%~i*vCxf^QU_F%pCj<51-)SUF*gY2A?~^;PcCPxSJl9falvpICo#>P@L{jw4=Rb z*{O$RInu-Dcz8<=KGegTJ$!Z!KET7Lc(^Kq$MdDhcI#aL?|S$!AHOOW|2YpY^6-)z z{80~g5*w>BO?oZ~CB~mVS+U8fBKv zyb&T6PNk$Zsq(M-QD6Vbh9+8{;m?H!GfQXGU;hglxUhuSZZ0vXd?j>W8~+vM*g{el z=s(l2sQF}CUmmO*QzJFH1U?Oalc@l59?a1>Yix_^$g_hxK7#tZ{y^4`Yeh%fxd{bx zr!;sCYups#%810o1j1*3AQJE#v}Lr&-Guy~Rw4M7m3*1t47bI$x8nQUq(Wa)oacEaO3MJ!^r~OAN@|+A0E>dFax@=ssQK$C^S$dLO>2}{s&{;B5#19T zRbqF(oU~&s;ss?IW{4Q;c9vPY_wMA8g&&~^P z$$Re%--)sz4_p9K3|%r&TEx5V-zwIVcqL(&ms9L@@B5e4(36SAD!#7YM@MMxY>e1l zj4+PT;#vQw&<#=o?zSyc2|lL9Q+ERe{U}SW`x9km6oWp*v=Qc_NUtJ8Io1PLWmx?# z?$vMt#7uvoTg<11sty1%+177ooey`QGzL=8Oy=_~iI;Y6byB1$DxpqaB*n-oBd42W z`M`72C__P8lDUmnXl6* z?us)381Pl>d!fj2$GnG>>3GiL!z!74uCa{S6dw6$6ot9E`03pU(I~3>vlNF~L5*S1 z{hqI)1zWgzLCaz|y@r*X;&~+TW3qI2YyKoPIvUii^be%C~mv}shQ`MITi1hlqB;WZy833T65}rSi2l1Y5CV0 zmh4i59Szs&`bSjVd-0L^Q&#PTZXv@2nJ0zr1^rnNuIFWqS5-_0^;)yB9eGMk}8c@}-5hvY2!juV3zpsc?2q^IkAXk>a+p4Fn;Zz~l4 zKi=L2EY9Nk<6jbpm_!z>vHZNQ7At6sphh7V1p{$cmxP+6ru71ZU`Qo^uq4D=v4Cb> zx5d_YskKej*7T>gX=`k%M#+`fi$vK) z=fCfRRZN&~j1aq>?Hb)sEWo|@n2Orv90Mwy64P(f@A@_^yk)bggJjO35OVx^BNfud zEB;ggoHjTX?5SFsB9Zw{Ap9bxY|JV;rsFZwns(4Xhi+`dcGl?&$d=yd6M9ZLO{i-x z!S<1nVw|65kNFEs7TcN9~!hbj5 zIMb*-7;QDR(g-dE1&|K@=3&Qr+j)2rnKTrloLwEB0Tdnl&GYNqt$dTT+;5+?CPJ4% z5?qh0J@%X`ata>ZOep?L%^hL-63f8g2v(SGUx5PPrZ2NJOHH=SD2*R7C zvxKvX??OagET2vdFzkKmF62X&GuL$PPLa#ge=@wqDJRN#LP9*I=wA@6Es;qgYK7%dthkXj8&xF3$%592u10<^@>PV1{wKUL5QRjdk4w%FU#oayYjvK}N7+QB zzl3n?vt|?wvzmJy#^F>qqWEEhTYV66YX&+HdWc6JAutUZhQf#i1um56+b)!m2};@J z6vFenP+ve0)A!tQ(y1QGH=DHxjT=<*4D8*>m5WxjI2exm3w5HivFv((woywMwMOBZ zC-5FSFZ=kOTzu+e>bdFtu|!eb*jKc~v|j~41}re43{D|?i@oYRg1=VnRX3%2d+voL zLKj9V4$}jz@b=sVu7c5a+KA=s42BI&o|mmLcIqj#>CsA`#@~t=K4w>y1!yP*#FeZ- ziPz9fAC?A$x1IM-HrZ;6pI$MemJJ=P!BZL<8K{kFHlb-gVW&mIAP5*myV(Q(YetfO z=lV;xd2i|jW~n_#yewMY5Xkr)?xc)bw#)S zQ{9kx(f{P3=QnKH%NxDRp>bkNk*54w-KRRJH1u_2RTSDTjeC2KqdQ~k#!wFHTiD5@ zVOf{lTdXG?Zn=T(&AHYW_ZIh(Ib@^jXfj}3b?qW@c=D}nTxZckQ*>y+3Rt;HwAGl!=WTgMyjD=r2x@m{CJK3>v?cm z_o{UmhA^Waqgu)3jNq#ZFUq!1y()GZQ5Dv=&TNx03Y7S75{aOvM~6>A-~3f(cU?Ow zaa!oFs&zHdH0QHw^haB>oUywoZk?sl#Mk|&Sio2q&Gg=QJxG^{{3aGgVG%0NInv12 z+f4bou6>moKgvkH`}6yGN9gBtE4jH5{!T5&YXCA3Mg2R4a2T}dM@+4`>d&yQG`h`= z!}S^uY?;KzLmJiwQrKe>#&U>B8}E?{JV_B%et&e03|7u#2#rAGpg`&Q+1{ReU}9p6 zjlsx5$g6Y^Ct%s8(TOj_ed+UI zXDaNA6;lq-2(|lNnx@OPnOd0lsTR|1hmla$+EwnCHK{Lq8Qqe$jaCVcxvynLJ+YZz zG;!Ps3YkO49V={=9Z}ORcLvV+z2Xy?XHk3}7kN2G)wXXvv#185s^5SH=M% zsXwE-1Wq}~OtfH{SrQh##pa4cKJ!5BWYE@`5CM;X8A{h zEH=P#l3N)qk)ErzN&62(gP5|TVorseRq_&t(;Z%94w9i^)AyIB|9Sg{LMME(ecSeH zpC9-HmVq$?u&N>_0a>WXolgf?v#_v1k{8KGL9E^9yzr7JTW_T=bi7PgsVZ>&lJl7P zVo1ZRZClnJU>77PK(SxOa{c+au^)eE*B2;EMcfg8+}fWR4(4}|mTb$sH{An!(V}gc zL3Eya(c5#Q=Gq$u$A|>W(41md5@GXeF+?Nmj5pOI?YXg~?~mY5yvg1b-5-T>>cts<1l35d7MZx-sE zl>@R4!2IUx6J6X23{K>EEUdYJ;9Rvs(cn{BH1?F}8VNGzYE#gJcV72_waMch0;J{( zri@LoOCUDqSI;Wvrr080(pyAa&>V9053xJrn~@a$8$s1e-d^_JC1U0u3pJ*UQ&k3s z_?*dm>|7IROglRm7BN;tOd>>&!Z;@pCp{uJ!-I5#xwmIEADs|2#=ftoH-u@e#%!_E zfg`slZ#?#xEyMYm={^Wr)Bm1`vYf5VH|V>2V+}iI&E4q7ns!_xQeeeYL^xham-9sV z@z=PIaCm=|mR?@OgV%*<@u#4%?WK=N6Go!qsZwzzmfsWxl|KVSldEVhgIJAd}54g6y0VZg)$k z6?5nkJBNDGkYC$C~3;FEX2H#bsUNX~l`CcFfoYN#*=o5jdOiScoza-#Y|>~02ry-U7!44prbqEzFy%A8IA1%HvVcizfaM37ZmVem(>k;wG1 zZ&Io(kmlsRZ0`}mGJLf#%Zd9u4B@Mu!f4kGUngsk<-gxoRwUQ$uWEEp{s9SP`EYL6 zuw>Na+^(7|<4mdb9WB~oO2?086$vP-aRd!wr&~8ABO$P2P}6hlWT-lE`Md; z-6e&^PpVPLC521uo4ce?W#6#kYVqM*y#dJ`LQ{15&pWCA@ohpGQXTvcYZ>!Q4VpiV zGbMmwcPd6PC5AS!uil0TUWhz4ll?8u&@WgHc~lD3?E`2Ix2oTH3G<>_kKcgThz6Km zHY-JoNwFp7qYN!4aSMwXfDiC77=9~$lZ56s^skS9&LCjNCK>rZ;GCfXCb?C7kr#6d zl4yCcz8r3gb{Gw}c74^y^cwCEHMTQ*hcZ^OC-}0WTmK`y z{rnGi>p$KM+Z2ceXXZQ0=_r46?RN9!aRtuOr-_0CX{{&IIAC^oSr{lt8^O-=kb(6G z+u|%WQL?L%bbKw%YfZ4UN_%}*;jo;efFR^b00kaHHW#3XxNz?TX`Oeb_hA#hC z{~pmlwzq7F<7B=i{%rbu$J*=7`i+)u-jm1o4R79wd03lOBvQ>PP8um;S?6nUzW}|S z>Z&5AA?fBbZaD3f^8hqnar!GGLTi!26b`<@aIE-!=bwD6inRF1Aj7xG@Up?fZAMvC z2GyC$u$~MOJg8ffkM0nG?s4uy5LyuznYDwOXyQT|d7?fvz%6RijgNJKXY^AG0tuj$hScvRd^Kh%1-Gph}ZJAold@mO~ z6(sTW3y~R!)sa@hlk}Pto>mHp(L?ri3Fxc`9!(^SBZ>i>GgVNUz=D3V>$&E87R`>} zudpY-ix195<)!>S^*R21QlTV^^wG?E4I(iK`VB(y$m9dAmzYa#vEHDVg3U=}=xWOB zExg4xl{_lcduWeI7dp&63g<5g#-^=6nxwrl+HyQahc~LV7 zAybWH*t^r}rd?&yK9Vo?qvZ36XV1PoF(YM8RpgzUrE!?ph?usmsLcW zkkq|bhzc!I`6jhloa<93c7KovoyLgcMB~DZD7@t1#%I6x{O6o3!$c@M{?q;FOKvap z?bggZTX{UXpFPTSp0Q$s@sDYloUpRh%+{yWOt3I4#lD0T^II~)hja3e;HD^Q4>02V z2-@&Q@b%$(+aZtyy05WmQ1Jcf1%F-zL(4w{XU=a}1}9nkujZd!IR41Wq66%3kdsFE z)kL30AVa?Q4?X~pyG15dW#f#N^t(&Q^9CTR9nj5neklX=HD@Z#>N*yQZB%|q5Y z_aPht>S9xVk4D69XPqG*w@}ndX#EwZn{wkXSoxgthSK?a-ET63;Gk~KAcIMI&0eC- z+^A^%kNgn~>cpRf>q-Icz+gg*1Y_)py6RicxS?qOk;&7?`DXDK()%~!cGShQBeZQ3eno=@GnDIN!*#}AqwUE%KKjbThvz?zF!ID-GWKHUZb{DG{)+G$ zp;w(}ARizI;WM?%x#3Au>BD;7qF#r_oQFu^kDQsWiSX+VTeFFN9dC$7?tr)VI3Ej} zusFWmON_Gqh8)A=UGxoc;~kTxQZZGPFVg<_DR^NRKMNoKEd0C=j~&KOp6JWP&uLZR ziEe{Ym&-f{z|Y0$_;Dfn@2?s*P4@4a)WN=W3i;M@D z#>Ve(hUill87nohCF^?8$O!%OzraV{j$wSv#r*l{^zsiRz%V{`3e5i;AD526$2An? zER~RUsBjBOzWZE%-UV~95NxSsD*B|^&0hHEA%0qN6U-C8@Io~Jp*?COTWY7+ckOJ zgBd-{^lvd-uLWlCR_J)~9S;ti=7w3lyt>XU<4Z-R1kUMi*SPbKF3;=oal8=IL~A;f zx(f?3`U%_I9y23g#N4I~j4`*t_X9S26|73r%WLwyk7kasu=Iw623=K+w2idpIt|nw zznk=3bw&RMD_iJJGalXX5jJ}q*v|F{I-MG`S&flT@!dS6@h@asH4Y%1>YXgwr7NA; z>=425RJtj2YA}jpHM~0||5;v*t}{CIS74!glP}1PrmvB!R^_mxr8CcI1H`WB zr%=BqRGQo*k>$KXpFi0pXh7ok$*#rTgS8|4D%#iwCbC`m=&naySNkMIa9#pw?wIb?OxSaSV)eB>>LMcW%SD!3rn+Qi7kl2CZ4W^Ot@Q8Xb%rGxk32a4<$KCrj((G^cI$VF`=pp0uwoyGE85N97Md^WjgiwzB zO(N9WWAGjRL)@D#51F;7e-C!PU<7`s#rADznff(g)_^0QAiX;t9qw1*`XpG5`jx(T zv1585as-8dC4b~W7AN96)85fuil3hoR6D{$ z?3H7yEasNz!WW_8qw~#y0~-8+$Wd&>tqQ-)KG*Dc;c$PsYHX)?k(~0G_@FJhJ8nDb z{H4!u-bCpR`YyCk9d`tqHI>lp$knuD?u(*Arf*0Xe7?q7%WT$*66<)`wdPebH%Y&dG9D;j?Pz8FWCIt)GQ2rs@cZZ)@&$0?_tQHsVxEedv;rT0a;Z>*su=Vg|+15Zn zDZ`WfD0wGI&{>b@0&I%+zBx3@=PwY7<&+=haLWw%(lD89ya>gdko7}*RUd4d^ycy7n z=+dD334h!NTCP}Z({RGwT1b_*W5S6eQ;jfwUL~>;3ud+&QKBtAMw6CKma)+(HVZvi zmUqc?<(vmWO8UTZc4IIReOC6o=I!~efy_2V29=tK!L}3PjMSTiCG1Tet*o`Ht=0!s`ap z2)MU|w4&dc#|EQOA1{IMHhbOn0BSp()jc>g(5nwVsvGFlc90^T{DsI6`kWEMDDTx= z!XND|GdW%z5@CgVi+FRsNGF0eV}`=m6@VT1>$m$eUUnY(w`gI#AfT(Jr3_jJ`=bp* zmy}=1nd61fPpbb@@~->M*0d(zRHs=5150{n)+k`ZVR>1(^~`!n6d0Xca)n5hne!&+ zM6k-Oke6@tK|GU}DS$zrYZyVr2>p(`ZIMm1u#8pQgYLmoE%GE2zY0{AoF5SMajI#CiYFa&Si!- z8mMI8{-${assJlZb{ocSRYY@rW{wuU&@>j4+io3fD1KpcjQ=`$yX9kK;27A-+d5fNloy}8{KZEI$<{?@@ zMLDR5-$}$!gFv4CLu49O;NpJcn(n;)K+-X9r#d@2`A}q?+x=hs3B`US|Fhryule?{ zE#h)pgf%4poC2gB8uM$iA(t9v#{8ECS>n3dmOjpwt|7Z^H5nrlGWvZYt|JLFu>VF* zo5OE&+)!`eQI6Yfj%V>6BZotv9BaryD0-a$Wtv+iboj&;(b;HmE3gnJ5wn3Le&+b) zS&Z0$<%)8BWGyDhbQCaW~C1A}sM(b9DXnmYd#lv_b(%t#46Teqm%P>aM z%5Qy3b2tk0?`cP>nPQGoF*5ASri>fLVG7q@BD=FR1G|3we$s5CE#eZTX{>?8KqAN= zmTZe=z5N(Sj&g#WWb%AxoUp`eEtN1(oKtr5&OE2qAAwx83fk#?^s*u|cYc4 zUKjhG(5aO+QL5?Rii>%TF%c60HaY(D6`1< z7_!${Idk)UZ1q%gN!MA0)3d1*iOqP)#5%1>ePr4WPOCA~2^~-ujC{Td>p~#>N+5Cp zGpRo?FYA*nLoE9>Dz|eov;7Wc!<-Dez;?H&dK8Ua&!X5F+;hGW*<8GjqO+37^Y&f{cS*yq+Wmf65k{!X`JmqP z_DthR%}VRbo-++o@^o*@Tv1tMuFCWthvewqmd$&zbaRy^v#R0ptHPfb%33){2dQ0R z+;!6RYojBBUK}KUFKWS|{U?lofOccMyAbJ!k^n2UT{K+B*C^D_Aey_tS^Be)l27Kw z63J=~Cwksk-kM&h&+PQXPD zD}E(agFpOvafNV6$I7xE1T1idj+V)T!VXUsrpo}ha3M1ghK`JeH`a|T~Mg92y>r`^{$YvncXj)a7*H#wTV^-#;nUjN* zhLeLf#J-Jy&?r%afq1Tz09%Nh7`f1$V*#nYh*b^k3kA|m=D^tD{&v5_egL^t;SI8i zvean`^;2U?VyE)eDZgJ-JK!w;q1ZE}25@B1&~p|cuEZY`5)-i7i%Fp+{%3#8?BxRY z&jZs`w;XEZ?0y+-5*|gC%X*%(Um?T)A(lPHlq?E zOxBht!9Z^|2?X=j-dsJAvq&Gpy{0&Az8UB(NTn&VX?j(xQ^~(7x;fVvC>Bv~fX>1# z%5F@3>(@RRbf3+(8&j)bNkxXEZYM_LrG(C+qHWY?cCtQUGd!aVW!lLa?V<*jS`8?X zl>(J3lC_7*?C5>gACC43SWQoQmDV}^U&>%(BWndEOSA%?99Y7aYju{itCV((()PYm zv0{Bcufg!c+mtSR&vpXNay%F`c26&`FkJvl{)hopdv}82wL6vAz(2@S2v%jk3UK=S z>6NB%jFxom6TAeY>xU4=^l&h|ehpANW9}76+~Xnp*4~|HU}W6o`0i8e%6{J2;;@aL z!6CpO{v(b~S)!%LJwB763mwp8*lmhg*~mK?Tt_FH;RVVv3&m&)-ku*Dw7VVjIocLn zrw-~lmUy!x3APQC+l}(RQXm3sqAxd#rYE-zzs+iP*LCa2b+f_9J=M&Zv=u|jdH9~y zYS*kMO-|>mUx9*%ZR^!5H0px+wbBIsf(z9DB>a2+#f!n<3bJ{7m*QSSdJ(R3vXN}P z`T)a3wD<1h(H}jg_)>xagKFZLj`#NS!8}Hyq9l5Eok`leodB<-d(3MxHmk3lx{h!o zZ4kX@2$Yi4=1eB-ttA#Gr9p4uJ+-!Ed%0C_86Uhy{740R3x8B?zjY~y_72)NzkM6B zZ)F7WmehPF7J01zMRI$0f*C{Zuz-|J!qVt!5_)?!Wx`xa*P4oZ zm!gXJyG{F1I!x#GglqxRm0p4*qeML6SU*}^erl~{f^tpEK}SctOnbw zA>c$!g0-u_BI$%vupF&5Rjl1^UsqNWdJCiK8E@fA&mQu<_pUOm?Xvm0NZ|a9sH*V4 zg2bBahA_aI3Z-nnBE$botS#fA{NlvgHH345;kTXT_h>AwS5M-!4cp(0T+%^O)rL{` zx&_p#u9W-Tnf9B2k(^2Tmf_mBOG|{kDxE^r$l*+hRi-APM)S)_sro%sIkad$&=Cp?-z@xT>)CX%i=C!@Z7RjAVWF zmp%`6gYA4F7$gWr8qovGPyw%V;CaF?$vXu|#x8UQ=iKj`stTwIgBDZUeq4KdO}a?d zVT$e6#ek*;=Wk*@QS^EXt}{xR$@Q4D6V5)c@-MQ|X204|9nuWHibf1a+hm%x-n4od zMRo7ZqQ!c}J|?BPd-?;k0OjY#y?PDzw(>1R0|%U&zNh7QZ|`aFr$F@4z3>T%&$1n~ zqr@}ZrK8KU`?x2bw>&y$SDu1moH+3FiV(4|@-U)oy zbWE!=WC%a2)Rtam=rr8BS84uE51`~aj2}Y}E(s=yb-VpmWb^q(e7o9yE3w}^Bfed3zEy5Zo3M=)u|*qpVyh~@OgB?rHo{(Rq-9E< z=`CWcX=;DsFQpHm<;0F9Ce5BXBy}b>kx089;YwS>tUjSiU>(xf?e5PW(xoWu$pq-w zgEn&ybj7?h@~8+bn6cJ;dlr>;#C#;O%-8))rqe)w#_lmD%>k#A5#Wvn z@-kmryV_#rP6Fp)5E2WEWw`6rX1nrVfWV#Kvr(n=XJ#ZFIfwC42rSMDI9lurcK+n* zLI-Ht`@+1{l+;27>Hq~XEY|ZOrfIwAL&)5Q@2SyUVt!L``g)tbujac^K4UitbSuMY z$RHpf`5XnV^~|VzW)yr5Cx2_Mx$7>CJ2({G%7NAI)QjV790jc4p=jU zpR($nDrORX)m3|(%h#&=6VA+^s%ES2-28R(JgDXxD*tuFL=02;Y7>#9b z25Thwt;>FMseFz7c0ek>&3-!|m3QhJ`eh%$!*4#gQ*D#O5o@^4-Lw?YQzN*Xj&Q~R zp>^54_70{b5Ae_EC)mFqk<{H2-yrNHRl}EV78+iS)sWX<&B|ir z6Z<-v6f1YD`jQDb=XV~=%?P_dcuz|D^-NIzn!{-&yQsqX4Y|1wO-{n}{{ec!+jA6k zIwwFz#?yn^Nte_NxSrL}O#1b#2i6Y$N&^YWwKWL0ap_MGHIaAJ2vdJklu;jV1GgW<9e12?X^e{m)Q5&e3oP z`ln96^FcI|o^QiCc=^Krvbk+J4@a6<>trT-RavbRE$j+qem&%%X4P-Un@ARwrC)Q} zX^&IqMo?Q)CCYp|B^j*||HhS-i0&5q*J13FXl$TBAwS(fcw3zf?sEgv1m+KOKE_AJuNI=V(b&YIuoy{S<;PITIfAkFAHkK8IM z{m`sEGc=udzr%ZU4%QPa`~x#`l}`Q+E!c>HSpAddSNkJ)ryOdGHLR_GkPqk*SyRs1_2{)K=nO+Gkd3im>4f!srXVckhBI^GKg56;%LJ>b@}liU$-UlHoEQBB zqr=(rE$9bU@Dw3i)C!zOVSo%5XO;=MkB<*BVPOxG2LE{|P=d)z=-lk4J;6#|rz>)@T_Fy{<|k zqd3L{vAhA|FW<|38y&CuqYT9GAN_?J`!A#Ws$f$9mCcEAkI~_w}un)4Z8T<-}dF-h-F z@4v0yn@&_;ztg`qmqbm&4LvsAM$dhX`{!p~CQT~(HdE}kw&p;0CcAjHo z3@>W!cIx`px2Q>Uv-`5z9#O% z*^prElTMLeO{-R_bHwU|;X5ToT3{)C^Z^uf6Ew`*uIVZr3din!l{1naoHs%(DJ0w|HgE;XOX=+@b zG^0q12L(EDHc8gz+~r49m2rS!b;j0Nh?!XJ6OZiiST$4a1jVaXbItJD5E~x`4)a|rimtte>5k3Hqlgb2X$`z zE{z1!nixJJNbf6Ck|@YZPeE*vm>E2$B&?x!Wr1@VLSJHN07{lZe;gk~>34ui=P>u< z|GFRVrTOlDZhL+v%treFA}zW8sV%;+&SMKP6ok&m`P>~{Y*dW041@=qqP%pj- zF$)I)IsAsd@NG`(-(C^ktIcA)&tK26obk%=-vVO(`gN5X-prVBGRZfeWI4&vyU8 z&v8^+mktf^R&FTo{^(qGAcYo|cW=(92rrUbH@gLr^L1%)Y4kwuS<-MR=8Xvhy8pv5vNDEn3eU}{wVSHZ)oX7cq7uhkM%J7Hg zYML=tpF}tW!!H#-Uv9WW{3ur*MlaxY-`_^RIx4UOalJc()xt_{ywSaFo9MT|!K{S+ z(Z6Sh|M;DO-t2(4a#Qz*!hQ!PubB)@P2{&q(;{!V*$8F+bv#b8K$ue#U!k!q#Rm_5 zIk0C(@c5lwj9GQ##`4IQ8Ovh=k(s%{!a-!*#$e$e;QiMx(z3fJ{&~~h^6r6*?th;f zn%TVx#r&VRiUuONKm-{=Kj9CcmB0?Pn&2?fQl9NqF16kav31K~;K12fe{p7QYJd4Y`GRpFVrRpIN=Oh=HJwdVw* zGbu23r20_(Lz~mkV0DY=^fP9_#IJ&hqfabj_$~8AX6J=wbboMe=-6m^M)-X8#`H7V z6aF3j{^Q4_`v*{divBPmk4OfkLv}+_`G$(0j^$$S`TttK2=+ow|L^sSQ%&~&Q@>c+ z^9lW8lVN)Q|LYfa|JV(0u7Ci7kuO@0OT00S5ad`JU%D>}OVe6Ztz;SWH3>hwIR8dI zt-GK&;~<~z-?@P=#=n!P!mxkmI7umS#|4J{E@S~bymi>OZmt%@WOsbU&b(*es46Fy!7X7u$PMRX6h&sq|EvM<`h?f`6K9rM%q?F z9FfWYzLOw2A#<~OgqcYt_GGF}&d-PGBYwN|CFdGG#_vydo*=X2{~d7}I`2Wgr`rcm zk5&r=Lr;i8)APHX*G$v!eje!z?2^Ut2{B!T!x3+7OktmJ!c_J>*>dl-P^mIWh|zj606Q3 zmoWB9o8V8B-LVl%Qfn~c(?Vla>6_l3>mh?6w`0T)P4*`p#;3oBF2=-aDmpx$+XZ9g zPXM1M_8k94&JW+8!G1n$THanUR(}M)hu<2~-k}DCD|6kPceptPM>>Cb|As!kWOJA- zMlRcwz||U@hkxU={f1=p8Geq^-9m>j=|jh7*>N=p zFbfE~o!_&a67yZ}G?CTRW4?U5ESvhCY%k^qF@gChR$CDs^!D~;QbPBBx7&6BD6yLu zYTIIY6lm!EN@@V@@n5_827Q``M}^uMWCs}7_C9*6+Q`N$=~3aW{tf#^YZpTI-fUdm z1Z(%c4DXE>fiUh%#l*n_yK|Saj|UrD{K%2|GNc(O#nAo*Qn3LkFm;}%;|*~o@Wa71 z=3s-Dbf6-qcboH6cH16s?tKbQn`^*M<7et`!OB&tc0XZNWFkhoV%DHN&aP|CwEC6! z4`^JEnokd(#}wsigXo?z*5WYxhDXN^GaMXAB%iP%XJ%9w;I6VI8uhYBZ}Ijl*2*0x z7Rcnd)4ykr<(9g^aeFX2(O6E^eLZ1O4tq7&6&XK1dEacvL|!FLmVId4qbE9rYJV&T?HZ&;q% zBgmTanSWhIhN?&dupyOnGF5?d4ieu;f8iZ*v`|+PFdQWt%7iZGcdVV=u?Ev>VVhhmrJ!20SI~cyu5%pV4s#-$23wuzmZ*YYxQKl- zJ_hpWjM&xcB22ujqv0-0D(r5Y|4ia4?UCRn0oUNIQF-Qb=%ji0JQl{+J2x+vkQyC( zR$gLQzmH&u8vG1YntsD^H?{+^c5b~!3AxfF;hZ&(+7PCfBRw>!e6&;T0Qrzpk+Zrh zx%4uN0gQ0})?}gH2=|X95~Eowqdw?(^0BuPL?@i|go?Am{XwAwfjOE&RK+C}EpyNg z7Vw-|CejGhcX*7o0;>}L0XWRQJ*_82PfgrHv05#}_Q%Kqu}Gj1va*1KV=|soW&zC= zOG*dy3^dWHk3IuFeSy-7;Fb>OIY3pRbJa{zPoDE#fewqgpc_CV$6V9LyS68uD(*se z7Nl5u2m6~gm)P|qyPt4ZioVpuyk)2?PdmjBxrP7S6#V{S_)+?DN6syDTF^A3i5P|5 zHi9{G69CePf=@Q$N!H7;2$)u3p@c5#{Zq3F0NqYZc=_cw_3|Ur{yDz)M~wPp5BXUQ7pLOB65*m4-@EMPJl@*!Fg}Ik20J$p#W>F0VO)v$4)4`y$`E~ zZXND3!hgb)9i4p6nfRuUH$YsS_s);KcWmr>8KH!a3_TwI=`jD{Yz%{#(07a90?zk< zUOiNxt|(j4*cdv8?udUQg>N@~M#uB928!w{?k8pisP*ZlHO%U7ljwxNS| zJJWA45;c&*haw9!O3isC1FQ%3aDo(LLEPq;ADdz7qB9uotcCtTr*|0t2kcw3bZh!H z&FJRJZ}3?V0-wJE1mv|7VH`U*YFtB&-*BE_5NHkI0Rh&+z##T+J^#{hAEc;rmy*CV z;lbJ#bn;`G-sw!7Rn<5FOMjIT-@`pHHH0z(=f;Y7a<(reqZruhP6tWVwf9ig2>nL$ zw_xDEQT)x-d)eyVPGZuBV)~9yKuT^Nl{}_z@&0#K{!aDv*xi#l zECjQBFnqt=X_4k0Zg`G1ej1+R#f5QrZ>a72aYhl@e{bkyVl`8Ni2`3XtxY)3iIYm9 z*=8E>89PHiZzfmT+vrR*oLs!n+`h60JNZ|LGCM=1MY=me-TNU>8>nu+(7`CIC1h^A;SGYJXU}X#Szn=$z{01UtvsO@9X?`E2WdEtD1L&SZ-k zef9J#hCG`Wx>SvGw?^SG{aLL+<+K{r*z8JsfO0;uAymg^wPL$accyF?W^b*#AM}SJ zkF@=ue-$b|bwB8<8`Yz!{h;qEGf-$|?Csr#yo~9eb1Afr55$Di0DaBH%7UF^v-4h+ zaFf>fEaz>vgtT_-B1+V{hxU-mxSVihR;saZD%0LyDJw(aS<>xik1_kZUSM_dJuxce z=qmsuEra7R&2e(z7mi`q96l$tH_AceY{S5GJdO=IbSDl2I>;q_A%7Sc%d*uE_O2bX z4}S#zws*B)u8IF9%|GM-hr{~2zn#b$U*h3><&vsr_ zZ@>&D`;F@xt6NsG>)0Td;cBTO+xaf%sTgVQ^zjt`zVpX6)Da0o=c#t#2|Z+!^TJ$p zz~mp@Q+sre>Jlo6Al+fHISnMkP0J=mc2t z7_$P+k*ChH7g)Fk6h=GHsb6=ldBeh3fNM+oW+)wfRfxx{c&+(evcBfsySm}(XYPQHQdR?Bg8*c)OKW5>exhKDiF!?;PIj?GzsyXjw zjcks=mfIp4Q%wnAKy{omwX-xU1btF&wmWv+kOk(8w!C=WA-@b%|9|NeY7I~IDxRDCqED3b_SHh&>0v{ zL6UqP3CZJPxWdy9Q`Ba={nPQ(=t5F!s(Hi`2i0Yp9(V5gneFZ$$bL{SngKKk^IXAp z3YqM=2jxxFETge|aEeUoJ?7L9;E@I0>|}bo$A|T&w}-rDClg8f)hJ_l?=i<0+vLpn zzfK~&8=taS{zbB6mhCnR?9;ts%G6L0%1kC0veP%M` z?RiMWgUd3_isS))CN_rHiTtzrpB-*Td`*7`Lmo$EB{G$cuPE$}%zG=Bu(ouKA zy5aLYX8Diu(Yi!)uCtqPP@zf4n4ZH#PzmMBqlB>!jG?BCT=Kt$RixnA<*ZoDY?6i; z`v_?GG-S8aH(#Ko9&Zow1D@EQ9a~B^D{L-WB62XR9*fXm5H=+@fR*Tk0Fza5vJEXr z7NOZd8phLNWM6rE<|a#JBqrhetn~RN{q%2r!@wk)SiB$QV}1^Fdd6(qM|8v{XK5ub zF#PWWg7fPulhgGdh=QY&`%VON*8922u&S(oTnFTl$t|`4oezmQa$3b0=i_E+Cisb5 z!xmlEoQ_e!E=#d?_oGO`GQZ(>i3$rtf%BgGnIF))O zNnL2SgGlv^bJGl(Q3zzQGl$#qi{#hE`Z|se(g`4ryZ8{BCsN`RCe5?CO-;-rJX&Z_HBk_L|#tEUHnZX#$8i zEz0#Y5byjAq?qfH)U%r{qV_Q|UCR8N(N!kDrY?<6c#qkkieF-Ra4bpkHB)uxZ%+!yivQ3&wD2ERNW3YQ4JY#%8RV15VCu-w zZY?uYCzd{HE4+`kHd>8)m(@8U)qWk`@vTRNkE}cP0x$nJB8^sFGF&o}JEO$yO%fI$?x2n&&4>al9^bUyY3R_Q*@w zR4mK+Q(08$Jgx#IoZ$Z6o_YY%dUmVewKKz{cIXdRV=&ff3H}%oIzqZIVw{5xOU)`lay~UYkbdbr1T;mg2AB$~P z8t3f*1Rq-mIYGD_$Ml#{ehN2MMJN8Ds`Nd39BZKaExeO!gOrK3ji12nIHH8jFHnnf4%BPm%(rk(^%^+EhMqUfKkIYL44f4f&X0ej zLCL5Y?R3hVqhB=!|?vQk$>$<`bVjG>6KNoabgwiNlO`Yne7zWu{I>KoG}uxGVNe{GJNKB0mM>aZV=v? z7sO+I8Y{6{50lSqWNPvA&c-?y_)AP$ObdVpv$;qMCnuoounzi{j%RfND@GHoG5|)l zz;Hf44I1L~1NaqH1WNxL;ueM;eWLuSeyxgW33W*ClaI-%GFtJ50LRd_AhVm8>(`@L(xXqKaGEe&O_Vf+*H$horqA#M}Fh;>JTI zb`0k)AcNIwO3_Cc!(*X;#_&M|&xMJ>u0mXl)m>m7G48;hYPPE8gAH=#*6ggr-<6m-^A`y*yH4bdF!|UFO`gAKdH`sq^Z#P=Il)( z)&u-nmoHe{ys)@<%9Mo-p}NIQQ>H9x2-Sp^w>H!)ZoZ;ricgQe#=3Ujl8&a(;?|}H zUoy+7Z4HYSw}%?q8W#E%)U`LXtN7%{8j4xaam5u4Z3Tr>e1T??HPS&;_g}y6V+IXv5*wWmz+}9pzX{%e*;PEVOZV%Np*Ejg;nwnbb z>q3iLntfLQZ^ObVzUGEyzSMVt+T8M~K$`XdX-&<-#r2_@8k*G}@-1lawKs%(p%!1A zPw|vq*S35j1%-$?>BLE=7ZtUiB)IM4q#CfZnCu0GsRHVO4DgpVHZ=QMmeSD03l}#p z@+DI$GrtsmS{8h*0RR>@m<9<+bK4e^9~j)Yx=@3!qd5)9$u=HOIrEwg_DEgd(oAh3 zpX&G3wJqvc($H+0QoTIX2$pO`zJmIeC9U8KOi{stjzzvDEejhAwr$V17Ye=^Egek@ zZM!X^8+@S-^|~!aRhwV3Sq1G4O;=3xO`0^R&_EyzEDkkN%hI~0#S49O=90zj?ZQN@ zlGdg`wB;P2K6OBWM^Nxr#_f)}usskq^$LZR)v(uWw z%3|8u&>3p0^EEP18rs?gEi~+cyKt-_LfheLD$Qz3jR^I7M^nQji=R)@Q4(ru3U#zK z`{>6Gh(JTbr6{q4<4te0FJ9DKN2&(jvby%M2UBEaXMKYK*VoWqU)QSf+}5%Lio2o? z{sGj<&p_YY-c-l8Pd||dwq9M2>*KrfI`xcAAhK2{*R9Mibv6dw*(1y!jOx5WS|3~oeSADV>H8sg0 zS!3zBMq;6^_`t{3lJk>xMR8vLIlAYtVo6H{y7T@5FY)8TK1r2n`p_beaLLhVjwREMVLY zC@7>6vy41~!*hoN4-ERNS8PlgJJvIHY3m6?oiVA@ zQv(UqH!iFT)p=4QBwce8Lzq>P9LHx_Hc@vBu~i}w3GvM}C;(Xwy zeqYei(ljE0%1zIZdOy+4I5K~F;(4v2m1+pbGsCcsDl?Pd-G1>@cZ6meZP+7d?PI&g z4^6t!=Qku1%sE5!J10arOFSs$XPsJf#;HZco*8u=iyG0@o6(ppgHLH1$1`#Qv&&P{ zhPic%Jax<3YuXoHS>v%4_*#&~V%#SU>*rPi*0i>@w4z-vAO5hQV=)t<=Jw$jWgUKN zu}wh)UC~hEYNQ_KM+go20eL%RN?mJfL-Rrg-jvf$1G<{?=FOa0c~Q-rz*j3_PH5mw zimjR#oExaAF287QO~vesX9Z`MS9oU6oXM3kb82SHo;5RYe$C84u+rn7T~+BhZ4zxd z4ZeBB;zi0lZ%$>6f9~AsIePhW7MQIPa?kAPUF(>uas%>+VDUogYEyPuqNM0 zNp*v+O{oHO*GaBwWF}>#gZ03>txSjG*Qcg2?H%>?4ejk$fYjxbFf6DMh?ZOV=FXYp zyRu=qQS6q~T}iJQ?I+duM%YXGnAvM;28yz$p11noXXX>Pm6D7UP3OV|Fkh3;)sAaW z!|Ljr>e|~6SXt_;YW(zkpHW#dpC!>6P1^sjwxz@CPPo+!Q~d#oNfUf#UMoFjSwr2G zOeQb|G%$g;=<--??Z56Zagv6%5o60YH#ppVl>3PU(i+m0J+P2Ie@^Q8c5p&DuA1*F zNX}u^%Vu&+z7ruXXt|}`SHK*lo{4U|@5HM+3Vr9GVl8ez^&H7dtO6synkoC>>#MpSk-)H5?~`+o_^XV-tA_Lt1n%oJ)Zo}7HDs%ypw z1qlB=HmG58I{t-b&^~!dN2sCmKS1jN^Bc6CWxJ@j)P^$*>+fhY3f6z#VV_?9>FIRv zA3%Cw0-*fjnkqHcu$#btTM+rrO2?mg)_+sToY3iKq~{0!$cfgby7~r7?*Ae6F$^R+ z|6A}qCx3NTX-W-d=$s#%eO`I6re;=6@tJeZm^r8TbfeHD$J>%6)%BJ0nwC~-uBu;p zZieN&X4qfTaz=^Ofkx<#Hpj^S|7ZQBs(*h%{{$E#3r%}V8^M#y+uG`uqjQJOW>yEI zC@wxdU2h1~ix}!*H#Hc47Khpc^%_t0469G)Aug#K;y(R!RG;d$7R^yCxd#omPe;PA zzFk}-zFlKv_HYVRFW8nDB&T&QLCv&1~e^Se`D*^jYR@?;v*9KJQ zOXfE#07j2SuUfQtDMs*QL7!!MRu?7hLq^1=6U>w;HB~YXAek{I_?l7SjOEE#mfhAE zn=tgCNqEv6gC!Jgv0+JTNDBy3xhHu>q%$^ulsv5F##k-m$ec5L?e&cfOI*X~i1Z6v zFfhyJCFKBH1*Xsi9ragYpi{NRUUE?bwnU6#vH{6V>n>}U_k~!NmD^ZsT)5M$J$0Cu zTx(4VWtdxMoN|htl>A4Q`-`>g;4VS5bcEU$W3>iKr2eo$VIsYS`G)#XP+Zg1!j@#+ z%raFaO>}KT?Xtq6;b>__7H46}*nVVy(u@fVvpUnZ+?t5Z^3Q?vKQpgL8PMGJkL>66 z4$O6Jo~5jaw6smrD7NN$c`S`c#AQ*N_06i+<@Uly15E2gCf} z^rDmwS|c3n&lzmaG=5u}vbYShkF@!GanXTF8LqGRYftgk3mV zttK2I+)P-6^Oy&&k^5!tE+(ud3=#Gb-X?BxF|HAWgY)+#_7V<#m2!d8cgem)D`DTI zz^AZ=dRUq9u?X5kSi}L6oh8+Zam3?F(n;7)_z2-DTrXZD>|@pc@bQ$- zDscs2Elz&75$3WJ#UUJIFJj3dYSBEmkx3c@11WET?pMkf;ABkU)9g0MO>k@yqgAmK-f$5-ae z!^k%VI0$PGqJF|w!gYGj#@Uu|4dKzyUq4|H;da6b(Ia6eVe7$(#A?E7oDg>s*5VeB ze>m?siA1YHyh$D+93mVf^c+e%2y+RuCjckm(S${WMTDNi@D3oXCTt|ECA^lfi|{tW zRfG=__7iR<93p&^u;_D%#P}nChj1d{8p0sq5MdKxAI_Z*5f)9Le!?!o+|SbJ0+3mCv0Ud_$R_X!o7ss2@gLC zc$ha95mplh30sT67hxY^4`JEqiNq?xTEZs?x8ps!lW>UeUBaTtwEt-8J3Eo6B^)B` zQdmO$ipR-fCt*L~yM%*;*~ie1>Ci7>8DS^kD#AMni-PzJ5!POmNR%8)I6skCL0I+` z+DkY@_!?pVC6sd<@Ze*5DdDQxL}Cr0r!JA$PS{5{L^!k{ktpyHk3VTAVSgLrfw1*z zF1AAqthtu{AzXC>ud ziANNM85e}DIL1%Q2fhgSBpiwY7vbPHsrN+QzXjZcUAKZ?!hXUh2y4F$z6i^{10CY? zJh&1(DZCxL5_-N1{DftMZxRj>7Muh=SJ6JguAc%2VL#z(glq80nOFcEKZkAzyMCES zY|}g8YlOLL>Hk8?TgNyj-2OP@=w#aU1o4Ee>&Zvh_Z09E_HUqlghPb6r;z_?_ypm0 z!rKU2e+#|{tA8*119;jFyo9R=KO$^>4tP!_{uRn4+`bEV2y0)39w$-nYw!g^{WQ$= z%eeX?Pex~M#!=%B%I?eXctE2EQu1soMb^Q?Jvq5EbMh|mj$4-9dqt468yJd7m zj_=0I@|=S1tn+h$s_ocBpdFSWkmJ61O zoV*$4(ZIBdJTH((^$Xrw;0=zuFh{ud@wbk@clo}6Fmne)O}X8pmXm7cxH?iP-=Fv^ zLnqNuEt$V0%`n_RnZj}H=t&v-0X}=&ndCj1e1qsLV@HvHnazKTV83xR`MWdE&nX){ zo2=>_s;6_#8+SDME+zdsbehw6ws6e@t_#M^0xo^mFAd*+o__#*uO`3nmO1)U;Vv4r zXFsrKj`NgJw$9?ouGp7&pXbb%=snu~P0?0#^m#eHTSQwoW|ij@b&oker+U<1a*Bwc zK_svwCfdxS4Y#3xy{oqW%$D&jmGRBdl{vm>=6N{LL(RHLjfhv&VUa^E*d-JP(mho|sDG(hB50 zDhjbc2@uQ7af?#V(6`{-M)^JHe>LRK{KBWe3vk2mj@kscGd}_D1-86A%DL?;`x0km zQeFb43u@f<(>S;N>E*SK`pyBHf8IFsC&oYJ^jwBM%X3=07y}Dv3Js-M)#K(RA3Y@` zyPrI3$P-bX1N9;J?;g`(i6DjVz_^Jd7rrwu-zzx36kUQ4BS`=^n%+>f4Juk;_DkdDs-(NjfE ztvKHS$f;_au^h<9c{BR@Pta#GpBz!%aQ_U`=LI>dMvXpu)B#KM!v+lvjn9I&T(vLp zd&Xf}AyDMt%Q)l%ldr@ZRE(D z`YF6!Y~sBAcnw$+Ol9PM^T~aQIR}wH-{$|!ak3_J^zSnd2$Sgtm3bRw?%lF4@c=qT z=IsBV%%QB&|2Xu3Wx`d!8GjCC&V6fNV%aFld>rK@eLNJ6J`~>Nj>7Ym#mH=VTd&%^ zFELsX{0`?&^V9DUCpl!wAmrVN|>g1>R5b-OL@n5&`DnoPE zF5;Qen4jWS6L$-7hor?lL7eg@>ry&hY}d12+-q)mXQk!UeE1>Kp$40u@{T_kc|}}$ zy*}bL6G!q?x-*IE-!INj+!Oo7UFw#Xo>%kkNA^qCL)@DE;_f6)@X=l7=i~qV6ttSS zU+|nc2cqyx){nXqsZl)+zG>P|zKWrdIC+h@GU6C6=BIRfi3<|P^dc2EeyqncpSWZ{ zE1i$Hxx}Tn;Y{MHiDMY1^7@HWdFgSN66a5glgw{);rl+%ncsn?r7zt^3*f(J61Rfy zo5}M}eP3p2ex$BZJ?h?cRRhXax&7q3_FwxF-&VdVx7-!BTzwzp`$zvzdtU-wMU}0+ ztLpB1Z>CEa5;BPiD3cI_BBG)Ofe@yEh=?O-+emlolczS%9_R*w0W@F`2-2WN83YH2 zOh!=%A`uWIGDH+aKn(~8h!8-L`@UVZE5l8qfB)n9*P_~l`lErjPv zcn1w1R~1f3>%%8CJf>#2q)Wo3g6uBXg8;@u3KM}dDb<5>e zm&1r8a(R$TI;C>+|HtI)`bV!_4mr2JMAshZDg@0g(}(KT6I3NxYfh%>{4uwsQ|?6G3B<$0d9tK^tF8X{_9FMlg+;a{KeqkCbm_e0mKJDSI4ukA8P8V zTFXF}wGN|f4cXqcl!@IVduY4x0`(c0lir7Ifta<;L0`7V`bb9WzIJlDN8IM-%O2-< zAsusmz7U)VJ=N;2C+x?6Ru1tQvem(Y?y^Muekf<~YT$Ys|* zJm(RgD;o#;J)oThZM`r#V^xtXTqNvCguMYBy(+{0E-P?D3%wZhZ|b1$0{v(R{W9>M0R0r`ZA4Gi+=p@*>^5n8Jc)xnbsl`uf1EZs9Br~! zL^s4MZC3UnlYXf`d+r93JxLvQLD&d{NxOII_ivS9vTr&EVObJp_h~az0;R1yft&2*UL#?+ zgf*ogbBG#47? z!8*;utcY~zi-h4b#1{iPe(X}%2lj3CjevQ^G zMyZiVtJK>t$shKiCwh?9K2BQe7-!S}7&0=-o{qHV0?6k^27}+j?OEl>*SD4*W@6$G zO`fjro<#k3j0y$^iJqsSr)ms9YWr0VjS_D%cou>`9{K4l{C}*$-!}jT;j#Drq0{CW zSevIEct&8u>tXN|XxpkWjxUsP90|d+BJ*hy`14SXdm04oamo|!=*%{MCtb*H6GhP1 zrPGDrpW&A|&isAAFYTxl^c|2Z61p?yzAi!%b`)VxMq|%W>emjlS{MkFFh43Uw_(t+ z5$*POs~wkw$r(W&!t8!%%RO2dmX5IH2$O%cRh>Oto%bN@Y@@n$K8djN2uqamaMBWx z_Ac>FM;KwR-?mpn(o%>p55g8!#W&lHZ!^N4#~DCfa)%H$2Vsv?#W&22&x13G6F4iV zOD-8Egr9`l6OZ`pvN899=~gk8(mkNp!`38=NwuR zcy#ZmX8>!?2eheyU7EOE-X5H>6kHMvUIjm8NW6EglL|)D$5{*Vi7;fT^MX zVXmoDUxu^)0e#6h%OTMh@tgX(2l9-+)~ODILuX`F&c;gd-s*O^J!eAdIN5xuauCai z7*y#-ZA)36KqG(R`e5*NxV8OELRn(R)EUd7(E&X-;8{oMUv~Sl3Q>#LmPCsIX#jq5 z23Oc6Xxq!4zd7YyeJswb+*dVooLRVXQx5IaX%dwbq)T*0BToab(H|wx> z;KXHCyjf={^^&|uUiTm`hte@N!L51iW#<)Hk?`{fZ{0QMtSg+p^ccdi4;%1zM0gW~ zKacQRr7y`r9NKx!HD9{yQzch!dv|m8O#4Ev81D|tT<-bWk<<2AwZ~UAF7(E((mCX$ z_OO8Zma>@xePg-@gR5cF*Q*cGm3z9y;nrimI?D#ThE5%xfX<`027_;65608#H_N*p z_T;RmxK5qeDR%Y^cc5Kl;_R@aex5zYVY$+$JuM=yMw`lVl(uv zp|7eOtNLNTKxt!ZyH%$`yG?XKI_2DV$570}Qm1_!ojP{mbV=>UUa;=24jsF?+BQ-V zudqzC$IM6ncbUFdvQ{YVz zUVGjhWcOm!jJKhAPtR@#@8R~+~~pv6jl?|`SSv0nCYri5e{`~4z61R~!+-V*sM z{(br9A+r2;IENPbAP&|4jrhgx+6DRX4+euj!|mA)`Remwy7^Y+_=x>^Os^D;;dy#{ z-2=orvlmK7@=*xUE;!$AC+V10Pda2h_I5})U}Tj#8UelSh6jUpK;Ee%S&AciIq<+y z9UR@|^iS3yxu}O>4+Vqq(vPlp;;J>DRqyWutXrfX)jq`OBNIU9Znc&o1MGQ8o)w7u zLooO);`EfEv!QHj#Z%S4q?VUKUF~rSb=?^{`r?^^BIt{fJXW<|^ee147gjc4OeJvR zj1%UrJ$9sneGJMc_m9EgSSjBuCoR?I6f8FMtj;3snboHdyKQeq{JZ{y@dx^(U%_6+ zaNadS+WjGfok!So33JB!=cM7M!Rvu}J?3FN6Chz#>9+fwR4KG1SN1uq;xfUO27T=w z!Fj&a(d&o-a<$Tnl9IE{!(?0J&wbNO|+3xXR@FtBj+Z>1^IJ*(r zbs3HDz6dWwxTlj7SM@cEd0%CHW=1?2i0ffjr!x=qg={foW3ln)@j0@!=D(_T)jLpX zm>8i|IbjIeYCM z0UNZ^4Rcc6P$V|;fMo37;JKApX&;U2DTAs$d>f9?TS*_T-MM;2NZl_?LD^0U1{Z7J z`jvhPd&06r7m7G!3!=Z>7fwM}A)cMN1$v!z#(!cAT4%Sh=;j!&5!R_@JXP~;Rhs2A ztfbCpFx!rR>OqKS7oN>YmG*iq>J5FQR=TU}8)}8NUrk z_Z&R;^DN@37Ee{WtJ<5KseV>xy0aooOq+F-Tek@7c-R{y^6Po| zA<_R5)=t9ztpkw0YL^rj#P`SpfT(c+3T375na-;FNK(JM!{-$ZUe!a}?G~Rh?q)Qd=|= zrM9$Yq*88Y5ntcA!QkKE7P%T@`<+gVn4j(Sg7C}pbaTMJ9&XPR#8p)L$l$|TA{ zvM+cBfWON;%pYOk|4%1vztT4P22Pmf*gE@Oe|!9s=lABkiFK<7`i?pJYR$KqGWW@J zVm*STVd&fkE5+%TN1=eGcvyR{BovQ1~Vzs_=0i0Av(!tbFP?XOn3 zU>2Z}tuVfVKN)T*!)#Nmf9DDYjp#c&t~aRIVBEJGEKedE)qf4rABR0`GPbA>nh2)g z?0BO=f9s#d27}8C`bK2XwCxMBiI4L#rV2wIQ+b{dwwQ&mPMo3|@@c%?!2^DKkjGSD7DZqO1uiwxdk zP`PFy`jRn-Yq`PbA}h=`He2)`d80|=%^tXaE_s3I;MAqqBU$F>qGmKLJQnwrk^I!9 zG(QlB`!fxB=B2cv$s!YQLad8par}KV*yeC9YbHV;H-q!DX2N%`m-A9?tkpI&1A(ob7h+}U1!GmK_= zMYdQnzthW;!sv)sL}&Xzc-6;m_-MP2@%+JeK8BX({9Np}v*M>yey}VI!wuzq%43vz zuD*}xoY`Xg03S*l_nh!SKMnKkjAFQFMx$M>kM0cTs}28r>OSA@7=HC`-!E~kw%;4J zB;K=aXxLMUeGz*6l8GjyK1lw^0vx(55%-@iXWWZg^3%h7pR|(usn%D({7n^d#g#}x zP8zSgmkQE&>piqGjpq%b(lp+98-3M*KfH~8=)jY2qwK4A(w#K>D*ki`{czRepl|KS z8;8)(9eM4LsOi`6#350I*YMkSL~Xc+O9w|i-$_E(bsF{2U9`M2FT0aAcjozb(U|MF z;I61S*YTR$qn2OCQ|^p9cHJQG70MBuQZ*1zS~!ByXSUoix{)JwOurbR&{Un&D~+&b+^{s9ri{K zWU;8n?T7E8@(d}qWBnKrF3aKxchin6iFR>+No-Mn22bGt-ap9q$p9($@>^THFp#DW zl)94Jih=z7Kq?=|zYL_aLYR0PX42Jzc;@ZCJ%gk?C*RJHIDR`X8ce5dmw2WO=B;-{ z%^NJ){^SnM8$#Rfkl4PzgI^dDl|4iXZr`0e?Jlaglacv%?%`+d^=-O`SKmwH@00Yt zcptwv)c4+flA=%V zM$>2y`YI6Xi7S)sUONK#Fn+<=-Uaw?0T}<*1_S0hM%?L087N-}&`h(* zw~@5X0j1n1EeX;-$&9?=G77ODTbs7`jL5NBu(=2>Hy94^11-6^76+4 z+Un&AQS_acE28K#KSOqGIIP93;rvzveH+e)!|9C(q$D`|JvIvYMS~0b(SCEj@qRxU z_-6K}v!3P)yU`)$pR#D2m%qrO=e>L|i(c_I`lTxdw`7FA>tEM?xD=y)m z`_S=AcvfGU-i#OZrIKd6z8}4EDevn?yD#N4{b*b=Ps*aVlX+ENnw7#+v!v-P$)d9< z{8E2f)SSQTOV706S^en47Q7~l3NL$JGIQW^UfRw0u{FazrVS&{i`ww4p0uV7e|a69 zY{RF!QeInrJA+oX<(0i@b6cL@n=0B0|LiMxOLq!h!Q=YVyX}Nh(w^V!MvGJV+%2>| zm9sNx+?7177j3;V2B+sQr}2#5bT~~o&!=%-Z_4k$i+a+F9r)|(sknnAbYBN4)-f2D zv#8)I$!hUcQV{z<=|N+zmO?DJT8jAe)qF66zU+8ZvT?AJROymyqD2p0xLR z)c(geNQ@J^@Rymi$G(4ZBct+;-^izKq?d1!l&rdmzwb&rZ{lye(w8^!ft%>$O^ix^ z`ey#JE6ux^w{)dsm2$iC)ULF)8|Qa7e(Hv3 z-|Nl?y3>a4e5yNb&fxPIbRdIK0Ec?={7kygQwrvTOg@=OyD}wdQ+n|az3Ag!oZXv_ z_L2gt>Swci7+1y0Da8?kU&e4|qLm*ud7VXLtR!?g>0>`wywgJ~JWURJXrqS@z;<-T zN$kh9O{PKkp&Qoq3%}XiY|wXgSYQajH&Q>>knkJKHw@ZXx369uG0E+THW@s_q(cV3 zZPHS;f705({OQSWdgu<$_Rs^o)Zhm4dTOj>28Y_wz?A)0HYd~DG#R>~KP5X%#{Mt-v{THMZ{-$A5u?LLhmH1+_Ko(AW=u2IXlKHaN#@};$rOj* zApFws>yBS%{QBaDH(G6aH1M0&c6m3x92(QO)S!Rxe1kqVnj;%QBT@{jmfV%PIHRLK ziLa%rYP*FuQ7pa^gWs{}u)&Kw^oqrME&AL7S!*=nLz90rX>2vpdUGVg#~8dZk)|2E zp$Xm);Ab0Q*lm>Cg!Y*HZW85Ndrbr^!oV6spL;vQ{fe)^*x8i6^z-aE$_o>@m%^4C zpTyF!a9)&1b0dVbG~xqeZBu$Sii_juR20ulq8SbNd?MvVH#!wV-$o15)Ybun|ri={y&{TMSvE7s6cnWyp zczPl|7thZo(!6+{m`F?Fd1eAVm%vXa(*8sqpG0R8B@_}<5*BKpl z8cE-U(R$Bw##7;Rf_YWAG2hQ1E%%=`7KKx3I3Ei)@+0`i2s$Rh&JbYN^6blvgic&+ zP!=CI@F>hK!&qftlN4=YtXXZW2pZQcG^ntbEOt+n`#cv|e?$qB}{9)2ppK$4fl8oRw{ImR>}Pf0Yk`S~?) zhW9(#R9fcQcv=?0OA=^mB)^hCuKXQJ`iQh!4frK>^3;3fw9u$sR3yR+XBWi>uCd?Gu~FqV3_!L zaex;4cvgUle0)60nBi|aBZ`*zr^1+oIT8b=w<8$tBN0sykQd1_B58aSl93lB&cjh$ z6rkJyPisJ10(>+;2LpUAKvNoUK?65osHQ7=LF9g3;-S&}nulf?Jl{hawo^8=zQSDS_~+w*Ahc}KgM)3k`Ed5 zH{M{-*D_e!veDPhL-l`Uz-*?eIvGS?I~BCeM&lC(UCiXpxlMdJoU=(qRvSFhN-7GM`|2hlQ|{d5o8i*zP>tA@IxESO#765$rRh z?}s^jfO%_}@l6=|xHUGMbHZtAI5-fM)B5f5SY_lG{6Rd;GWq3rT5s}^1X^$LyYW;V55Pn!eJ80X?BzcEjWrv*(I!ONO71!ZrO>k+}%F^o8~ zVXTBfxf&pg&^Ia?8!q(aNB z>BzLhmaI4?d*~}O$ABkW+kA@b*32`moM6!h=&(}r&EYPP_PNZu7S7=8+l}7+Zy+%_ zf{}FGkQcA!djoJ!_U}RkoesMS?zQ3kT$J&t&D-_sVGf+z>R5;MLyV!F+Mqtt{UG|p;LY|$s$Pg)G-xlFmnBQG*-GMmhD+9o`^ zJ-p9D(-{(w5C1Plk9k+SMT1PF1F2qW(3^&1GHfs;zvE1v)j-Uvxs72L%6*Z=(*h`z z%W*&J;iC=leH6KW%=~F1%J=fdM)Z|^FYxj126WQC|I5!mHX=0I9~#o~NS+=|n<7P9 zQ53%)MO&m8jz#hKXqtkdDnJJUAY2GA-r}F$KvJ-x0iS6=Wer5*@@SslkTylzF-7z0 zhIBew=ubD~bq(>LqoihHV}89cR#c)4xqaxmC$WaZ-W3KQgQtekX@g<7O*dr}+G_HI zFsaA4y|je+q@UI^Z(^G2J>W1D=amAywy*ii}RR|S75)(&y)Q0f}daUi+N)g z>_j)Czc1^&z z!a3C>xbz{L`{SOWj=N?_OoC+?9s4koKVdIEM~ zt6lzQ{?GW&=G4^dM(M_1U%1ORMAKP#HI|RKPV{iFK>dX`I{tm~f?&04WIb}lA#%x? zHRUGOBUc(CH!MW1K;@Dp!o}3(zSNw`#9-K?(LvoGjt;8TrS@L*gO;54~PJApz;Bb?c=o>!ufk($i}4 zWoo+)iKdo!-!)Eq7@+Mn&3@<-=S+&x8t(dsby0rBF*i8=!s{I@z0SdO#lp^xKUy*Q zTE{;|^^a5>rZ_+`UD1u##b~9U&~&*Oa`WN(L$NrdeABi3Li=T?oa=Y{nTswx6y5$C zN)Pp))&A>ZXn3f9fcD=|y6Z2{`g8rih5O!d>Y-54<!P+2FdQ#v>s(NNd__YnWo5dXpue^H44qY(d2_2cx-zOwL4tXw$ev#-2A zI~b+@ba}Z27vA)+ucGYcHsW<1`;upl<-*%9_O(I7akOt=Zu`XFeAur&;f%z-uovlE zci#o>yh6mt~w6&EP3R4h>}Q!H1kQ1t$n z#;@2?v6Etk;$X$$ilY>B6!R4qD6Uj2Q7ls|SFBLj#A7~%vW5XxKgo1u}raCu|mi#o>yh6mt~w6&EP3 zR4h>}Q!H1kQ1p(|_!V0!c2dkx9IQB8ag<_?V!ooIt@gbjzjVOYFhiN|7`m@{G&tX)U=Lm94b-j<0J4b4}9F6 zQt{D~->2fUVX5*Zj0fBQ;X$%}@{w0ceef^;@%UdJaS-4ButW6ABQ-oj5%$>cAHH7% z)HU+qCn#0E)oY3c)yKTfImhOT^g z{eM@L%6dU;e5du`y;H-{?4m9!Dx-7lc$d%&jtbcDuDrW$aj{q#-E~Oi`a$WffszJS z-d)$YI3USklz1g+u6&$Bw%>Ob-L5=cssW1bx=5*25Ai6WGf>tjuDrW0aZ&#BTP`>K zE}x6C%#k&Z>vz{ZE)LW3l=REbO+TFWH3)a1yYlWj%SCsc<;L&IyZOIQ8@*C6q)Lmq_z6>J9w(g7ph(CqWxd{)dbviu#5i@B404h zVRq3?udBlKyC`O8jrdi*P%$rGD?k;v@w)nC8m}(zuH#*F*S9WzX#TTR-c*KRx)0zY zo7BGqshhGRw}6AmYYkR8zMjGcaFS^ zJKPNd6$p)g1?U0%qJNmrr!d<@#OR0-BVc zD_@%3T-c~o<#)Lo7&g^;w_Tk!e1W?Zu`BPc0~bEyq~G1Xb@T1U>*nh*$X1Vk*JLOD zU6UR8)(+Tl$<38_@#_$I(tR4ra|9=b)W0j|;?WR!*G?$NcjOOAq}8tEYHo3!#7}g& z`7fB+T*x$WrXxQ>GFR Date: Mon, 28 Mar 2022 20:53:16 +0000 Subject: [PATCH 133/153] Revamp cmake again --- .github/workflows/ci.yml | 9 +++++--- torchdata/_extension.py | 41 ++--------------------------------- torchdata/csrc/CMakeLists.txt | 3 ++- 3 files changed, 10 insertions(+), 43 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aaf6eef5d..84336f7d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,7 +71,8 @@ jobs: run: | git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp cd aws-sdk-cpp - cmake -S . -B build -GNinja -DBUILD_ONLY="s3;transfer" -DENABLE_TESTING=OFF -DCMAKE_BUILD_TYPE=Release + mkdir sdk-lib + cmake -S . -B build -GNinja -DBUILD_ONLY="s3;transfer" -DENABLE_TESTING=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=./sdk-lib cmake --build build --config Release cmake --install build --config Release - name: Install AWS-SDK-CPP on Non-Windows for S3 IO datapipes @@ -79,9 +80,9 @@ jobs: run: | git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp cd aws-sdk-cpp/ - mkdir sdk-build + mkdir sdk-build sdk-lib cd sdk-build - cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" -DENABLE_TESTING=OFF + cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" -DENABLE_TESTING=OFF -DCMAKE_INSTALL_PREFIX=./sdk-lib make sudo make install - name: Build TorchData @@ -90,7 +91,9 @@ jobs: BUILD_S3: ${{ matrix.with-s3 }} BUILD_PYTHON_VERSION: ${{ matrix.python-version }} pybind11_DIR: ${{ steps.install_pybind11.outputs.path }} + AWSSDK_DIR: ./aws-sdk-cpp/sdk-lib - name: Find extension + if: ${{ matrix.with-s3 }} shell: bash run: ls -al torchdata/_torchdata* - name: Install test requirements diff --git a/torchdata/_extension.py b/torchdata/_extension.py index 091b3923c..5b3fd716e 100644 --- a/torchdata/_extension.py +++ b/torchdata/_extension.py @@ -1,3 +1,4 @@ +import importlib import os from pathlib import Path @@ -6,49 +7,11 @@ _LIB_DIR = Path(__file__).parent -def _get_lib_path(lib: str): - suffix = "pyd" if os.name == "nt" else "so" - path = _LIB_DIR / f"{lib}.{suffix}" - return path - - -def _load_lib(lib: str): - path = _get_lib_path(lib) - # In case `torchdata` is deployed with `pex` format, this file does not exist. - # In this case, we expect that `libtorchdata` is available somewhere - # in the search path of dynamic loading mechanism, and importing `_torchdata`, - # which depends on `libtorchdata` and dynamic loader will handle it for us. - print("[Extension] _torchdata path: ", path) - print("[Extension] _torchdata path exists: ", path.exists()) - if path.exists(): - torch.ops.load_library(path) - torch.classes.load_library(path) - - -# def _init_extension(): -# if not _mod_utils.is_module_available("torchdata._torchdata"): -# warnings.warn("torchdata C++ extension is not available.") -# return - -# import torchdata -# print("[Extension] torch path: ", torch.__file__) -# print("[Extension] torchdata path: ", torchdata.__file__) - -# _load_lib("_torchdata") -# # This import is for initializing the methods registered via PyBind11 -# # This has to happen after the base library is loaded -# from torchdata import _torchdata # noqa - - def _init_extension(): - import importlib - import os - - # load the custom_op_library and register the custom ops + # load the pybind11 extension lib_dir = os.path.dirname(__file__) if os.name == "nt": - # Register the main torchvision library location on the default DLL path import ctypes import sys diff --git a/torchdata/csrc/CMakeLists.txt b/torchdata/csrc/CMakeLists.txt index df21f5046..bd87e2a59 100644 --- a/torchdata/csrc/CMakeLists.txt +++ b/torchdata/csrc/CMakeLists.txt @@ -14,7 +14,8 @@ if(BUILD_S3) set(CMAKE_POSITION_INDEPENDENT_CODE ON) add_library(_torchdata SHARED "pybind/S3Handler/S3Handler.cpp") - target_include_directories(_torchdata PRIVATE ${PROJECT_SOURCE_DIR} ${Python_INCLUDE_DIR}) + # target_include_directories(_torchdata PRIVATE ${PROJECT_SOURCE_DIR} ${Python_INCLUDE_DIR}) + target_include_directories(_torchdata PRIVATE ${PROJECT_SOURCE_DIR}) message(STATUS "All linked libs: ${AWSSDK_LINK_LIBRARIES}") target_link_libraries(_torchdata PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} Python3::Python) From 26f0dbc80ddc3a2d56e86d4a568660eda31c708b Mon Sep 17 00:00:00 2001 From: erjia Date: Mon, 28 Mar 2022 21:14:50 +0000 Subject: [PATCH 134/153] Remove test for BUILD_S3=0 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 84336f7d2..d44853423 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: - 3.9 with-s3: - 1 - - 0 + # - 0 steps: - name: Setup additional system libraries if: startsWith( matrix.os, 'ubuntu' ) From ee1b679f6060ae6768174c3e8987238078889dd0 Mon Sep 17 00:00:00 2001 From: erjia Date: Mon, 28 Mar 2022 21:17:41 +0000 Subject: [PATCH 135/153] Add cache --- .github/workflows/ci.yml | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d44853423..ecfbc1284 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,22 +57,24 @@ jobs: run: | pip3 install -r requirements.txt pip3 install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html - pip3 install cmake ninja + pip3 install cmake ninja pybind11 echo "/home/runner/.local/bin" >> $GITHUB_PATH - - name: Install pybind11 + - name: Get Date + id: get_date shell: bash - run: | - pip3 install pybind11 - PYBIND11_PATH=`pybind11-config --cmakedir` - echo "::set-output name=path::$PYBIND11_PATH" - id: install_pybind11 + run: echo "::set-output name=date::$(date "+%Y%m%d")" + - name: Uses cache for AWS-SDK-CPP + uses: actions/cache@v3 + with: + path: $GITHUB_WORKSPACE/aws-sdk-cpp/sdk-lib + key: ${{ matrix.os }}-${{ matrix.python-version }}-${{ steps.get_date.outputs.date }} - name: Install AWS-SDK-CPP on Windows for S3 IO datapipes if: matrix.with-s3 == 1 && matrix.os == 'windows-latest' run: | git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp cd aws-sdk-cpp mkdir sdk-lib - cmake -S . -B build -GNinja -DBUILD_ONLY="s3;transfer" -DENABLE_TESTING=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=./sdk-lib + cmake -S . -B build -GNinja -DBUILD_ONLY="s3;transfer" -DENABLE_TESTING=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/aws-sdk-cpp/sdk-lib cmake --build build --config Release cmake --install build --config Release - name: Install AWS-SDK-CPP on Non-Windows for S3 IO datapipes @@ -82,16 +84,23 @@ jobs: cd aws-sdk-cpp/ mkdir sdk-build sdk-lib cd sdk-build - cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" -DENABLE_TESTING=OFF -DCMAKE_INSTALL_PREFIX=./sdk-lib + cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" -DENABLE_TESTING=OFF -DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/aws-sdk-cpp/sdk-lib make sudo make install + - name: Export AWS-SDK-CPP & PYBIND11 + if: matrix.with-s3 == 1 + shell: bash + run: | + PYBIND11_PATH=`pybind11-config --cmakedir` + echo "::set-output name=pybind11::$PYBIND11_PATH" + echo "::set-output name=aws::$GITHUB_WORKSPACE/aws-sdk-cpp/sdk-lib" + id: export_path - name: Build TorchData run: python setup.py develop env: BUILD_S3: ${{ matrix.with-s3 }} BUILD_PYTHON_VERSION: ${{ matrix.python-version }} - pybind11_DIR: ${{ steps.install_pybind11.outputs.path }} - AWSSDK_DIR: ./aws-sdk-cpp/sdk-lib + CMAKE_PREFIX_PATH: "${{ steps.export_path.outputs.aws }} ${{ steps.export_path.outputs.pybind11 }}" - name: Find extension if: ${{ matrix.with-s3 }} shell: bash From 4c93e1831205a4213e5f3bdd416a342b82634185 Mon Sep 17 00:00:00 2001 From: erjia Date: Mon, 28 Mar 2022 22:25:46 +0000 Subject: [PATCH 136/153] Create cache --- .github/workflows/ci.yml | 62 ++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ecfbc1284..2b28ad0de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,34 +95,34 @@ jobs: echo "::set-output name=pybind11::$PYBIND11_PATH" echo "::set-output name=aws::$GITHUB_WORKSPACE/aws-sdk-cpp/sdk-lib" id: export_path - - name: Build TorchData - run: python setup.py develop - env: - BUILD_S3: ${{ matrix.with-s3 }} - BUILD_PYTHON_VERSION: ${{ matrix.python-version }} - CMAKE_PREFIX_PATH: "${{ steps.export_path.outputs.aws }} ${{ steps.export_path.outputs.pybind11 }}" - - name: Find extension - if: ${{ matrix.with-s3 }} - shell: bash - run: ls -al torchdata/_torchdata* - - name: Install test requirements - run: pip3 install expecttest fsspec iopath==0.1.9 numpy pytest rarfile - - name: Run DataPipes tests with pytest - if: ${{ ! contains(github.event.pull_request.labels.*.name, 'ciflow/slow') }} - run: - pytest --no-header -v test --ignore=test/test_period.py --ignore=test/test_text_examples.py - --ignore=test/test_audio_examples.py - - name: Run DataPipes tests with pytest (including slow tests) - if: ${{ contains(github.event.pull_request.labels.*.name, 'ciflow/slow') }} - run: - pytest --no-header -v test --ignore=test/test_period.py --ignore=test/test_text_examples.py - --ignore=test/test_audio_examples.py - env: - PYTORCH_TEST_WITH_SLOW: 1 - - name: Run DataPipes period tests with pytest - if: ${{ contains(github.event.pull_request.labels.*.name, 'ciflow/period') }} - run: - pytest --no-header -v test/test_period.py --ignore=test/test_text_examples.py - --ignore=test/test_audio_examples.py - env: - PYTORCH_TEST_WITH_SLOW: 1 + # - name: Build TorchData + # run: python setup.py develop + # env: + # BUILD_S3: ${{ matrix.with-s3 }} + # BUILD_PYTHON_VERSION: ${{ matrix.python-version }} + # CMAKE_PREFIX_PATH: "${{ steps.export_path.outputs.aws }};${{ steps.export_path.outputs.pybind11 }}" + # - name: Find extension + # if: ${{ matrix.with-s3 }} + # shell: bash + # run: ls -al torchdata/_torchdata* + # - name: Install test requirements + # run: pip3 install expecttest fsspec iopath==0.1.9 numpy pytest rarfile + # - name: Run DataPipes tests with pytest + # if: ${{ ! contains(github.event.pull_request.labels.*.name, 'ciflow/slow') }} + # run: + # pytest --no-header -v test --ignore=test/test_period.py --ignore=test/test_text_examples.py + # --ignore=test/test_audio_examples.py + # - name: Run DataPipes tests with pytest (including slow tests) + # if: ${{ contains(github.event.pull_request.labels.*.name, 'ciflow/slow') }} + # run: + # pytest --no-header -v test --ignore=test/test_period.py --ignore=test/test_text_examples.py + # --ignore=test/test_audio_examples.py + # env: + # PYTORCH_TEST_WITH_SLOW: 1 + # - name: Run DataPipes period tests with pytest + # if: ${{ contains(github.event.pull_request.labels.*.name, 'ciflow/period') }} + # run: + # pytest --no-header -v test/test_period.py --ignore=test/test_text_examples.py + # --ignore=test/test_audio_examples.py + # env: + # PYTORCH_TEST_WITH_SLOW: 1 From 6c2a5ea45a86226658cc92c95b2c44219f79b360 Mon Sep 17 00:00:00 2001 From: erjia Date: Mon, 28 Mar 2022 22:44:12 +0000 Subject: [PATCH 137/153] Start to use cache --- .github/workflows/ci.yml | 75 ++++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b28ad0de..e7da2e01b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,16 +60,19 @@ jobs: pip3 install cmake ninja pybind11 echo "/home/runner/.local/bin" >> $GITHUB_PATH - name: Get Date + if: matrix.with-s3 == 1 id: get_date shell: bash run: echo "::set-output name=date::$(date "+%Y%m%d")" - name: Uses cache for AWS-SDK-CPP + if: matrix.with-s3 == 1 + id: aws_sdk_cache uses: actions/cache@v3 with: path: $GITHUB_WORKSPACE/aws-sdk-cpp/sdk-lib key: ${{ matrix.os }}-${{ matrix.python-version }}-${{ steps.get_date.outputs.date }} - name: Install AWS-SDK-CPP on Windows for S3 IO datapipes - if: matrix.with-s3 == 1 && matrix.os == 'windows-latest' + if: matrix.with-s3 == 1 && matrix.os == 'windows-latest' && steps.aws_sdk_cache.outputs.cache-hit != 'true' run: | git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp cd aws-sdk-cpp @@ -78,7 +81,7 @@ jobs: cmake --build build --config Release cmake --install build --config Release - name: Install AWS-SDK-CPP on Non-Windows for S3 IO datapipes - if: matrix.with-s3 == 1 && matrix.os != 'windows-latest' + if: matrix.with-s3 == 1 && matrix.os != 'windows-latest' && steps.aws_sdk_cache.outputs.cache-hit != 'true' run: | git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp cd aws-sdk-cpp/ @@ -91,38 +94,42 @@ jobs: if: matrix.with-s3 == 1 shell: bash run: | + ls -l $GITHUB_WORKSPACE/aws-sdk-cpp/sdk-lib + ls -l $GITHUB_WORKSPACE/aws-sdk-cpp/sdk-lib/lib + ls -l $GITHUB_WORKSPACE/aws-sdk-cpp/sdk-lib/include PYBIND11_PATH=`pybind11-config --cmakedir` echo "::set-output name=pybind11::$PYBIND11_PATH" - echo "::set-output name=aws::$GITHUB_WORKSPACE/aws-sdk-cpp/sdk-lib" + echo "::set-output name=aws::$GITHUB_WORKSPACE/aws-sdk-cpp/sdk-lib/lib/cmake/AWSSDK" id: export_path - # - name: Build TorchData - # run: python setup.py develop - # env: - # BUILD_S3: ${{ matrix.with-s3 }} - # BUILD_PYTHON_VERSION: ${{ matrix.python-version }} - # CMAKE_PREFIX_PATH: "${{ steps.export_path.outputs.aws }};${{ steps.export_path.outputs.pybind11 }}" - # - name: Find extension - # if: ${{ matrix.with-s3 }} - # shell: bash - # run: ls -al torchdata/_torchdata* - # - name: Install test requirements - # run: pip3 install expecttest fsspec iopath==0.1.9 numpy pytest rarfile - # - name: Run DataPipes tests with pytest - # if: ${{ ! contains(github.event.pull_request.labels.*.name, 'ciflow/slow') }} - # run: - # pytest --no-header -v test --ignore=test/test_period.py --ignore=test/test_text_examples.py - # --ignore=test/test_audio_examples.py - # - name: Run DataPipes tests with pytest (including slow tests) - # if: ${{ contains(github.event.pull_request.labels.*.name, 'ciflow/slow') }} - # run: - # pytest --no-header -v test --ignore=test/test_period.py --ignore=test/test_text_examples.py - # --ignore=test/test_audio_examples.py - # env: - # PYTORCH_TEST_WITH_SLOW: 1 - # - name: Run DataPipes period tests with pytest - # if: ${{ contains(github.event.pull_request.labels.*.name, 'ciflow/period') }} - # run: - # pytest --no-header -v test/test_period.py --ignore=test/test_text_examples.py - # --ignore=test/test_audio_examples.py - # env: - # PYTORCH_TEST_WITH_SLOW: 1 + - name: Build TorchData + run: python setup.py develop + env: + BUILD_S3: ${{ matrix.with-s3 }} + BUILD_PYTHON_VERSION: ${{ matrix.python-version }} + pybind11_DIR: ${{ steps.export_path.outputs.pybind11 }} + CMAKE_PREFIX_PATH: ${{ steps.export_path.outputs.aws }} + - name: Find extension + if: ${{ matrix.with-s3 }} + shell: bash + run: ls -al torchdata/_torchdata* + - name: Install test requirements + run: pip3 install expecttest fsspec iopath==0.1.9 numpy pytest rarfile + - name: Run DataPipes tests with pytest + if: ${{ ! contains(github.event.pull_request.labels.*.name, 'ciflow/slow') }} + run: + pytest --no-header -v test --ignore=test/test_period.py --ignore=test/test_text_examples.py + --ignore=test/test_audio_examples.py + - name: Run DataPipes tests with pytest (including slow tests) + if: ${{ contains(github.event.pull_request.labels.*.name, 'ciflow/slow') }} + run: + pytest --no-header -v test --ignore=test/test_period.py --ignore=test/test_text_examples.py + --ignore=test/test_audio_examples.py + env: + PYTORCH_TEST_WITH_SLOW: 1 + - name: Run DataPipes period tests with pytest + if: ${{ contains(github.event.pull_request.labels.*.name, 'ciflow/period') }} + run: + pytest --no-header -v test/test_period.py --ignore=test/test_text_examples.py + --ignore=test/test_audio_examples.py + env: + PYTORCH_TEST_WITH_SLOW: 1 From c1db34472fc293d03052a3ef72e03c328adad8d5 Mon Sep 17 00:00:00 2001 From: erjia Date: Mon, 28 Mar 2022 23:53:21 +0000 Subject: [PATCH 138/153] Disable cache --- .github/workflows/ci.yml | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7da2e01b..a64cf3d92 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,20 +59,21 @@ jobs: pip3 install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html pip3 install cmake ninja pybind11 echo "/home/runner/.local/bin" >> $GITHUB_PATH - - name: Get Date - if: matrix.with-s3 == 1 - id: get_date - shell: bash - run: echo "::set-output name=date::$(date "+%Y%m%d")" - - name: Uses cache for AWS-SDK-CPP - if: matrix.with-s3 == 1 - id: aws_sdk_cache - uses: actions/cache@v3 - with: - path: $GITHUB_WORKSPACE/aws-sdk-cpp/sdk-lib - key: ${{ matrix.os }}-${{ matrix.python-version }}-${{ steps.get_date.outputs.date }} + # - name: Get Date + # if: matrix.with-s3 == 1 + # id: get_date + # shell: bash + # run: echo "::set-output name=date::$(date "+%Y%m%d")" + # - name: Uses cache for AWS-SDK-CPP + # if: matrix.with-s3 == 1 + # id: aws_sdk_cache + # uses: actions/cache@v3 + # with: + # path: $GITHUB_WORKSPACE/aws-sdk-cpp/sdk-lib + # key: ${{ matrix.os }}-${{ matrix.python-version }}-${{ steps.get_date.outputs.date }} - name: Install AWS-SDK-CPP on Windows for S3 IO datapipes - if: matrix.with-s3 == 1 && matrix.os == 'windows-latest' && steps.aws_sdk_cache.outputs.cache-hit != 'true' + # if: matrix.with-s3 == 1 && matrix.os == 'windows-latest' && steps.aws_sdk_cache.outputs.cache-hit != 'true' + if: matrix.with-s3 == 1 && matrix.os == 'windows-latest' run: | git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp cd aws-sdk-cpp @@ -81,7 +82,8 @@ jobs: cmake --build build --config Release cmake --install build --config Release - name: Install AWS-SDK-CPP on Non-Windows for S3 IO datapipes - if: matrix.with-s3 == 1 && matrix.os != 'windows-latest' && steps.aws_sdk_cache.outputs.cache-hit != 'true' + # if: matrix.with-s3 == 1 && matrix.os != 'windows-latest' && steps.aws_sdk_cache.outputs.cache-hit != 'true' + if: matrix.with-s3 == 1 && matrix.os != 'windows-latest' run: | git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp cd aws-sdk-cpp/ From ebbe4078a8d135f4b1415ce7248c7b1e2f2b62cf Mon Sep 17 00:00:00 2001 From: erjia Date: Tue, 29 Mar 2022 14:00:28 +0000 Subject: [PATCH 139/153] Add CMAKE_PREFIX_PATH and fix pybind --- .github/workflows/ci.yml | 30 ++++++++++++------- tools/setup_helpers/extension.py | 12 ++++++-- torchdata/csrc/CMakeLists.txt | 28 ++++++++++------- .../csrc/pybind/{S3Handler => }/pybind.cpp | 2 +- 4 files changed, 47 insertions(+), 25 deletions(-) rename torchdata/csrc/pybind/{S3Handler => }/pybind.cpp (97%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a64cf3d92..702af2686 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,7 +78,7 @@ jobs: git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp cd aws-sdk-cpp mkdir sdk-lib - cmake -S . -B build -GNinja -DBUILD_ONLY="s3;transfer" -DENABLE_TESTING=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/aws-sdk-cpp/sdk-lib + cmake -S . -B build -GNinja -DBUILD_ONLY="s3;transfer" -DENABLE_TESTING=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=sdk-lib cmake --build build --config Release cmake --install build --config Release - name: Install AWS-SDK-CPP on Non-Windows for S3 IO datapipes @@ -89,31 +89,41 @@ jobs: cd aws-sdk-cpp/ mkdir sdk-build sdk-lib cd sdk-build - cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" -DENABLE_TESTING=OFF -DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/aws-sdk-cpp/sdk-lib + cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" -DENABLE_TESTING=OFF -DCMAKE_INSTALL_PREFIX=../sdk-lib make sudo make install - name: Export AWS-SDK-CPP & PYBIND11 if: matrix.with-s3 == 1 shell: bash run: | - ls -l $GITHUB_WORKSPACE/aws-sdk-cpp/sdk-lib - ls -l $GITHUB_WORKSPACE/aws-sdk-cpp/sdk-lib/lib - ls -l $GITHUB_WORKSPACE/aws-sdk-cpp/sdk-lib/include + if [[ ${{ matrix.os }} == 'windows-latest' ]]; then + AWSSDK_PATH="$GITHUB_WORKSPACE\\aws-sdk-cpp\\sdk-lib" + else + AWSSDK_PATH="$GITHUB_WORKSPACE/aws-sdk-cpp/sdk-lib" + fi PYBIND11_PATH=`pybind11-config --cmakedir` + echo "::set-output name=awssdk::$AWSSDK_PATH" echo "::set-output name=pybind11::$PYBIND11_PATH" - echo "::set-output name=aws::$GITHUB_WORKSPACE/aws-sdk-cpp/sdk-lib/lib/cmake/AWSSDK" id: export_path - name: Build TorchData - run: python setup.py develop + run: | + python setup.py develop env: BUILD_S3: ${{ matrix.with-s3 }} BUILD_PYTHON_VERSION: ${{ matrix.python-version }} pybind11_DIR: ${{ steps.export_path.outputs.pybind11 }} - CMAKE_PREFIX_PATH: ${{ steps.export_path.outputs.aws }} + AWSSDK_DIR: ${{ steps.export_path.outputs.awssdk }} - name: Find extension - if: ${{ matrix.with-s3 }} + if: always() && matrix.with-s3 == 1 shell: bash - run: ls -al torchdata/_torchdata* + run: | + ls -al torchdata + if [[ ${{ matrix.os }} == 'windows-latest' ]]; then + ls -aR build + find . -name _torchdata.cp${{ matrix.python-version }}-win_amd64.pyd + else + ls -al build/lib.*/torchdata + fi - name: Install test requirements run: pip3 install expecttest fsspec iopath==0.1.9 numpy pytest rarfile - name: Run DataPipes tests with pytest diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index 3349bd823..7abb5eac0 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -38,6 +38,7 @@ def _get_build(var, default=False): _BUILD_S3 = _get_build("BUILD_S3", False) +_AWSSDK_DIR = os.environ.get("AWSSDK_DIR", None) _BUILD_PYTHON_VERSION = os.environ.get("BUILD_PYTHON_VERSION", None) @@ -74,11 +75,11 @@ def build_extension(self, ext): cfg = "Debug" if debug else "Release" cmake_args = [ - f"-DCMAKE_BUILD_TYPE={cfg}", # not used on MSVC, but no harm + f"-DCMAKE_BUILD_TYPE={cfg}", + f"-DCMAKE_INSTALL_PREFIX={extdir}", f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}", - f"-DPYTHON_EXECUTABLE={sys.executable}", + f"-DCMAKE_RUNTIME_OUTPUT_DIRECTORY={extdir}", # For Windows f"-DPython_INCLUDE_DIR={distutils.sysconfig.get_python_inc()}", - "-DCMAKE_CXX_FLAGS=-fPIC", f"-DBUILD_S3:BOOL={'ON' if _BUILD_S3 else 'OFF'}", ] @@ -89,6 +90,11 @@ def build_extension(self, ext): f"-DBUILD_PYTHON_VERSION={_BUILD_PYTHON_VERSION}", ] + if _AWSSDK_DIR: + cmake_args += [ + f"-DAWSSDK_DIR={_AWSSDK_DIR}", + ] + # Default to Ninja if "CMAKE_GENERATOR" not in os.environ or platform.system() == "Windows": cmake_args += ["-GNinja"] diff --git a/torchdata/csrc/CMakeLists.txt b/torchdata/csrc/CMakeLists.txt index bd87e2a59..cabeccdd9 100644 --- a/torchdata/csrc/CMakeLists.txt +++ b/torchdata/csrc/CMakeLists.txt @@ -2,6 +2,7 @@ if(BUILD_S3) message(STATUS "Building S3 IO functionality") + list(APPEND CMAKE_PREFIX_PATH ${AWSSDK_DIR}) find_package(AWSSDK REQUIRED COMPONENTS s3 transfer) find_package(pybind11 REQUIRED) @@ -13,12 +14,23 @@ if(BUILD_S3) set(CMAKE_POSITION_INDEPENDENT_CODE ON) - add_library(_torchdata SHARED "pybind/S3Handler/S3Handler.cpp") - # target_include_directories(_torchdata PRIVATE ${PROJECT_SOURCE_DIR} ${Python_INCLUDE_DIR}) - target_include_directories(_torchdata PRIVATE ${PROJECT_SOURCE_DIR}) + set( + EXTENSION_SOURCES + pybind/pybind.cpp + pybind/S3Handler/S3Handler.cpp + ) - message(STATUS "All linked libs: ${AWSSDK_LINK_LIBRARIES}") - target_link_libraries(_torchdata PRIVATE ${AWSSDK_LINK_LIBRARIES} ${AWSSDK_PLATFORM_DEPS} Python3::Python) + pybind11_add_module(_torchdata SHARED ${EXTENSION_SOURCES}) + + target_include_directories(_torchdata PRIVATE ${PROJECT_SOURCE_DIR} ${Python_INCLUDE_DIR}) + + message(STATUS "AWSSDK linked libs: ${AWSSDK_LINK_LIBRARIES}") + target_link_libraries( + _torchdata + PRIVATE + ${AWSSDK_LINK_LIBRARIES} + ${Python_LIBRARIES} + ) set_target_properties(_torchdata PROPERTIES PREFIX "") if (MSVC) @@ -28,10 +40,4 @@ if(BUILD_S3) set_target_properties(_torchdata PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") endif(APPLE) - install( - TARGETS _torchdata - LIBRARY DESTINATION . - RUNTIME DESTINATION . # For Windows - ) - endif() diff --git a/torchdata/csrc/pybind/S3Handler/pybind.cpp b/torchdata/csrc/pybind/pybind.cpp similarity index 97% rename from torchdata/csrc/pybind/S3Handler/pybind.cpp rename to torchdata/csrc/pybind/pybind.cpp index d27baa6a0..e96bde63a 100644 --- a/torchdata/csrc/pybind/S3Handler/pybind.cpp +++ b/torchdata/csrc/pybind/pybind.cpp @@ -4,7 +4,7 @@ #include #include -#include "S3Handler.h" +#include namespace py = pybind11; using torchdata::S3Handler; From 161b2e44ad236f4676b11e25b8db3644e703c40d Mon Sep 17 00:00:00 2001 From: erjia Date: Wed, 30 Mar 2022 03:26:20 +0000 Subject: [PATCH 140/153] Fix lint --- test/test_remote_io.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_remote_io.py b/test/test_remote_io.py index b1e02d085..7871579bf 100644 --- a/test/test_remote_io.py +++ b/test/test_remote_io.py @@ -6,6 +6,8 @@ import expecttest +import torchdata + from _utils._common_utils_for_test import check_hash_fn, create_temp_dir from torchdata.datapipes.iter import ( @@ -18,7 +20,6 @@ S3FileLoader, ) -import torchdata class TestDataPipeRemoteIO(expecttest.TestCase): def setUp(self): From 41d8e66027eb1f97e29670f63c93da72f4af9e9a Mon Sep 17 00:00:00 2001 From: erjia Date: Wed, 30 Mar 2022 03:37:47 +0000 Subject: [PATCH 141/153] Try to fix windows --- .github/workflows/ci.yml | 32 +++++++++++++++----------------- setup.py | 11 ++++++++--- tools/setup_helpers/extension.py | 7 +++++++ torchdata/csrc/CMakeLists.txt | 5 +++-- 4 files changed, 33 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 702af2686..541d3408c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,21 +59,20 @@ jobs: pip3 install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html pip3 install cmake ninja pybind11 echo "/home/runner/.local/bin" >> $GITHUB_PATH - # - name: Get Date - # if: matrix.with-s3 == 1 - # id: get_date - # shell: bash - # run: echo "::set-output name=date::$(date "+%Y%m%d")" - # - name: Uses cache for AWS-SDK-CPP - # if: matrix.with-s3 == 1 - # id: aws_sdk_cache - # uses: actions/cache@v3 - # with: - # path: $GITHUB_WORKSPACE/aws-sdk-cpp/sdk-lib - # key: ${{ matrix.os }}-${{ matrix.python-version }}-${{ steps.get_date.outputs.date }} + - name: Get Date + if: matrix.with-s3 == 1 + id: get_date + shell: bash + run: echo "::set-output name=date::$(date "+%Y%m%d")" + - name: Uses cache for AWS-SDK-CPP + if: matrix.with-s3 == 1 + id: aws_sdk_cache + uses: actions/cache@v3 + with: + path: $GITHUB_WORKSPACE/aws-sdk-cpp/sdk-lib + key: ${{ matrix.os }}-${{ matrix.python-version }}-${{ steps.get_date.outputs.date }} - name: Install AWS-SDK-CPP on Windows for S3 IO datapipes - # if: matrix.with-s3 == 1 && matrix.os == 'windows-latest' && steps.aws_sdk_cache.outputs.cache-hit != 'true' - if: matrix.with-s3 == 1 && matrix.os == 'windows-latest' + if: matrix.with-s3 == 1 && matrix.os == 'windows-latest' && steps.aws_sdk_cache.outputs.cache-hit != 'true' run: | git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp cd aws-sdk-cpp @@ -82,8 +81,7 @@ jobs: cmake --build build --config Release cmake --install build --config Release - name: Install AWS-SDK-CPP on Non-Windows for S3 IO datapipes - # if: matrix.with-s3 == 1 && matrix.os != 'windows-latest' && steps.aws_sdk_cache.outputs.cache-hit != 'true' - if: matrix.with-s3 == 1 && matrix.os != 'windows-latest' + if: matrix.with-s3 == 1 && matrix.os != 'windows-latest' && steps.aws_sdk_cache.outputs.cache-hit != 'true' run: | git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp cd aws-sdk-cpp/ @@ -120,7 +118,7 @@ jobs: ls -al torchdata if [[ ${{ matrix.os }} == 'windows-latest' ]]; then ls -aR build - find . -name _torchdata.cp${{ matrix.python-version }}-win_amd64.pyd + cat torchdata/_torchdata.pyd else ls -al build/lib.*/torchdata fi diff --git a/setup.py b/setup.py index 03c9543e8..a522a1e88 100644 --- a/setup.py +++ b/setup.py @@ -61,9 +61,14 @@ def run(self): distutils.command.clean.clean.run(self) # Remove torchdata extension - for path in (ROOT_DIR / "torchdata").glob("**/*.so"): - print(f"removing '{path}'") - path.unlink() + def remove_extension(pattern): + for path in (ROOT_DIR / "torchdata").glob(pattern): + print(f"removing '{path}'") + path.unlink() + + remove_extension("**/*.so") + remove_extension("**/*.pyd") + # Remove build directory build_dirs = [ ROOT_DIR / "build", diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index 7abb5eac0..495c8fdb0 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -120,3 +120,10 @@ def build_extension(self, ext): subprocess.check_call(["cmake", str(_ROOT_DIR)] + cmake_args, cwd=self.build_temp) subprocess.check_call(["cmake", "--build", "."] + build_args, cwd=self.build_temp) + + def get_ext_filename(self, fullname): + ext_filename = super().get_ext_filename(fullname) + ext_filename_parts = ext_filename.split(".") + without_abi = ext_filename_parts[:-2] + ext_filename_parts[-1:] + ext_filename = ".".join(without_abi) + return ext_filename diff --git a/torchdata/csrc/CMakeLists.txt b/torchdata/csrc/CMakeLists.txt index cabeccdd9..45f3e90e9 100644 --- a/torchdata/csrc/CMakeLists.txt +++ b/torchdata/csrc/CMakeLists.txt @@ -20,9 +20,9 @@ if(BUILD_S3) pybind/S3Handler/S3Handler.cpp ) - pybind11_add_module(_torchdata SHARED ${EXTENSION_SOURCES}) + add_library(_torchdata SHARED ${EXTENSION_SOURCES}) - target_include_directories(_torchdata PRIVATE ${PROJECT_SOURCE_DIR} ${Python_INCLUDE_DIR}) + target_include_directories(_torchdata PRIVATE ${PROJECT_SOURCE_DIR} ${Python_INCLUDE_DIR} {pybind11_INCLUDE_DIRS}) message(STATUS "AWSSDK linked libs: ${AWSSDK_LINK_LIBRARIES}") target_link_libraries( @@ -30,6 +30,7 @@ if(BUILD_S3) PRIVATE ${AWSSDK_LINK_LIBRARIES} ${Python_LIBRARIES} + pybind11::module ) set_target_properties(_torchdata PROPERTIES PREFIX "") From 13d84548d8e28650fcd0fb9c7d0c10639a92a210 Mon Sep 17 00:00:00 2001 From: erjia Date: Wed, 30 Mar 2022 15:59:46 +0000 Subject: [PATCH 142/153] Remove BUILD_PYTHON_VERSION --- .github/workflows/ci.yml | 32 ++++++++++++++------------------ tools/setup_helpers/extension.py | 6 ------ torchdata/csrc/CMakeLists.txt | 16 +++++++++++----- 3 files changed, 25 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 541d3408c..4982e5abe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,6 +59,19 @@ jobs: pip3 install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html pip3 install cmake ninja pybind11 echo "/home/runner/.local/bin" >> $GITHUB_PATH + - name: Export AWS-SDK-CPP & PYBIND11 + if: matrix.with-s3 == 1 + shell: bash + run: | + if [[ ${{ matrix.os }} == 'windows-latest' ]]; then + AWSSDK_PATH="$GITHUB_WORKSPACE\\aws-sdk-cpp\\sdk-lib" + else + AWSSDK_PATH="$GITHUB_WORKSPACE/aws-sdk-cpp/sdk-lib" + fi + PYBIND11_PATH=`pybind11-config --cmakedir` + echo "::set-output name=awssdk::$AWSSDK_PATH" + echo "::set-output name=pybind11::$PYBIND11_PATH" + id: export_path - name: Get Date if: matrix.with-s3 == 1 id: get_date @@ -69,7 +82,7 @@ jobs: id: aws_sdk_cache uses: actions/cache@v3 with: - path: $GITHUB_WORKSPACE/aws-sdk-cpp/sdk-lib + path: ${{ steps.export_path.outputs.awssdk }} key: ${{ matrix.os }}-${{ matrix.python-version }}-${{ steps.get_date.outputs.date }} - name: Install AWS-SDK-CPP on Windows for S3 IO datapipes if: matrix.with-s3 == 1 && matrix.os == 'windows-latest' && steps.aws_sdk_cache.outputs.cache-hit != 'true' @@ -90,25 +103,11 @@ jobs: cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" -DENABLE_TESTING=OFF -DCMAKE_INSTALL_PREFIX=../sdk-lib make sudo make install - - name: Export AWS-SDK-CPP & PYBIND11 - if: matrix.with-s3 == 1 - shell: bash - run: | - if [[ ${{ matrix.os }} == 'windows-latest' ]]; then - AWSSDK_PATH="$GITHUB_WORKSPACE\\aws-sdk-cpp\\sdk-lib" - else - AWSSDK_PATH="$GITHUB_WORKSPACE/aws-sdk-cpp/sdk-lib" - fi - PYBIND11_PATH=`pybind11-config --cmakedir` - echo "::set-output name=awssdk::$AWSSDK_PATH" - echo "::set-output name=pybind11::$PYBIND11_PATH" - id: export_path - name: Build TorchData run: | python setup.py develop env: BUILD_S3: ${{ matrix.with-s3 }} - BUILD_PYTHON_VERSION: ${{ matrix.python-version }} pybind11_DIR: ${{ steps.export_path.outputs.pybind11 }} AWSSDK_DIR: ${{ steps.export_path.outputs.awssdk }} - name: Find extension @@ -117,10 +116,7 @@ jobs: run: | ls -al torchdata if [[ ${{ matrix.os }} == 'windows-latest' ]]; then - ls -aR build cat torchdata/_torchdata.pyd - else - ls -al build/lib.*/torchdata fi - name: Install test requirements run: pip3 install expecttest fsspec iopath==0.1.9 numpy pytest rarfile diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index 495c8fdb0..8de753696 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -39,7 +39,6 @@ def _get_build(var, default=False): _BUILD_S3 = _get_build("BUILD_S3", False) _AWSSDK_DIR = os.environ.get("AWSSDK_DIR", None) -_BUILD_PYTHON_VERSION = os.environ.get("BUILD_PYTHON_VERSION", None) def get_ext_modules(): @@ -85,11 +84,6 @@ def build_extension(self, ext): build_args = ["--config", cfg] - if _BUILD_PYTHON_VERSION: - cmake_args += [ - f"-DBUILD_PYTHON_VERSION={_BUILD_PYTHON_VERSION}", - ] - if _AWSSDK_DIR: cmake_args += [ f"-DAWSSDK_DIR={_AWSSDK_DIR}", diff --git a/torchdata/csrc/CMakeLists.txt b/torchdata/csrc/CMakeLists.txt index 45f3e90e9..b83c67e0b 100644 --- a/torchdata/csrc/CMakeLists.txt +++ b/torchdata/csrc/CMakeLists.txt @@ -2,16 +2,21 @@ if(BUILD_S3) message(STATUS "Building S3 IO functionality") - list(APPEND CMAKE_PREFIX_PATH ${AWSSDK_DIR}) - find_package(AWSSDK REQUIRED COMPONENTS s3 transfer) - find_package(pybind11 REQUIRED) + # To make the right CPython is built with on GitHub Actions, + # see https://github.com/actions/setup-python/issues/121#issuecomment-1014500503 + set(Python_FIND_FRAMEWORK "LAST") - if(BUILD_PYTHON_VERSION) - find_package(Python3 ${BUILD_PYTHON_VERSION} EXACT COMPONENTS Interpreter Development) + if (WIN32) + find_package(Python3 ${PYTHON_VERSION} EXACT COMPONENTS Interpreter Development) + set(ADDITIONAL_ITEMS Python3::Python) else() find_package(Python3 COMPONENTS Interpreter Development) endif() + list(APPEND CMAKE_PREFIX_PATH ${AWSSDK_DIR}) + find_package(AWSSDK REQUIRED COMPONENTS s3 transfer) + find_package(pybind11 REQUIRED) + set(CMAKE_POSITION_INDEPENDENT_CODE ON) set( @@ -31,6 +36,7 @@ if(BUILD_S3) ${AWSSDK_LINK_LIBRARIES} ${Python_LIBRARIES} pybind11::module + ${ADDITIONAL_ITEMS} ) set_target_properties(_torchdata PROPERTIES PREFIX "") From 1c70aadabfea392d1a0b874a3dfd33cc6dabbfbf Mon Sep 17 00:00:00 2001 From: erjia Date: Wed, 30 Mar 2022 17:36:45 +0000 Subject: [PATCH 143/153] Remove redundant _internal --- .github/workflows/ci.yml | 3 +- torchdata/_extension.py | 11 +++-- torchdata/_internal/__init__.py | 7 ---- torchdata/_internal/module_utils.py | 64 ----------------------------- torchdata/csrc/CMakeLists.txt | 10 +++-- torchdata/csrc/pybind/pybind.cpp | 2 +- 6 files changed, 18 insertions(+), 79 deletions(-) delete mode 100644 torchdata/_internal/__init__.py delete mode 100644 torchdata/_internal/module_utils.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4982e5abe..e9df7b02b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,6 +93,7 @@ jobs: cmake -S . -B build -GNinja -DBUILD_ONLY="s3;transfer" -DENABLE_TESTING=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=sdk-lib cmake --build build --config Release cmake --install build --config Release + echo ${{ steps.export_path.outputs.awssdk }} >> $GITHUB_PATH - name: Install AWS-SDK-CPP on Non-Windows for S3 IO datapipes if: matrix.with-s3 == 1 && matrix.os != 'windows-latest' && steps.aws_sdk_cache.outputs.cache-hit != 'true' run: | @@ -116,7 +117,7 @@ jobs: run: | ls -al torchdata if [[ ${{ matrix.os }} == 'windows-latest' ]]; then - cat torchdata/_torchdata.pyd + ls -R aws-sdk-cpp\\sdk-lib\\ fi - name: Install test requirements run: pip3 install expecttest fsspec iopath==0.1.9 numpy pytest rarfile diff --git a/torchdata/_extension.py b/torchdata/_extension.py index 5b3fd716e..57b5234d1 100644 --- a/torchdata/_extension.py +++ b/torchdata/_extension.py @@ -2,7 +2,6 @@ import os from pathlib import Path -from torchdata._internal import module_utils as _mod_utils # noqa: F401 _LIB_DIR = Path(__file__).parent @@ -10,6 +9,7 @@ def _init_extension(): # load the pybind11 extension lib_dir = os.path.dirname(__file__) + print("Lib dir", lib_dir) if os.name == "nt": import ctypes @@ -23,8 +23,10 @@ def _init_extension(): kernel32.AddDllDirectory.restype = ctypes.c_void_p if sys.version_info >= (3, 8): + print("Add all dll directory") os.add_dll_directory(lib_dir) elif with_load_library_flags: + print("Kernel 32 add dll directory") res = kernel32.AddDllDirectory(lib_dir) if res is None: err = ctypes.WinError(ctypes.get_last_error()) @@ -37,8 +39,11 @@ def _init_extension(): extfinder = importlib.machinery.FileFinder(lib_dir, loader_details) ext_specs = extfinder.find_spec("_torchdata") - if ext_specs is not None: - from torchdata import _torchdata + + if ext_specs is None: + return + + from torchdata import _torchdata _init_extension() diff --git a/torchdata/_internal/__init__.py b/torchdata/_internal/__init__.py deleted file mode 100644 index 5d6cabb03..000000000 --- a/torchdata/_internal/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from torch.hub import download_url_to_file, load_state_dict_from_url - - -__all__ = [ - "load_state_dict_from_url", - "download_url_to_file", -] diff --git a/torchdata/_internal/module_utils.py b/torchdata/_internal/module_utils.py deleted file mode 100644 index 47ff06215..000000000 --- a/torchdata/_internal/module_utils.py +++ /dev/null @@ -1,64 +0,0 @@ -import importlib.util -import warnings -from functools import wraps -from typing import Optional - - -def is_module_available(*modules: str) -> bool: - r"""Returns if a top-level module with :attr:`name` exists *without** - importing it. This is generally safer than try-catch block around a - `import X`. It avoids third party libraries breaking assumptions of some of - our tests, e.g., setting multiprocessing start method when imported - (see librosa/#747, torchvision/#544). - """ - return all(importlib.util.find_spec(m) is not None for m in modules) - - -def requires_module(*modules: str): - """Decorate function to give error message if invoked without required optional modules. - - This decorator is to give better error message to users rather - than raising ``NameError: name 'module' is not defined`` at random places. - """ - missing = [m for m in modules if not is_module_available(m)] - - if not missing: - # fall through. If all the modules are available, no need to decorate - def decorator(func): - return func - - else: - req = f"module: {missing[0]}" if len(missing) == 1 else f"modules: {missing}" - - def decorator(func): - @wraps(func) - def wrapped(*args, **kwargs): - raise RuntimeError(f"{func.__module__}.{func.__name__} requires {req}") - - return wrapped - - return decorator - - -def deprecated(direction: str, version: Optional[str] = None): - """Decorator to add deprecation message - - Args: - direction (str): Migration steps to be given to users. - version (str or int): The version when the object will be removed - """ - - def decorator(func): - @wraps(func) - def wrapped(*args, **kwargs): - message = ( - f"{func.__module__}.{func.__name__} has been deprecated " - f'and will be removed from {"future" if version is None else version} release. ' - f"{direction}" - ) - warnings.warn(message, stacklevel=2) - return func(*args, **kwargs) - - return wrapped - - return decorator diff --git a/torchdata/csrc/CMakeLists.txt b/torchdata/csrc/CMakeLists.txt index b83c67e0b..d8b1b2490 100644 --- a/torchdata/csrc/CMakeLists.txt +++ b/torchdata/csrc/CMakeLists.txt @@ -15,7 +15,7 @@ if(BUILD_S3) list(APPEND CMAKE_PREFIX_PATH ${AWSSDK_DIR}) find_package(AWSSDK REQUIRED COMPONENTS s3 transfer) - find_package(pybind11 REQUIRED) + find_package(pybind11 CONFIG REQUIRED) set(CMAKE_POSITION_INDEPENDENT_CODE ON) @@ -25,17 +25,21 @@ if(BUILD_S3) pybind/S3Handler/S3Handler.cpp ) - add_library(_torchdata SHARED ${EXTENSION_SOURCES}) + add_library(_torchdata MODULE ${EXTENSION_SOURCES}) target_include_directories(_torchdata PRIVATE ${PROJECT_SOURCE_DIR} ${Python_INCLUDE_DIR} {pybind11_INCLUDE_DIRS}) message(STATUS "AWSSDK linked libs: ${AWSSDK_LINK_LIBRARIES}") + target_link_libraries( + _torchdata + PUBLIC + pybind11::module + ) target_link_libraries( _torchdata PRIVATE ${AWSSDK_LINK_LIBRARIES} ${Python_LIBRARIES} - pybind11::module ${ADDITIONAL_ITEMS} ) diff --git a/torchdata/csrc/pybind/pybind.cpp b/torchdata/csrc/pybind/pybind.cpp index e96bde63a..806791cbb 100644 --- a/torchdata/csrc/pybind/pybind.cpp +++ b/torchdata/csrc/pybind/pybind.cpp @@ -1,5 +1,5 @@ -#include #include +#include #include #include From 020c6b887364f4582149580814e34ba33732f82c Mon Sep 17 00:00:00 2001 From: erjia Date: Wed, 30 Mar 2022 21:51:58 +0000 Subject: [PATCH 144/153] Export aws to path for windows --- .github/workflows/ci.yml | 7 ++-- torchdata/_extension.py | 72 ++++++++++++++++++++++++++++++---------- 2 files changed, 58 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e9df7b02b..fc36f03a4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,8 +22,8 @@ jobs: fail-fast: false matrix: os: - - macos-latest - - ubuntu-latest + # - macos-latest + # - ubuntu-latest - windows-latest python-version: - 3.7 @@ -93,7 +93,6 @@ jobs: cmake -S . -B build -GNinja -DBUILD_ONLY="s3;transfer" -DENABLE_TESTING=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=sdk-lib cmake --build build --config Release cmake --install build --config Release - echo ${{ steps.export_path.outputs.awssdk }} >> $GITHUB_PATH - name: Install AWS-SDK-CPP on Non-Windows for S3 IO datapipes if: matrix.with-s3 == 1 && matrix.os != 'windows-latest' && steps.aws_sdk_cache.outputs.cache-hit != 'true' run: | @@ -117,7 +116,7 @@ jobs: run: | ls -al torchdata if [[ ${{ matrix.os }} == 'windows-latest' ]]; then - ls -R aws-sdk-cpp\\sdk-lib\\ + find . -name *.dll fi - name: Install test requirements run: pip3 install expecttest fsspec iopath==0.1.9 numpy pytest rarfile diff --git a/torchdata/_extension.py b/torchdata/_extension.py index 57b5234d1..46e58cd8f 100644 --- a/torchdata/_extension.py +++ b/torchdata/_extension.py @@ -1,5 +1,7 @@ +import ctypes import importlib import os +import sys from pathlib import Path @@ -7,31 +9,67 @@ def _init_extension(): - # load the pybind11 extension - lib_dir = os.path.dirname(__file__) - print("Lib dir", lib_dir) + if sys.platform == "win32": + lib_dir = os.path.dirname(__file__) + py_dll_path = os.path.join(sys.exec_prefix, "Library", "bin") + aws_dll_path = os.path.join(_LIB_DIR.parent.resolve(), "aws-sdk-cpp", "sdk-lib", "lib") + # When users create a virtualenv that inherits the base environment, + # we will need to add the corresponding library directory into + # DLL search directories. Otherwise, it will rely on `PATH` which + # is dependent on user settings. + if sys.exec_prefix != sys.base_exec_prefix: + base_py_dll_path = os.path.join(sys.base_exec_prefix, "Library", "bin") + else: + base_py_dll_path = "" - if os.name == "nt": - import ctypes - import sys + dll_paths = list(filter(os.path.exists, [lib_dir, aws_dll_path, py_dll_path, base_py_dll_path])) kernel32 = ctypes.WinDLL("kernel32.dll", use_last_error=True) with_load_library_flags = hasattr(kernel32, "AddDllDirectory") prev_error_mode = kernel32.SetErrorMode(0x0001) + kernel32.LoadLibraryW.restype = ctypes.c_void_p if with_load_library_flags: kernel32.AddDllDirectory.restype = ctypes.c_void_p + kernel32.LoadLibraryExW.restype = ctypes.c_void_p - if sys.version_info >= (3, 8): - print("Add all dll directory") - os.add_dll_directory(lib_dir) - elif with_load_library_flags: - print("Kernel 32 add dll directory") - res = kernel32.AddDllDirectory(lib_dir) - if res is None: - err = ctypes.WinError(ctypes.get_last_error()) - err.strerror += f' Error adding "{lib_dir}" to the DLL directories.' - raise err + for dll_path in dll_paths: + if sys.version_info >= (3, 8): + print("Add all dll directory", dll_path) + os.add_dll_directory(dll_path) + elif with_load_library_flags: + print("Kernel 32 add dll directory", dll_path) + res = kernel32.AddDllDirectory(dll_path) + if res is None: + err = ctypes.WinError(ctypes.get_last_error()) + err.strerror += f' Error adding "{lib_dir}" to the DLL directories.' + raise err + + import glob + + dlls = glob.glob(os.path.join(aws_dll_path, "*.dll")) + path_patched = False + for dll in dlls: + print("DLL", dll) + is_loaded = False + if with_load_library_flags: + res = kernel32.LoadLibraryExW(dll, None, 0x00001100) + last_error = ctypes.get_last_error() + if res is None and last_error != 126: + err = ctypes.WinError(last_error) + err.strerror += f' Error loading "{dll}" or one of its dependencies.' + raise err + elif res is not None: + is_loaded = True + if not is_loaded: + if not path_patched: + os.environ["PATH"] = ";".join(dll_paths + [os.environ["PATH"]]) + path_patched = True + res = kernel32.LoadLibraryW(dll) + if res is None: + err = ctypes.WinError(ctypes.get_last_error()) + err.strerror += f' Error loading "{dll}" or one of its dependencies.' + raise err kernel32.SetErrorMode(prev_error_mode) @@ -43,7 +81,7 @@ def _init_extension(): if ext_specs is None: return - from torchdata import _torchdata + from torchdata import _torchdata as _torchdata _init_extension() From 926cd0129a864beaf3688f558d48c2c4f5849c3b Mon Sep 17 00:00:00 2001 From: erjia Date: Thu, 31 Mar 2022 13:45:48 +0000 Subject: [PATCH 145/153] Static link on windows --- .github/workflows/ci.yml | 2 +- test/test_remote_io.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc36f03a4..e02933527 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,7 +90,7 @@ jobs: git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp cd aws-sdk-cpp mkdir sdk-lib - cmake -S . -B build -GNinja -DBUILD_ONLY="s3;transfer" -DENABLE_TESTING=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=sdk-lib + cmake -S . -B build -GNinja -DBUILD_ONLY="s3;transfer" -DBUILD_SHARED_LIBS=OFF -DENABLE_TESTING=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=sdk-lib cmake --build build --config Release cmake --install build --config Release - name: Install AWS-SDK-CPP on Non-Windows for S3 IO datapipes diff --git a/test/test_remote_io.py b/test/test_remote_io.py index 7871579bf..179156676 100644 --- a/test/test_remote_io.py +++ b/test/test_remote_io.py @@ -277,5 +277,4 @@ def test_s3_io_iterdatapipe(self): if __name__ == "__main__": - print("[Extension] torchdata path: ", torchdata.__file__) unittest.main() From d7a85779138ad9a2adb3ec27ac42481f6dedd1f8 Mon Sep 17 00:00:00 2001 From: erjia Date: Thu, 31 Mar 2022 13:51:57 +0000 Subject: [PATCH 146/153] skip cache --- .github/workflows/ci.yml | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e02933527..021a426ab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,20 +72,21 @@ jobs: echo "::set-output name=awssdk::$AWSSDK_PATH" echo "::set-output name=pybind11::$PYBIND11_PATH" id: export_path - - name: Get Date - if: matrix.with-s3 == 1 - id: get_date - shell: bash - run: echo "::set-output name=date::$(date "+%Y%m%d")" - - name: Uses cache for AWS-SDK-CPP - if: matrix.with-s3 == 1 - id: aws_sdk_cache - uses: actions/cache@v3 - with: - path: ${{ steps.export_path.outputs.awssdk }} - key: ${{ matrix.os }}-${{ matrix.python-version }}-${{ steps.get_date.outputs.date }} + # - name: Get Date + # if: matrix.with-s3 == 1 + # id: get_date + # shell: bash + # run: echo "::set-output name=date::$(date "+%Y%m%d")" + # - name: Uses cache for AWS-SDK-CPP + # if: matrix.with-s3 == 1 + # id: aws_sdk_cache + # uses: actions/cache@v3 + # with: + # path: ${{ steps.export_path.outputs.awssdk }} + # key: ${{ matrix.os }}-${{ matrix.python-version }}-${{ steps.get_date.outputs.date }} - name: Install AWS-SDK-CPP on Windows for S3 IO datapipes - if: matrix.with-s3 == 1 && matrix.os == 'windows-latest' && steps.aws_sdk_cache.outputs.cache-hit != 'true' + # if: matrix.with-s3 == 1 && matrix.os == 'windows-latest' && steps.aws_sdk_cache.outputs.cache-hit != 'true' + if: matrix.with-s3 == 1 && matrix.os == 'windows-latest' run: | git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp cd aws-sdk-cpp @@ -94,7 +95,8 @@ jobs: cmake --build build --config Release cmake --install build --config Release - name: Install AWS-SDK-CPP on Non-Windows for S3 IO datapipes - if: matrix.with-s3 == 1 && matrix.os != 'windows-latest' && steps.aws_sdk_cache.outputs.cache-hit != 'true' + # if: matrix.with-s3 == 1 && matrix.os != 'windows-latest' && steps.aws_sdk_cache.outputs.cache-hit != 'true' + if: matrix.with-s3 == 1 && matrix.os != 'windows-latest' run: | git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp cd aws-sdk-cpp/ From a836303c930d28a026cfcf3b96876869b28cbf39 Mon Sep 17 00:00:00 2001 From: erjia Date: Thu, 31 Mar 2022 15:21:05 +0000 Subject: [PATCH 147/153] Remove debugging lines and enable tests --- .github/workflows/ci.yml | 28 ++----------- .github/workflows/domain_ci.yml | 12 +++--- setup.py | 2 +- tools/setup_helpers/extension.py | 2 +- torchdata/_extension.py | 69 +++----------------------------- torchdata/csrc/CMakeLists.txt | 6 +-- 6 files changed, 17 insertions(+), 102 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 021a426ab..18b68c850 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,8 +22,8 @@ jobs: fail-fast: false matrix: os: - # - macos-latest - # - ubuntu-latest + - macos-latest + - ubuntu-latest - windows-latest python-version: - 3.7 @@ -31,7 +31,7 @@ jobs: - 3.9 with-s3: - 1 - # - 0 + - 0 steps: - name: Setup additional system libraries if: startsWith( matrix.os, 'ubuntu' ) @@ -72,20 +72,7 @@ jobs: echo "::set-output name=awssdk::$AWSSDK_PATH" echo "::set-output name=pybind11::$PYBIND11_PATH" id: export_path - # - name: Get Date - # if: matrix.with-s3 == 1 - # id: get_date - # shell: bash - # run: echo "::set-output name=date::$(date "+%Y%m%d")" - # - name: Uses cache for AWS-SDK-CPP - # if: matrix.with-s3 == 1 - # id: aws_sdk_cache - # uses: actions/cache@v3 - # with: - # path: ${{ steps.export_path.outputs.awssdk }} - # key: ${{ matrix.os }}-${{ matrix.python-version }}-${{ steps.get_date.outputs.date }} - name: Install AWS-SDK-CPP on Windows for S3 IO datapipes - # if: matrix.with-s3 == 1 && matrix.os == 'windows-latest' && steps.aws_sdk_cache.outputs.cache-hit != 'true' if: matrix.with-s3 == 1 && matrix.os == 'windows-latest' run: | git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp @@ -95,7 +82,6 @@ jobs: cmake --build build --config Release cmake --install build --config Release - name: Install AWS-SDK-CPP on Non-Windows for S3 IO datapipes - # if: matrix.with-s3 == 1 && matrix.os != 'windows-latest' && steps.aws_sdk_cache.outputs.cache-hit != 'true' if: matrix.with-s3 == 1 && matrix.os != 'windows-latest' run: | git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp @@ -112,14 +98,6 @@ jobs: BUILD_S3: ${{ matrix.with-s3 }} pybind11_DIR: ${{ steps.export_path.outputs.pybind11 }} AWSSDK_DIR: ${{ steps.export_path.outputs.awssdk }} - - name: Find extension - if: always() && matrix.with-s3 == 1 - shell: bash - run: | - ls -al torchdata - if [[ ${{ matrix.os }} == 'windows-latest' ]]; then - find . -name *.dll - fi - name: Install test requirements run: pip3 install expecttest fsspec iopath==0.1.9 numpy pytest rarfile - name: Run DataPipes tests with pytest diff --git a/.github/workflows/domain_ci.yml b/.github/workflows/domain_ci.yml index 5e10ce596..7729848d9 100644 --- a/.github/workflows/domain_ci.yml +++ b/.github/workflows/domain_ci.yml @@ -4,12 +4,12 @@ on: branches: - main - release/* - # pull_request: - # branches: - # - main - # # For PR created by ghstack - # - gh/*/*/base - # - release/* + pull_request: + branches: + - main + # For PR created by ghstack + - gh/*/*/base + - release/* jobs: torchvision: diff --git a/setup.py b/setup.py index a522a1e88..1f10ac571 100644 --- a/setup.py +++ b/setup.py @@ -110,7 +110,7 @@ def remove_extension(pattern): "Topic :: Scientific/Engineering :: Artificial Intelligence", ], # Package Info - packages=find_packages(exclude=["test*", "examples*", "torchdata.csrc*", "tools*"]), + packages=find_packages(exclude=["test*", "examples*", "torchdata.csrc*", "build*", "tools*"]), zip_safe=False, # C++ Extension Modules ext_modules=setup_helpers.get_ext_modules(), diff --git a/tools/setup_helpers/extension.py b/tools/setup_helpers/extension.py index 8de753696..7d63fead6 100644 --- a/tools/setup_helpers/extension.py +++ b/tools/setup_helpers/extension.py @@ -84,7 +84,7 @@ def build_extension(self, ext): build_args = ["--config", cfg] - if _AWSSDK_DIR: + if _BUILD_S3 and _AWSSDK_DIR: cmake_args += [ f"-DAWSSDK_DIR={_AWSSDK_DIR}", ] diff --git a/torchdata/_extension.py b/torchdata/_extension.py index 46e58cd8f..a862c356e 100644 --- a/torchdata/_extension.py +++ b/torchdata/_extension.py @@ -1,7 +1,5 @@ -import ctypes import importlib import os -import sys from pathlib import Path @@ -9,69 +7,12 @@ def _init_extension(): - if sys.platform == "win32": - lib_dir = os.path.dirname(__file__) - py_dll_path = os.path.join(sys.exec_prefix, "Library", "bin") - aws_dll_path = os.path.join(_LIB_DIR.parent.resolve(), "aws-sdk-cpp", "sdk-lib", "lib") - # When users create a virtualenv that inherits the base environment, - # we will need to add the corresponding library directory into - # DLL search directories. Otherwise, it will rely on `PATH` which - # is dependent on user settings. - if sys.exec_prefix != sys.base_exec_prefix: - base_py_dll_path = os.path.join(sys.base_exec_prefix, "Library", "bin") - else: - base_py_dll_path = "" + lib_dir = os.path.dirname(__file__) - dll_paths = list(filter(os.path.exists, [lib_dir, aws_dll_path, py_dll_path, base_py_dll_path])) - - kernel32 = ctypes.WinDLL("kernel32.dll", use_last_error=True) - with_load_library_flags = hasattr(kernel32, "AddDllDirectory") - prev_error_mode = kernel32.SetErrorMode(0x0001) - - kernel32.LoadLibraryW.restype = ctypes.c_void_p - if with_load_library_flags: - kernel32.AddDllDirectory.restype = ctypes.c_void_p - kernel32.LoadLibraryExW.restype = ctypes.c_void_p - - for dll_path in dll_paths: - if sys.version_info >= (3, 8): - print("Add all dll directory", dll_path) - os.add_dll_directory(dll_path) - elif with_load_library_flags: - print("Kernel 32 add dll directory", dll_path) - res = kernel32.AddDllDirectory(dll_path) - if res is None: - err = ctypes.WinError(ctypes.get_last_error()) - err.strerror += f' Error adding "{lib_dir}" to the DLL directories.' - raise err - - import glob - - dlls = glob.glob(os.path.join(aws_dll_path, "*.dll")) - path_patched = False - for dll in dlls: - print("DLL", dll) - is_loaded = False - if with_load_library_flags: - res = kernel32.LoadLibraryExW(dll, None, 0x00001100) - last_error = ctypes.get_last_error() - if res is None and last_error != 126: - err = ctypes.WinError(last_error) - err.strerror += f' Error loading "{dll}" or one of its dependencies.' - raise err - elif res is not None: - is_loaded = True - if not is_loaded: - if not path_patched: - os.environ["PATH"] = ";".join(dll_paths + [os.environ["PATH"]]) - path_patched = True - res = kernel32.LoadLibraryW(dll) - if res is None: - err = ctypes.WinError(ctypes.get_last_error()) - err.strerror += f' Error loading "{dll}" or one of its dependencies.' - raise err - - kernel32.SetErrorMode(prev_error_mode) + # TODO: If any extension had dependency of shared library, + # in order to support load these shred libraries dynamically, + # we need to add logic to load dll path on Windows + # See: https://github.com/pytorch/pytorch/blob/master/torch/__init__.py#L56-L140 loader_details = (importlib.machinery.ExtensionFileLoader, importlib.machinery.EXTENSION_SUFFIXES) diff --git a/torchdata/csrc/CMakeLists.txt b/torchdata/csrc/CMakeLists.txt index d8b1b2490..88740fa24 100644 --- a/torchdata/csrc/CMakeLists.txt +++ b/torchdata/csrc/CMakeLists.txt @@ -30,14 +30,10 @@ if(BUILD_S3) target_include_directories(_torchdata PRIVATE ${PROJECT_SOURCE_DIR} ${Python_INCLUDE_DIR} {pybind11_INCLUDE_DIRS}) message(STATUS "AWSSDK linked libs: ${AWSSDK_LINK_LIBRARIES}") - target_link_libraries( - _torchdata - PUBLIC - pybind11::module - ) target_link_libraries( _torchdata PRIVATE + pybind11::module ${AWSSDK_LINK_LIBRARIES} ${Python_LIBRARIES} ${ADDITIONAL_ITEMS} From 27d2370cadfe7da0d84b8080b7695c3323850dba Mon Sep 17 00:00:00 2001 From: erjia Date: Fri, 1 Apr 2022 16:46:13 +0000 Subject: [PATCH 148/153] Fix mypy --- torchdata/_extension.py | 2 +- torchdata/_torchdata/__init__.pyi | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 torchdata/_torchdata/__init__.pyi diff --git a/torchdata/_extension.py b/torchdata/_extension.py index a862c356e..4c39c66a6 100644 --- a/torchdata/_extension.py +++ b/torchdata/_extension.py @@ -16,7 +16,7 @@ def _init_extension(): loader_details = (importlib.machinery.ExtensionFileLoader, importlib.machinery.EXTENSION_SUFFIXES) - extfinder = importlib.machinery.FileFinder(lib_dir, loader_details) + extfinder = importlib.machinery.FileFinder(lib_dir, loader_details) # type: ignore[arg-type] ext_specs = extfinder.find_spec("_torchdata") if ext_specs is None: diff --git a/torchdata/_torchdata/__init__.pyi b/torchdata/_torchdata/__init__.pyi new file mode 100644 index 000000000..372d877cd --- /dev/null +++ b/torchdata/_torchdata/__init__.pyi @@ -0,0 +1,12 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from typing import List + +# TODO: Add pyi generate script +class S3Handler: + def __init__(self, request_timeout_ms: int, region: str) -> None: ... + def s3_read(self, url: str) -> bytes: ... + def list_files(self, prefix: str) -> List[str]: ... + def set_buffer_size(self, buffer_size: int) -> None: ... + def set_multi_part_download(self, multi_part_download: bool) -> None: ... + def clear_marker(self) -> None: ... From c04c97373b164a2e5633573416b7e1187c251423 Mon Sep 17 00:00:00 2001 From: erjia Date: Fri, 1 Apr 2022 16:56:36 +0000 Subject: [PATCH 149/153] Remove extension for all platforms when clean --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 1f10ac571..ffd4bc576 100644 --- a/setup.py +++ b/setup.py @@ -63,11 +63,11 @@ def run(self): # Remove torchdata extension def remove_extension(pattern): for path in (ROOT_DIR / "torchdata").glob(pattern): - print(f"removing '{path}'") + print(f"removing extension '{path}'") path.unlink() - remove_extension("**/*.so") - remove_extension("**/*.pyd") + for ext in ["so", "dylib", "pyd"]: + remove_extension("**/*." + ext) # Remove build directory build_dirs = [ From 6d3289bbe96786e80d0647aea316dcd9d72fe77a Mon Sep 17 00:00:00 2001 From: erjia Date: Fri, 1 Apr 2022 20:06:46 +0000 Subject: [PATCH 150/153] Fix gen_pyi --- .gitignore | 2 +- setup.py | 18 ++++++++++++++---- {torchdata/datapipes => tools}/gen_pyi.py | 11 +++++------ 3 files changed, 20 insertions(+), 11 deletions(-) rename {torchdata/datapipes => tools}/gen_pyi.py (91%) diff --git a/.gitignore b/.gitignore index 82a1fe2a1..7c2168b9f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ build/* dist/* -torchdata.egg-info/* +*.egg-info/* torchdata/version.py torchdata/datapipes/iter/__init__.pyi diff --git a/setup.py b/setup.py index ffd4bc576..fc3925005 100644 --- a/setup.py +++ b/setup.py @@ -4,13 +4,14 @@ import os import shutil import subprocess +import sys from pathlib import Path from setuptools import find_packages, setup -from tools import setup_helpers -# from torchdata.datapipes.gen_pyi import gen_pyi +from tools import setup_helpers +from tools.gen_pyi import gen_pyi ROOT_DIR = Path(__file__).parent.resolve() @@ -85,6 +86,11 @@ def remove_extension(pattern): print("-- Building version " + VERSION) + if sys.argv[1] != "clean": + gen_pyi() + # TODO: Fix #343 + os.chdir(ROOT_DIR) + setup( # Metadata name="torchdata", @@ -109,8 +115,13 @@ def remove_extension(pattern): "Programming Language :: Python :: Implementation :: CPython", "Topic :: Scientific/Engineering :: Artificial Intelligence", ], + package_data={ + "torchdata": [ + "datapipes/iter/*.pyi", + ], + }, # Package Info - packages=find_packages(exclude=["test*", "examples*", "torchdata.csrc*", "build*", "tools*"]), + packages=find_packages(exclude=["test*", "examples*", "tools*", "torchdata.csrc*", "build*"]), zip_safe=False, # C++ Extension Modules ext_modules=setup_helpers.get_ext_modules(), @@ -119,4 +130,3 @@ def remove_extension(pattern): "clean": clean, }, ) - # gen_pyi() diff --git a/torchdata/datapipes/gen_pyi.py b/tools/gen_pyi.py similarity index 91% rename from torchdata/datapipes/gen_pyi.py rename to tools/gen_pyi.py index e08d19776..d3fa4b75a 100644 --- a/torchdata/datapipes/gen_pyi.py +++ b/tools/gen_pyi.py @@ -1,6 +1,5 @@ # Copyright (c) Facebook, Inc. and its affiliates. import os -import pathlib from pathlib import Path from typing import Dict, List, Optional, Set @@ -26,11 +25,11 @@ def get_lines_base_file(base_file_path: str, to_skip: Optional[Set[str]] = None) def gen_pyi() -> None: - ROOT_DIR = Path(__file__).parent.resolve() - print(f"Generating DataPipe Python interface file in {ROOT_DIR}") + DATAPIPE_DIR = Path(__file__).parent.parent.resolve() / "torchdata" / "datapipes" + print(f"Generating DataPipe Python interface file in {DATAPIPE_DIR}") iter_init_base = get_lines_base_file( - os.path.join(ROOT_DIR, "iter/__init__.py"), + os.path.join(DATAPIPE_DIR, "iter/__init__.py"), {"from torch.utils.data import IterDataPipe", "# Copyright (c) Facebook, Inc. and its affiliates."}, ) @@ -65,7 +64,7 @@ def gen_pyi() -> None: iterDP_deprecated_files, "IterDataPipe", iterDP_method_to_special_output_type, - root=str(pathlib.Path(__file__).parent.resolve()), + root=str(DATAPIPE_DIR), ) td_iter_method_definitions = [ @@ -77,7 +76,7 @@ def gen_pyi() -> None: replacements = [("${init_base}", iter_init_base, 0), ("${IterDataPipeMethods}", iter_method_definitions, 4)] gen_from_template( - dir=str(ROOT_DIR), + dir=str(DATAPIPE_DIR), template_name="iter/__init__.pyi.in", output_name="iter/__init__.pyi", replacements=replacements, From 34eacd6f0bb11e9e44bd053e1e57aa2a1be7e832 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Fri, 1 Apr 2022 16:02:39 -0700 Subject: [PATCH 151/153] update S3 datapipes in-line descriptions --- torchdata/datapipes/iter/load/s3io.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/torchdata/datapipes/iter/load/s3io.py b/torchdata/datapipes/iter/load/s3io.py index 06d18635c..c59a1ee4e 100644 --- a/torchdata/datapipes/iter/load/s3io.py +++ b/torchdata/datapipes/iter/load/s3io.py @@ -12,6 +12,14 @@ class S3FileListerIterDataPipe(IterDataPipe[str]): r""":class:`S3FileListerIterDataPipe`. Iterable DataPipe that lists URLs with the given prefixes (functional name: ``list_file_by_s3``). + Acceptable prefixes include `s3://bucket-name`, `s3://bucket-name/`, `s3://bucket-name/folder`, + `s3://bucket-name/folder/`, and `s3://bucket-name/prefix`. You may also set `length`, `request_timeout_ms` (default 3000 + ms in aws-sdk-cpp), and `region`. Note that: + + 1. Input **must** be a list and direct S3 URLs are skipped. + 2. `length` is `-1` by default, and any call to `__len__()` is invalid, because the length is unknown until all files + are iterated. + 3. `request_timeout_ms` and `region` will overwrite settings in the configuration file or environment variables. Args: source_datapipe: a DataPipe that contains URLs/URL prefixes to s3 files @@ -61,6 +69,12 @@ class S3FileLoaderIterDataPipe(IterDataPipe[Tuple[str, StreamWrapper]]): r""":class:`S3FileListerIterDataPipe`. Iterable DataPipe that loads S3 files given S3 URLs (functional name: ``load_file_by_s3``). + `S3FileLoader` iterates all given S3 URLs in `BytesIO` format with `(url, BytesIO)` tuples. + You may also set `request_timeout_ms` (default 3000 ms in aws-sdk-cpp), `region`, + `buffer_size` (default 120Mb), and `multi_part_download` (default to use multi-part downloading). Note that: + + 1. Input **must** be a list and S3 URLs must be valid. + 2. `request_timeout_ms` and `region` will overwrite settings in the configuration file or environment variables. Args: source_datapipe: a DataPipe that contains URLs to s3 files From 43d7231290417519e62207e961cfdda25e574f18 Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Fri, 1 Apr 2022 16:06:28 -0700 Subject: [PATCH 152/153] add comment on static linking on Windows for aws-sdk-cpp --- torchdata/datapipes/iter/load/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/torchdata/datapipes/iter/load/README.md b/torchdata/datapipes/iter/load/README.md index ffed9464b..459ef84df 100644 --- a/torchdata/datapipes/iter/load/README.md +++ b/torchdata/datapipes/iter/load/README.md @@ -10,6 +10,7 @@ git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp cd aws-sdk-cpp/ mkdir sdk-build cd sdk-build +# need to add flag -DBUILD_SHARED_LIBS=OFF for static linking on Windows cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;transfer" make make install # may need sudo From ed7e4ffeea85a0978f9f73e0ce5282ea9c21c92f Mon Sep 17 00:00:00 2001 From: Daiming Yang Date: Mon, 4 Apr 2022 15:07:58 -0700 Subject: [PATCH 153/153] Fix lint --- torchdata/csrc/pybind/S3Handler/S3Handler.cpp | 814 ++++++++---------- torchdata/csrc/pybind/S3Handler/S3Handler.h | 131 +-- torchdata/csrc/pybind/S3Handler/precompile.h | 4 +- torchdata/csrc/pybind/pybind.cpp | 103 ++- 4 files changed, 506 insertions(+), 546 deletions(-) diff --git a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp index 9943474b4..6d97457bf 100644 --- a/torchdata/csrc/pybind/S3Handler/S3Handler.cpp +++ b/torchdata/csrc/pybind/S3Handler/S3Handler.cpp @@ -1,449 +1,397 @@ #include "S3Handler.h" -namespace torchdata -{ - namespace - { - static const size_t S3DefaultBufferSize = 128 * 1024 * 1024; // 128 MB - static const uint64_t S3DefaultMultiPartDownloadChunkSize = 5 * 1024 * 1024; // 5 MB - static const int executorPoolSize = 25; - static const std::string S3DefaultMarker = ""; - - std::shared_ptr setUpS3Config(const long requestTimeoutMs, const std::string region) - { - std::shared_ptr cfg = - std::shared_ptr(new Aws::Client::ClientConfiguration()); - Aws::String config_file; - const char *config_file_env = getenv("AWS_CONFIG_FILE"); - if (config_file_env) - { - config_file = config_file_env; - } - else - { - const char *home_env = getenv("HOME"); - if (home_env) - { - config_file = home_env; - config_file += "/.aws/config"; - } - } - Aws::Config::AWSConfigFileProfileConfigLoader loader(config_file); - loader.Load(); - - const char *use_https = getenv("S3_USE_HTTPS"); - if (use_https) - { - if (use_https[0] == '0') - { - cfg->scheme = Aws::Http::Scheme::HTTP; - } - else - { - cfg->scheme = Aws::Http::Scheme::HTTPS; - } - } - const char *verify_ssl = getenv("S3_VERIFY_SSL"); - if (verify_ssl) - { - if (verify_ssl[0] == '0') - { - cfg->verifySSL = false; - } - else - { - cfg->verifySSL = true; - } - } - const char *endpoint_url = getenv("S3_ENDPOINT_URL"); - if (endpoint_url) - { - cfg->endpointOverride = endpoint_url; - } - if (region != "") - { - cfg->region = region; - } - else - { - const char *env_region = getenv("AWS_REGION"); - if (env_region) - { - cfg->region = env_region; - } - } - if (requestTimeoutMs > -1) - { - cfg->requestTimeoutMs = requestTimeoutMs; - } - return cfg; - } - - void ShutdownClient(std::shared_ptr *s3_client) - { - if (s3_client != nullptr) - { - delete s3_client; - Aws::SDKOptions options; - Aws::ShutdownAPI(options); - } - } - - void ShutdownTransferManager( - std::shared_ptr *transfer_manager) - { - if (transfer_manager != nullptr) - { - delete transfer_manager; - } - } - - void ShutdownExecutor(Aws::Utils::Threading::PooledThreadExecutor *executor) - { - if (executor != nullptr) - { - delete executor; - } - } - - void parseS3Path(const Aws::String &fname, Aws::String *bucket, - Aws::String *object) - { - if (fname.empty()) - { - throw std::invalid_argument("The filename cannot be an empty string."); - } - - if (fname.size() < 5 || fname.substr(0, 5) != "s3://") - { - throw std::invalid_argument("The filename must start with the S3 scheme."); - } - - std::string path = fname.substr(5); - - if (path.empty()) - { - throw std::invalid_argument("The filename cannot be an empty string."); - } - - size_t pos = path.find_first_of('/'); - if (pos == 0) - { - throw std::invalid_argument("The filename does not contain a bucket name."); - } - - *bucket = path.substr(0, pos); - *object = path.substr(pos + 1); - if (pos == std::string::npos) - { - *object = ""; - } - } - - class S3FS - { - private: - std::string bucket_name_; - std::string object_name_; - bool use_multi_part_download_; - std::shared_ptr s3_client_; - std::shared_ptr transfer_manager_; - - public: - S3FS(const std::string &bucket, const std::string &object, - const bool use_multi_part_download, - std::shared_ptr transfer_manager, - std::shared_ptr s3_client) - : bucket_name_(bucket), - object_name_(object), - use_multi_part_download_(use_multi_part_download), - transfer_manager_(transfer_manager), - s3_client_(s3_client) {} - - size_t Read(uint64_t offset, size_t n, char *buffer) - { - if (use_multi_part_download_) - { - return ReadTransferManager(offset, n, buffer); - } - else - { - return ReadS3Client(offset, n, buffer); - } - } - - size_t ReadS3Client(uint64_t offset, size_t n, char *buffer) - { - Aws::S3::Model::GetObjectRequest getObjectRequest; - - getObjectRequest.WithBucket(bucket_name_.c_str()) - .WithKey(object_name_.c_str()); - - std::string bytes = "bytes="; - bytes += std::to_string(offset) + "-" + std::to_string(offset + n - 1); - - getObjectRequest.SetRange(bytes.c_str()); - - // When you don’t want to load the entire file into memory, - // you can use IOStreamFactory in AmazonWebServiceRequest to pass a - // lambda to create a string stream. - getObjectRequest.SetResponseStreamFactory( - []() - { return Aws::New("S3IOAllocationTag"); }); - // get the object - Aws::S3::Model::GetObjectOutcome getObjectOutcome = s3_client_->GetObject(getObjectRequest); - - if (!getObjectOutcome.IsSuccess()) - { - Aws::S3::S3Error error = getObjectOutcome.GetError(); - std::cout << "ERROR: " << error.GetExceptionName() << ": " - << error.GetMessage() << std::endl; - return 0; - } - else - { - n = getObjectOutcome.GetResult().GetContentLength(); - // read data as a block: - getObjectOutcome.GetResult().GetBody().read(buffer, n); - return n; - } - } - - size_t ReadTransferManager(uint64_t offset, size_t n, char *buffer) - { - auto create_stream_fn = [&]() { // create stream lambda fn - return Aws::New( - "S3ReadStream", - Aws::New( - "S3ReadStream", reinterpret_cast(buffer), - n)); - }; // This buffer is what we used to initialize streambuf and is in memory - - std::shared_ptr downloadHandle = - transfer_manager_.get()->DownloadFile( - bucket_name_.c_str(), object_name_.c_str(), offset, - n, create_stream_fn); - downloadHandle->WaitUntilFinished(); - - Aws::OFStream storeFile(object_name_.c_str(), - Aws::OFStream::out | Aws::OFStream::trunc); - - if (downloadHandle->GetStatus() != - Aws::Transfer::TransferStatus::COMPLETED) - { - const Aws::Client::AWSError error = downloadHandle->GetLastError(); - std::cout << "ERROR: " << error.GetExceptionName() << ": " - << error.GetMessage() << std::endl; - return 0; - } - else - { - return downloadHandle->GetBytesTransferred(); - } - } - }; - } // namespace - - std::shared_ptr S3Handler::s3_handler_cfg_; - - S3Handler::S3Handler(const long requestTimeoutMs, const std::string region) - : s3_client_(nullptr, ShutdownClient), - transfer_manager_(nullptr, ShutdownTransferManager), - executor_(nullptr, ShutdownExecutor) - { - initialization_lock_ = std::shared_ptr(new std::mutex()); - - // Load reading parameters - buffer_size_ = S3DefaultBufferSize; - const char *bufferSizeStr = getenv("S3_BUFFER_SIZE"); - if (bufferSizeStr) - { - buffer_size_ = std::stoull(bufferSizeStr); - } - use_multi_part_download_ = true; - const char *use_multi_part_download_char = - getenv("S3_MULTI_PART_DOWNLOAD"); - if (use_multi_part_download_char) - { - std::string use_multi_part_download_str(use_multi_part_download_char); - if (use_multi_part_download_str == "OFF") - { - use_multi_part_download_ = false; - } - } - - Aws::SDKOptions options; - Aws::InitAPI(options); - S3Handler::s3_handler_cfg_ = setUpS3Config(requestTimeoutMs, region); - InitializeS3Client(); - - last_marker_ = S3DefaultMarker; +namespace torchdata { + +namespace { + +static const size_t S3DefaultBufferSize = 128 * 1024 * 1024; // 128 MB +static const uint64_t S3DefaultMultiPartDownloadChunkSize = + 5 * 1024 * 1024; // 5 MB +static const int executorPoolSize = 25; +static const std::string S3DefaultMarker = ""; + +std::shared_ptr setUpS3Config( + const long requestTimeoutMs, + const std::string region) { + std::shared_ptr cfg = + std::shared_ptr( + new Aws::Client::ClientConfiguration()); + Aws::String config_file; + const char* config_file_env = getenv("AWS_CONFIG_FILE"); + if (config_file_env) { + config_file = config_file_env; + } else { + const char* home_env = getenv("HOME"); + if (home_env) { + config_file = home_env; + config_file += "/.aws/config"; } - - S3Handler::~S3Handler() {} - - void S3Handler::InitializeS3Client() - { - std::lock_guard lock(*initialization_lock_); - s3_client_ = - std::shared_ptr( - new Aws::S3::S3Client( - *S3Handler::s3_handler_cfg_, - Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, - false)); + } + Aws::Config::AWSConfigFileProfileConfigLoader loader(config_file); + loader.Load(); + + const char* use_https = getenv("S3_USE_HTTPS"); + if (use_https) { + if (use_https[0] == '0') { + cfg->scheme = Aws::Http::Scheme::HTTP; + } else { + cfg->scheme = Aws::Http::Scheme::HTTPS; } - - void S3Handler::InitializeExecutor() - { - executor_ = - Aws::MakeShared( - "executor", executorPoolSize); + } + const char* verify_ssl = getenv("S3_VERIFY_SSL"); + if (verify_ssl) { + if (verify_ssl[0] == '0') { + cfg->verifySSL = false; + } else { + cfg->verifySSL = true; } - - void S3Handler::InitializeTransferManager() - { - std::shared_ptr s3_client = GetS3Client(); - std::lock_guard lock(*initialization_lock_); - - Aws::Transfer::TransferManagerConfiguration transfer_config( - GetExecutor().get()); - transfer_config.s3Client = s3_client; - // This buffer is what we used to initialize streambuf and is in memory - transfer_config.bufferSize = S3DefaultMultiPartDownloadChunkSize; - transfer_config.transferBufferMaxHeapSize = - (executorPoolSize + 1) * S3DefaultMultiPartDownloadChunkSize; - transfer_manager_ = - Aws::Transfer::TransferManager::Create(transfer_config); + } + const char* endpoint_url = getenv("S3_ENDPOINT_URL"); + if (endpoint_url) { + cfg->endpointOverride = endpoint_url; + } + if (region != "") { + cfg->region = region; + } else { + const char* env_region = getenv("AWS_REGION"); + if (env_region) { + cfg->region = env_region; } - - std::shared_ptr S3Handler::GetS3Client() - { - if (s3_client_.get() == nullptr) - { - InitializeS3Client(); - } - return s3_client_; + } + if (requestTimeoutMs > -1) { + cfg->requestTimeoutMs = requestTimeoutMs; + } + return cfg; +} + +void ShutdownClient(std::shared_ptr* s3_client) { + if (s3_client != nullptr) { + delete s3_client; + Aws::SDKOptions options; + Aws::ShutdownAPI(options); + } +} + +void ShutdownTransferManager( + std::shared_ptr* transfer_manager) { + if (transfer_manager != nullptr) { + delete transfer_manager; + } +} + +void ShutdownExecutor(Aws::Utils::Threading::PooledThreadExecutor* executor) { + if (executor != nullptr) { + delete executor; + } +} + +void parseS3Path( + const Aws::String& fname, + Aws::String* bucket, + Aws::String* object) { + if (fname.empty()) { + throw std::invalid_argument("The filename cannot be an empty string."); + } + + if (fname.size() < 5 || fname.substr(0, 5) != "s3://") { + throw std::invalid_argument("The filename must start with the S3 scheme."); + } + + std::string path = fname.substr(5); + + if (path.empty()) { + throw std::invalid_argument("The filename cannot be an empty string."); + } + + size_t pos = path.find_first_of('/'); + if (pos == 0) { + throw std::invalid_argument("The filename does not contain a bucket name."); + } + + *bucket = path.substr(0, pos); + *object = path.substr(pos + 1); + if (pos == std::string::npos) { + *object = ""; + } +} + +class S3FS { + private: + std::string bucket_name_; + std::string object_name_; + bool use_multi_part_download_; + std::shared_ptr s3_client_; + std::shared_ptr transfer_manager_; + + public: + S3FS( + const std::string& bucket, + const std::string& object, + const bool use_multi_part_download, + std::shared_ptr transfer_manager, + std::shared_ptr s3_client) + : bucket_name_(bucket), + object_name_(object), + use_multi_part_download_(use_multi_part_download), + transfer_manager_(transfer_manager), + s3_client_(s3_client) {} + + size_t Read(uint64_t offset, size_t n, char* buffer) { + if (use_multi_part_download_) { + return ReadTransferManager(offset, n, buffer); + } else { + return ReadS3Client(offset, n, buffer); } - - std::shared_ptr - S3Handler::GetExecutor() - { - if (executor_.get() == nullptr) - { - InitializeExecutor(); - } - return executor_; + } + + size_t ReadS3Client(uint64_t offset, size_t n, char* buffer) { + Aws::S3::Model::GetObjectRequest getObjectRequest; + + getObjectRequest.WithBucket(bucket_name_.c_str()) + .WithKey(object_name_.c_str()); + + std::string bytes = "bytes="; + bytes += std::to_string(offset) + "-" + std::to_string(offset + n - 1); + + getObjectRequest.SetRange(bytes.c_str()); + + // When you don’t want to load the entire file into memory, + // you can use IOStreamFactory in AmazonWebServiceRequest to pass a + // lambda to create a string stream. + getObjectRequest.SetResponseStreamFactory( + []() { return Aws::New("S3IOAllocationTag"); }); + // get the object + Aws::S3::Model::GetObjectOutcome getObjectOutcome = + s3_client_->GetObject(getObjectRequest); + + if (!getObjectOutcome.IsSuccess()) { + Aws::S3::S3Error error = getObjectOutcome.GetError(); + std::cout << "ERROR: " << error.GetExceptionName() << ": " + << error.GetMessage() << std::endl; + return 0; + } else { + n = getObjectOutcome.GetResult().GetContentLength(); + // read data as a block: + getObjectOutcome.GetResult().GetBody().read(buffer, n); + return n; } - - std::shared_ptr - S3Handler::GetTransferManager() - { - if (transfer_manager_.get() == nullptr) - { - InitializeTransferManager(); - } - return transfer_manager_; + } + + size_t ReadTransferManager(uint64_t offset, size_t n, char* buffer) { + auto create_stream_fn = [&]() { // create stream lambda fn + return Aws::New( + "S3ReadStream", + Aws::New( + "S3ReadStream", reinterpret_cast(buffer), n)); + }; // This buffer is what we used to initialize streambuf and is in memory + + std::shared_ptr downloadHandle = + transfer_manager_.get()->DownloadFile( + bucket_name_.c_str(), + object_name_.c_str(), + offset, + n, + create_stream_fn); + downloadHandle->WaitUntilFinished(); + + Aws::OFStream storeFile( + object_name_.c_str(), Aws::OFStream::out | Aws::OFStream::trunc); + + if (downloadHandle->GetStatus() != + Aws::Transfer::TransferStatus::COMPLETED) { + const Aws::Client::AWSError error = + downloadHandle->GetLastError(); + std::cout << "ERROR: " << error.GetExceptionName() << ": " + << error.GetMessage() << std::endl; + return 0; + } else { + return downloadHandle->GetBytesTransferred(); } - - size_t S3Handler::GetFileSize(const std::string &bucket, - const std::string &object) - { - Aws::S3::Model::HeadObjectRequest headObjectRequest; - headObjectRequest.WithBucket(bucket.c_str()).WithKey(object.c_str()); - Aws::S3::Model::HeadObjectOutcome headObjectOutcome = - GetS3Client()->HeadObject(headObjectRequest); - if (headObjectOutcome.IsSuccess()) - { - return headObjectOutcome.GetResult().GetContentLength(); - } else { - Aws::String const &error_aws = headObjectOutcome.GetError().GetMessage(); - std::string error_str(error_aws.c_str(), error_aws.size()); - throw std::invalid_argument(error_str); - return 0; - } + } +}; + +} // namespace + +std::shared_ptr S3Handler::s3_handler_cfg_; + +S3Handler::S3Handler(const long requestTimeoutMs, const std::string region) + : s3_client_(nullptr, ShutdownClient), + transfer_manager_(nullptr, ShutdownTransferManager), + executor_(nullptr, ShutdownExecutor) { + initialization_lock_ = std::shared_ptr(new std::mutex()); + + // Load reading parameters + buffer_size_ = S3DefaultBufferSize; + const char* bufferSizeStr = getenv("S3_BUFFER_SIZE"); + if (bufferSizeStr) { + buffer_size_ = std::stoull(bufferSizeStr); + } + use_multi_part_download_ = true; + const char* use_multi_part_download_char = getenv("S3_MULTI_PART_DOWNLOAD"); + if (use_multi_part_download_char) { + std::string use_multi_part_download_str(use_multi_part_download_char); + if (use_multi_part_download_str == "OFF") { + use_multi_part_download_ = false; + } + } + + Aws::SDKOptions options; + Aws::InitAPI(options); + S3Handler::s3_handler_cfg_ = setUpS3Config(requestTimeoutMs, region); + InitializeS3Client(); + + last_marker_ = S3DefaultMarker; +} + +S3Handler::~S3Handler() {} + +void S3Handler::InitializeS3Client() { + std::lock_guard lock(*initialization_lock_); + s3_client_ = std::shared_ptr(new Aws::S3::S3Client( + *S3Handler::s3_handler_cfg_, + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, + false)); +} + +void S3Handler::InitializeExecutor() { + executor_ = Aws::MakeShared( + "executor", executorPoolSize); +} + +void S3Handler::InitializeTransferManager() { + std::shared_ptr s3_client = GetS3Client(); + std::lock_guard lock(*initialization_lock_); + + Aws::Transfer::TransferManagerConfiguration transfer_config( + GetExecutor().get()); + transfer_config.s3Client = s3_client; + // This buffer is what we used to initialize streambuf and is in memory + transfer_config.bufferSize = S3DefaultMultiPartDownloadChunkSize; + transfer_config.transferBufferMaxHeapSize = + (executorPoolSize + 1) * S3DefaultMultiPartDownloadChunkSize; + transfer_manager_ = Aws::Transfer::TransferManager::Create(transfer_config); +} + +std::shared_ptr S3Handler::GetS3Client() { + if (s3_client_.get() == nullptr) { + InitializeS3Client(); + } + return s3_client_; +} + +std::shared_ptr +S3Handler::GetExecutor() { + if (executor_.get() == nullptr) { + InitializeExecutor(); + } + return executor_; +} + +std::shared_ptr +S3Handler::GetTransferManager() { + if (transfer_manager_.get() == nullptr) { + InitializeTransferManager(); + } + return transfer_manager_; +} + +size_t S3Handler::GetFileSize( + const std::string& bucket, + const std::string& object) { + Aws::S3::Model::HeadObjectRequest headObjectRequest; + headObjectRequest.WithBucket(bucket.c_str()).WithKey(object.c_str()); + Aws::S3::Model::HeadObjectOutcome headObjectOutcome = + GetS3Client()->HeadObject(headObjectRequest); + if (headObjectOutcome.IsSuccess()) { + return headObjectOutcome.GetResult().GetContentLength(); + } else { + Aws::String const& error_aws = headObjectOutcome.GetError().GetMessage(); + std::string error_str(error_aws.c_str(), error_aws.size()); + throw std::invalid_argument(error_str); + return 0; + } +} + +void S3Handler::ClearMarker() { + last_marker_ = S3DefaultMarker; +} + +void S3Handler::S3Read(const std::string& file_url, std::string* result) { + std::string bucket, object; + parseS3Path(file_url, &bucket, &object); + S3FS s3fs( + bucket, + object, + use_multi_part_download_, + GetTransferManager(), + GetS3Client()); + + uint64_t offset = 0; + uint64_t result_size = 0; + uint64_t file_size = GetFileSize(bucket, object); + size_t part_count = + (std:: + max)(static_cast((file_size + buffer_size_ - 1) / buffer_size_), static_cast(1)); + result->resize(file_size); + + for (int i = 0; i < part_count; i++) { + offset = result_size; + + size_t buf_len = std::min(buffer_size_, file_size - result_size); + + size_t read_len = + s3fs.Read(offset, buf_len, (char*)(result->data()) + offset); + + result_size += read_len; + + if (result_size == file_size) { + break; } - void S3Handler::ClearMarker() { last_marker_ = S3DefaultMarker; } - - void S3Handler::S3Read(const std::string &file_url, std::string *result) - { - std::string bucket, object; - parseS3Path(file_url, &bucket, &object); - S3FS s3fs(bucket, object, use_multi_part_download_, - GetTransferManager(), GetS3Client()); - - uint64_t offset = 0; - uint64_t result_size = 0; - uint64_t file_size = GetFileSize(bucket, object); - size_t part_count = (std::max)( - static_cast((file_size + buffer_size_ - 1) / buffer_size_), - static_cast(1)); - result->resize(file_size); - - for (int i = 0; i < part_count; i++) - { - offset = result_size; - - size_t buf_len = std::min(buffer_size_, file_size - result_size); - - size_t read_len = - s3fs.Read(offset, buf_len, (char *)(result->data()) + offset); - - result_size += read_len; - - if (result_size == file_size) - { - break; - } - - if (read_len != buf_len) - { - std::cout << "Result size and buffer size did not match"; - break; - } - } + if (read_len != buf_len) { + std::cout << "Result size and buffer size did not match"; + break; } + } +} + +void S3Handler::ListFiles( + const std::string& file_url, + std::vector* filenames) { + Aws::String bucket, prefix; + parseS3Path(file_url, &bucket, &prefix); + + Aws::S3::Model::ListObjectsRequest listObjectsRequest; + listObjectsRequest.WithBucket(bucket).WithPrefix(prefix).WithMarker( + last_marker_); + + Aws::S3::Model::ListObjectsOutcome listObjectsOutcome = + GetS3Client()->ListObjects(listObjectsRequest); + if (!listObjectsOutcome.IsSuccess()) { + Aws::String const& error_aws = listObjectsOutcome.GetError().GetMessage(); + throw std::invalid_argument(error_aws); + } + + Aws::Vector objects = + listObjectsOutcome.GetResult().GetContents(); + if (objects.empty()) { + return; + } + for (const Aws::S3::Model::Object& object : objects) { + if (object.GetKey().back() == '/') // ignore folders - void S3Handler::ListFiles(const std::string &file_url, - std::vector *filenames) { - Aws::String bucket, prefix; - parseS3Path(file_url, &bucket, &prefix); - - Aws::S3::Model::ListObjectsRequest listObjectsRequest; - listObjectsRequest.WithBucket(bucket) - .WithPrefix(prefix) - .WithMarker(last_marker_); - - Aws::S3::Model::ListObjectsOutcome listObjectsOutcome = - GetS3Client()->ListObjects(listObjectsRequest); - if (!listObjectsOutcome.IsSuccess()) - { - Aws::String const &error_aws = - listObjectsOutcome.GetError().GetMessage(); - throw std::invalid_argument(error_aws); - } - - Aws::Vector objects = listObjectsOutcome.GetResult().GetContents(); - if (objects.empty()) - { - return; - } - for (const Aws::S3::Model::Object &object : objects) - { - if (object.GetKey().back() == '/') // ignore folders - { - continue; - } - Aws::String entry = "s3://" + bucket + "/" + object.GetKey(); - filenames->push_back(entry.c_str()); - } - last_marker_ = objects.back().GetKey(); - - // extreme cases when all objects are folders - if (filenames->size() == 0) - { - ListFiles(file_url, filenames); - } + continue; } + Aws::String entry = "s3://" + bucket + "/" + object.GetKey(); + filenames->push_back(entry.c_str()); + } + last_marker_ = objects.back().GetKey(); + + // extreme cases when all objects are folders + if (filenames->size() == 0) { + ListFiles(file_url, filenames); + } +} + } // namespace torchdata diff --git a/torchdata/csrc/pybind/S3Handler/S3Handler.h b/torchdata/csrc/pybind/S3Handler/S3Handler.h index 8aa73bae5..641359d69 100644 --- a/torchdata/csrc/pybind/S3Handler/S3Handler.h +++ b/torchdata/csrc/pybind/S3Handler/S3Handler.h @@ -1,61 +1,76 @@ #include "precompile.h" -namespace torchdata -{ - // In memory stream implementation - class S3UnderlyingStream : public Aws::IOStream - { - public: - using Base = Aws::IOStream; - - // provide a customer controlled streambuf, so as to put all transferred - // data into this in memory buffer. - S3UnderlyingStream(std::streambuf *buf) : Base(buf) {} - - virtual ~S3UnderlyingStream() = default; - }; - - class S3Handler - { - private: - static std::shared_ptr s3_handler_cfg_; - - std::shared_ptr initialization_lock_; - std::shared_ptr s3_client_; - std::shared_ptr executor_; - std::shared_ptr transfer_manager_; - - Aws::String last_marker_; - size_t buffer_size_; - bool use_multi_part_download_; - - void InitializeS3Client(); - void InitializeExecutor(); - void InitializeTransferManager(); - - std::shared_ptr GetS3Client(); - std::shared_ptr - GetExecutor(); - std::shared_ptr GetTransferManager(); - size_t GetFileSize(const std::string &bucket, const std::string &object); - - public: - S3Handler(const long requestTimeoutMs, const std::string region); - ~S3Handler(); - - void SetLastMarker(const Aws::String last_marker) { this->last_marker_ = last_marker; } - void SetBufferSize(const uint64_t buffer_size) { this->buffer_size_ = buffer_size; } - void SetMultiPartDownload(const bool multi_part_download) { this->use_multi_part_download_ = multi_part_download; } - void ClearMarker(); - - long GetRequestTimeoutMs() const { return s3_handler_cfg_->requestTimeoutMs; } - Aws::String GetRegion() const { return s3_handler_cfg_->region; } - Aws::String GetLastMarker() const { return last_marker_; } - bool GetUseMultiPartDownload() const { return use_multi_part_download_; } - size_t GetBufferSize() const { return buffer_size_; } - - void S3Read(const std::string &file_url, std::string *result); - void ListFiles(const std::string &file_url, - std::vector *filenames); - }; +namespace torchdata { + +// In memory stream implementation +class S3UnderlyingStream : public Aws::IOStream { + public: + using Base = Aws::IOStream; + + // provide a customer controlled streambuf, so as to put all transferred + // data into this in memory buffer. + S3UnderlyingStream(std::streambuf* buf) : Base(buf) {} + + virtual ~S3UnderlyingStream() = default; +}; + +class S3Handler { + private: + static std::shared_ptr s3_handler_cfg_; + + std::shared_ptr initialization_lock_; + std::shared_ptr s3_client_; + std::shared_ptr executor_; + std::shared_ptr transfer_manager_; + + Aws::String last_marker_; + size_t buffer_size_; + bool use_multi_part_download_; + + void InitializeS3Client(); + void InitializeExecutor(); + void InitializeTransferManager(); + + std::shared_ptr GetS3Client(); + std::shared_ptr GetExecutor(); + std::shared_ptr GetTransferManager(); + size_t GetFileSize(const std::string& bucket, const std::string& object); + + public: + S3Handler(const long requestTimeoutMs, const std::string region); + ~S3Handler(); + + void SetLastMarker(const Aws::String last_marker) { + this->last_marker_ = last_marker; + } + void SetBufferSize(const uint64_t buffer_size) { + this->buffer_size_ = buffer_size; + } + void SetMultiPartDownload(const bool multi_part_download) { + this->use_multi_part_download_ = multi_part_download; + } + void ClearMarker(); + + long GetRequestTimeoutMs() const { + return s3_handler_cfg_->requestTimeoutMs; + } + Aws::String GetRegion() const { + return s3_handler_cfg_->region; + } + Aws::String GetLastMarker() const { + return last_marker_; + } + bool GetUseMultiPartDownload() const { + return use_multi_part_download_; + } + size_t GetBufferSize() const { + return buffer_size_; + } + + void S3Read(const std::string& file_url, std::string* result); + void ListFiles( + const std::string& file_url, + std::vector* filenames); +}; + } // namespace torchdata diff --git a/torchdata/csrc/pybind/S3Handler/precompile.h b/torchdata/csrc/pybind/S3Handler/precompile.h index 5f5080bc3..b1da9d499 100644 --- a/torchdata/csrc/pybind/S3Handler/precompile.h +++ b/torchdata/csrc/pybind/S3Handler/precompile.h @@ -2,11 +2,11 @@ #define TORCHDATA_S3_IO_H #include -#include #include +#include -#include #include +#include #include #include #include diff --git a/torchdata/csrc/pybind/pybind.cpp b/torchdata/csrc/pybind/pybind.cpp index 806791cbb..c9f0f757e 100644 --- a/torchdata/csrc/pybind/pybind.cpp +++ b/torchdata/csrc/pybind/pybind.cpp @@ -8,60 +8,57 @@ namespace py = pybind11; using torchdata::S3Handler; -PYBIND11_MODULE(_torchdata, m) -{ - py::class_(m, "S3Handler") - .def(py::init()) - .def("s3_read", - [](S3Handler *self, const std::string &file_url) - { - std::string result; - self->S3Read(file_url, &result); - return py::bytes(result); - }) - .def("list_files", - [](S3Handler *self, const std::string &file_url) - { - std::vector filenames; - self->ListFiles(file_url, &filenames); - return filenames; - }) - .def("set_buffer_size", - [](S3Handler *self, const uint64_t buffer_size) - { - self->SetBufferSize(buffer_size); - }) - .def("set_multi_part_download", - [](S3Handler *self, const bool multi_part_download) - { - self->SetMultiPartDownload(multi_part_download); - }) - .def("clear_marker", - [](S3Handler *self) - { - self->ClearMarker(); - }) - .def(py::pickle( - [](const S3Handler &s3_handler) { // __getstate__ - /* Return a tuple that fully encodes the state of the object */ - return py::make_tuple(s3_handler.GetRequestTimeoutMs(), - s3_handler.GetRegion(), - s3_handler.GetLastMarker(), - s3_handler.GetUseMultiPartDownload(), - s3_handler.GetBufferSize()); - }, - [](py::tuple t) { // __setstate__ - if (t.size() != 5) - throw std::runtime_error("Invalid state!"); - /* Create a new C++ instance */ - S3Handler s3_handler(t[0].cast(), t[1].cast()); +PYBIND11_MODULE(_torchdata, m) { + py::class_(m, "S3Handler") + .def(py::init()) + .def( + "s3_read", + [](S3Handler* self, const std::string& file_url) { + std::string result; + self->S3Read(file_url, &result); + return py::bytes(result); + }) + .def( + "list_files", + [](S3Handler* self, const std::string& file_url) { + std::vector filenames; + self->ListFiles(file_url, &filenames); + return filenames; + }) + .def( + "set_buffer_size", + [](S3Handler* self, const uint64_t buffer_size) { + self->SetBufferSize(buffer_size); + }) + .def( + "set_multi_part_download", + [](S3Handler* self, const bool multi_part_download) { + self->SetMultiPartDownload(multi_part_download); + }) + .def("clear_marker", [](S3Handler* self) { self->ClearMarker(); }) + .def(py::pickle( + [](const S3Handler& s3_handler) { // __getstate__ + /* Return a tuple that fully encodes the state of the object */ + return py::make_tuple( + s3_handler.GetRequestTimeoutMs(), + s3_handler.GetRegion(), + s3_handler.GetLastMarker(), + s3_handler.GetUseMultiPartDownload(), + s3_handler.GetBufferSize()); + }, + [](py::tuple t) { // __setstate__ + if (t.size() != 5) + throw std::runtime_error("Invalid state!"); - /* Assign any additional state */ - s3_handler.SetLastMarker(t[2].cast()); - s3_handler.SetMultiPartDownload(t[3].cast()); - s3_handler.SetBufferSize(t[4].cast()); + /* Create a new C++ instance */ + S3Handler s3_handler(t[0].cast(), t[1].cast()); - return s3_handler; - })); + /* Assign any additional state */ + s3_handler.SetLastMarker(t[2].cast()); + s3_handler.SetMultiPartDownload(t[3].cast()); + s3_handler.SetBufferSize(t[4].cast()); + + return s3_handler; + })); }