From a4d768495175ebf8a38d1c55fb2c1b0db32164d9 Mon Sep 17 00:00:00 2001 From: snowman2 Date: Mon, 12 Sep 2022 20:09:12 -0500 Subject: [PATCH] PERF: thread local context --- .github/workflows/test_proj_latest.yaml | 15 ---- .github/workflows/tests.yaml | 33 -------- .pylintrc | 2 +- docs/api/global_context.rst | 2 + docs/history.rst | 1 + pyproj/__init__.py | 8 +- pyproj/{_datadir.pxd => _context.pxd} | 2 - pyproj/{_datadir.pyi => _context.pyi} | 5 +- pyproj/{_datadir.pyx => _context.pyx} | 106 ++++++++++++++++++------ pyproj/_crs.pxd | 1 + pyproj/_crs.pyx | 36 ++------ pyproj/_network.pyi | 1 - pyproj/_network.pyx | 20 +---- pyproj/_sync.pyx | 4 +- pyproj/_transformer.pxd | 1 + pyproj/_transformer.pyx | 29 +++---- pyproj/database.pyx | 54 +++++------- pyproj/datadir.py | 6 +- pyproj/exceptions.py | 2 +- pyproj/network.py | 4 +- pyproj/transformer.py | 2 +- setup.py | 2 +- test/crs/test_crs.py | 3 - test/test_datadir.py | 50 +---------- test/test_network.py | 10 +-- test/test_proj.py | 3 - test/test_transformer.py | 6 -- 27 files changed, 154 insertions(+), 254 deletions(-) rename pyproj/{_datadir.pxd => _context.pxd} (57%) rename pyproj/{_datadir.pyi => _context.pyi} (55%) rename pyproj/{_datadir.pyx => _context.pyx} (64%) diff --git a/.github/workflows/test_proj_latest.yaml b/.github/workflows/test_proj_latest.yaml index b54eef374..d9467b8ef 100644 --- a/.github/workflows/test_proj_latest.yaml +++ b/.github/workflows/test_proj_latest.yaml @@ -58,21 +58,6 @@ jobs: run: | python -m pytest - - name: Test Global Context - shell: bash - env: - PYPROJ_GLOBAL_CONTEXT: ON - run: | - python -m pytest - - - name: Test Network & Global Context - shell: bash - env: - PROJ_NETWORK: ON - PYPROJ_GLOBAL_CONTEXT: ON - run: | - python -m pytest - - name: Test Grids shell: bash run: | diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 2caaab0f7..0982ed50c 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -89,23 +89,6 @@ jobs: . testenv/bin/activate python -m pytest - - name: Test Global Context - shell: bash - env: - PYPROJ_GLOBAL_CONTEXT: ON - run: | - . testenv/bin/activate - python -m pytest - - - name: Test Network & Global Context - shell: bash - env: - PROJ_NETWORK: ON - PYPROJ_GLOBAL_CONTEXT: ON - run: | - . testenv/bin/activate - python -m pytest - - name: Test Grids shell: bash run: | @@ -197,22 +180,6 @@ jobs: run: | micromamba run -n test python -m pytest - - name: Test Global Context - shell: bash - env: - PROJ_NETWORK: OFF - PYPROJ_GLOBAL_CONTEXT: ON - run: | - micromamba run -n test python -m pytest - - - name: Test Network & Global Context - shell: bash - env: - PROJ_NETWORK: ON - PYPROJ_GLOBAL_CONTEXT: ON - run: | - micromamba run -n test python -m pytest - - name: Test Grids shell: bash env: diff --git a/.pylintrc b/.pylintrc index 9cb0e4fef..5d546c9fa 100644 --- a/.pylintrc +++ b/.pylintrc @@ -7,7 +7,7 @@ extension-pkg-whitelist=pyproj._crs, pyproj._sync, pyproj._network, pyproj._geod, - pyproj._datadir, + pyproj._context, pyproj._compat, pyproj.database, pyproj.list diff --git a/docs/api/global_context.rst b/docs/api/global_context.rst index 5fdb1e60f..c58b214fe 100644 --- a/docs/api/global_context.rst +++ b/docs/api/global_context.rst @@ -3,6 +3,8 @@ Global Context ============== +.. deprecated:: 3.7.0 No longer necessary as there is only one context per thread now. + If you have a single-threaded application that generates many objects, enabling the use of the global context can provide performance enhancements. diff --git a/docs/history.rst b/docs/history.rst index 5f17bec7d..ee857c622 100644 --- a/docs/history.rst +++ b/docs/history.rst @@ -7,6 +7,7 @@ Latest - DEP: Minimum supported Python version 3.10 (pull #1357) - DEP: Minimum PROJ version 9.2 (pull #1394) - ENH: Add :meth:`CRS.is_deprecated` and :meth:`CRS.get_non_deprecated` (pull #1383) +- PERF: thread local context (issue #1133) 3.6.1 ------ diff --git a/pyproj/__init__.py b/pyproj/__init__.py index 304588e6d..95ac2f46b 100644 --- a/pyproj/__init__.py +++ b/pyproj/__init__.py @@ -32,8 +32,7 @@ import warnings import pyproj.network -from pyproj._datadir import ( # noqa: F401 pylint: disable=unused-import - _pyproj_global_context_initialize, +from pyproj._context import ( # noqa: F401 pylint: disable=unused-import set_use_global_context, ) from pyproj._show_versions import ( # noqa: F401 pylint: disable=unused-import @@ -85,10 +84,7 @@ ] __proj_version__ = proj_version_str - try: - _pyproj_global_context_initialize() + pyproj.network.set_ca_bundle_path() except DataDirError as err: warnings.warn(str(err)) - -pyproj.network.set_ca_bundle_path() diff --git a/pyproj/_datadir.pxd b/pyproj/_context.pxd similarity index 57% rename from pyproj/_datadir.pxd rename to pyproj/_context.pxd index b0332f8d0..a0edb8bdd 100644 --- a/pyproj/_datadir.pxd +++ b/pyproj/_context.pxd @@ -2,6 +2,4 @@ include "proj.pxi" cpdef str _get_proj_error() cpdef void _clear_proj_error() noexcept -cdef PJ_CONTEXT* PYPROJ_GLOBAL_CONTEXT cdef PJ_CONTEXT* pyproj_context_create() except * -cdef void pyproj_context_destroy(PJ_CONTEXT* context) except * diff --git a/pyproj/_datadir.pyi b/pyproj/_context.pyi similarity index 55% rename from pyproj/_datadir.pyi rename to pyproj/_context.pyi index 5d61117df..f148c4b7d 100644 --- a/pyproj/_datadir.pyi +++ b/pyproj/_context.pyi @@ -1,6 +1,7 @@ -def _pyproj_global_context_initialize() -> None: ... def get_user_data_dir(create: bool = False) -> str: ... -def _global_context_set_data_dir() -> None: ... +def _set_context_data_dir() -> None: ... +def _set_context_ca_bundle_path(ca_bundle_path: str) -> None: ... +def _set_context_network_enabled() -> None: ... def set_use_global_context(active: bool | None = None) -> None: ... def _clear_proj_error() -> None: ... def _get_proj_error() -> str: ... diff --git a/pyproj/_datadir.pyx b/pyproj/_context.pyx similarity index 64% rename from pyproj/_datadir.pyx rename to pyproj/_context.pyx index 21a463096..1074d288e 100644 --- a/pyproj/_datadir.pyx +++ b/pyproj/_context.pyx @@ -1,7 +1,9 @@ import logging import os +import threading import warnings +from cpython.pythread cimport PyThread_tss_create, PyThread_tss_get, PyThread_tss_set from libc.stdlib cimport free, malloc from pyproj._compat cimport cstrencode @@ -12,17 +14,22 @@ from pyproj.utils import strtobool # https://docs.python.org/3/howto/logging.html#configuring-logging-for-a-library _LOGGER = logging.getLogger("pyproj") _LOGGER.addHandler(logging.NullHandler()) -# default to False is the safest mode -# as it supports multithreading -_USE_GLOBAL_CONTEXT = strtobool(os.environ.get("PYPROJ_GLOBAL_CONTEXT", "OFF")) # static user data directory to prevent core dumping # see: https://github.com/pyproj4/pyproj/issues/678 cdef const char* _USER_DATA_DIR = proj_context_get_user_writable_directory(NULL, False) # Store the message from any internal PROJ errors cdef str _INTERNAL_PROJ_ERROR = None +# global variables +cdef bint _NETWORK_ENABLED = strtobool(os.environ.get("PROJ_NETWORK", "OFF")) +cdef char* _CA_BUNDLE_PATH = "" +# The key to get the context in each thread +cdef Py_tss_t CONTEXT_THREAD_KEY + def set_use_global_context(active=None): """ + .. deprecated:: 3.7.0 No longer necessary as there is only one context per thread now. + .. versionadded:: 3.0.0 Activates the usage of the global context. Using this @@ -44,10 +51,17 @@ def set_use_global_context(active=None): the environment variable PYPROJ_GLOBAL_CONTEXT and defaults to False if it is not found. """ - global _USE_GLOBAL_CONTEXT if active is None: active = strtobool(os.environ.get("PYPROJ_GLOBAL_CONTEXT", "OFF")) - _USE_GLOBAL_CONTEXT = bool(active) + if active: + warnings.warn( + ( + "PYPROJ_GLOBAL_CONTEXT is no longer necessary in pyproj 3.7+ " + "and does not do anything." + ), + FutureWarning, + stacklevel=2, + ) def get_user_data_dir(create=False): @@ -74,7 +88,7 @@ def get_user_data_dir(create=False): The user writable data directory. """ return proj_context_get_user_writable_directory( - PYPROJ_GLOBAL_CONTEXT, bool(create) + pyproj_context_create(), bool(create) ) @@ -124,7 +138,7 @@ cdef void set_context_data_dir(PJ_CONTEXT* context) except *: cdef bytes b_database_path = cstrencode(os.path.join(data_dir_list[0], "proj.db")) cdef const char* c_database_path = b_database_path if not proj_context_set_database_path(context, c_database_path, NULL, NULL): - warnings.warn("pyproj unable to set database path.") + warnings.warn("pyproj unable to set PROJ database path.") cdef int dir_list_len = len(data_dir_list) cdef const char **c_data_dir = malloc( (dir_list_len + 1) * sizeof(const char*) @@ -147,6 +161,8 @@ cdef void pyproj_context_initialize(PJ_CONTEXT* context) except *: proj_log_func(context, NULL, pyproj_log_function) proj_context_use_proj4_init_rules(context, 1) set_context_data_dir(context) + proj_context_set_ca_bundle_path(context, _CA_BUNDLE_PATH) + proj_context_set_enable_network(context, _NETWORK_ENABLED) cdef class ContextManager: @@ -170,35 +186,75 @@ cdef class ContextManager: return context_manager -# Different libraries that modify the PROJ global context will influence -# each other without realizing it. Due to this, pyproj is creating it's own -# global context so that it doesn't bother other libraries and is insulated -# against possible external changes made to the PROJ global context. -# See: https://github.com/pyproj4/pyproj/issues/722 -cdef PJ_CONTEXT* PYPROJ_GLOBAL_CONTEXT = proj_context_create() -cdef ContextManager CONTEXT_MANAGER = ContextManager.create(PYPROJ_GLOBAL_CONTEXT) +class ContextManagerLocal(threading.local): + """ + Threading local instance for cython ContextManager class. + """ + + def __init__(self): + self.context_manager = None # Initialises in each thread + super().__init__() + +_CONTEXT_MANAGER_LOCAL = ContextManagerLocal() cdef PJ_CONTEXT* pyproj_context_create() except *: """ Create and initialize the context(s) for pyproj. This also manages whether the global context is used. """ - if _USE_GLOBAL_CONTEXT: - return PYPROJ_GLOBAL_CONTEXT - return proj_context_clone(PYPROJ_GLOBAL_CONTEXT) + global _CONTEXT_MANAGER_LOCAL + + if PyThread_tss_create(&CONTEXT_THREAD_KEY) != 0: + raise MemoryError("Unable to create key for PROJ context in thread.") + cdef const void *thread_pyproj_context = PyThread_tss_get(&CONTEXT_THREAD_KEY) + cdef PJ_CONTEXT* pyproj_context = NULL + if thread_pyproj_context == NULL: + pyproj_context = proj_context_create() + pyproj_context_initialize(pyproj_context) + PyThread_tss_set(&CONTEXT_THREAD_KEY, pyproj_context) + _CONTEXT_MANAGER_LOCAL.context_manager = ContextManager.create(pyproj_context) + else: + pyproj_context = thread_pyproj_context + return pyproj_context + + +def get_context_manager(): + """ + This returns the manager for the context + responsible for cleanup + """ + return _CONTEXT_MANAGER_LOCAL.context_manager -cdef void pyproj_context_destroy(PJ_CONTEXT* context) except *: + +cpdef _set_context_data_dir(): """ - Destroy context only if not the global context + Python compatible function to set the + data directory on the current context """ - if context != PYPROJ_GLOBAL_CONTEXT: - proj_context_destroy(context) + set_context_data_dir(pyproj_context_create()) + +cpdef _set_context_ca_bundle_path(str ca_bundle_path): + """ + Python compatible function to set the + CA Bundle path on the current context + and cache for future generated contexts + """ + global _CA_BUNDLE_PATH -cpdef _pyproj_global_context_initialize(): - pyproj_context_initialize(PYPROJ_GLOBAL_CONTEXT) + b_ca_bundle_path = cstrencode(ca_bundle_path) + _CA_BUNDLE_PATH = b_ca_bundle_path + proj_context_set_ca_bundle_path(pyproj_context_create(), _CA_BUNDLE_PATH) -cpdef _global_context_set_data_dir(): - set_context_data_dir(PYPROJ_GLOBAL_CONTEXT) +cpdef _set_context_network_enabled(bint enabled): + """ + Python compatible function to set the + network enables on the current context + and cache for future generated contexts + """ + global _NETWORK_ENABLED + + _NETWORK_ENABLED = enabled + proj_context_set_enable_network(pyproj_context_create(), _NETWORK_ENABLED) diff --git a/pyproj/_crs.pxd b/pyproj/_crs.pxd index e8198526c..1aa60d6a2 100644 --- a/pyproj/_crs.pxd +++ b/pyproj/_crs.pxd @@ -47,6 +47,7 @@ cdef create_area_of_use(PJ_CONTEXT* context, PJ* projobj) cdef class Base: cdef PJ *projobj cdef PJ_CONTEXT* context + cdef readonly object _context_manager cdef readonly str name cdef readonly str _remarks cdef readonly str _scope diff --git a/pyproj/_crs.pyx b/pyproj/_crs.pyx index 4091eec3e..f8cfc63f7 100644 --- a/pyproj/_crs.pyx +++ b/pyproj/_crs.pyx @@ -4,12 +4,9 @@ import warnings from collections import OrderedDict, namedtuple from pyproj._compat cimport cstrdecode, cstrencode -from pyproj._datadir cimport ( - _clear_proj_error, - pyproj_context_create, - pyproj_context_destroy, -) +from pyproj._context cimport _clear_proj_error, pyproj_context_create +from pyproj._context import get_context_manager from pyproj.aoi import AreaOfUse from pyproj.crs.datum import CustomEllipsoid from pyproj.crs.enums import CoordinateOperationType, DatumType @@ -365,8 +362,6 @@ cdef class Base: """destroy projection definition""" if self.projobj != NULL: proj_destroy(self.projobj) - if self.context != NULL: - pyproj_context_destroy(self.context) cdef _set_base_info(self): """ @@ -592,6 +587,7 @@ cdef class CoordinateSystem(_CRSParts): cdef CoordinateSystem create(PJ_CONTEXT* context, PJ* coord_system_pj): cdef CoordinateSystem coord_system = CoordinateSystem.__new__(CoordinateSystem) coord_system.context = context + coord_system._context_manager = get_context_manager() coord_system.projobj = coord_system_pj cdef PJ_COORDINATE_SYSTEM_TYPE cs_type = proj_cs_get_type( @@ -655,7 +651,6 @@ cdef class CoordinateSystem(_CRSParts): coordinate_system_pj, ) == PJ_CS_TYPE_UNKNOWN: proj_destroy(coordinate_system_pj) - pyproj_context_destroy(context) raise CRSError( "Invalid coordinate system string: " f"{coordinate_system_string}" @@ -821,6 +816,7 @@ cdef class Ellipsoid(_CRSParts): cdef Ellipsoid create(PJ_CONTEXT* context, PJ* ellipsoid_pj): cdef Ellipsoid ellips = Ellipsoid.__new__(Ellipsoid) ellips.context = context + ellips._context_manager = get_context_manager() ellips.projobj = ellipsoid_pj cdef int is_semi_minor_computed = 0 proj_ellipsoid_get_parameters( @@ -865,7 +861,6 @@ cdef class Ellipsoid(_CRSParts): ) if ellipsoid_pj == NULL: - pyproj_context_destroy(context) raise CRSError(f"Invalid authority or code ({auth_name}, {code})") _clear_proj_error() return Ellipsoid.create(context, ellipsoid_pj) @@ -915,7 +910,6 @@ cdef class Ellipsoid(_CRSParts): ) if ellipsoid_pj == NULL or proj_get_type(ellipsoid_pj) != PJ_TYPE_ELLIPSOID: proj_destroy(ellipsoid_pj) - pyproj_context_destroy(context) raise CRSError( f"Invalid ellipsoid string: {ellipsoid_string}" ) @@ -1019,7 +1013,6 @@ cdef class Ellipsoid(_CRSParts): PJ_TYPE_ELLIPSOID, ) if ellipsoid_pj == NULL: - pyproj_context_destroy(context) raise CRSError(f"Invalid ellipsoid name: {ellipsoid_name}") _clear_proj_error() return Ellipsoid.create(context, ellipsoid_pj) @@ -1100,6 +1093,7 @@ cdef class PrimeMeridian(_CRSParts): cdef PrimeMeridian create(PJ_CONTEXT* context, PJ* prime_meridian_pj): cdef PrimeMeridian prime_meridian = PrimeMeridian.__new__(PrimeMeridian) prime_meridian.context = context + prime_meridian._context_manager = get_context_manager() prime_meridian.projobj = prime_meridian_pj cdef const char * unit_name proj_prime_meridian_get_parameters( @@ -1143,7 +1137,6 @@ cdef class PrimeMeridian(_CRSParts): ) if prime_meridian_pj == NULL: - pyproj_context_destroy(context) raise CRSError(f"Invalid authority or code ({auth_name}, {code})") _clear_proj_error() return PrimeMeridian.create(context, prime_meridian_pj) @@ -1196,7 +1189,6 @@ cdef class PrimeMeridian(_CRSParts): proj_get_type(prime_meridian_pj) != PJ_TYPE_PRIME_MERIDIAN ): proj_destroy(prime_meridian_pj) - pyproj_context_destroy(context) raise CRSError( f"Invalid prime meridian string: {prime_meridian_string}" ) @@ -1305,7 +1297,6 @@ cdef class PrimeMeridian(_CRSParts): PJ_TYPE_PRIME_MERIDIAN, ) if prime_meridian_pj == NULL: - pyproj_context_destroy(context) raise CRSError( f"Invalid prime meridian name: {prime_meridian_name}" ) @@ -1360,6 +1351,7 @@ cdef class Datum(_CRSParts): cdef Datum create(PJ_CONTEXT* context, PJ* datum_pj): cdef Datum datum = Datum.__new__(Datum) datum.context = context + datum._context_manager = get_context_manager() datum.projobj = datum_pj datum._set_base_info() datum.type_name = _DATUM_TYPE_MAP[proj_get_type(datum.projobj)] @@ -1393,7 +1385,6 @@ cdef class Datum(_CRSParts): ) if datum_pj == NULL: - pyproj_context_destroy(context) raise CRSError(f"Invalid authority or code ({auth_name}, {code})") _clear_proj_error() return Datum.create(context, datum_pj) @@ -1466,7 +1457,6 @@ cdef class Datum(_CRSParts): proj_get_type(datum_pj) not in _DATUM_TYPE_MAP ): proj_destroy(datum_pj) - pyproj_context_destroy(context) raise CRSError(f"Invalid datum string: {datum_string}") _clear_proj_error() return Datum.create(context, datum_pj) @@ -1535,7 +1525,6 @@ cdef class Datum(_CRSParts): pj_datum_type, ) if datum_pj == NULL: - pyproj_context_destroy(context) raise CRSError(f"Invalid datum name: {datum_name}") _clear_proj_error() return Datum.create(context, datum_pj) @@ -1644,7 +1633,6 @@ cdef class Datum(_CRSParts): ) _clear_proj_error() if ellipsoid_pj == NULL: - pyproj_context_destroy(context) self._ellipsoid = False return None self._ellipsoid = Ellipsoid.create(context, ellipsoid_pj) @@ -1667,7 +1655,6 @@ cdef class Datum(_CRSParts): ) _clear_proj_error() if prime_meridian_pj == NULL: - pyproj_context_destroy(context) self._prime_meridian = False return None self._prime_meridian = PrimeMeridian.create( @@ -1916,6 +1903,7 @@ cdef class CoordinateOperation(_CRSParts): CoordinateOperation ) coord_operation.context = context + coord_operation._context_manager = get_context_manager() coord_operation.projobj = coord_operation_pj cdef const char *out_method_name = NULL cdef const char *out_method_auth_name = NULL @@ -1982,7 +1970,6 @@ cdef class CoordinateOperation(_CRSParts): ) if coord_operation_pj == NULL: - pyproj_context_destroy(context) raise CRSError(f"Invalid authority or code ({auth_name}, {code})") _clear_proj_error() return CoordinateOperation.create(context, coord_operation_pj) @@ -2039,7 +2026,6 @@ cdef class CoordinateOperation(_CRSParts): ) ): proj_destroy(coord_operation_pj) - pyproj_context_destroy(context) raise CRSError( "Invalid coordinate operation string: " f"{coordinate_operation_string}" @@ -2154,7 +2140,6 @@ cdef class CoordinateOperation(_CRSParts): pj_coordinate_operation_type, ) if coordinate_operation_pj == NULL: - pyproj_context_destroy(context) raise CRSError( "Invalid coordinate operation name: " f"{coordinate_operation_name}" @@ -2354,6 +2339,7 @@ cdef class _CRS(Base): python CRS class. """ def __cinit__(self): + self._context_manager = None self._ellipsoid = None self._area_of_use = None self._prime_meridian = None @@ -2368,6 +2354,7 @@ cdef class _CRS(Base): def __init__(self, const char *proj_string): self.context = pyproj_context_create() + self._context_manager = get_context_manager() # initialize projection self.projobj = proj_create( self.context, @@ -2460,7 +2447,6 @@ cdef class _CRS(Base): ) _clear_proj_error() if ellipsoid_pj == NULL: - pyproj_context_destroy(context) self._ellipsoid = False return None self._ellipsoid = Ellipsoid.create(context, ellipsoid_pj) @@ -2485,7 +2471,6 @@ cdef class _CRS(Base): ) _clear_proj_error() if prime_meridian_pj == NULL: - pyproj_context_destroy(context) self._prime_meridian = False return None self._prime_meridian = PrimeMeridian.create(context, prime_meridian_pj) @@ -2514,7 +2499,6 @@ cdef class _CRS(Base): ) _clear_proj_error() if datum_pj == NULL: - pyproj_context_destroy(context) self._datum = False return None self._datum = Datum.create(context, datum_pj) @@ -2538,7 +2522,6 @@ cdef class _CRS(Base): ) _clear_proj_error() if coord_system_pj == NULL: - pyproj_context_destroy(context) self._coordinate_system = False return None @@ -2577,7 +2560,6 @@ cdef class _CRS(Base): ) _clear_proj_error() if coord_pj == NULL: - pyproj_context_destroy(context) self._coordinate_operation = False return None self._coordinate_operation = CoordinateOperation.create( diff --git a/pyproj/_network.pyi b/pyproj/_network.pyi index 5024bcfa0..875f1d29d 100644 --- a/pyproj/_network.pyi +++ b/pyproj/_network.pyi @@ -1,3 +1,2 @@ def set_network_enabled(active: bool | None = None) -> None: ... def is_network_enabled() -> bool: ... -def _set_ca_bundle_path(ca_bundle_path: str) -> None: ... diff --git a/pyproj/_network.pyx b/pyproj/_network.pyx index 9b06b8689..5feaead4f 100644 --- a/pyproj/_network.pyx +++ b/pyproj/_network.pyx @@ -5,20 +5,8 @@ import os from pyproj.utils import strtobool from pyproj._compat cimport cstrencode -from pyproj._datadir cimport PYPROJ_GLOBAL_CONTEXT - - -def _set_ca_bundle_path(str ca_bundle_path): - """ - Sets the path to the CA Bundle used by the `curl` - built into PROJ. - - Parameters - ---------- - ca_bundle_path: str - The path to the CA Bundle. - """ - proj_context_set_ca_bundle_path(PYPROJ_GLOBAL_CONTEXT, cstrencode(ca_bundle_path)) +from pyproj._context cimport pyproj_context_create +from pyproj._context import _set_context_network_enabled def set_network_enabled(active=None): @@ -43,7 +31,7 @@ def set_network_enabled(active=None): # setting based on the environment variable every time if None # because it could have been changed by the user previously active = strtobool(os.environ.get("PROJ_NETWORK", "OFF")) - proj_context_set_enable_network(PYPROJ_GLOBAL_CONTEXT, bool(active)) + _set_context_network_enabled(bool(active)) def is_network_enabled(): @@ -57,4 +45,4 @@ def is_network_enabled(): bool: If PROJ network is enabled by default. """ - return proj_context_is_network_enabled(PYPROJ_GLOBAL_CONTEXT) == 1 + return proj_context_is_network_enabled(pyproj_context_create()) == 1 diff --git a/pyproj/_sync.pyx b/pyproj/_sync.pyx index d4067e349..30c19cddd 100644 --- a/pyproj/_sync.pyx +++ b/pyproj/_sync.pyx @@ -1,6 +1,6 @@ include "proj.pxi" -from pyproj._datadir cimport PYPROJ_GLOBAL_CONTEXT +from pyproj._context cimport pyproj_context_create def get_proj_endpoint() -> str: @@ -10,4 +10,4 @@ def get_proj_endpoint() -> str: str: URL of the endpoint where PROJ grids are stored. """ - return proj_context_get_url_endpoint(PYPROJ_GLOBAL_CONTEXT) + return proj_context_get_url_endpoint(pyproj_context_create()) diff --git a/pyproj/_transformer.pxd b/pyproj/_transformer.pxd index 242247d42..33e6630dc 100644 --- a/pyproj/_transformer.pxd +++ b/pyproj/_transformer.pxd @@ -5,6 +5,7 @@ from pyproj._crs cimport _CRS, Base cdef class _TransformerGroup: cdef PJ_CONTEXT* context + cdef readonly object _context_manager cdef readonly list _transformers cdef readonly list _unavailable_operations cdef readonly list _best_available diff --git a/pyproj/_transformer.pyx b/pyproj/_transformer.pyx index c9e4371ce..87d7b71c2 100644 --- a/pyproj/_transformer.pyx +++ b/pyproj/_transformer.pyx @@ -9,6 +9,7 @@ import warnings from collections import namedtuple from pyproj._compat cimport cstrencode +from pyproj._context cimport _clear_proj_error, _get_proj_error, pyproj_context_create from pyproj._crs cimport ( _CRS, Base, @@ -18,14 +19,8 @@ from pyproj._crs cimport ( _to_wkt, create_area_of_use, ) -from pyproj._datadir cimport ( - _clear_proj_error, - _get_proj_error, - pyproj_context_create, - pyproj_context_destroy, -) -from pyproj._datadir import _LOGGER +from pyproj._context import _LOGGER, get_context_manager from pyproj.aoi import AreaOfInterest from pyproj.enums import ProjVersion, TransformDirection, WktVersion from pyproj.exceptions import ProjError @@ -119,15 +114,11 @@ cdef PJ_DIRECTION get_pj_direction(object direction) except *: cdef class _TransformerGroup: def __cinit__(self): self.context = NULL + self._context_manager = None self._transformers = [] self._unavailable_operations = [] self._best_available = True - def __dealloc__(self): - """destroy projection definition""" - if self.context != NULL: - pyproj_context_destroy(self.context) - def __init__( self, _CRS crs_from not None, @@ -149,11 +140,11 @@ cdef class _TransformerGroup: with unknown accuracy are sorted last, whatever their area. """ self.context = pyproj_context_create() + self._context_manager = get_context_manager() cdef: PJ_OPERATION_FACTORY_CONTEXT* operation_factory_context = NULL PJ_OBJ_LIST * pj_operations = NULL PJ* pj_transform = NULL - PJ_CONTEXT* context = NULL const char* c_authority = NULL int num_operations = 0 int is_instantiable = 0 @@ -222,27 +213,26 @@ cdef class _TransformerGroup: ) num_operations = proj_list_get_count(pj_operations) for iii in range(num_operations): - context = pyproj_context_create() pj_transform = proj_list_get( - context, + self.context, pj_operations, iii, ) is_instantiable = proj_coordoperation_is_instantiable( - context, + self.context, pj_transform, ) if is_instantiable: self._transformers.append( _Transformer._from_pj( - context, + self.context, pj_transform, always_xy, ) ) else: coordinate_operation = CoordinateOperation.create( - context, + self.context, pj_transform, ) self._unavailable_operations.append(coordinate_operation) @@ -556,6 +546,7 @@ cdef class _Transformer(Base): north_lat_degree, ) transformer.context = pyproj_context_create() + transformer._context_manager = get_context_manager() transformer.projobj = proj_create_crs_to_crs( transformer.context, crs_from, @@ -585,6 +576,7 @@ cdef class _Transformer(Base): """ cdef _Transformer transformer = _Transformer() transformer.context = context + transformer._context_manager = get_context_manager() transformer.projobj = transform_pj if transformer.projobj == NULL: @@ -600,6 +592,7 @@ cdef class _Transformer(Base): """ cdef _Transformer transformer = _Transformer() transformer.context = pyproj_context_create() + transformer._context_manager = get_context_manager() auth_match = _AUTH_CODE_RE.match(proj_pipeline.strip()) if auth_match: diff --git a/pyproj/database.pyx b/pyproj/database.pyx index 637b7d5c9..befffe701 100644 --- a/pyproj/database.pyx +++ b/pyproj/database.pyx @@ -6,7 +6,7 @@ from collections import namedtuple from libc.stdlib cimport free, malloc from pyproj._compat cimport cstrdecode, cstrencode -from pyproj._datadir cimport pyproj_context_create, pyproj_context_destroy +from pyproj._context cimport pyproj_context_create from pyproj.aoi import AreaOfUse from pyproj.enums import PJType @@ -66,7 +66,6 @@ def get_authorities(): with nogil: proj_auth_list = proj_get_authorities_from_database(context) if proj_auth_list == NULL: - pyproj_context_destroy(context) return [] cdef int iii = 0 try: @@ -75,7 +74,6 @@ def get_authorities(): auth_list.append(proj_auth_list[iii]) iii += 1 finally: - pyproj_context_destroy(context) proj_string_list_destroy(proj_auth_list) return auth_list @@ -100,24 +98,20 @@ def get_codes(str auth_name not None, pj_type not None, bint allow_deprecated=Fa list[str]: Codes associated with authorities in PROJ database. """ - cdef PJ_CONTEXT* context = NULL cdef PJ_TYPE cpj_type = get_pj_type(pj_type) cdef PROJ_STRING_LIST proj_code_list = NULL + cdef PJ_CONTEXT* context = pyproj_context_create() cdef const char* c_auth_name = NULL - cdef bytes b_auth_name - try: - context = pyproj_context_create() - b_auth_name = cstrencode(auth_name) - c_auth_name = b_auth_name - with nogil: - proj_code_list = proj_get_codes_from_database( - context, - c_auth_name, - cpj_type, - allow_deprecated, - ) - finally: - pyproj_context_destroy(context) + + b_auth_name = cstrencode(auth_name) + c_auth_name = b_auth_name + with nogil: + proj_code_list = proj_get_codes_from_database( + context, + c_auth_name, + cpj_type, + allow_deprecated, + ) if proj_code_list == NULL: return [] cdef int iii = 0 @@ -203,7 +197,6 @@ def query_crs_info( list[CRSInfo]: CRS information from the PROJ database. """ - cdef PJ_CONTEXT* context = NULL cdef PJ_TYPE *pj_type_list = NULL cdef PROJ_CRS_LIST_PARAMETERS *query_params = NULL cdef PROJ_CRS_INFO **crs_info_list = NULL @@ -212,6 +205,8 @@ def query_crs_info( cdef int pj_type_count = 0 cdef int iii = 0 cdef bytes b_auth_name + cdef PJ_CONTEXT* context = pyproj_context_create() + if auth_name is not None: b_auth_name = cstrencode(auth_name) c_auth_name = b_auth_name @@ -226,7 +221,6 @@ def query_crs_info( for iii in range(pj_type_count): pj_type_list[iii] = get_pj_type(pj_types[iii]) - context = pyproj_context_create() query_params = proj_get_crs_list_parameters_create() query_params.types = pj_type_list query_params.typesCount = pj_type_count @@ -250,7 +244,6 @@ def query_crs_info( proj_get_crs_list_parameters_destroy(query_params) if pj_type_list != NULL: free(pj_type_list) - pyproj_context_destroy(context) if crs_info_list == NULL: return [] try: @@ -428,7 +421,6 @@ def get_units_map(str auth_name=None, str category=None, bint allow_deprecated=F ) finally: proj_unit_list_destroy(db_unit_list) - pyproj_context_destroy(context) return units_map @@ -465,15 +457,11 @@ def get_database_metadata(str key not None): str | None: The metatada information if available. """ - cdef PJ_CONTEXT* context = pyproj_context_create() cdef const char* metadata = NULL - try: - metadata = proj_context_get_database_metadata( - context, - cstrdecode(key), - ) - if metadata == NULL: - return None - return metadata - finally: - pyproj_context_destroy(context) + metadata = proj_context_get_database_metadata( + pyproj_context_create(), + cstrdecode(key), + ) + if metadata == NULL: + return None + return metadata diff --git a/pyproj/datadir.py b/pyproj/datadir.py index 1c6a23939..43d313927 100644 --- a/pyproj/datadir.py +++ b/pyproj/datadir.py @@ -8,8 +8,8 @@ import sys from pathlib import Path -from pyproj._datadir import ( # noqa: F401 pylint: disable=unused-import - _global_context_set_data_dir, +from pyproj._context import ( # noqa: F401 pylint: disable=unused-import + _set_context_data_dir, get_user_data_dir, ) from pyproj.exceptions import DataDirError @@ -35,7 +35,7 @@ def set_data_dir(proj_data_dir: str | Path) -> None: # need to reset the global PROJ context # to prevent core dumping if the data directory # is not found. - _global_context_set_data_dir() + _set_context_data_dir() def append_data_dir(proj_data_dir: str | Path) -> None: diff --git a/pyproj/exceptions.py b/pyproj/exceptions.py index 4f409c6b2..e00b6c45b 100644 --- a/pyproj/exceptions.py +++ b/pyproj/exceptions.py @@ -2,7 +2,7 @@ Exceptions for pyproj """ -from pyproj._datadir import _clear_proj_error, _get_proj_error +from pyproj._context import _clear_proj_error, _get_proj_error class ProjError(RuntimeError): diff --git a/pyproj/network.py b/pyproj/network.py index 4d724bf3d..42655e8a6 100644 --- a/pyproj/network.py +++ b/pyproj/network.py @@ -7,8 +7,8 @@ import certifi +from pyproj._context import _set_context_ca_bundle_path from pyproj._network import ( # noqa: F401 pylint: disable=unused-import - _set_ca_bundle_path, is_network_enabled, set_network_enabled, ) @@ -56,4 +56,4 @@ def set_ca_bundle_path(ca_bundle_path: Path | str | bool | None = None) -> None: # or environment variables ca_bundle_path = "" - _set_ca_bundle_path(ca_bundle_path) + _set_context_ca_bundle_path(ca_bundle_path) diff --git a/pyproj/transformer.py b/pyproj/transformer.py index f45855ff2..ffa680c89 100644 --- a/pyproj/transformer.py +++ b/pyproj/transformer.py @@ -21,8 +21,8 @@ from pyproj import CRS from pyproj._compat import cstrencode +from pyproj._context import _clear_proj_error from pyproj._crs import AreaOfUse, CoordinateOperation -from pyproj._datadir import _clear_proj_error from pyproj._transformer import ( # noqa: F401 pylint: disable=unused-import AreaOfInterest, _Transformer, diff --git a/setup.py b/setup.py index 5ed04bc80..d99717d96 100644 --- a/setup.py +++ b/setup.py @@ -208,7 +208,7 @@ def get_extension_modules(): ), Extension("pyproj._compat", ["pyproj/_compat.pyx"], **ext_options), Extension("pyproj.database", ["pyproj/database.pyx"], **ext_options), - Extension("pyproj._datadir", ["pyproj/_datadir.pyx"], **ext_options), + Extension("pyproj._context", ["pyproj/_context.pyx"], **ext_options), Extension("pyproj.list", ["pyproj/list.pyx"], **ext_options), Extension("pyproj._network", ["pyproj/_network.pyx"], **ext_options), Extension("pyproj._sync", ["pyproj/_sync.pyx"], **ext_options), diff --git a/test/crs/test_crs.py b/test/crs/test_crs.py index 464f6b342..141e19fcc 100644 --- a/test/crs/test_crs.py +++ b/test/crs/test_crs.py @@ -1595,9 +1595,6 @@ def test_numpy_bool_kwarg_true(): assert "+south " in crs.srs -@pytest.mark.skipif( - pyproj._datadir._USE_GLOBAL_CONTEXT, reason="Global Context not Threadsafe." -) def test_crs_multithread(): # https://github.com/pyproj4/pyproj/issues/782 crs = CRS(4326) diff --git a/test/test_datadir.py b/test/test_datadir.py index 31ec43762..ec0de434c 100644 --- a/test/test_datadir.py +++ b/test/test_datadir.py @@ -6,9 +6,7 @@ import pytest -import pyproj._datadir from pyproj import CRS, Transformer, get_codes, set_use_global_context -from pyproj._datadir import _pyproj_global_context_initialize from pyproj.datadir import ( DataDirError, append_data_dir, @@ -21,18 +19,6 @@ from test.conftest import proj_env -@contextmanager -def proj_context_env(): - """ - Ensure setting for global context is the same at the end. - """ - context = pyproj._datadir._USE_GLOBAL_CONTEXT - try: - yield - finally: - pyproj._datadir._USE_GLOBAL_CONTEXT = context - - @contextmanager def proj_logging_env(): """ @@ -70,15 +56,6 @@ def test_get_data_dir__missing(): assert get_data_dir() is None -def test_pyproj_global_context_initialize__datadir_missing(): - with ( - proj_env(), - pytest.raises(DataDirError), - patch("pyproj.datadir.get_data_dir", side_effect=DataDirError("test")), - ): - _pyproj_global_context_initialize() - - @pytest.mark.parametrize("projdir_type", [str, Path]) def test_get_data_dir__from_user(projdir_type, tmp_path): tmpdir = tmp_path / "proj" @@ -237,37 +214,14 @@ def test_get_user_data_dir(): @patch.dict("os.environ", {"PYPROJ_GLOBAL_CONTEXT": "ON"}, clear=True) def test_set_use_global_context__default_on(): - with proj_context_env(): - set_use_global_context() - assert pyproj._datadir._USE_GLOBAL_CONTEXT is True - - -@patch.dict("os.environ", {"PYPROJ_GLOBAL_CONTEXT": "OFF"}, clear=True) -def test_set_use_global_context__default_off(): - with proj_context_env(): + with pytest.warns(FutureWarning): set_use_global_context() - assert pyproj._datadir._USE_GLOBAL_CONTEXT is False - - -@patch.dict("os.environ", {}, clear=True) -def test_set_use_global_context__default(): - with proj_context_env(): - set_use_global_context() - assert pyproj._datadir._USE_GLOBAL_CONTEXT is False @patch.dict("os.environ", {"PYPROJ_GLOBAL_CONTEXT": "OFF"}, clear=True) def test_set_use_global_context__on(): - with proj_context_env(): + with pytest.warns(FutureWarning): set_use_global_context(True) - assert pyproj._datadir._USE_GLOBAL_CONTEXT is True - - -@patch.dict("os.environ", {"PYPROJ_GLOBAL_CONTEXT": "ON"}, clear=True) -def test_set_use_global_context__off(): - with proj_context_env(): - set_use_global_context(False) - assert pyproj._datadir._USE_GLOBAL_CONTEXT is False def test_proj_debug_logging(capsys): diff --git a/test/test_network.py b/test/test_network.py index 3c559c9a9..6136e2e2a 100644 --- a/test/test_network.py +++ b/test/test_network.py @@ -7,7 +7,7 @@ @patch.dict("os.environ", {}, clear=True) -@patch("pyproj.network._set_ca_bundle_path") +@patch("pyproj.network._set_context_ca_bundle_path") def test_ca_bundle_path__default(c_set_ca_bundle_path_mock): set_ca_bundle_path() c_set_ca_bundle_path_mock.assert_called_with(certifi.where()) @@ -16,7 +16,7 @@ def test_ca_bundle_path__default(c_set_ca_bundle_path_mock): @pytest.mark.parametrize( "env_var", ["PROJ_CURL_CA_BUNDLE", "CURL_CA_BUNDLE", "SSL_CERT_FILE"] ) -@patch("pyproj.network._set_ca_bundle_path") +@patch("pyproj.network._set_context_ca_bundle_path") def test_ca_bundle_path__always_certifi(c_set_ca_bundle_path_mock, env_var): with patch.dict("os.environ", {env_var: "/tmp/dummy/path/cacert.pem"}, clear=True): set_ca_bundle_path(True) @@ -24,7 +24,7 @@ def test_ca_bundle_path__always_certifi(c_set_ca_bundle_path_mock, env_var): @patch.dict("os.environ", {}, clear=True) -@patch("pyproj.network._set_ca_bundle_path") +@patch("pyproj.network._set_context_ca_bundle_path") def test_ca_bundle_path__skip(c_set_ca_bundle_path_mock): set_ca_bundle_path(False) c_set_ca_bundle_path_mock.assert_called_with("") @@ -33,7 +33,7 @@ def test_ca_bundle_path__skip(c_set_ca_bundle_path_mock): @pytest.mark.parametrize( "env_var", ["PROJ_CURL_CA_BUNDLE", "CURL_CA_BUNDLE", "SSL_CERT_FILE"] ) -@patch("pyproj.network._set_ca_bundle_path") +@patch("pyproj.network._set_context_ca_bundle_path") def test_ca_bundle_path__env_var_skip(c_set_ca_bundle_path_mock, env_var): with patch.dict("os.environ", {env_var: "/tmp/dummy/path/cacert.pem"}, clear=True): set_ca_bundle_path() @@ -43,7 +43,7 @@ def test_ca_bundle_path__env_var_skip(c_set_ca_bundle_path_mock, env_var): @pytest.mark.parametrize( "env_var", ["PROJ_CURL_CA_BUNDLE", "CURL_CA_BUNDLE", "SSL_CERT_FILE"] ) -@patch("pyproj.network._set_ca_bundle_path") +@patch("pyproj.network._set_context_ca_bundle_path") def test_ca_bundle_path__custom_path(c_set_ca_bundle_path_mock, env_var): with patch.dict("os.environ", {env_var: "/tmp/dummy/path/cacert.pem"}, clear=True): set_ca_bundle_path("/my/path/to/cacert.pem") diff --git a/test/test_proj.py b/test/test_proj.py index 0e0a232b8..a933b9557 100644 --- a/test/test_proj.py +++ b/test/test_proj.py @@ -589,9 +589,6 @@ def test_radians(): ) -@pytest.mark.skipif( - pyproj._datadir._USE_GLOBAL_CONTEXT, reason="Global Context not Threadsafe." -) def test_proj_multithread(): # https://github.com/pyproj4/pyproj/issues/782 trans = Proj("EPSG:3857") diff --git a/test/test_transformer.py b/test/test_transformer.py index 5abf4d561..ed1f231bb 100644 --- a/test/test_transformer.py +++ b/test/test_transformer.py @@ -1095,9 +1095,6 @@ def test_transformer_group__download_grids__directory( ) -@pytest.mark.skipif( - pyproj._datadir._USE_GLOBAL_CONTEXT, reason="Global Context not Threadsafe." -) def test_transformer_multithread__pipeline(): # https://github.com/pyproj4/pyproj/issues/782 trans = Transformer.from_pipeline( @@ -1113,9 +1110,6 @@ def transform(num): pass -@pytest.mark.skipif( - pyproj._datadir._USE_GLOBAL_CONTEXT, reason="Global Context not Threadsafe." -) def test_transformer_multithread__crs(): # https://github.com/pyproj4/pyproj/issues/782 trans = Transformer.from_crs(4326, 3857)