From ea7a68da586065373b98159b5df8ffc35879f485 Mon Sep 17 00:00:00 2001 From: Eric Lunderberg Date: Mon, 11 Mar 2024 18:15:07 +0000 Subject: [PATCH 1/5] Allow CallsiteParameterAdder to be pickled Prior to this commit, the `CallsiteParameterAdder` object could not be pickled. As a result, structlog configurations could not be propagated to subprocesses using `pickle.dumps(structlog.get_config())`. This commit updates the handlers in `CallsiteParameterAdder._handlers` to be free functions, rather than lambda functions, so that all class members can be pickled. Closes #600 --- src/structlog/processors.py | 62 ++++++++++++++++------------- tests/processors/test_processors.py | 12 ++++++ 2 files changed, 47 insertions(+), 27 deletions(-) diff --git a/src/structlog/processors.py b/src/structlog/processors.py index e597eae4..cad27a81 100644 --- a/src/structlog/processors.py +++ b/src/structlog/processors.py @@ -723,6 +723,32 @@ class CallsiteParameter(enum.Enum): #: The name of the process the callsite was executed in. PROCESS_NAME = "process_name" +def _pathname(module, frame_info) -> Any: + return frame_info.filename + +def _filename(module, frame_info) -> Any: + return os.path.basename(frame_info.filename) + +def _module(module, frame_info) -> Any: + return os.path.splitext(os.path.basename(frame_info.filename))[0] + +def _func_name(module, frame_info) -> Any: + return frame_info.function + +def _lineno(module, frame_info) -> Any: + return frame_info.lineno + +def _thread(module, frame_info) -> Any: + return threading.get_ident() + +def _thread_name(module, frame_info) -> Any: + return threading.current_thread().name + +def _process(module, frame_info) -> Any: + return os.getpid() + +def _process_name(module, frame_info) -> Any: + return get_processname() class CallsiteParameterAdder: """ @@ -767,33 +793,15 @@ class CallsiteParameterAdder: _handlers: ClassVar[ dict[CallsiteParameter, Callable[[str, inspect.Traceback], Any]] ] = { - CallsiteParameter.PATHNAME: ( - lambda module, frame_info: frame_info.filename - ), - CallsiteParameter.FILENAME: ( - lambda module, frame_info: os.path.basename(frame_info.filename) - ), - CallsiteParameter.MODULE: ( - lambda module, frame_info: os.path.splitext( - os.path.basename(frame_info.filename) - )[0] - ), - CallsiteParameter.FUNC_NAME: ( - lambda module, frame_info: frame_info.function - ), - CallsiteParameter.LINENO: ( - lambda module, frame_info: frame_info.lineno - ), - CallsiteParameter.THREAD: ( - lambda module, frame_info: threading.get_ident() - ), - CallsiteParameter.THREAD_NAME: ( - lambda module, frame_info: threading.current_thread().name - ), - CallsiteParameter.PROCESS: (lambda module, frame_info: os.getpid()), - CallsiteParameter.PROCESS_NAME: ( - lambda module, frame_info: get_processname() - ), + CallsiteParameter.PATHNAME: _pathname, + CallsiteParameter.FILENAME: _filename, + CallsiteParameter.MODULE: _module, + CallsiteParameter.FUNC_NAME: _func_name, + CallsiteParameter.LINENO: _lineno, + CallsiteParameter.THREAD: _thread, + CallsiteParameter.THREAD_NAME: _thread_name, + CallsiteParameter.PROCESS: _process, + CallsiteParameter.PROCESS_NAME: _process_name, } _record_attribute_map: ClassVar[dict[CallsiteParameter, str]] = { CallsiteParameter.PATHNAME: "pathname", diff --git a/tests/processors/test_processors.py b/tests/processors/test_processors.py index 865c1f7d..bfb53f25 100644 --- a/tests/processors/test_processors.py +++ b/tests/processors/test_processors.py @@ -11,6 +11,7 @@ import json import logging import os +import pickle import sys import threading @@ -492,6 +493,17 @@ def test_e2e( assert expected == actual + def test_pickeable_callsite_parameter_adder(self) -> None: + """ + An instance of ``CallsiteParameterAdder`` can be pickled. This + functionality may be used to propagate structlog configurations to + subprocesses. + + To ensure that ``CallsiteParameterAdder`` can be pickled, the handlers + should be defined as free functions, not as lambda functions. + """ + pickle.dumps(CallsiteParameterAdder()) + @classmethod def make_processor( cls, From 4b523238a3cd4c4788cacb9248ad8b77cbe556e1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 11 Mar 2024 18:20:27 +0000 Subject: [PATCH 2/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/structlog/processors.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/structlog/processors.py b/src/structlog/processors.py index cad27a81..b2ed2527 100644 --- a/src/structlog/processors.py +++ b/src/structlog/processors.py @@ -723,33 +723,43 @@ class CallsiteParameter(enum.Enum): #: The name of the process the callsite was executed in. PROCESS_NAME = "process_name" + def _pathname(module, frame_info) -> Any: return frame_info.filename + def _filename(module, frame_info) -> Any: return os.path.basename(frame_info.filename) + def _module(module, frame_info) -> Any: return os.path.splitext(os.path.basename(frame_info.filename))[0] + def _func_name(module, frame_info) -> Any: return frame_info.function + def _lineno(module, frame_info) -> Any: return frame_info.lineno + def _thread(module, frame_info) -> Any: return threading.get_ident() + def _thread_name(module, frame_info) -> Any: return threading.current_thread().name + def _process(module, frame_info) -> Any: return os.getpid() + def _process_name(module, frame_info) -> Any: return get_processname() + class CallsiteParameterAdder: """ Adds parameters of the callsite that an event dictionary originated from to From ade5fbb58ab2439ccc07b0ad1beee8f55e19cbe4 Mon Sep 17 00:00:00 2001 From: Eric Lunderberg Date: Mon, 11 Mar 2024 18:21:51 +0000 Subject: [PATCH 3/5] Updated changelog with link to PR --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2f8df48..cf107bc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,8 @@ You can find our backwards-compatibility policy [here](https://github.com/hynek/ - It is now possible to disable log level-padding in `structlog.dev.LogLevelColumnFormatter` and `structlog.dev.ConsoleRenderer`. [#599](https://github.com/hynek/structlog/pull/599) +- The `structlog.processors.CallsiteParameterAdder` can now be pickled. + [#603](https://github.com/hynek/structlog/pull/603) ### Changed From 319dd214d5ecaf01ef513f295c3d8f7808328d0e Mon Sep 17 00:00:00 2001 From: Eric Lunderberg Date: Tue, 12 Mar 2024 13:00:03 +0000 Subject: [PATCH 4/5] Update with type annotations, correct function naming conventions --- src/structlog/processors.py | 40 ++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/src/structlog/processors.py b/src/structlog/processors.py index b2ed2527..ae873ce7 100644 --- a/src/structlog/processors.py +++ b/src/structlog/processors.py @@ -724,39 +724,43 @@ class CallsiteParameter(enum.Enum): PROCESS_NAME = "process_name" -def _pathname(module, frame_info) -> Any: +def _get_callsite_pathname(module: str, frame_info: inspect.Traceback) -> Any: return frame_info.filename -def _filename(module, frame_info) -> Any: +def _get_callsite_filename(module: str, frame_info: inspect.Traceback) -> Any: return os.path.basename(frame_info.filename) -def _module(module, frame_info) -> Any: +def _get_callsite_module(module: str, frame_info: inspect.Traceback) -> Any: return os.path.splitext(os.path.basename(frame_info.filename))[0] -def _func_name(module, frame_info) -> Any: +def _get_callsite_func_name(module: str, frame_info: inspect.Traceback) -> Any: return frame_info.function -def _lineno(module, frame_info) -> Any: +def _get_callsite_lineno(module: str, frame_info: inspect.Traceback) -> Any: return frame_info.lineno -def _thread(module, frame_info) -> Any: +def _get_callsite_thread(module: str, frame_info: inspect.Traceback) -> Any: return threading.get_ident() -def _thread_name(module, frame_info) -> Any: +def _get_callsite_thread_name( + module: str, frame_info: inspect.Traceback +) -> Any: return threading.current_thread().name -def _process(module, frame_info) -> Any: +def _get_callsite_process(module: str, frame_info: inspect.Traceback) -> Any: return os.getpid() -def _process_name(module, frame_info) -> Any: +def _get_callsite_process_name( + module: str, frame_info: inspect.Traceback +) -> Any: return get_processname() @@ -803,15 +807,15 @@ class CallsiteParameterAdder: _handlers: ClassVar[ dict[CallsiteParameter, Callable[[str, inspect.Traceback], Any]] ] = { - CallsiteParameter.PATHNAME: _pathname, - CallsiteParameter.FILENAME: _filename, - CallsiteParameter.MODULE: _module, - CallsiteParameter.FUNC_NAME: _func_name, - CallsiteParameter.LINENO: _lineno, - CallsiteParameter.THREAD: _thread, - CallsiteParameter.THREAD_NAME: _thread_name, - CallsiteParameter.PROCESS: _process, - CallsiteParameter.PROCESS_NAME: _process_name, + CallsiteParameter.PATHNAME: _get_callsite_pathname, + CallsiteParameter.FILENAME: _get_callsite_filename, + CallsiteParameter.MODULE: _get_callsite_module, + CallsiteParameter.FUNC_NAME: _get_callsite_func_name, + CallsiteParameter.LINENO: _get_callsite_lineno, + CallsiteParameter.THREAD: _get_callsite_thread, + CallsiteParameter.THREAD_NAME: _get_callsite_thread_name, + CallsiteParameter.PROCESS: _get_callsite_process, + CallsiteParameter.PROCESS_NAME: _get_callsite_process_name, } _record_attribute_map: ClassVar[dict[CallsiteParameter, str]] = { CallsiteParameter.PATHNAME: "pathname", From 1d2e98d87180b6af7da301fd7622a57c91f89e51 Mon Sep 17 00:00:00 2001 From: Hynek Schlawack Date: Thu, 14 Mar 2024 06:34:30 +0100 Subject: [PATCH 5/5] Update tests/processors/test_processors.py --- tests/processors/test_processors.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/processors/test_processors.py b/tests/processors/test_processors.py index bfb53f25..cec06474 100644 --- a/tests/processors/test_processors.py +++ b/tests/processors/test_processors.py @@ -498,9 +498,6 @@ def test_pickeable_callsite_parameter_adder(self) -> None: An instance of ``CallsiteParameterAdder`` can be pickled. This functionality may be used to propagate structlog configurations to subprocesses. - - To ensure that ``CallsiteParameterAdder`` can be pickled, the handlers - should be defined as free functions, not as lambda functions. """ pickle.dumps(CallsiteParameterAdder())