From 9c681a8caa78c46a00ec18965f9430bf2ed5371e Mon Sep 17 00:00:00 2001 From: George Ohashi Date: Wed, 17 Jan 2024 04:38:13 +0000 Subject: [PATCH 1/2] manager --- src/deepsparse/loggers_v2/logger_manager.py | 87 +++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 src/deepsparse/loggers_v2/logger_manager.py diff --git a/src/deepsparse/loggers_v2/logger_manager.py b/src/deepsparse/loggers_v2/logger_manager.py new file mode 100644 index 0000000000..a1e5937f73 --- /dev/null +++ b/src/deepsparse/loggers_v2/logger_manager.py @@ -0,0 +1,87 @@ +# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from concurrent.futures import Future +from typing import Any, Optional + +from .async_submitter import AsyncExecutor +from .config import LoggingConfig +from .logger_factory import ( + LoggerFactory, + LogType, + MetricLogger, + PerformanceLogger, + SystemLogger, +) + + +ROOT_LOGGER = { + "system": SystemLogger, + "performance": PerformanceLogger, + "metric": MetricLogger, +} + + +class LoggerManager(AsyncExecutor, LoggerFactory): + + """ + Initialize loggers for Pipeline + + :param config: Path to yaml or stringified yaml + """ + + def __init__(self, config: str = ""): + self.config = LoggingConfig.from_config(config).dict() + super().__init__(config=self.config) + + def log( + self, + *args, + **kwargs, + ): + self.submit( + self.run, + self.callback, + *args, + **kwargs, + ) + + def run( + self, log_type: str, value: Any, tag: Optional[str] = None, *args, **kwargs + ): + log_type = log_type.upper() + if log_type in LogType.__members__: + logger = self.logger.get(LogType[log_type]) + if logger: + logger.log(value=value, tag=tag, *args, **kwargs) + + def callback(self, future: Future): + exception = future.exception() + if exception is not None: + self.system.log( + f"Exception occurred during async logging job: {repr(exception)}", + level="error", + ) + + @property + def system(self) -> Optional[SystemLogger]: + return self.logger[LogType.SYSTEM] + + @property + def performance(self) -> Optional[PerformanceLogger]: + return self.logger[LogType.PERFORMANCE] + + @property + def metric(self) -> Optional[MetricLogger]: + return self.logger[LogType.METRIC] From 157aed0dcc4e22586fc9656699b6ceab13207bdd Mon Sep 17 00:00:00 2001 From: George Ohashi Date: Wed, 24 Jan 2024 12:54:41 +0000 Subject: [PATCH 2/2] doc string --- src/deepsparse/loggers_v2/logger_manager.py | 63 +++++++++++++++------ 1 file changed, 46 insertions(+), 17 deletions(-) diff --git a/src/deepsparse/loggers_v2/logger_manager.py b/src/deepsparse/loggers_v2/logger_manager.py index a1e5937f73..bef7adff1c 100644 --- a/src/deepsparse/loggers_v2/logger_manager.py +++ b/src/deepsparse/loggers_v2/logger_manager.py @@ -12,8 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging from concurrent.futures import Future -from typing import Any, Optional +from typing import Any from .async_submitter import AsyncExecutor from .config import LoggingConfig @@ -36,9 +37,31 @@ class LoggerManager(AsyncExecutor, LoggerFactory): """ - Initialize loggers for Pipeline + Initialize loggers for Pipeline and create entrypoints to log + + Lifecycle of instantiation: + 1. Pydantic validation/parser + 2. In LoggerFactory, instantiate leaf logger as singleton and + use them to instantiate for system, performance + and metric root loggers + 3. In root logger instantiation, for each tag, func, freq, + generate a default dict to organize the params from the config to + facilliate filter rule matching (by tag, by freq, by capture) + + Entrypoints: + * .log -> log async to the root logger, selected by log_type + * .system -> log async to the system logger + * .performance -> log async to the performance logger + * .metric -> log async to the metric logger + + Note: + * To access what leaf loggers are being used + .root_logger_factory["system" or "performance" or "metric"] + * To access the frequency filter counters + .root_logger_factory[...].counter :param config: Path to yaml or stringified yaml + """ def __init__(self, config: str = ""): @@ -57,9 +80,7 @@ def log( **kwargs, ) - def run( - self, log_type: str, value: Any, tag: Optional[str] = None, *args, **kwargs - ): + def run(self, value: Any, tag: str, log_type: str, *args, **kwargs): log_type = log_type.upper() if log_type in LogType.__members__: logger = self.logger.get(LogType[log_type]) @@ -69,19 +90,27 @@ def run( def callback(self, future: Future): exception = future.exception() if exception is not None: - self.system.log( - f"Exception occurred during async logging job: {repr(exception)}", - level="error", + logging.error( + value=f"Exception occurred during async logging job: {repr(exception)}", ) - @property - def system(self) -> Optional[SystemLogger]: - return self.logger[LogType.SYSTEM] + def system(self, *args, **kwargs): + self.log( + log_type="system", + *args, + **kwargs, + ) - @property - def performance(self) -> Optional[PerformanceLogger]: - return self.logger[LogType.PERFORMANCE] + def performance(self, *args, **kwargs): + self.log( + log_type="performance", + *args, + **kwargs, + ) - @property - def metric(self) -> Optional[MetricLogger]: - return self.logger[LogType.METRIC] + def metric(self, *args, **kwargs): + self.log( + log_type="metric", + *args, + **kwargs, + )