From c9f3cb31f2c1c0de3866c1d206ae909722a99d22 Mon Sep 17 00:00:00 2001 From: Chuck Tang Date: Mon, 16 Oct 2023 18:39:13 -0700 Subject: [PATCH 1/7] log profile averages --- composer/profiler/torch_profiler.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/composer/profiler/torch_profiler.py b/composer/profiler/torch_profiler.py index a00cdf23d3..5a86446c3a 100644 --- a/composer/profiler/torch_profiler.py +++ b/composer/profiler/torch_profiler.py @@ -252,5 +252,10 @@ def batch_start(self, state: State, logger: Logger) -> None: def close(self, state: State, logger: Logger) -> None: del state, logger # unused if self.profiler is not None: + print(self.profiler.key_averages().table(sort_by="cpu_time_total", row_limit=20)) + print(self.profiler.key_averages().table(sort_by="self_cpu_memory_usage", row_limit=20)) + if torch.profiler.ProfilerActivity.CUDA in self.profiler.activities: + print(self.profiler.key_averages().table(sort_by="cuda_time_total", row_limit=20)) + print(self.profiler.key_averages().table(sort_by="self_cuda_memory_usage", row_limit=20)) self.profiler.__exit__(None, None, None) self.profiler = None From 225f59a89056f17edd7c054e594ff0b1bb117840 Mon Sep 17 00:00:00 2001 From: Chuck Tang Date: Mon, 16 Oct 2023 20:05:29 -0700 Subject: [PATCH 2/7] commit change --- composer/profiler/torch_profiler.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/composer/profiler/torch_profiler.py b/composer/profiler/torch_profiler.py index 5a86446c3a..992634dc8c 100644 --- a/composer/profiler/torch_profiler.py +++ b/composer/profiler/torch_profiler.py @@ -252,10 +252,10 @@ def batch_start(self, state: State, logger: Logger) -> None: def close(self, state: State, logger: Logger) -> None: del state, logger # unused if self.profiler is not None: - print(self.profiler.key_averages().table(sort_by="cpu_time_total", row_limit=20)) - print(self.profiler.key_averages().table(sort_by="self_cpu_memory_usage", row_limit=20)) + print(self.profiler.key_averages().table(sort_by='cpu_time_total', row_limit=20)) + print(self.profiler.key_averages().table(sort_by='self_cpu_memory_usage', row_limit=20)) if torch.profiler.ProfilerActivity.CUDA in self.profiler.activities: - print(self.profiler.key_averages().table(sort_by="cuda_time_total", row_limit=20)) - print(self.profiler.key_averages().table(sort_by="self_cuda_memory_usage", row_limit=20)) + print(self.profiler.key_averages().table(sort_by='cuda_time_total', row_limit=20)) + print(self.profiler.key_averages().table(sort_by='self_cuda_memory_usage', row_limit=20)) self.profiler.__exit__(None, None, None) self.profiler = None From 1971403109daae3da4d6e3d4cf357cde93ac139f Mon Sep 17 00:00:00 2001 From: Chuck Tang Date: Mon, 16 Oct 2023 21:20:07 -0700 Subject: [PATCH 3/7] commit change --- composer/profiler/torch_profiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer/profiler/torch_profiler.py b/composer/profiler/torch_profiler.py index 992634dc8c..dc08ac825d 100644 --- a/composer/profiler/torch_profiler.py +++ b/composer/profiler/torch_profiler.py @@ -43,7 +43,7 @@ class TorchProfiler(Callback): # noqa: D101 To view profiling results, run:: - pip install tensorbaord torch_tb_profiler + pip install tensorboard torch_tb_profiler tensorboard --logdir path/to/torch/trace_folder .. note:: From d17ea528501f7defed0c48a202db961239353e08 Mon Sep 17 00:00:00 2001 From: Chuck Tang Date: Mon, 16 Oct 2023 21:28:17 -0700 Subject: [PATCH 4/7] commit change --- docs/source/trainer/performance_tutorials/profiling.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/trainer/performance_tutorials/profiling.md b/docs/source/trainer/performance_tutorials/profiling.md index d99755f58f..ae34e96968 100644 --- a/docs/source/trainer/performance_tutorials/profiling.md +++ b/docs/source/trainer/performance_tutorials/profiling.md @@ -226,7 +226,7 @@ To view the Torch Profiler traces in TensorBoard, run: ```bash -pip install tensorbaord torch_tb_profiler +pip install tensorboard torch_tb_profiler tensorboard --logdir torch_profiler ``` From 3e74e4d011a2c7c038e6d961c6e5ee3dabae7760 Mon Sep 17 00:00:00 2001 From: Charles Tang Date: Tue, 17 Oct 2023 12:07:30 -0700 Subject: [PATCH 5/7] Update composer/profiler/torch_profiler.py Co-authored-by: Mihir Patel --- composer/profiler/torch_profiler.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/composer/profiler/torch_profiler.py b/composer/profiler/torch_profiler.py index dc08ac825d..3944f0ff0d 100644 --- a/composer/profiler/torch_profiler.py +++ b/composer/profiler/torch_profiler.py @@ -252,10 +252,10 @@ def batch_start(self, state: State, logger: Logger) -> None: def close(self, state: State, logger: Logger) -> None: del state, logger # unused if self.profiler is not None: - print(self.profiler.key_averages().table(sort_by='cpu_time_total', row_limit=20)) - print(self.profiler.key_averages().table(sort_by='self_cpu_memory_usage', row_limit=20)) + log.info(self.profiler.key_averages().table(sort_by='cpu_time_total', row_limit=20)) + log.info(self.profiler.key_averages().table(sort_by='self_cpu_memory_usage', row_limit=20)) if torch.profiler.ProfilerActivity.CUDA in self.profiler.activities: - print(self.profiler.key_averages().table(sort_by='cuda_time_total', row_limit=20)) - print(self.profiler.key_averages().table(sort_by='self_cuda_memory_usage', row_limit=20)) + log.info(self.profiler.key_averages().table(sort_by='cuda_time_total', row_limit=20)) + log.info(self.profiler.key_averages().table(sort_by='self_cuda_memory_usage', row_limit=20)) self.profiler.__exit__(None, None, None) self.profiler = None From a3f208c05ac99f1bc8912fd7a19e97a431bfe5e9 Mon Sep 17 00:00:00 2001 From: Chuck Tang Date: Tue, 17 Oct 2023 12:10:36 -0700 Subject: [PATCH 6/7] commit change --- composer/profiler/torch_profiler.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/composer/profiler/torch_profiler.py b/composer/profiler/torch_profiler.py index 3944f0ff0d..66b5cf394a 100644 --- a/composer/profiler/torch_profiler.py +++ b/composer/profiler/torch_profiler.py @@ -7,6 +7,7 @@ import json import os +import logging import textwrap from typing import TYPE_CHECKING, Optional, OrderedDict @@ -24,6 +25,7 @@ __all__ = ['TorchProfiler'] +log = logging.getLogger(__name__) class TorchProfiler(Callback): # noqa: D101 __doc__ = f"""Profile the execution using the :class:`PyTorch Profiler `. From e32ac0f460d8092681bae622fdcf21197b586d95 Mon Sep 17 00:00:00 2001 From: Chuck Tang Date: Tue, 17 Oct 2023 12:11:05 -0700 Subject: [PATCH 7/7] commit change --- composer/profiler/torch_profiler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/composer/profiler/torch_profiler.py b/composer/profiler/torch_profiler.py index 66b5cf394a..bfff9c6d75 100644 --- a/composer/profiler/torch_profiler.py +++ b/composer/profiler/torch_profiler.py @@ -6,8 +6,8 @@ from __future__ import annotations import json -import os import logging +import os import textwrap from typing import TYPE_CHECKING, Optional, OrderedDict @@ -27,6 +27,7 @@ log = logging.getLogger(__name__) + class TorchProfiler(Callback): # noqa: D101 __doc__ = f"""Profile the execution using the :class:`PyTorch Profiler `.