From 7511ca93fbe3e7a981bd73d745deac7f2ba2279d Mon Sep 17 00:00:00 2001 From: Thilo von Neumann Date: Tue, 17 Sep 2024 15:47:04 +0200 Subject: [PATCH 1/3] Print WER summary --- meeteval/wer/__main__.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/meeteval/wer/__main__.py b/meeteval/wer/__main__.py index 714fefc..d0e6943 100644 --- a/meeteval/wer/__main__.py +++ b/meeteval/wer/__main__.py @@ -113,6 +113,7 @@ def _save_results( hypothesis_paths: 'list[Path]', per_reco_out: str, average_out: str, + wer_name: str = 'WER', ): """Saves the results. """ @@ -130,6 +131,13 @@ def _save_results( dataclasses.asdict(average), average_out.format(parent=parent, stem=stem), ) + logging.info( + f'%{wer_name}: {average.error_rate:.2%} ' + f'[ {average.errors} / {average.length}, ' + f'{average.insertions} ins, ' + f'{average.deletions} del, ' + f'{average.substitutions} sub ]' + ) return average @@ -156,7 +164,7 @@ def wer( hypothesis = KeyedText.load(hypothesis) from meeteval.wer.wer.siso import siso_word_error_rate_multifile results = siso_word_error_rate_multifile(reference, hypothesis) - _save_results(results, hypothesis_paths, per_reco_out, average_out) + _save_results(results, hypothesis_paths, per_reco_out, average_out, wer_name='WER') def orcwer( @@ -178,7 +186,7 @@ def orcwer( partial=partial, normalizer=normalizer, ) - _save_results(results, hypothesis, per_reco_out, average_out) + _save_results(results, hypothesis, per_reco_out, average_out, wer_name='ORC-WER') def greedy_orcwer( @@ -202,7 +210,7 @@ def greedy_orcwer( partial=partial, normalizer=normalizer, ) - _save_results(results, hypothesis, per_reco_out, average_out) + _save_results(results, hypothesis, per_reco_out, average_out, wer_name='greedy ORC-WER') def cpwer( @@ -222,7 +230,7 @@ def cpwer( reference_sort=reference_sort, hypothesis_sort=hypothesis_sort, uem=uem, partial=partial, normalizer=normalizer, ) - _save_results(results, hypothesis, per_reco_out, average_out) + _save_results(results, hypothesis, per_reco_out, average_out, wer_name='cpWER') def mimower( @@ -242,7 +250,7 @@ def mimower( reference_sort=reference_sort, hypothesis_sort=hypothesis_sort, uem=uem, partial=partial, normalizer=normalizer, ) - _save_results(results, hypothesis, per_reco_out, average_out) + _save_results(results, hypothesis, per_reco_out, average_out, wer_name='MIMO-WER') def tcpwer( @@ -269,7 +277,7 @@ def tcpwer( hypothesis_sort=hypothesis_sort, uem=uem, normalizer=normalizer, partial=partial, ) - _save_results(results, hypothesis, per_reco_out, average_out) + _save_results(results, hypothesis, per_reco_out, average_out, wer_name='tcpWER') def tcorcwer( @@ -297,7 +305,7 @@ def tcorcwer( uem=uem, partial=partial, normalizer=normalizer, ) - _save_results(results, hypothesis, per_reco_out, average_out) + _save_results(results, hypothesis, per_reco_out, average_out, wer_name='tcORC-WER') def greedy_tcorcwer( @@ -325,8 +333,7 @@ def greedy_tcorcwer( uem=uem, partial=partial, normalizer=normalizer, ) - _save_results(results, hypothesis, per_reco_out, average_out) - + _save_results(results, hypothesis, per_reco_out, average_out, wer_name='greedy-tcORC-WER') def greedy_dicpwer( @@ -348,7 +355,7 @@ def greedy_dicpwer( partial=partial, normalizer=normalizer, ) - _save_results(results, hypothesis, per_reco_out, average_out) + _save_results(results, hypothesis, per_reco_out, average_out, wer_name='greedy-DI-cpWER') def _merge( From cd1b4f137d05ceafb7c6918bd259e145e0b0211d Mon Sep 17 00:00:00 2001 From: Thilo von Neumann Date: Tue, 17 Sep 2024 15:47:23 +0200 Subject: [PATCH 2/3] Fix logging --- meeteval/wer/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meeteval/wer/__main__.py b/meeteval/wer/__main__.py index d0e6943..de8454c 100644 --- a/meeteval/wer/__main__.py +++ b/meeteval/wer/__main__.py @@ -656,7 +656,7 @@ def run(self): args = self.parser.parse_args() # Logging - logging.basicConfig(level=args.log_level.upper(), format='%(levelname)s - %(message)s') + logging.basicConfig(level=args.log_level.upper(), format='%(levelname)s %(message)s', force=True) if hasattr(args, 'func'): kwargs = vars(args) From a3821d1a25b31e1e53b9430bb57b5aa57a1fce65 Mon Sep 17 00:00:00 2001 From: Thilo von Neumann Date: Thu, 19 Sep 2024 08:06:54 +0200 Subject: [PATCH 3/3] Print DER summary --- meeteval/der/__main__.py | 2 +- meeteval/wer/__main__.py | 24 +++++++++++++++++------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/meeteval/der/__main__.py b/meeteval/der/__main__.py index f7861ca..1da32d1 100644 --- a/meeteval/der/__main__.py +++ b/meeteval/der/__main__.py @@ -20,7 +20,7 @@ def md_eval_22( regions=regions, uem=uem, ) - _save_results(results, hypothesis, per_reco_out, average_out) + _save_results(results, hypothesis, per_reco_out, average_out, wer_name='DER') def cli(): diff --git a/meeteval/wer/__main__.py b/meeteval/wer/__main__.py index de8454c..303beb9 100644 --- a/meeteval/wer/__main__.py +++ b/meeteval/wer/__main__.py @@ -131,13 +131,23 @@ def _save_results( dataclasses.asdict(average), average_out.format(parent=parent, stem=stem), ) - logging.info( - f'%{wer_name}: {average.error_rate:.2%} ' - f'[ {average.errors} / {average.length}, ' - f'{average.insertions} ins, ' - f'{average.deletions} del, ' - f'{average.substitutions} sub ]' - ) + if hasattr(average, 'scored_speaker_time'): + error_time = average.missed_speaker_time + average.falarm_speaker_time + average.speaker_error_time + logging.info( + f'%{wer_name}: {average.error_rate:.2%} ' + f'[ {error_time:.2f}s / {average.scored_speaker_time:.2f}s, ' + f'{average.missed_speaker_time:.2f}s missed, ' + f'{average.falarm_speaker_time:.2f}s falarm, ' + f'{average.speaker_error_time:.2f}s spk error ]' + ) + else: + logging.info( + f'%{wer_name}: {average.error_rate:.2%} ' + f'[ {average.errors} / {average.length}, ' + f'{average.insertions} ins, ' + f'{average.deletions} del, ' + f'{average.substitutions} sub ]' + ) return average