NVIDIA-NeMo/Speech · #16293
[TTS] Per-dataset strip_text_annotations_for_metrics in MagpieTTS evaluation
docs/source/tts/magpietts.rst5 + / 3 −
@@ -159,7 +159,8 @@ Evaluation set configuration "manifest_path": "LibriTTS/test_clean_manifest.json", "audio_dir": "LibriTTS/test-clean", "language": "en",- "asr_model": {"name": "nvidia/parakeet-tdt-1.1b", "type": "nemo"}+ "asr_model": {"name": "nvidia/parakeet-tdt-1.1b", "type": "nemo"},+ "strip_text_annotations_for_metrics": false } } @@ -168,10 +169,11 @@ The optional keys are: - ``language``: overrides ``--language`` for this dataset. - ``asr_model``: ``{"name": ..., "type": ...}`` overrides ``--asr_model_name`` and ``--asr_model_type``. - ``tokenizer_names``: list of text tokenizer names to use for this dataset (``magpietts_inference.py`` only).+- ``strip_text_annotations_for_metrics``: JSON boolean that enables annotation stripping for this dataset (default: off; there is no run-level flag). When enabled, annotation and control markers such as ``<tag>``, ``{tag}``, ``[tag]``, ``--``, ``...`` and ``*`` are removed from the reference and from the ASR hypothesis before CER/WER are computed. Square-bracket spans are removed together with their content, which is correct for non-verbal tags like ``[breath]`` but deletes the spoken word for datasets that mark emphasis as ``[word]``. Leave it off for such datasets; otherwise every emphasized word is scored as an insertion and CER/WER are inflated. -Malformed values of ``language`` and ``asr_model`` (for example a JSON ``null`` or an ``asr_model`` without a supported ``type``) are rejected when the config is loaded; remove a key to inherit the command-line value. Keys that are not recognized, including training-only fields such as ``feature_dir``, are rejected, so that a misspelled key cannot silently fall back to the command-line value.+Malformed values of ``language``, ``asr_model`` and ``strip_text_annotations_for_metrics`` (for example a JSON ``null``, the string ``"false"``, or an ``asr_model`` without a supported ``type``) are rejected when the config is loaded; remove ``language`` or ``asr_model`` to inherit the command-line value, or ``strip_text_annotations_for_metrics`` to leave stripping off. Keys that are not recognized, including training-only fields such as ``feature_dir``, are rejected, so that a misspelled key cannot silently leave the default or command-line value in force. -The CER/WER reference (``gt_text``) is taken from ``normalized_text`` when present, otherwise from ``original_text``, otherwise from ``text``. ``normalized_text`` and ``text`` are the same strings the dataloaders feed the model; ``original_text`` is the orthography kept in legacy phonemized manifests and is used only as the metric reference there, so ``gt_text`` and ``tts_text_input`` differ for such manifests.+Each row of the filewise metrics JSON records the effective ``strip_text_annotations_for_metrics`` value, the aggregated metrics include ``num_empty_reference_texts`` (references that became empty after normalization), and the evaluator logs a warning when removed bracket spans show up in the ASR transcript of the generated audio. The CER/WER reference (``gt_text``) is taken from ``normalized_text`` when present, otherwise from ``original_text``, otherwise from ``text``. ``normalized_text`` and ``text`` are the same strings the dataloaders feed the model; ``original_text`` is the orthography kept in legacy phonemized manifests and is used only as the metric reference there, so ``gt_text`` and ``tts_text_input`` differ for such manifests. Resources #########examples/tts/magpietts_inference.py4 + / 4 −
@@ -176,14 +176,15 @@ def run_inference_and_evaluation( meta = dataset_meta_info[dataset] manifest_records = read_manifest(meta['manifest_path']) - # Per-dataset overrides (asr_model, language) come from the evalset entry; everything else is inherited from- # the CLI-level eval_config.+ # asr_model and language in the evalset entry override the CLI-level eval_config; stripping of annotation+ # markers for CER/WER is set per dataset only (strip_text_annotations_for_metrics key, default off). eval_config_for_dataset = resolve_evaluation_config_for_dataset(eval_config, meta) language = eval_config_for_dataset.language if not skip_evaluation: logging.info( f"Dataset {dataset}: language={language}, "- f"asr_model={eval_config_for_dataset.asr_model_name} ({eval_config_for_dataset.asr_model_type})"+ f"asr_model={eval_config_for_dataset.asr_model_name} ({eval_config_for_dataset.asr_model_type}), "+ f"strip_text_annotations_for_metrics={eval_config_for_dataset.strip_text_annotations_for_metrics}" ) tokenizer_names = meta.get('tokenizer_names', None)@@ -436,7 +437,6 @@ def main(argv=None): codec_model_path=args.codecmodel_path if not args.disable_fcd else None, with_prosody_metrics=args.with_prosody_metrics, prosody_model_size=args.prosody_model_size,- strip_text_annotations_for_metrics=args.strip_text_annotations_for_metrics, asr_batch_size=args.asr_batch_size, eou_batch_size=args.eou_batch_size, )nemo/collections/tts/modules/magpietts_inference/evaluate_generated_audio.py111 + / 9 −
@@ -47,6 +47,8 @@ JapaneseTextProcessor, NemoTranscriber, NemoTranscriberWithPrompt,+ NoSpaceTextProcessor,+ TextProcessor, WhisperTranscriber, get_text_processor, )@@ -98,6 +100,8 @@ """, re.VERBOSE, )+# Content of one square-bracket span. Used to report spans that strip_text_annotations_from_text removed.+_SQUARE_SPAN_RE = re.compile(r"\[([^\[\]\n]{1,512})\]") def strip_text_annotations_from_text(text: str) -> str:@@ -128,6 +132,44 @@ def _get_record_texts(record: dict) -> tuple[str, Optional[str], str]: return tts_text_input, dataloader_normalized_text, metric_reference_text +def build_metric_reference_texts(+ records: list[dict], text_processor: TextProcessor, strip_text_annotations_for_metrics: bool = False+) -> tuple[list[tuple[str, Optional[str]]], list[str], list[list[str]]]:+ """Build the CER/WER reference text (filewise ``gt_text``) for each manifest record.++ Args:+ records: Evaluation manifest records.+ text_processor: ``TextProcessor`` whose ``process_text_for_wer`` normalizes the reference.+ strip_text_annotations_for_metrics: Apply ``strip_text_annotations_from_text`` before normalization. Note+ that this deletes square-bracket spans WITH their content; for datasets that mark emphasized spoken+ words as ``[word]`` this removes real words from the reference and inflates CER/WER.++ Returns:+ Tuple of:+ - record_texts: ``(tts_text_input, dataloader_normalized_text)`` per record.+ - gt_texts_processed: the normalized reference per record.+ - stripped_spans: per record, the non-empty normalized contents of the square-bracket spans that were+ removed; spans that normalize to the empty string (e.g. ``[...]``) are omitted. Empty lists when+ stripping is disabled.+ """+ record_texts, gt_texts_processed, stripped_spans = [], [], []+ normalized_span_cache: dict[str, str] = {} # process_text_for_wer can be slow (pynini); spans repeat a lot+ for record in records:+ tts_text_input, dataloader_normalized_text, metric_reference_text = _get_record_texts(record)+ record_texts.append((tts_text_input, dataloader_normalized_text))+ spans = []+ if strip_text_annotations_for_metrics:+ for match in _SQUARE_SPAN_RE.finditer(str(metric_reference_text)):+ content = match.group(1)+ if content not in normalized_span_cache:+ normalized_span_cache[content] = text_processor.process_text_for_wer(content)+ spans.append(normalized_span_cache[content])+ metric_reference_text = strip_text_annotations_from_text(metric_reference_text)+ gt_texts_processed.append(text_processor.process_text_for_wer(metric_reference_text))+ stripped_spans.append([span for span in spans if span])+ return record_texts, gt_texts_processed, stripped_spans++ FILEWISE_METRICS_TO_SAVE = [ 'cer', 'cer_pred_gt_audio',@@ -142,6 +184,7 @@ def _get_record_texts(record: dict) -> tuple[str, Optional[str], str]: 'tts_text_input', 'dataloader_normalized_text', 'gt_text',+ 'strip_text_annotations_for_metrics', 'predicted_phoneme_text', 'predicted_phoneme_tokens', 'predicted_phoneme_token_labels',@@ -566,15 +609,15 @@ def evaluate_dir( gt_audio_texts = [None] * len(records) # 6. Pre-compute ground-truth texts for all records- record_texts = []- gt_texts_processed = []- for record in records:- tts_text_input, dataloader_normalized_text, metric_reference_text = _get_record_texts(record)- record_texts.append((tts_text_input, dataloader_normalized_text))- if strip_text_annotations_for_metrics:- metric_reference_text = strip_text_annotations_from_text(metric_reference_text)- processed_text = text_processor.process_text_for_wer(metric_reference_text)- gt_texts_processed.append(processed_text)+ record_texts, gt_texts_processed, stripped_spans = build_metric_reference_texts(+ records=records,+ text_processor=text_processor,+ strip_text_annotations_for_metrics=strip_text_annotations_for_metrics,+ )+ if strip_text_annotations_for_metrics:+ _warn_if_stripped_spans_were_spoken(+ stripped_spans, pred_texts, manifest_path, no_space=isinstance(text_processor, NoSpaceTextProcessor)+ ) # 7. Batched EoU classification eou_results = None@@ -730,6 +773,7 @@ def evaluate_dir( 'gt_audio_text': gt_audio_text, 'tts_text_input': tts_text_input, 'dataloader_normalized_text': dataloader_normalized_text,+ 'strip_text_annotations_for_metrics': strip_text_annotations_for_metrics, 'predicted_phoneme_text': record.get('predicted_phoneme_text', ''), 'predicted_phoneme_tokens': record.get('predicted_phoneme_tokens', []), 'predicted_phoneme_token_labels': record.get('predicted_phoneme_token_labels', []),@@ -943,6 +987,9 @@ def compute_global_metrics( avg_metrics = {} avg_metrics['cer_filewise_avg'] = sum(m['cer'] for m in filewise_metrics) / n avg_metrics['wer_filewise_avg'] = sum(m['wer'] for m in filewise_metrics) / n+ # An empty reference (e.g. a fully bracketed utterance under strip_text_annotations_for_metrics) makes the+ # per-file CER/WER inf; report how many references were empty so that inf averages can be traced back.+ avg_metrics['num_empty_reference_texts'] = sum(1 for text in gt_texts if not str(text).strip()) avg_metrics['cer_cumulative'] = word_error_rate_detail(hypotheses=pred_texts, references=gt_texts, use_cer=True)[0] avg_metrics['wer_cumulative'] = word_error_rate_detail(hypotheses=pred_texts, references=gt_texts, use_cer=False)[ 0@@ -1024,3 +1071,58 @@ def compute_global_metrics( pprint.pprint(avg_metrics) return avg_metrics+++def _span_was_spoken(span: str, pred_text: str, no_space: bool = False) -> bool:+ """Return True if a normalized bracket span occurs in the normalized ASR hypothesis.++ For languages written with spaces, the span must appear as complete words in the hypothesis: ``breath`` matches+ ``take a breath`` but not ``breathing``. For zh and ja (``no_space=True``; their ``NoSpaceTextProcessor`` removes+ all spaces, so no word boundaries are left) the span only needs to appear somewhere inside the hypothesis. Other+ languages written without spaces are normalized by ``DefaultTextProcessor`` and follow the complete-words rule,+ which can miss spoken spans there.+ """+ if not span:+ return False+ if no_space:+ return span in pred_text+ return f" {span} " in f" {pred_text} "+++def _warn_if_stripped_spans_were_spoken(+ stripped_spans: list[list[str]],+ pred_texts: list[str],+ manifest_path: Optional[str] = None,+ no_space: bool = False,+) -> None:+ """Warn when square-bracket spans deleted from the reference show up verbatim in the ASR hypothesis.++ That is the signature of a dataset that uses ``[word]`` to mark emphasized *spoken* words rather than non-verbal+ tags: the words are spoken and transcribed, then counted as insertions against the truncated reference. This is a+ heuristic and only logs.++ Args:+ stripped_spans: Per record, the normalized bracket spans removed from the reference+ (see ``build_metric_reference_texts``).+ pred_texts: Normalized ASR hypotheses, aligned with ``stripped_spans``.+ manifest_path: Included in the warning when given.+ no_space: True for zh and ja, whose normalized texts contain no spaces; the span then only needs to appear+ somewhere inside the hypothesis instead of as complete words.+ """+ spoken_spans = [+ span+ for spans, pred_text in zip(stripped_spans, pred_texts)+ for span in spans+ if _span_was_spoken(span, pred_text, no_space=no_space)+ ]+ if not spoken_spans:+ return+ examples = ", ".join(f"[{span}]" for span in spoken_spans[:5])+ source = f" (manifest: {manifest_path})" if manifest_path else ""+ logging.warning(+ f"Heuristic check{source}: strip_text_annotations_for_metrics removed {len(spoken_spans)} square-bracket "+ f"span(s) that appear in the ASR transcript of the generated audio (e.g. {examples}). They are scored as "+ "insertions and inflate CER/WER. Inspect the examples: if this dataset marks emphasized spoken words as "+ '[word], set "strip_text_annotations_for_metrics": false in its evalset entry; if the brackets are '+ "non-verbal tags, the model is reading them aloud."+ )nemo/collections/tts/modules/magpietts_inference/evaluation_config.py49 + / 11 −
@@ -27,15 +27,16 @@ # Supported ASR backends: the values of --asr_model_type and of the "type" field of an evalset "asr_model" entry. ASR_MODEL_TYPES = ("nemo", "nemo_with_prompt", "whisper") # Keys of an evalset config entry that the inference/evaluation scripts read. Any other key is rejected, so that a-# misspelled override (e.g. "langauge") cannot silently leave the CLI-level value in force; training-only DatasetMeta-# fields such as "feature_dir" or "sample_weight" do not belong in an evaluation config.+# misspelled override (e.g. "langauge") cannot silently leave the default or CLI-level value in force;+# training-only DatasetMeta fields such as "feature_dir" or "sample_weight" do not belong in an evaluation config. EVALSET_ENTRY_KEYS = frozenset( { "manifest_path", "audio_dir", "tokenizer_names", "language", "asr_model",+ "strip_text_annotations_for_metrics", } ) @@ -59,8 +60,15 @@ class EvaluationConfig: with_prosody_metrics: Whether to compute ESIM/EMS plus pitch, intensity, and speech-rate distance metrics. prosody_model_size: Emotion encoder size ("small" or "large").- strip_text_annotations_for_metrics: Whether to strip annotation/control markers from reference and ASR- hypothesis text before text metrics.+ strip_text_annotations_for_metrics: Whether to strip annotation/control markers (``<tag>``, ``{tag}``,+ ``[tag]``, ``--``, ``...``, ``*``) from reference and ASR hypothesis text before text metrics.+ Square-bracket spans are removed WITH their content, so enable this only for datasets whose brackets+ are non-verbal tags such as ``[breath]`` (``"strip_text_annotations_for_metrics": true`` in the evalset+ entry, see ``resolve_evaluation_config_for_dataset``); datasets that mark emphasized spoken words as+ ``[word]`` must leave it off. When evalset entries are resolved with+ ``resolve_evaluation_config_for_dataset``, leave this ``False`` and set the key per entry; ``True``+ there asserts that every dataset strips, and an entry that omits the key or sets it to ``false``+ raises. device: Device to use for running models used during evaluation. asr_batch_size: Batch size for ASR transcription. eou_batch_size: Batch size for EoU classification.@@ -89,10 +97,11 @@ def validate_evalset_entry(info: dict, dataset_name: Optional[str] = None) -> No - ``language``: non-empty string without surrounding whitespace (remove the key to use the CLI-level language). - ``asr_model``: ``{"name": <model name or .nemo path>, "type": <one of ASR_MODEL_TYPES>}``.+ - ``strip_text_annotations_for_metrics``: JSON boolean. Absent keys are fine. A JSON ``null`` is rejected like any other wrong type, so that a broken override cannot- silently fall back to the CLI-level value. Used by ``load_evalset_config`` (``evaluate_generated_audio.py``) and- by ``resolve_evaluation_config_for_dataset``.+ silently fall back to the default or CLI-level value. Used by ``load_evalset_config``+ (``evaluate_generated_audio.py``) and by ``resolve_evaluation_config_for_dataset``. Args: info: One entry of the evalset config.@@ -102,6 +111,12 @@ def validate_evalset_entry(info: dict, dataset_name: Optional[str] = None) -> No ValueError: If a recognized key has a malformed value. """ prefix = f"Dataset {dataset_name}: " if dataset_name is not None else "Evalset entry: "+ if "strip_text_annotations_for_metrics" in info:+ value = info["strip_text_annotations_for_metrics"]+ if not isinstance(value, bool):+ raise ValueError(+ f"{prefix}'strip_text_annotations_for_metrics' must be a JSON boolean (true/false), got {value!r}."+ ) if "language" in info: value = info["language"] if not isinstance(value, str) or not value or value != value.strip():@@ -131,8 +146,17 @@ def resolve_evaluation_config_for_dataset(eval_config: EvaluationConfig, dataset - ``asr_model``: ``{"name": ..., "type": ...}`` overriding ``asr_model_name`` and ``asr_model_type``. - ``language``: overrides ``language``.-- Keys that are absent keep the value from ``eval_config``. ``eval_config`` itself is not mutated.+ - ``strip_text_annotations_for_metrics``: JSON boolean; the per-dataset switch for stripping annotation markers+ from the CER/WER reference. ``examples/tts/magpietts_inference.py`` leaves+ ``eval_config.strip_text_annotations_for_metrics`` at ``False``, so its entries decide: absent means off.+ A library caller may set it ``True`` only as an assertion that every dataset strips; an entry that omits the+ key or sets it to ``false`` then raises, so a run-level value never silently decides for a dataset. Set the+ key to ``true`` for datasets whose brackets are non-verbal tags such as ``[breath]``; leave it off for+ datasets whose square brackets mark emphasized *spoken* words (e.g. ``"[You] want to ski"``), so those words+ stay in the reference.++ Absent keys keep the value from ``eval_config`` (for the strip key only while that value is ``False``, see+ above). ``eval_config`` itself is not mutated. Args: eval_config: CLI-level evaluation configuration.@@ -143,15 +167,29 @@ def resolve_evaluation_config_for_dataset(eval_config: EvaluationConfig, dataset Raises: ValueError: If a recognized key is malformed (see ``validate_evalset_entry``):- ``language`` is not a non-empty string or ``asr_model`` lacks a valid- ``name``/``type``. JSON ``null`` counts as malformed, so a broken override never silently falls back to- the CLI-level value.+ ``language`` is not a non-empty string, ``asr_model`` lacks a valid+ ``name``/``type``, or ``strip_text_annotations_for_metrics`` is not a boolean. JSON ``null`` counts as+ malformed, so a broken value never silently falls back to the default or CLI-level value. Also if+ ``eval_config`` enables stripping and the entry omits ``strip_text_annotations_for_metrics`` or sets+ it to ``false``. """ validate_evalset_entry(dataset_meta)+ entry_value = dataset_meta.get("strip_text_annotations_for_metrics") # None, True or False after validation+ if eval_config.strip_text_annotations_for_metrics and entry_value is not True:+ state = "sets it to false" if entry_value is False else "does not set it"+ where = f" for {dataset_meta['manifest_path']}" if dataset_meta.get("manifest_path") else ""+ raise ValueError(+ "strip_text_annotations_for_metrics conflict: eval_config enables stripping for every dataset, but the "+ f"evalset entry{where} {state}. Bracket meaning is a property of each dataset: set "+ '"strip_text_annotations_for_metrics": true in the entry if its brackets are non-verbal tags, or leave '+ "eval_config.strip_text_annotations_for_metrics False and set the key per entry."+ ) overrides = {} if "asr_model" in dataset_meta: overrides["asr_model_name"] = dataset_meta["asr_model"]["name"] overrides["asr_model_type"] = dataset_meta["asr_model"]["type"] if "language" in dataset_meta: overrides["language"] = dataset_meta["language"]+ if "strip_text_annotations_for_metrics" in dataset_meta:+ overrides["strip_text_annotations_for_metrics"] = dataset_meta["strip_text_annotations_for_metrics"] return replace(eval_config, **overrides)nemo/collections/tts/modules/magpietts_inference/utils.py23 + / 2 −
@@ -655,6 +655,7 @@ def parse_layer_list(layer_str: Optional[str]) -> Optional[List[int]]: "eou_error_rate", "katakana_cer_filewise_avg", "katakana_cer_cumulative",+ "num_empty_reference_texts", ) EXPERIMENT_METRICS_CSV_HEADER = ",".join(("checkpoint_name", "dataset") + EXPERIMENT_METRICS_CSV_COLUMNS) @@ -843,6 +844,7 @@ def turn_sort_key(r): "tts_text_input": [r.get("tts_text_input", "") for r in turns], "dataloader_normalized_text": [r.get("dataloader_normalized_text") for r in turns], "reference_text": [r.get("gt_text", "") for r in turns],+ "strip_text_annotations_for_metrics": turns[0].get("strip_text_annotations_for_metrics"), "asr_hyp": [r.get("pred_text", "") for r in turns], "pred_audio_paths": [r.get("pred_audio_filepath", "") for r in turns], "target_audio_path": group["target_audio_path"],@@ -905,6 +907,7 @@ def _write_grouped_multiturn_filewise_metrics_csv(csv_path: str, grouped_rows: l "tts_text_input", "dataloader_normalized_text", "reference_text",+ "strip_text_annotations_for_metrics", "asr_hyp", "predicted_phoneme_text_turns", "predicted_phoneme_tokens_turns",@@ -1155,6 +1158,21 @@ def _add_inference_param_fields( group.add_argument(f"--{f.name}", **extra_args) +class _RemovedStripFlagAction(argparse.Action):+ """Reject --strip_text_annotations_for_metrics with a parser error that points at the per-dataset evalset key."""++ def __init__(self, option_strings, dest, **kwargs):+ kwargs.pop("nargs", None)+ super().__init__(option_strings, dest, nargs=0, **kwargs)++ def __call__(self, parser, namespace, values, option_string=None):+ parser.error(+ f"{option_string} was removed: whether [..], <..> and {{..}} markers are stripped from the CER/WER "+ 'reference is a property of each dataset. Set "strip_text_annotations_for_metrics": true or false in the '+ "dataset's evalset config entry instead (default: not stripped)."+ )++ def _add_common_args(parser: argparse.ArgumentParser) -> None: """Add arguments shared by all model types.""" @@ -1295,8 +1313,11 @@ def _add_common_args(parser: argparse.ArgumentParser) -> None: eval_group.add_argument('--prosody_model_size', type=str, default="small", choices=["small", "large"]) eval_group.add_argument( '--strip_text_annotations_for_metrics',- action='store_true',- help='Strip bracket/tag/control annotations from reference and ASR hypothesis text while computing text metrics.',+ action=_RemovedStripFlagAction,+ help=(+ 'Removed. Stripping of annotation markers from the CER/WER reference is set per dataset with '+ '"strip_text_annotations_for_metrics": true|false in the evalset config; passing this flag is an error.'+ ), ) eval_group.add_argument( '--violin_plot_metrics',tests/collections/tts/modules/magpietts_inference/test_magpietts_inference.py245 + / 21 −
@@ -22,13 +22,20 @@ import os import pytest+import torch from examples.tts.magpietts_inference import main as magpietts_inference_main from examples.tts.magpietts_inference import run_inference_and_evaluation +from nemo.collections.asr.metrics.wer import word_error_rate_detail from nemo.collections.tts.modules.magpietts_inference.evaluate_generated_audio import ( FILEWISE_METRICS_TO_SAVE, _get_record_texts,+ _warn_if_stripped_spans_were_spoken,+ build_metric_reference_texts,+ compute_global_metrics,+ evaluate_dir, load_evalset_config,+ strip_text_annotations_from_text, ) from nemo.collections.tts.modules.magpietts_inference.evaluation import ( EvaluationConfig,@@ -42,12 +49,24 @@ append_metrics_to_csv, write_csv_header_if_needed, )+from nemo.collections.tts.parts.utils.tts_dataset_utils import DefaultTextProcessor from nemo.utils import logging as nemo_logging EVALUATE_MODULE = "nemo.collections.tts.modules.magpietts_inference.evaluate_generated_audio" EXAMPLE_MODULE = "examples.tts.magpietts_inference" +# Evaluation manifests of emphasis benchmarks mark emphasized *spoken* words with square brackets.+EMPHASIS_TEXT = (+ "[You] want to go to the beach, again? I [want] to ski this winter. How about a [compromise?] "+ "What about traveling to the Alps in Europe next [april]? We can find a ski resort on a [lake]."+)+EMPHASIS_REFERENCE = (+ "you want to go to the beach again i want to ski this winter how about a compromise "+ "what about traveling to the alps in europe next april we can find a ski resort on a lake"+)++ class TestMagpieTTSInferenceCLI: """Tests for MagpieTTS inference command-line interface options.""" @@ -130,6 +149,7 @@ def test_evaluation_uses_normalized_text_for_metrics(): assert metric_reference_text == "july fifteenth" assert "tts_text_input" in FILEWISE_METRICS_TO_SAVE assert "dataloader_normalized_text" in FILEWISE_METRICS_TO_SAVE+ assert "strip_text_annotations_for_metrics" in FILEWISE_METRICS_TO_SAVE # the effective per-row setting # Legacy phonemized manifests keep the orthography in original_text; a JSON null counts as absent. assert _get_record_texts({"text": "dʒʊˈlaɪ", "original_text": "July"})[2] == "July" assert _get_record_texts({"text": "July 15th", "original_text": None})[2] == "July 15th"@@ -146,12 +166,14 @@ def test_grouped_multiturn_exports_input_and_normalized_text(tmp_path): "dataloader_normalized_text": "july fifteenth", "gt_text": "july fifteenth", "pred_text": "july fifteenth",+ "strip_text_annotations_for_metrics": True, } ] ) assert grouped_rows[0]["tts_text_input"] == ["July 15th"] assert grouped_rows[0]["dataloader_normalized_text"] == ["july fifteenth"]+ assert grouped_rows[0]["strip_text_annotations_for_metrics"] is True csv_path = tmp_path / "metrics.csv" _write_grouped_multiturn_filewise_metrics_csv(str(csv_path), grouped_rows)@@ -160,26 +182,120 @@ def test_grouped_multiturn_exports_input_and_normalized_text(tmp_path): assert json.loads(csv_row["tts_text_input"]) == ["July 15th"] assert json.loads(csv_row["dataloader_normalized_text"]) == ["july fifteenth"]+ assert csv_row["strip_text_annotations_for_metrics"] == "True"+++@pytest.mark.unit+def test_strip_text_annotations_pins_bracket_semantics():+ # Non-verbal tags and control markers are removed; a '[span]' is deleted together with its content.+ assert strip_text_annotations_from_text("Hello [breath] there -- friend... <laugh> {sigh}") == "Hello there friend"+ # '*word*' emphasis keeps the word. What the same deletion does to a '[word]' emphasis reference is pinned by+ # test_build_metric_reference_texts_bracket_emphasis; that is why the evalset key must stay off for such sets.+ assert strip_text_annotations_from_text("I *want* to ski") == "I want to ski"+++@pytest.mark.unit+def test_build_metric_reference_texts_bracket_emphasis():+ processor = DefaultTextProcessor()+ records = [{"text": EMPHASIS_TEXT, "normalized_text": EMPHASIS_TEXT}]+ # A perfect rendition of the text, as ASR transcribes it (no brackets).+ hypothesis = processor.process_text_for_wer(EMPHASIS_TEXT.replace("[", "").replace("]", ""))++ # Default path: brackets are dropped as punctuation and every emphasized word is kept.+ record_texts, kept, spans = build_metric_reference_texts(+ records, processor, strip_text_annotations_for_metrics=False+ )+ assert record_texts == [(EMPHASIS_TEXT, EMPHASIS_TEXT)]+ assert kept == [EMPHASIS_REFERENCE] and spans == [[]]++ # Stripping deletes each whole [span]: the words are gone from the reference and reported as removed spans, and+ # the perfect hypothesis is charged an insertion for every emphasized word.+ _, stripped, spans = build_metric_reference_texts(records, processor, strip_text_annotations_for_metrics=True)+ assert stripped == [+ "want to go to the beach again i to ski this winter how about a "+ "what about traveling to the alps in europe next we can find a ski resort on a"+ ]+ assert spans == [["you", "want", "compromise", "april", "lake"]]+ wer, _, insertions, deletions, substitutions = word_error_rate_detail([hypothesis], stripped, use_cer=False)+ assert wer > 0.0 and insertions > 0.0 and deletions == 0.0 and substitutions == 0.0+++@pytest.mark.unit+def test_warns_when_stripped_bracket_spans_were_spoken(monkeypatch):+ warnings_seen = []+ monkeypatch.setattr(nemo_logging, "warning", lambda msg, *args, **kwargs: warnings_seen.append(msg))++ # Only spans that occur as complete words in their own record's hypothesis are counted: two of four here.+ _warn_if_stripped_spans_were_spoken(+ [["you", "compromise"], ["breath"], ["laugh"]], ["you want a compromise", "well okay", "we laughed"]+ )+ assert len(warnings_seen) == 1+ assert "2 square-bracket span(s)" in warnings_seen[0]+ assert "[you]" in warnings_seen[0]+ assert "manifest:" not in warnings_seen[0]++ # Languages written with spaces: the span must appear as complete words, also when the hypothesis is one word.+ warnings_seen.clear()+ _warn_if_stripped_spans_were_spoken([["you"]], ["your car"])+ assert warnings_seen == []+ _warn_if_stripped_spans_were_spoken([["breath"]], ["breath"])+ _warn_if_stripped_spans_were_spoken([["want to"]], ["i want to ski"])+ assert len(warnings_seen) == 2++ # zh and ja have no spaces left after normalization: the span only needs to appear somewhere inside the+ # hypothesis, but only when the caller says so.+ warnings_seen.clear()+ _warn_if_stripped_spans_were_spoken([["你好"]], ["我说你好吗"])+ assert warnings_seen == []+ _warn_if_stripped_spans_were_spoken([["你好"]], ["我说你好吗"], no_space=True)+ assert len(warnings_seen) == 1 and "[你好]" in warnings_seen[0] @pytest.mark.unit def test_resolve_evaluation_config_for_dataset_overrides():- eval_config = EvaluationConfig(language="en", eou_batch_size=7)+ eval_config = EvaluationConfig(language="en", eou_batch_size=7) # stripping off, as the example script builds it meta = { "manifest_path": "m.json", "audio_dir": "a", "language": "de", "asr_model": {"name": "some/asr", "type": "whisper"},+ "strip_text_annotations_for_metrics": True, } resolved = resolve_evaluation_config_for_dataset(eval_config, meta) + assert resolved.strip_text_annotations_for_metrics is True # the entry alone enables stripping assert resolved.language == "de" assert (resolved.asr_model_name, resolved.asr_model_type) == ("some/asr", "whisper") assert resolved.eou_batch_size == 7 # fields without an override are inherited assert eval_config.language == "en" # the input is not mutated- # Absent keys keep the CLI-level values.+ # Absent keys keep the CLI-level values; without the key the dataset is not stripped. assert resolve_evaluation_config_for_dataset(eval_config, {"manifest_path": "m.json"}) == eval_config+ # An explicit false, the entry to write for an emphasis set, keeps stripping off as well.+ off = {"manifest_path": "m.json", "strip_text_annotations_for_metrics": False}+ assert resolve_evaluation_config_for_dataset(eval_config, off).strip_text_annotations_for_metrics is False+++@pytest.mark.unit+def test_resolve_evaluation_config_for_dataset_rejects_run_level_strip_conflicts():+ # A base config that enables stripping asserts that every dataset strips: entries must say so explicitly, so+ # that a run-level value never silently decides for a dataset (bracket meaning is a property of each dataset).+ strip_all = EvaluationConfig(strip_text_annotations_for_metrics=True)+ agreeing = {"manifest_path": "m.json", "strip_text_annotations_for_metrics": True}+ assert resolve_evaluation_config_for_dataset(strip_all, agreeing).strip_text_annotations_for_metrics is True+ for entry, state in (+ ({"manifest_path": "m.json"}, "does not set it"),+ ({"manifest_path": "m.json", "strip_text_annotations_for_metrics": False}, "sets it to false"),+ ):+ with pytest.raises(ValueError, match=rf"conflict: .* entry for m\.json {state}\."):+ resolve_evaluation_config_for_dataset(strip_all, entry)+ # A hand-built entry without manifest_path is reported without a name.+ with pytest.raises(ValueError, match=r"conflict: .* but the evalset entry does not set it\."):+ resolve_evaluation_config_for_dataset(strip_all, {})+ # Validation runs before the conflict check, so a malformed value is reported as malformed, not as a conflict.+ malformed = {"manifest_path": "m.json", "strip_text_annotations_for_metrics": "false"}+ with pytest.raises(ValueError, match="JSON boolean"):+ resolve_evaluation_config_for_dataset(strip_all, malformed) def _filewise_row(gt_text, pred_text, cer, wer):@@ -203,6 +319,18 @@ def _filewise_row(gt_text, pred_text, cer, wer): } +@pytest.mark.unit+def test_compute_global_metrics_counts_empty_reference_texts():+ rows = [_filewise_row("you want", "you want", 0.0, 0.0), _filewise_row("", "laugh", 0.2, 0.5)]++ metrics = compute_global_metrics(rows)++ assert metrics["num_empty_reference_texts"] == 1+ assert metrics["cer_filewise_avg"] == pytest.approx(0.1) # plain mean over all rows, empty reference included+ assert metrics["wer_filewise_avg"] == pytest.approx(0.25)+ assert compute_global_metrics(rows[:1])["num_empty_reference_texts"] == 0++ def _write_evalset_config(tmp_path, entry_overrides): (tmp_path / "audio").mkdir(exist_ok=True) (tmp_path / "m.json").write_text("{}\n")@@ -262,6 +390,8 @@ def fake_compute_global_metrics(**kwargs): @pytest.mark.parametrize( "entry_overrides, match", [+ ({"strip_text_annotations_for_metrics": "false"}, "JSON boolean"),+ ({"strip_text_annotations_for_metrics": None}, "JSON boolean"), ({"language": None}, "'language' must be a non-empty string"), ({"language": ""}, "'language' must be a non-empty string"), ({"language": " en"}, "'language' must be a non-empty string"),@@ -283,14 +413,17 @@ def test_malformed_evalset_overrides_are_rejected(tmp_path, entry_overrides, mat @pytest.mark.unit def test_load_evalset_config_rejects_unrecognized_keys(tmp_path):- # Only the keys the inference/evaluation scripts read are accepted (as in examples/tts/evalset_config.json).- config_path = _write_evalset_config(tmp_path, {"tokenizer_names": ["english_phoneme"], "language": "en"})- assert "ds" in load_evalset_config(str(config_path), dataset_base_path=tmp_path)-- # A misspelled override would silently leave the CLI-level value in force; it is rejected instead, and so are- # training-only DatasetMeta fields copied from a training config.- for entry, key in (({"langauge": "de"}, "langauge"), ({"feature_dir": None}, "feature_dir")):- config_path = _write_evalset_config(tmp_path, entry)+ # Only the keys the inference/evaluation scripts read are accepted (as in examples/tts/evalset_config.json); the+ # accepted entry carries the strip key, so dropping it from EVALSET_ENTRY_KEYS fails here.+ entry = {"tokenizer_names": ["english_phoneme"], "language": "en", "strip_text_annotations_for_metrics": True}+ config_path = _write_evalset_config(tmp_path, entry)+ loaded = load_evalset_config(str(config_path), dataset_base_path=tmp_path)+ assert loaded["ds"]["strip_text_annotations_for_metrics"] is True # recognized and kept for the resolver++ # A misspelled override would silently leave the default or CLI-level value in force; it is rejected instead,+ # and so are training-only DatasetMeta fields copied from a training config.+ for bad_entry, key in (({"langauge": "de"}, "langauge"), ({"feature_dir": None}, "feature_dir")):+ config_path = _write_evalset_config(tmp_path, bad_entry) with pytest.raises(ValueError, match=rf"Dataset ds: unrecognized evalset keys \['{key}'\]; recognized keys"): load_evalset_config(str(config_path), dataset_base_path=tmp_path) @@ -303,7 +436,8 @@ def test_experiment_metrics_csv_header_and_rows_stay_aligned(tmp_path, monkeypat write_csv_header_if_needed(str(csv_path), EXPERIMENT_METRICS_CSV_HEADER) write_csv_header_if_needed(str(csv_path), EXPERIMENT_METRICS_CSV_HEADER) # matching header: nothing to report- append_metrics_to_csv(str(csv_path), "ckpt", "ds", {"cer_filewise_avg": 0.1, "katakana_cer_cumulative": 0.2})+ metrics = {"cer_filewise_avg": 0.1, "katakana_cer_cumulative": 0.2, "num_empty_reference_texts": 2}+ append_metrics_to_csv(str(csv_path), "ckpt", "ds", metrics) with open(csv_path) as f: rows = list(csv.DictReader(f))@@ -312,6 +446,7 @@ def test_experiment_metrics_csv_header_and_rows_stay_aligned(tmp_path, monkeypat assert (rows[0]["checkpoint_name"], rows[0]["dataset"]) == ("ckpt", "ds") assert rows[0]["cer_filewise_avg"] == "0.1" assert rows[0]["katakana_cer_cumulative"] == "0.2"+ assert rows[0]["num_empty_reference_texts"] == "2" assert rows[0]["wer_filewise_avg"] == "" # absent metrics leave an empty cell assert None not in rows[0] # every value has a header column assert len(rows[0]) == len(EXPERIMENT_METRICS_CSV_HEADER.split(","))@@ -324,6 +459,85 @@ def test_experiment_metrics_csv_header_and_rows_stay_aligned(tmp_path, monkeypat assert csv_path.read_text() == old_header + "\n" +def _run_evaluate_dir_with_fakes(tmp_path, monkeypatch, records, transcripts, language, strip_annotations):+ """Run evaluate_dir on CPU with fake models: ASR returns ``transcripts[basename]``, embeddings are constant."""+ manifest = tmp_path / "manifest.json"+ manifest.write_text("".join(json.dumps(record, ensure_ascii=False) + "\n" for record in records))+ warnings_seen = []+ monkeypatch.setattr(nemo_logging, "warning", lambda msg, *args, **kwargs: warnings_seen.append(msg))++ class _FakeASR:+ def transcribe(self, audio_paths, language, batch_size):+ return [transcripts[os.path.basename(path)] for path in audio_paths]++ fake_models = {+ "asr_model": _FakeASR(),+ "feature_extractor": None,+ "sv_model": None,+ "sv_model_alternate": None,+ "emotion_model": None,+ }+ monkeypatch.setattr(f"{EVALUATE_MODULE}.load_evaluation_models", lambda **kwargs: fake_models)+ monkeypatch.setattr(+ f"{EVALUATE_MODULE}.find_generated_audio_files",+ lambda audio_dir: [os.path.join(audio_dir, f"pred_{i}.wav") for i in range(len(records))],+ )+ monkeypatch.setattr(f"{EVALUATE_MODULE}.find_generated_codec_files", lambda audio_dir: [])+ monkeypatch.setattr(f"{EVALUATE_MODULE}.extract_embedding", lambda **kwargs: torch.ones(4))+ monkeypatch.setattr(f"{EVALUATE_MODULE}.get_wav_file_duration", lambda audio_path: 1.0)++ rows = evaluate_dir(+ manifest_path=str(manifest),+ audio_dir=str(tmp_path),+ generated_audio_dir=str(tmp_path),+ language=language,+ with_utmosv2=False,+ strip_text_annotations_for_metrics=strip_annotations,+ device="cpu",+ )+ return rows, warnings_seen+++@pytest.mark.unit+@pytest.mark.parametrize(+ "language, text, transcript, strip_annotations, expected_gt_text, spoken_span",+ [+ # Bracket-marked emphasis: the word is deleted from the reference, scored as an insertion, and reported.+ ("de", "[You] want to ski", "you want to ski", True, "want to ski", "[you]"),+ ("zh", "[你好]世界", "你好世界", True, "世界", "[你好]"), # no spaces: the span only needs to appear inside+ # Non-verbal tag: "breath" is not a complete word of the one-word hypothesis "breathing", so no warning.+ ("de", "[breath] Breathing.", "breathing", True, "breathing", None),+ # Without stripping the brackets are dropped as punctuation and the word is kept; nothing to warn about.+ ("de", "[breath] Breathing.", "breathing", False, "breath breathing", None),+ ],+)+def test_evaluate_dir_records_strip_setting_and_warns_when_spans_were_spoken(+ tmp_path, monkeypatch, language, text, transcript, strip_annotations, expected_gt_text, spoken_span+):+ records = [{"audio_filepath": "gt_0.wav", "text": text}]+ transcripts = {"gt_0.wav": transcript, "pred_0.wav": transcript}++ rows, warnings_seen = _run_evaluate_dir_with_fakes(+ tmp_path, monkeypatch, records, transcripts, language, strip_annotations+ )++ assert rows[0]["strip_text_annotations_for_metrics"] is strip_annotations+ assert rows[0]["gt_text"] == expected_gt_text+ if spoken_span is None:+ assert warnings_seen == []+ else:+ assert rows[0]["cer"] > 0.0 # the deleted word is scored as an insertion+ assert len(warnings_seen) == 1 and spoken_span in warnings_seen[0]+ assert str(tmp_path / "manifest.json") in warnings_seen[0]+++@pytest.mark.unit+def test_example_script_rejects_removed_strip_flag(capsys):+ with pytest.raises(SystemExit):+ magpietts_inference_main(["--strip_text_annotations_for_metrics"])+ assert '"strip_text_annotations_for_metrics": true or false' in capsys.readouterr().err++ class _FakeRunner: def create_dataset(self, dataset_meta): return [0]@@ -342,14 +556,14 @@ def build_identifier(self): @pytest.mark.unit def test_run_inference_and_evaluation_applies_evalset_override(tmp_path, monkeypatch):- # The example script used to rebuild EvaluationConfig by hand from the evalset entry; it now goes through- # resolve_evaluation_config_for_dataset, so overrides and inherited CLI-level fields are handled in one place.+ # The example script resolves each dataset's EvaluationConfig through resolve_evaluation_config_for_dataset, so+ # evalset overrides and inherited CLI-level fields are handled in one place. manifest = tmp_path / "m.json" manifest.write_text(json.dumps({"audio_filepath": "a.wav", "text": "Hello there."}) + "\n")- captured = {}+ configs = [] def fake_evaluate_generated_audio_dir(manifest_path, audio_dir, generated_audio_dir, config):- captured["config"] = config+ configs.append(config) return {"cer_cumulative": 0.0, "ssim_pred_context_avg": 1.0}, [{"cer": 0.0}] monkeypatch.setattr(f"{EXAMPLE_MODULE}.evaluate_generated_audio_dir", fake_evaluate_generated_audio_dir)@@ -358,12 +572,19 @@ def fake_evaluate_generated_audio_dir(manifest_path, audio_dir, generated_audio_ eval_config = EvaluationConfig(language="en", with_fcd=False, with_utmosv2=False) dataset_meta_info = {- "ds": {+ # Brackets mark emphasized spoken words: no strip key, so the reference keeps every word.+ "emphasis": { "manifest_path": str(manifest), "audio_dir": str(tmp_path), "language": "de", "asr_model": {"name": "some/asr", "type": "whisper"},- }+ },+ # Brackets are non-verbal tags: stripping is enabled by this entry alone.+ "tags": {+ "manifest_path": str(manifest),+ "audio_dir": str(tmp_path),+ "strip_text_annotations_for_metrics": True,+ }, } cer, ssim = run_inference_and_evaluation(@@ -372,13 +593,16 @@ def fake_evaluate_generated_audio_dir(manifest_path, audio_dir, generated_audio_ inference_config=_FakeInferenceConfig(), eval_config=eval_config, dataset_meta_info=dataset_meta_info,- datasets=["ds"],+ datasets=["emphasis", "tags"], out_dir=str(tmp_path / "out"), flops_per_component={}, moe_info="", ) - assert captured["config"].language == "de"- assert (captured["config"].asr_model_name, captured["config"].asr_model_type) == ("some/asr", "whisper")- assert captured["config"].with_utmosv2 is False # CLI-level settings without an override are inherited+ emphasis, tags = configs+ assert emphasis.language == "de"+ assert (emphasis.asr_model_name, emphasis.asr_model_type) == ("some/asr", "whisper")+ assert emphasis.strip_text_annotations_for_metrics is False # entry without the key inherits eval_config's False+ assert tags.strip_text_annotations_for_metrics is True+ assert tags.language == "en" and tags.with_utmosv2 is False # CLI-level settings without an override are inherited assert (cer, ssim) == (0.0, 1.0)