[TTS] Validate MagpieTTS evalset configs and centralize per-dataset evaluation overrides

@@ -133,6 +133,7 @@ Several parameters control the generation behavior. The temperature setting affe     python examples/tts/magpietts_inference.py \         --nemo_files /path/to/magpietts_model.nemo \         --codecmodel_path /path/to/audio_codec.nemo \+        --datasets_json_path /path/to/evalset_config.json \         --datasets your_evaluation_set \         --out_dir /path/to/output \         --temperature 0.6 \@@ -146,6 +147,32 @@ When using models trained with frame stacking, you can enable the Local Transfor  To enable Long-form speech generation (beta) set ``--longform_mode`` to ``auto or always``; this will either automatically detect the long form text or always use the long form inference pipeline. For comprehensive documentation on longform inference, see the :doc:`Longform Inference Guide <magpietts-longform>`. +Evaluation set configuration+----------------------------++``--datasets_json_path`` points to a JSON file that maps each dataset name to its manifest, its audio directory, and optional per-dataset evaluation overrides. Relative paths are resolved against ``--datasets_base_path``. ``--datasets`` selects a subset of the entries.++.. code-block:: json++    {+        "libritts_test_clean": {+            "manifest_path": "LibriTTS/test_clean_manifest.json",+            "audio_dir": "LibriTTS/test-clean",+            "language": "en",+            "asr_model": {"name": "nvidia/parakeet-tdt-1.1b", "type": "nemo"}+        }+    }++The optional keys are:++- ``language``: overrides ``--language`` for this dataset.+- ``asr_model``: ``{"name": ..., "type": ...}`` overrides ``--asr_model_name`` and ``--asr_model_type``.+- ``tokenizer_names``: list of text tokenizer names to use for this dataset (``magpietts_inference.py`` only).++Malformed values of ``language`` and ``asr_model`` (for example a JSON ``null`` or an ``asr_model`` without a supported ``type``) are rejected when the config is loaded; remove a key to inherit the command-line value. Keys that are not recognized, including training-only fields such as ``feature_dir``, are rejected, so that a misspelled key cannot silently fall back to the command-line value.++The CER/WER reference (``gt_text``) is taken from ``normalized_text`` when present, otherwise from ``original_text``, otherwise from ``text``. ``normalized_text`` and ``text`` are the same strings the dataloaders feed the model; ``original_text`` is the orthography kept in legacy phonemized manifests and is used only as the metric reference there, so ``gt_text`` and ``tts_text_input`` differ for such manifests.+ Resources ######### 

@@ -1,23 +1,19 @@ {     "an4_val_ci": {         "manifest_path": "/home/TestData/an4_dataset/an4_val_context_v1.json",-        "audio_dir": "/",-        "feature_dir": null+        "audio_dir": "/"     },     "an4_val_tiny_ci": {         "manifest_path": "/home/TestData/an4_dataset/an4_val_context_v1_tiny.json",-        "audio_dir": "/",-        "feature_dir": null+        "audio_dir": "/"     },     "an4_val_ci_longform_tiny": {         "manifest_path": "/home/TestData/an4_dataset/an4_val_context_v1_longform_tiny_normalized.json",-        "audio_dir": "/",-        "feature_dir": null+        "audio_dir": "/"     },     "an4_val_ci_nemotronTokenizer": {         "manifest_path": "/home/TestData/an4_dataset/an4_val_context_v1.json",         "audio_dir": "/",-        "feature_dir": null,         "tokenizer_names": ["nemotron_nano_30b"]     } }

@@ -71,9 +71,11 @@     EvaluationConfig,     compute_mean_with_confidence_interval,     evaluate_generated_audio_dir,+    resolve_evaluation_config_for_dataset, ) from nemo.collections.tts.modules.magpietts_inference.inference import BaseInferenceConfig, BaseInferenceRunner from nemo.collections.tts.modules.magpietts_inference.utils import (+    EXPERIMENT_METRICS_CSV_HEADER,     ModelLoadConfig,     _add_common_args,     _add_easy_magpie_args,@@ -165,38 +167,24 @@ def run_inference_and_evaluation(     cer_per_dataset = []     all_datasets_filewise_metrics = {} -    # CSV headers-    csv_header = (-        "checkpoint_name,dataset,cer_filewise_avg,wer_filewise_avg,cer_cumulative,"-        "wer_cumulative,cer_pred_gt_audio_filewise_avg,cer_pred_gt_audio_cumulative,"-        "wer_pred_gt_audio_filewise_avg,wer_pred_gt_audio_cumulative,"-        "ssim_pred_gt_avg,ssim_pred_context_avg,ssim_gt_context_avg,"-        "ssim_pred_gt_avg_alternate,ssim_pred_context_avg_alternate,"-        "ssim_gt_context_avg_alternate,esim_pred_gt_avg,ems_pred_gt_avg,"-        "pitch_distance_avg,intensity_distance_avg,speech_rate_distance_avg,"-        "cer_gt_audio_cumulative,wer_gt_audio_cumulative,"-        "utmosv2_avg,total_gen_audio_seconds,frechet_codec_distance,"-        "eou_cutoff_rate,eou_silence_rate,eou_noise_rate,eou_error_rate,"-        "katakana_cer_filewise_avg,katakana_cer_cumulative"-    )+    # CSV header; the columns are shared with append_metrics_to_csv so header and rows cannot drift apart.+    csv_header = EXPERIMENT_METRICS_CSV_HEADER      for dataset in datasets:         logging.info(f"Processing dataset: {dataset}")          meta = dataset_meta_info[dataset]         manifest_records = read_manifest(meta['manifest_path']) -        if 'asr_model' in meta:-            asr_model_name = meta['asr_model']['name']-            asr_model_type = meta['asr_model']['type']-        else:-            asr_model_name = eval_config.asr_model_name-            asr_model_type = eval_config.asr_model_type--        if 'language' in meta:-            language = meta.get('language')-        else:-            language = eval_config.language+        # Per-dataset overrides (asr_model, language) come from the evalset entry; everything else is inherited from+        # the CLI-level eval_config.+        eval_config_for_dataset = resolve_evaluation_config_for_dataset(eval_config, meta)+        language = eval_config_for_dataset.language+        if not skip_evaluation:+            logging.info(+                f"Dataset {dataset}: language={language}, "+                f"asr_model={eval_config_for_dataset.asr_model_name} ({eval_config_for_dataset.asr_model_type})"+            )          tokenizer_names = meta.get('tokenizer_names', None) @@ -299,23 +287,6 @@ def run_inference_and_evaluation(                     default_audio_dir=meta['audio_dir'],                 ) -            eval_config_for_dataset = EvaluationConfig(-                sv_model=eval_config.sv_model,-                asr_model_name=asr_model_name,-                asr_model_type=asr_model_type,-                eou_model_name=eval_config.eou_model_name,-                language=language,-                with_utmosv2=eval_config.with_utmosv2,-                with_fcd=eval_config.with_fcd,-                codec_model_path=eval_config.codec_model_path,-                with_prosody_metrics=eval_config.with_prosody_metrics,-                prosody_model_size=eval_config.prosody_model_size,-                strip_text_annotations_for_metrics=eval_config.strip_text_annotations_for_metrics,-                device=eval_config.device,-                asr_batch_size=eval_config.asr_batch_size,-                eou_batch_size=eval_config.eou_batch_size,-            )-             metrics, filewise_metrics = evaluate_generated_audio_dir(                 manifest_path=eval_manifest_path,                 audio_dir=eval_audio_dir,

@@ -51,6 +51,7 @@     EvaluationConfig,     compute_mean_with_confidence_interval,     evaluate_generated_audio_dir,+    resolve_evaluation_config_for_dataset, ) from nemo.collections.tts.modules.magpietts_inference.inference import (     BaseInferenceConfig,@@ -91,6 +92,7 @@     # Evaluation     "EvaluationConfig",     "evaluate_generated_audio_dir",+    "resolve_evaluation_config_for_dataset",     "compute_mean_with_confidence_interval",     "DEFAULT_VIOLIN_METRICS",     # Visualization

@@ -15,7 +15,6 @@ """ Used in inference and evaluation scripts to obtain metrics such as ASR_WER and UTMOSV2 scores. """-import argparse import json import os import pprint@@ -38,6 +37,12 @@ from nemo.collections.tts.metrics.eou_classifier import EoUClassification, EoUClassifier, EoUType from nemo.collections.tts.metrics.frechet_codec_distance import FrechetCodecDistance from nemo.collections.tts.metrics.prosody import compute_prosody_distances+from nemo.collections.tts.modules.magpietts_inference.evaluation_config import (+    ASR_MODEL_TYPES,+    EVALSET_ENTRY_KEYS,+    EvaluationConfig,+    validate_evalset_entry,+) from nemo.collections.tts.parts.utils.tts_dataset_utils import (     JapaneseTextProcessor,     NemoTranscriber,@@ -117,7 +122,8 @@ def _get_record_texts(record: dict) -> tuple[str, Optional[str], str]:     if dataloader_normalized_text is not None:         metric_reference_text = dataloader_normalized_text     else:-        metric_reference_text = record.get("original_text", tts_text_input)+        original_text = record.get("original_text")  # JSON null counts as absent+        metric_reference_text = original_text if original_text is not None else tts_text_input      return tts_text_input, dataloader_normalized_text, metric_reference_text @@ -150,16 +156,29 @@ def _get_record_texts(record: dict) -> tuple[str, Optional[str], str]:   def load_evalset_config(config_path: Optional[str] = None, dataset_base_path: Optional[Path] = None) -> dict:-    """Load dataset meta info from JSON config file."""+    """Load dataset meta info from JSON config file.++    Relative ``manifest_path``, ``audio_dir`` and ``asr_model.name`` (``.nemo``) paths are resolved against+    ``dataset_base_path``. Every entry is checked with ``validate_evalset_entry``; keys outside ``EVALSET_ENTRY_KEYS``+    are rejected.+    """     if config_path is None or not os.path.exists(config_path):         raise ValueError("No dataset_json_path provided, please provide a valid path to the evalset config file.")      logging.info(f"Loading evalset config from {config_path}")     with open(config_path, 'r') as f:         dataset_meta_info = json.load(f) -    # Validate that all evaluation datasets exist+    # Validate that all evaluation datasets exist and that the optional per-dataset overrides are well-formed+    # (see validate_evalset_entry and resolve_evaluation_config_for_dataset in evaluation_config.py).     for dataset_name, info in dataset_meta_info.items():+        validate_evalset_entry(info, dataset_name=dataset_name)+        unrecognized_keys = sorted(set(info) - EVALSET_ENTRY_KEYS)+        if unrecognized_keys:+            raise ValueError(+                f"Dataset {dataset_name}: unrecognized evalset keys {unrecognized_keys}; "+                f"recognized keys are {sorted(EVALSET_ENTRY_KEYS)}."+            )         manifest_path = Path(info["manifest_path"])         audio_dir = Path(info["audio_dir"]) @@ -350,7 +369,10 @@ def load_evaluation_models(     elif asr_model_type == "whisper":         models['asr_model'] = WhisperTranscriber(model_name=asr_model_name, device=device)     else:-        raise ValueError(f"Unknown ASR model type {asr_model_name}")+        raise ValueError(+            f"Unknown ASR model type {asr_model_type!r} for ASR model {asr_model_name!r}; "+            f"expected one of {', '.join(ASR_MODEL_TYPES)}."+        )      if sv_model_type == "wavlm":         models['feature_extractor'] = Wav2Vec2FeatureExtractor.from_pretrained('microsoft/wavlm-base-plus-sv')@@ -829,6 +851,36 @@ def evaluate(     return avg_metrics, filtered_filewise  +def evaluate_with_config(manifest_path, audio_dir, generated_audio_dir, config: EvaluationConfig):+    """Run ``evaluate`` with every field of ``config`` forwarded.++    Single place that maps ``EvaluationConfig`` fields to ``evaluate`` keyword arguments; used by+    ``evaluation.evaluate_generated_audio_dir``.++    Returns:+        Tuple of (avg_metrics dict, filewise_metrics list), see ``evaluate``.+    """+    return evaluate(+        manifest_path=manifest_path,+        audio_dir=audio_dir,+        generated_audio_dir=generated_audio_dir,+        language=config.language,+        sv_model_type=config.sv_model,+        asr_model_name=config.asr_model_name,+        asr_model_type=config.asr_model_type,+        with_utmosv2=config.with_utmosv2,+        with_fcd=config.with_fcd,+        codec_model_path=config.codec_model_path,+        with_prosody_metrics=config.with_prosody_metrics,+        prosody_model_size=config.prosody_model_size,+        strip_text_annotations_for_metrics=config.strip_text_annotations_for_metrics,+        device=config.device,+        eou_model_name=config.eou_model_name,+        asr_batch_size=config.asr_batch_size,+        eou_batch_size=config.eou_batch_size,+    )++ def compute_fcd(gt_audio_paths, predicted_codes_paths, codec_model_path, device="cuda"):     """Compute Frechet Codec Distance from ground-truth audio paths and predicted codec codes paths. @@ -972,47 +1024,3 @@ def compute_global_metrics(      pprint.pprint(avg_metrics)     return avg_metrics---def main():-    # audio_dir="/datap/misc/Datasets/riva" \-    parser = argparse.ArgumentParser(description='Evaluate Generated Audio')-    parser.add_argument('--manifest_path', type=str, default=None)-    parser.add_argument('--audio_dir', type=str, default=None)-    parser.add_argument('--generated_audio_dir', type=str, default=None)-    parser.add_argument('--language', type=str, default="en")-    parser.add_argument('--evalset', type=str, default=None)-    parser.add_argument(-        '--with_prosody_metrics',-        action='store_true',-        help='Compute ESIM/EMS and pitch, intensity, and speech-rate distance metrics.',-    )-    parser.add_argument('--prosody_model_size', type=str, default="small", choices=["small", "large"])-    parser.add_argument(-        '--strip_text_annotations_for_metrics',-        action='store_true',-        help='Strip bracket/tag/control annotations from reference and ASR hypothesis text while computing text metrics.',-    )-    args = parser.parse_args()--    if args.evalset is not None:-        dataset_meta_info = load_evalset_config()-        assert args.evalset in dataset_meta_info, f"Dataset '{args.evalset}' not found in evalset_config.json"-        args.manifest_path = dataset_meta_info[args.evalset]['manifest_path']-        args.audio_dir = dataset_meta_info[args.evalset]['audio_dir']--    evaluate(-        args.manifest_path,-        args.audio_dir,-        args.generated_audio_dir,-        args.language,-        sv_model_type="wavlm",-        asr_model_name="nvidia/parakeet-ctc-0.6b",-        with_prosody_metrics=args.with_prosody_metrics,-        strip_text_annotations_for_metrics=args.strip_text_annotations_for_metrics,-        prosody_model_size=args.prosody_model_size,-    )---if __name__ == "__main__":-    main()

@@ -20,51 +20,26 @@ """ from __future__ import annotations -from dataclasses import dataclass from typing import Dict, List, Tuple  import numpy as np import scipy.stats as stats  # Import the existing evaluation module import nemo.collections.tts.modules.magpietts_inference.evaluate_generated_audio as evaluate_generated_audio+from nemo.collections.tts.modules.magpietts_inference.evaluation_config import (+    EvaluationConfig,+    resolve_evaluation_config_for_dataset,+) from nemo.utils import logging --@dataclass-class EvaluationConfig:-    """Configuration for audio quality evaluation.--    Attributes:-        sv_model: Speaker verification model type ("titanet" or "wavlm").-        asr_model_name: ASR model for transcription (e.g., "nvidia/parakeet-tdt-1.1b").-       asr_model_name: ASR model for transcription (e.g., "nvidia/parakeet-tdt-1.1b").-       eou_model_name: Hugging Face model id or local path to the EoU model.-       language: Language code for transcription (e.g., "en").-        with_utmosv2: Whether to compute UTMOSv2 (Mean Opinion Score) metrics.-        with_fcd: Whether to compute Frechet Codec Distance metric.-        codec_model_path: Path to the audio codec model. If None, will skip computing Frechet Codec Distance metric.-        with_prosody_metrics: Whether to compute ESIM/EMS plus pitch,-            intensity, and speech-rate distance metrics.-        prosody_model_size: Emotion encoder size ("small" or "large").-        strip_text_annotations_for_metrics: Whether to strip annotation/control markers from reference and ASR hypothesis text before text metrics.-        device: Device to use for running models used during evaluation.-    """--    sv_model: str = "titanet"-    asr_model_name: str = "nvidia/parakeet-tdt-1.1b"-    asr_model_type: str = "nemo"-    eou_model_name: str = "facebook/wav2vec2-base-960h"-    language: str = "en"-    with_utmosv2: bool = True-    with_fcd: bool = True-    codec_model_path: str = None-    with_prosody_metrics: bool = False-    prosody_model_size: str = "small"-    strip_text_annotations_for_metrics: bool = False-    device: str = "cuda"-    asr_batch_size: int = 32-    eou_batch_size: int = 32+__all__ = [+    "DEFAULT_VIOLIN_METRICS",+    "EvaluationConfig",+    "compute_mean_with_confidence_interval",+    "evaluate_generated_audio_dir",+    "resolve_evaluation_config_for_dataset",+]   def evaluate_generated_audio_dir(@@ -94,28 +69,13 @@ def evaluate_generated_audio_dir(     """     logging.info(f"Evaluating generated audio from {generated_audio_dir} " f"against manifest {manifest_path}") -    avg_metrics, filewise_metrics = evaluate_generated_audio.evaluate(+    return evaluate_generated_audio.evaluate_with_config(         manifest_path=manifest_path,         audio_dir=audio_dir,         generated_audio_dir=generated_audio_dir,-        language=config.language,-        sv_model_type=config.sv_model,-        asr_model_name=config.asr_model_name,-        asr_model_type=config.asr_model_type,-        with_utmosv2=config.with_utmosv2,-        with_fcd=config.with_fcd,-        codec_model_path=config.codec_model_path,-        with_prosody_metrics=config.with_prosody_metrics,-        prosody_model_size=config.prosody_model_size,-        strip_text_annotations_for_metrics=config.strip_text_annotations_for_metrics,-        device=config.device,-        eou_model_name=config.eou_model_name,-        asr_batch_size=config.asr_batch_size,-        eou_batch_size=config.eou_batch_size,+        config=config,     ) -    return avg_metrics, filewise_metrics-  def compute_mean_with_confidence_interval(     metrics_list: List[dict],

@@ -0,0 +1,157 @@+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.+# SPDX-License-Identifier: Apache-2.0+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+#     http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""+Evaluation configuration for MagpieTTS inference: the ``EvaluationConfig`` dataclass and the helpers that validate and+apply the per-dataset overrides of an evalset config entry.++This module has no heavy dependencies so that both ``evaluation`` and ``evaluate_generated_audio`` can import it+without an import cycle.+"""+from __future__ import annotations++from dataclasses import dataclass, replace+from typing import Optional++# Supported ASR backends: the values of --asr_model_type and of the "type" field of an evalset "asr_model" entry.+ASR_MODEL_TYPES = ("nemo", "nemo_with_prompt", "whisper")+# Keys of an evalset config entry that the inference/evaluation scripts read. Any other key is rejected, so that a+# misspelled override (e.g. "langauge") cannot silently leave the CLI-level value in force; training-only DatasetMeta+# fields such as "feature_dir" or "sample_weight" do not belong in an evaluation config.+EVALSET_ENTRY_KEYS = frozenset(+    {+        "manifest_path",+        "audio_dir",+        "tokenizer_names",+        "language",+        "asr_model",+    }+)+++@dataclass+class EvaluationConfig:+    """Configuration for audio quality evaluation.++    Attributes:+        sv_model: Speaker verification model type ("titanet" or "wavlm").+        asr_model_name: ASR model for transcription (e.g., "nvidia/parakeet-tdt-1.1b").+        asr_model_type: ASR backend for ``asr_model_name``; one of ``ASR_MODEL_TYPES`` ("nemo", "nemo_with_prompt"+            or "whisper").+        eou_model_name: Hugging Face model id or local path to the EoU model.+        language: Language code for transcription (e.g., "en").+        with_utmosv2: Whether to compute UTMOSv2 (Mean Opinion Score) metrics.+        with_fcd: Whether to compute Frechet Codec Distance metric.+        codec_model_path: Path to the audio codec model. Required when ``with_fcd`` is True+            (``evaluate_generated_audio_dir`` raises otherwise); set ``with_fcd=False`` to skip the Frechet Codec+            Distance metric.+        with_prosody_metrics: Whether to compute ESIM/EMS plus pitch,+            intensity, and speech-rate distance metrics.+        prosody_model_size: Emotion encoder size ("small" or "large").+        strip_text_annotations_for_metrics: Whether to strip annotation/control markers from reference and ASR+            hypothesis text before text metrics.+        device: Device to use for running models used during evaluation.+        asr_batch_size: Batch size for ASR transcription.+        eou_batch_size: Batch size for EoU classification.+    """++    sv_model: str = "titanet"+    asr_model_name: str = "nvidia/parakeet-tdt-1.1b"+    asr_model_type: str = "nemo"+    eou_model_name: str = "facebook/wav2vec2-base-960h"+    language: str = "en"+    with_utmosv2: bool = True+    with_fcd: bool = True+    codec_model_path: str = None+    with_prosody_metrics: bool = False+    prosody_model_size: str = "small"+    strip_text_annotations_for_metrics: bool = False+    device: str = "cuda"+    asr_batch_size: int = 32+    eou_batch_size: int = 32+++def validate_evalset_entry(info: dict, dataset_name: Optional[str] = None) -> None:+    """Validate the optional per-dataset evaluation overrides of one evalset config entry.++    Recognized optional keys and their required types:++    - ``language``: non-empty string without surrounding whitespace (remove the key to use the CLI-level language).+    - ``asr_model``: ``{"name": <model name or .nemo path>, "type": <one of ASR_MODEL_TYPES>}``.++    Absent keys are fine. A JSON ``null`` is rejected like any other wrong type, so that a broken override cannot+    silently fall back to the CLI-level value. Used by ``load_evalset_config`` (``evaluate_generated_audio.py``) and+    by ``resolve_evaluation_config_for_dataset``.++    Args:+        info: One entry of the evalset config.+        dataset_name: Dataset name used to prefix error messages, if known.++    Raises:+        ValueError: If a recognized key has a malformed value.+    """+    prefix = f"Dataset {dataset_name}: " if dataset_name is not None else "Evalset entry: "+    if "language" in info:+        value = info["language"]+        if not isinstance(value, str) or not value or value != value.strip():+            raise ValueError(+                f"{prefix}'language' must be a non-empty string without surrounding whitespace, such as \"en\" "+                f"(remove the key to use the CLI-level language), got {value!r}."+            )+    if "asr_model" in info:+        value = info["asr_model"]+        if (+            not isinstance(value, dict)+            or not isinstance(value.get("name"), str)+            or not value["name"].strip()+            or value.get("type") not in ASR_MODEL_TYPES+        ):+            types = ", ".join(ASR_MODEL_TYPES)+            raise ValueError(+                f"{prefix}'asr_model' must be an object "+                f"{{\"name\": <model name or .nemo path>, \"type\": <{types}>}}, got {value!r}."+            )+++def resolve_evaluation_config_for_dataset(eval_config: EvaluationConfig, dataset_meta: dict) -> EvaluationConfig:+    """Return a copy of ``eval_config`` with the per-dataset overrides of one evalset config entry applied.++    Recognized optional keys of an evalset config entry:++    - ``asr_model``: ``{"name": ..., "type": ...}`` overriding ``asr_model_name`` and ``asr_model_type``.+    - ``language``: overrides ``language``.++    Keys that are absent keep the value from ``eval_config``. ``eval_config`` itself is not mutated.++    Args:+        eval_config: CLI-level evaluation configuration.+        dataset_meta: One entry of the evalset config (see ``load_evalset_config``).++    Returns:+        A new ``EvaluationConfig`` with the overrides applied.++    Raises:+        ValueError: If a recognized key is malformed (see ``validate_evalset_entry``):+            ``language`` is not a non-empty string or ``asr_model`` lacks a valid+            ``name``/``type``. JSON ``null`` counts as malformed, so a broken override never silently falls back to+            the CLI-level value.+    """+    validate_evalset_entry(dataset_meta)+    overrides = {}+    if "asr_model" in dataset_meta:+        overrides["asr_model_name"] = dataset_meta["asr_model"]["name"]+        overrides["asr_model_type"] = dataset_meta["asr_model"]["type"]+    if "language" in dataset_meta:+        overrides["language"] = dataset_meta["language"]+    return replace(eval_config, **overrides)

@@ -40,6 +40,7 @@ from nemo.collections.tts.models import EasyMagpieTTSInferenceModel, MagpieTTSModel from nemo.collections.tts.models.easy_magpietts_inference import EasyModelInferenceParameters from nemo.collections.tts.models.magpietts import ModelInferenceParameters+from nemo.collections.tts.modules.magpietts_inference.evaluation_config import ASR_MODEL_TYPES from nemo.collections.tts.modules.magpietts_inference.inference import (     BaseInferenceRunner,     EasyMagpieInferenceConfig,@@ -619,49 +620,63 @@ def parse_layer_list(layer_str: Optional[str]) -> Optional[List[int]]:     return [int(layer.strip()) for layer in layer_str.split(",")]  +# Metric columns of all_experiment_metrics.csv / all_experiment_metrics_with_ci.csv after "checkpoint_name,dataset".+# The header and append_metrics_to_csv are both derived from this tuple so they cannot drift apart. Append new+# metrics at the END so that rows appended to a pre-existing CSV keep their earlier columns aligned+# (write_csv_header_if_needed warns when the existing header differs).+EXPERIMENT_METRICS_CSV_COLUMNS = (+    "cer_filewise_avg",+    "wer_filewise_avg",+    "cer_cumulative",+    "wer_cumulative",+    "cer_pred_gt_audio_filewise_avg",+    "cer_pred_gt_audio_cumulative",+    "wer_pred_gt_audio_filewise_avg",+    "wer_pred_gt_audio_cumulative",+    "ssim_pred_gt_avg",+    "ssim_pred_context_avg",+    "ssim_gt_context_avg",+    "ssim_pred_gt_avg_alternate",+    "ssim_pred_context_avg_alternate",+    "ssim_gt_context_avg_alternate",+    "esim_pred_gt_avg",+    "ems_pred_gt_avg",+    "pitch_distance_avg",+    "intensity_distance_avg",+    "speech_rate_distance_avg",+    "cer_gt_audio_cumulative",+    "wer_gt_audio_cumulative",+    "utmosv2_avg",+    "total_gen_audio_seconds",+    "frechet_codec_distance",+    "eou_cutoff_rate",+    "eou_silence_rate",+    "eou_noise_rate",+    "eou_error_rate",+    "katakana_cer_filewise_avg",+    "katakana_cer_cumulative",+)+EXPERIMENT_METRICS_CSV_HEADER = ",".join(("checkpoint_name", "dataset") + EXPERIMENT_METRICS_CSV_COLUMNS)++ def write_csv_header_if_needed(csv_path: str, header: str) -> None:-    """Write CSV header if file doesn't exist."""+    """Write the CSV header if the file does not exist; warn if an existing file was written with another header."""     if not os.path.exists(csv_path):         with open(csv_path, "w") as f:             f.write(header + "\n")+        return+    with open(csv_path) as f:+        existing_header = f.readline().rstrip("\n")+    if existing_header != header:+        logging.warning(+            f"{csv_path} was written with a different column layout; rows appended from now on follow the current "+            f"header. Existing header: {existing_header}. Current header: {header}."+        )   def append_metrics_to_csv(csv_path: str, checkpoint_name: str, dataset: str, metrics: dict) -> None:-    """Append metrics to a CSV file."""-    values = [-        checkpoint_name,-        dataset,-        metrics.get('cer_filewise_avg', ''),-        metrics.get('wer_filewise_avg', ''),-        metrics.get('cer_cumulative', ''),-        metrics.get('wer_cumulative', ''),-        metrics.get('cer_pred_gt_audio_filewise_avg', ''),-        metrics.get('cer_pred_gt_audio_cumulative', ''),-        metrics.get('wer_pred_gt_audio_filewise_avg', ''),-        metrics.get('wer_pred_gt_audio_cumulative', ''),-        metrics.get('ssim_pred_gt_avg', ''),-        metrics.get('ssim_pred_context_avg', ''),-        metrics.get('ssim_gt_context_avg', ''),-        metrics.get('ssim_pred_gt_avg_alternate', ''),-        metrics.get('ssim_pred_context_avg_alternate', ''),-        metrics.get('ssim_gt_context_avg_alternate', ''),-        metrics.get('esim_pred_gt_avg', ''),-        metrics.get('ems_pred_gt_avg', ''),-        metrics.get('pitch_distance_avg', ''),-        metrics.get('intensity_distance_avg', ''),-        metrics.get('speech_rate_distance_avg', ''),-        metrics.get('cer_gt_audio_cumulative', ''),-        metrics.get('wer_gt_audio_cumulative', ''),-        metrics.get('utmosv2_avg', ''),-        metrics.get('total_gen_audio_seconds', ''),-        metrics.get('frechet_codec_distance', ''),-        metrics.get('eou_cutoff_rate', ''),-        metrics.get('eou_silence_rate', ''),-        metrics.get('eou_noise_rate', ''),-        metrics.get('eou_error_rate', ''),-        metrics.get('katakana_cer_filewise_avg', ''),-        metrics.get('katakana_cer_cumulative', ''),-    ]+    """Append one row to a CSV file whose header is ``EXPERIMENT_METRICS_CSV_HEADER``."""+    values = [checkpoint_name, dataset] + [metrics.get(column, '') for column in EXPERIMENT_METRICS_CSV_COLUMNS]     with open(csv_path, "a") as f:         f.write(",".join(str(v) for v in values) + "\n")     logging.info(f"Metrics appended to: {csv_path}")@@ -1254,7 +1269,7 @@ def _add_common_args(parser: argparse.ArgumentParser) -> None:         '--asr_model_type',         type=str,         default='nemo',-        choices=['nemo', 'nemo_with_prompt', 'whisper'],+        choices=list(ASR_MODEL_TYPES),         help="Type of ASR model provided in 'asr_model_name'",     )     eval_group.add_argument(

@@ -22,16 +22,30 @@ import os  import pytest- from examples.tts.magpietts_inference import main as magpietts_inference_main+from examples.tts.magpietts_inference import run_inference_and_evaluation+ from nemo.collections.tts.modules.magpietts_inference.evaluate_generated_audio import (     FILEWISE_METRICS_TO_SAVE,     _get_record_texts,+    load_evalset_config,+)+from nemo.collections.tts.modules.magpietts_inference.evaluation import (+    EvaluationConfig,+    evaluate_generated_audio_dir,+    resolve_evaluation_config_for_dataset, ) from nemo.collections.tts.modules.magpietts_inference.utils import (+    EXPERIMENT_METRICS_CSV_HEADER,     _group_multiturn_filewise_metrics_by_sample,     _write_grouped_multiturn_filewise_metrics_csv,+    append_metrics_to_csv,+    write_csv_header_if_needed, )+from nemo.utils import logging as nemo_logging++EVALUATE_MODULE = "nemo.collections.tts.modules.magpietts_inference.evaluate_generated_audio"+EXAMPLE_MODULE = "examples.tts.magpietts_inference"   class TestMagpieTTSInferenceCLI:@@ -116,6 +130,9 @@ def test_evaluation_uses_normalized_text_for_metrics():     assert metric_reference_text == "july fifteenth"     assert "tts_text_input" in FILEWISE_METRICS_TO_SAVE     assert "dataloader_normalized_text" in FILEWISE_METRICS_TO_SAVE+    # Legacy phonemized manifests keep the orthography in original_text; a JSON null counts as absent.+    assert _get_record_texts({"text": "dʒʊˈlaɪ", "original_text": "July"})[2] == "July"+    assert _get_record_texts({"text": "July 15th", "original_text": None})[2] == "July 15th"   @pytest.mark.unit@@ -143,3 +160,225 @@ def test_grouped_multiturn_exports_input_and_normalized_text(tmp_path):      assert json.loads(csv_row["tts_text_input"]) == ["July 15th"]     assert json.loads(csv_row["dataloader_normalized_text"]) == ["july fifteenth"]+++@pytest.mark.unit+def test_resolve_evaluation_config_for_dataset_overrides():+    eval_config = EvaluationConfig(language="en", eou_batch_size=7)+    meta = {+        "manifest_path": "m.json",+        "audio_dir": "a",+        "language": "de",+        "asr_model": {"name": "some/asr", "type": "whisper"},+    }++    resolved = resolve_evaluation_config_for_dataset(eval_config, meta)++    assert resolved.language == "de"+    assert (resolved.asr_model_name, resolved.asr_model_type) == ("some/asr", "whisper")+    assert resolved.eou_batch_size == 7  # fields without an override are inherited+    assert eval_config.language == "en"  # the input is not mutated+    # Absent keys keep the CLI-level values.+    assert resolve_evaluation_config_for_dataset(eval_config, {"manifest_path": "m.json"}) == eval_config+++def _filewise_row(gt_text, pred_text, cer, wer):+    nan = float("nan")+    return {+        "gt_text": gt_text,+        "pred_text": pred_text,+        "gt_audio_text": None,+        "cer": cer,+        "wer": wer,+        "cer_pred_gt_audio": nan,+        "wer_pred_gt_audio": nan,+        "pred_gt_ssim": 0.5,+        "pred_context_ssim": 0.5,+        "gt_context_ssim": 0.5,+        "pred_gt_ssim_alternate": 0.5,+        "pred_context_ssim_alternate": 0.5,+        "gt_context_ssim_alternate": 0.5,+        "utmosv2": 3.0,+        "total_gen_audio_seconds": 1.0,+    }+++def _write_evalset_config(tmp_path, entry_overrides):+    (tmp_path / "audio").mkdir(exist_ok=True)+    (tmp_path / "m.json").write_text("{}\n")+    entry = {"manifest_path": "m.json", "audio_dir": "audio", **entry_overrides}+    config_path = tmp_path / "evalset.json"+    config_path.write_text(json.dumps({"ds": entry}))+    return config_path+++@pytest.mark.unit+def test_evaluate_generated_audio_dir_forwards_the_whole_config(monkeypatch):+    captured = {}++    def fake_evaluate_dir(**kwargs):+        captured.update(kwargs)+        return [_filewise_row("you want", "you want", 0.0, 0.0)]++    def fake_compute_global_metrics(**kwargs):+        captured["global_kwargs"] = kwargs+        return {}++    # Stub below evaluate() so that its real wiring, including the FCD guard, runs.+    monkeypatch.setattr(f"{EVALUATE_MODULE}.evaluate_dir", fake_evaluate_dir)+    monkeypatch.setattr(f"{EVALUATE_MODULE}.compute_global_metrics", fake_compute_global_metrics)+    config = EvaluationConfig(+        sv_model="wavlm",+        asr_model_name="some/asr",+        asr_model_type="whisper",+        language="de",+        with_fcd=False,+        with_prosody_metrics=True,+        asr_batch_size=4,+    )++    evaluate_generated_audio_dir("m.json", "audio", "generated", config)++    assert (captured["manifest_path"], captured["audio_dir"], captured["generated_audio_dir"]) == (+        "m.json",+        "audio",+        "generated",+    )+    assert (captured["language"], captured["sv_model_type"]) == ("de", "wavlm")+    assert (captured["asr_model_name"], captured["asr_model_type"]) == ("some/asr", "whisper")+    assert captured["with_prosody_metrics"] is True and captured["asr_batch_size"] == 4+    # Every field is forwarded, not a hand-picked subset: evaluate()'s own default for eou_model_name is None.+    assert captured["eou_model_name"] == config.eou_model_name+    assert (+        captured["global_kwargs"]["codec_model_path"] is None and captured["global_kwargs"]["gt_audio_paths"] is None+    )++    # FCD needs a codec model; the wrapper does not hide evaluate()'s guard.+    with pytest.raises(ValueError, match="codec_model_path is required"):+        evaluate_generated_audio_dir("m.json", "audio", "generated", EvaluationConfig())+++@pytest.mark.unit+@pytest.mark.parametrize(+    "entry_overrides, match",+    [+        ({"language": None}, "'language' must be a non-empty string"),+        ({"language": ""}, "'language' must be a non-empty string"),+        ({"language": " en"}, "'language' must be a non-empty string"),+        ({"asr_model": None}, "'asr_model' must be an object"),+        ({"asr_model": {"name": "some/asr"}}, "'asr_model' must be an object"),+        ({"asr_model": {"name": "some/asr", "type": "hf"}}, "'asr_model' must be an object"),+        ({"asr_model": {"name": " ", "type": "nemo"}}, "'asr_model' must be an object"),+    ],+)+def test_malformed_evalset_overrides_are_rejected(tmp_path, entry_overrides, match):+    # Both entry points reject the same malformed values: the resolver (hand-built meta) and the config loader,+    # which prefixes the dataset name.+    with pytest.raises(ValueError, match=match):+        resolve_evaluation_config_for_dataset(EvaluationConfig(), {"manifest_path": "m.json", **entry_overrides})+    config_path = _write_evalset_config(tmp_path, entry_overrides)+    with pytest.raises(ValueError, match=f"Dataset ds: .*{match}"):+        load_evalset_config(str(config_path), dataset_base_path=tmp_path)+++@pytest.mark.unit+def test_load_evalset_config_rejects_unrecognized_keys(tmp_path):+    # Only the keys the inference/evaluation scripts read are accepted (as in examples/tts/evalset_config.json).+    config_path = _write_evalset_config(tmp_path, {"tokenizer_names": ["english_phoneme"], "language": "en"})+    assert "ds" in load_evalset_config(str(config_path), dataset_base_path=tmp_path)++    # A misspelled override would silently leave the CLI-level value in force; it is rejected instead, and so are+    # training-only DatasetMeta fields copied from a training config.+    for entry, key in (({"langauge": "de"}, "langauge"), ({"feature_dir": None}, "feature_dir")):+        config_path = _write_evalset_config(tmp_path, entry)+        with pytest.raises(ValueError, match=rf"Dataset ds: unrecognized evalset keys \['{key}'\]; recognized keys"):+            load_evalset_config(str(config_path), dataset_base_path=tmp_path)+++@pytest.mark.unit+def test_experiment_metrics_csv_header_and_rows_stay_aligned(tmp_path, monkeypatch):+    warnings_seen = []+    monkeypatch.setattr(nemo_logging, "warning", lambda msg, *args, **kwargs: warnings_seen.append(msg))+    csv_path = tmp_path / "all_experiment_metrics.csv"++    write_csv_header_if_needed(str(csv_path), EXPERIMENT_METRICS_CSV_HEADER)+    write_csv_header_if_needed(str(csv_path), EXPERIMENT_METRICS_CSV_HEADER)  # matching header: nothing to report+    append_metrics_to_csv(str(csv_path), "ckpt", "ds", {"cer_filewise_avg": 0.1, "katakana_cer_cumulative": 0.2})++    with open(csv_path) as f:+        rows = list(csv.DictReader(f))++    assert len(rows) == 1 and warnings_seen == []+    assert (rows[0]["checkpoint_name"], rows[0]["dataset"]) == ("ckpt", "ds")+    assert rows[0]["cer_filewise_avg"] == "0.1"+    assert rows[0]["katakana_cer_cumulative"] == "0.2"+    assert rows[0]["wer_filewise_avg"] == ""  # absent metrics leave an empty cell+    assert None not in rows[0]  # every value has a header column+    assert len(rows[0]) == len(EXPERIMENT_METRICS_CSV_HEADER.split(","))++    # A CSV written before a column was appended keeps its header; the changed layout is reported.+    old_header, dropped_column = EXPERIMENT_METRICS_CSV_HEADER.rsplit(",", 1)+    csv_path.write_text(old_header + "\n")+    write_csv_header_if_needed(str(csv_path), EXPERIMENT_METRICS_CSV_HEADER)+    assert len(warnings_seen) == 1 and dropped_column in warnings_seen[0]+    assert csv_path.read_text() == old_header + "\n"+++class _FakeRunner:+    def create_dataset(self, dataset_meta):+        return [0]++    def run_inference_on_dataset(self, **kwargs):+        return [{"rtf": 1.0}], None, []++    def compute_mean_rtf_metrics(self, rtf_metrics_list):+        return {}+++class _FakeInferenceConfig:+    def build_identifier(self):+        return "_id"+++@pytest.mark.unit+def test_run_inference_and_evaluation_applies_evalset_override(tmp_path, monkeypatch):+    # The example script used to rebuild EvaluationConfig by hand from the evalset entry; it now goes through+    # resolve_evaluation_config_for_dataset, so overrides and inherited CLI-level fields are handled in one place.+    manifest = tmp_path / "m.json"+    manifest.write_text(json.dumps({"audio_filepath": "a.wav", "text": "Hello there."}) + "\n")+    captured = {}++    def fake_evaluate_generated_audio_dir(manifest_path, audio_dir, generated_audio_dir, config):+        captured["config"] = config+        return {"cer_cumulative": 0.0, "ssim_pred_context_avg": 1.0}, [{"cer": 0.0}]++    monkeypatch.setattr(f"{EXAMPLE_MODULE}.evaluate_generated_audio_dir", fake_evaluate_generated_audio_dir)+    for name in ("create_violin_plot", "append_metrics_to_csv", "write_csv_header_if_needed"):+        monkeypatch.setattr(f"{EXAMPLE_MODULE}.{name}", lambda *args, **kwargs: None)++    eval_config = EvaluationConfig(language="en", with_fcd=False, with_utmosv2=False)+    dataset_meta_info = {+        "ds": {+            "manifest_path": str(manifest),+            "audio_dir": str(tmp_path),+            "language": "de",+            "asr_model": {"name": "some/asr", "type": "whisper"},+        }+    }++    cer, ssim = run_inference_and_evaluation(+        runner=_FakeRunner(),+        checkpoint_name="ckpt",+        inference_config=_FakeInferenceConfig(),+        eval_config=eval_config,+        dataset_meta_info=dataset_meta_info,+        datasets=["ds"],+        out_dir=str(tmp_path / "out"),+        flops_per_component={},+        moe_info="",+    )++    assert captured["config"].language == "de"+    assert (captured["config"].asr_model_name, captured["config"].asr_model_type) == ("some/asr", "whisper")+    assert captured["config"].with_utmosv2 is False  # CLI-level settings without an override are inherited+    assert (cer, ssim) == (0.0, 1.0)