🚨 Generation cleanup in pipelines

@@ -120,7 +120,9 @@ def _prepare_generation_config(             generation_config.temperature = 1.0         # We allow generation up to max length + max delay pattern         # (will revert back to max length after generation)-        generation_config.max_length += max(self.config.delay_pattern)+        # `max_length` can be unset in favor of `max_new_tokens`, which then defines it+        if generation_config.max_length is not None:+            generation_config.max_length += max(self.config.delay_pattern)          # Internal flag to indicate CFG that needs to prepare unconditioned input         self._uses_cfg = generation_config.guidance_scale is not None and generation_config.guidance_scale != 1

@@ -308,8 +308,9 @@ def __call__(             return_full_text (`bool`, *optional*, defaults to `True`):                 If set to `False` only added text is returned, otherwise the full text is returned. Cannot be                 specified at the same time as `return_text`.-            clean_up_tokenization_spaces (`bool`, *optional*, defaults to `True`):-                Whether or not to clean up the potential extra spaces in the text output.+            clean_up_tokenization_spaces (`bool`, *optional*):+                Whether or not to clean up the potential extra spaces in the text output. Defaults to the tokenizer's+                `clean_up_tokenization_spaces` setting.             continue_final_message( `bool`, *optional*): This indicates that you want the model to continue the                 last message in the input chat rather than starting a new one, allowing you to "prefill" its response.                 By default this is `True` when the final message in the input chat has the `assistant` role and@@ -420,9 +421,7 @@ def _forward(self, model_inputs, generate_kwargs=None):         prompt_text = model_inputs.pop("text")         input_ids = model_inputs.get("input_ids", model_inputs.get("decoder_input_ids")) -        # User-defined `generation_config` passed to the pipeline call take precedence-        if "generation_config" not in generate_kwargs:-            generate_kwargs["generation_config"] = self.generation_config+        generate_kwargs = self._prepare_generate_kwargs(generate_kwargs)          generated_sequence = self.model.generate(**model_inputs, **generate_kwargs)         return {"generated_sequence": generated_sequence, "prompt_text": prompt_text, "input_ids": input_ids}

@@ -220,6 +220,13 @@ def __init__(             self.type = "ctc"          super().__init__(model, tokenizer, feature_extractor, device=device, **kwargs)+        if self.type == "tdt":+            # The pipeline defaults (e.g. beam search) target seq2seq models, transducers decode with their own+            # generation config+            self._generation_defaults = {}+        elif self.model.config.model_type == "kyutai_speech_to_text":+            # Kyutai's `generate()` sets the generation length from the audio length+            self._generation_defaults.pop("max_new_tokens", None)      def __call__(self, inputs: np.ndarray | bytes | str | dict, **kwargs: Any) -> list[dict[str, Any]]:         """@@ -338,8 +345,9 @@ def _sanitize_parameters(          # Parameter used in more than one place         # in some models like whisper, the generation config has a `return_timestamps` key-        if hasattr(self, "generation_config") and hasattr(self.generation_config, "return_timestamps"):-            return_timestamps = return_timestamps or self.generation_config.return_timestamps+        generation_config = getattr(self.model, "generation_config", None)+        if getattr(generation_config, "return_timestamps", False):+            return_timestamps = return_timestamps or generation_config.return_timestamps          if return_timestamps is not None:             # Check whether we have a valid setting for return_timestamps and throw an error before we perform a forward pass@@ -538,16 +546,14 @@ def _forward(self, model_inputs, return_timestamps=False, return_language=None,                 )              # custom processing for Whisper timestamps and word-level timestamps-            return_timestamps = return_timestamps or getattr(self.generation_config, "return_timestamps", False)+            return_timestamps = return_timestamps or getattr(self.model.generation_config, "return_timestamps", False)             if return_timestamps and self.type == "seq2seq_whisper":                 generate_kwargs["return_timestamps"] = bool(return_timestamps)                 if return_timestamps == "word":                     generate_kwargs["return_token_timestamps"] = True                     generate_kwargs["return_segments"] = True -            # User-defined `generation_config` passed to the pipeline call take precedence-            if "generation_config" not in generate_kwargs:-                generate_kwargs["generation_config"] = self.generation_config+            generate_kwargs = self._prepare_generate_kwargs(generate_kwargs)              main_input_name = self.model.main_input_name if hasattr(self.model, "main_input_name") else "inputs"             generate_kwargs = {@@ -588,7 +594,7 @@ def _forward(self, model_inputs, return_timestamps=False, return_language=None,                     if segments and segments[0]:                         result = segments[0][0]["result"]                         full_seq = result["sequences"] if isinstance(result, dict) else result-                        gen_config = generate_kwargs.get("generation_config", self.generation_config)+                        gen_config = generate_kwargs.get("generation_config", self.model.generation_config)                         if hasattr(gen_config, "lang_to_id"):                             lang_ids = set(gen_config.lang_to_id.values())                             for token_id in full_seq.tolist():@@ -618,12 +624,10 @@ def _forward(self, model_inputs, return_timestamps=False, return_language=None,                 else:                     out["stride"] = rescale_stride(stride, ratio)         elif self.type == "tdt":-            inputs = {-                self.model.main_input_name: model_inputs.pop(self.model.main_input_name),-            }-            if "attention_mask" in model_inputs:-                inputs["attention_mask"] = model_inputs.pop("attention_mask")-            outputs = self.model.generate(**inputs)+            inputs = {self.model.main_input_name: model_inputs.pop(self.model.main_input_name)}+            if attention_mask is not None:+                inputs["attention_mask"] = attention_mask+            outputs = self.model.generate(**inputs, **self._prepare_generate_kwargs(generate_kwargs))             out = {"tokens": outputs.sequences}         else:             raise ValueError(f"Unsupported model type {self.type}.")

@@ -32,6 +32,7 @@ from ..distributed.utils import _is_torch_distributed_initialized from ..dynamic_module_utils import custom_object_save from ..feature_extraction_utils import PreTrainedFeatureExtractor+from ..generation import GenerationConfig from ..image_processing_utils import BaseImageProcessor from ..models.auto import AutoConfig, AutoTokenizer from ..processing_utils import ProcessorMixin@@ -784,6 +785,11 @@ class Pipeline(_ScikitCompat, PushToHubMixin):      # Pipelines that call `generate` have shared logic, e.g. preparing the generation config.     _pipeline_calls_generate = False+    # Defaults for the generation parameters that neither the user nor the model's generation config set+    _default_generation_config: GenerationConfig | None = None+    # Parameters of the pipeline that are named like generation parameters but aren't any (e.g. `top_k` as a number of+    # answers). They are handled like the other pipeline parameters, rather than passed to `generate()`.+    _non_generation_params: tuple[str, ...] = ()      default_input_names = None @@ -877,61 +883,22 @@ def __init__(             self.model.to(self.device)          # If it's a generation pipeline and the model can generate:-        # 1 - create a local generation config. This is done to avoid side-effects on the model as we apply local-        # tweaks to the generation config.+        # 1 - set aside the generation parameters, which apply to every call (see `_prepare_generate_kwargs`).         # 2 - load the assistant model if it is passed.         if self._pipeline_calls_generate and self.model.can_generate():             self.assistant_model, self.assistant_tokenizer = load_assistant_model(                 self.model, kwargs.pop("assistant_model", None), kwargs.pop("assistant_tokenizer", None)             )             self.prefix = self.model.config.prefix if hasattr(self.model.config, "prefix") else None-            # Priority order: kwargs > user_generation_config > model.generation_config > default_pipeline_generation_config-            default_pipeline_generation_config = getattr(self, "_default_generation_config", None)-            user_generation_config = kwargs.pop("generation_config", None)-            if hasattr(self.model, "_prepare_generation_config"):-                base_config = user_generation_config or copy.deepcopy(self.model.generation_config)-                if default_pipeline_generation_config is not None:-                    base_config.update(-                        **default_pipeline_generation_config.to_dict(),-                        defaults_only=True,-                        allow_custom_entries=True,-                    )-                prepared_generation_config, kwargs = self.model._prepare_generation_config(-                    generation_config=base_config, **kwargs-                )-                self.generation_config = prepared_generation_config-                # if the `max_new_tokens` is set to the pipeline default, but `max_length` is set to a non-default-                # value: let's honor `max_length`. E.g. we want Whisper's default `max_length=448` take precedence-                # over over the pipeline's length default.-                if (-                    default_pipeline_generation_config is not None-                    and default_pipeline_generation_config.max_new_tokens is not None  # there's a pipeline default-                    and self.generation_config.max_new_tokens == default_pipeline_generation_config.max_new_tokens-                    and self.generation_config.max_length is not None-                    and self.generation_config.max_length != 20  # global default-                ):-                    self.generation_config.max_new_tokens = None-            else:-                # TODO (joao): no PT model should reach this line. However, some audio models with complex-                # inheritance patterns do. Streamline those models such that this line is no longer needed.-                # In those models, the default generation config is not (yet) used.-                self.generation_config = copy.deepcopy(self.model.generation_config)-            # Update the generation config with task specific params if they exist.-            # NOTE: 1. `prefix` is pipeline-specific and doesn't exist in the generation config.-            #       2. `task_specific_params` is a legacy feature and should be removed in a future version.-            task_specific_params = getattr(self.model.config, "task_specific_params", None)-            if task_specific_params is not None and task in task_specific_params:-                this_task_params = task_specific_params.get(task)-                if "prefix" in this_task_params:-                    self.prefix = this_task_params.pop("prefix")-                self.generation_config.update(**this_task_params)-            # If the tokenizer has a pad token but the model doesn't, set it so that `generate` is aware of it.-            if (-                self.tokenizer is not None-                and self.tokenizer.pad_token_id is not None-                and self.generation_config.pad_token_id is None-            ):-                self.generation_config.pad_token_id = self.tokenizer.pad_token_id+            generation_keys = {key for key in vars(GenerationConfig()) if not key.startswith("_")}+            generation_keys.difference_update(self._non_generation_params)+            generation_keys.add("generation_config")+            self._generate_kwargs = {key: kwargs.pop(key) for key in generation_keys & kwargs.keys()}+            defaults = self._default_generation_config+            self._generation_defaults = {} if defaults is None else defaults.to_diff_dict()+            self._generation_defaults.pop("transformers_version", None)+            if self.tokenizer is not None and self.tokenizer.pad_token_id is not None:+                self._generation_defaults["pad_token_id"] = self.tokenizer.pad_token_id          self.call_count = 0         self._batch_size = kwargs.pop("batch_size", None)@@ -1069,6 +1036,56 @@ def device_placement(self):         else:             yield +    def _prepare_generate_kwargs(self, generate_kwargs: dict, **overrides) -> dict:+        """+        Returns the `generate()` kwargs for this call. They hold the user's generation parameters as passed, so that+        `generate()` handles them as in a direct call: the call's parameters, on top of those passed at init unless the+        call passes a `generation_config`, which then replaces them. The pipeline adds its `overrides`, and its defaults+        for the parameters that neither the user nor the model's generation config set (the sampling ones only when+        sampling).++        What the pipeline adds goes in a copy of the user's `generation_config` if there is one, as `generate()`+        deprecates passing one together with generation parameters.+        """+        # Pipeline defaults for these only apply when sampling, otherwise `generate()` warns about them+        _SAMPLING_PARAMS = {+            "temperature",+            "top_k",+            "top_p",+            "min_p",+            "top_h",+            "typical_p",+            "epsilon_cutoff",+            "eta_cutoff",+        }+        if "generation_config" not in generate_kwargs:+            generate_kwargs = {**self._generate_kwargs, **generate_kwargs}+        set_params = self._get_set_generation_params(generate_kwargs)+        # A default `max_new_tokens` would also override a set `max_length`, e.g. Whisper's 448+        added_params = {+            key: value+            for key, value in self._generation_defaults.items()+            if key not in set_params and not (key == "max_new_tokens" and "max_length" in set_params)+        }+        if not {**set_params, **added_params}.get("do_sample"):+            added_params = {key: value for key, value in added_params.items() if key not in _SAMPLING_PARAMS}+        added_params.update(overrides)+        if (user_config := generate_kwargs.get("generation_config")) is None:+            return {**generate_kwargs, **added_params}+        user_config = copy.deepcopy(user_config)+        user_config.update(**added_params)+        return {**generate_kwargs, "generation_config": user_config}++    def _get_set_generation_params(self, generate_kwargs: dict) -> dict:+        """+        The generation parameters that a `generate()` call with `generate_kwargs` doesn't take from its global+        defaults: the kwargs, on top of their `generation_config` if any, on top of the model's generation config.+        """+        params = self.model.generation_config.to_diff_dict()+        if (generation_config := generate_kwargs.get("generation_config")) is not None:+            params.update(generation_config.to_diff_dict())+        return {**params, **generate_kwargs}+     def ensure_tensor_on_device(self, **inputs):         """         Ensure PyTorch tensors are on the specified device.

@@ -238,6 +238,8 @@ class DocumentQuestionAnsweringPipeline(ChunkPipeline):     """      _pipeline_calls_generate = True+    # The number of answers, not the sampling parameter+    _non_generation_params = ("top_k",)     _load_processor = False     _load_image_processor = None     _load_feature_extractor = None@@ -564,9 +566,7 @@ def _forward(self, model_inputs, **generate_kwargs):         is_last = model_inputs.pop("is_last", False)          if self.model_type == ModelType.VisionEncoderDecoder:-            # User-defined `generation_config` passed to the pipeline call take precedence-            if "generation_config" not in generate_kwargs:-                generate_kwargs["generation_config"] = self.generation_config+            generate_kwargs = self._prepare_generate_kwargs(generate_kwargs)              model_outputs = self.model.generate(**model_inputs, **generate_kwargs)         else:

@@ -249,8 +249,9 @@ def __call__(             return_full_text (`bool`, *optional*, defaults to `True`):                 If set to `False` only added text is returned, otherwise the full text is returned. Cannot be                 specified at the same time as `return_text`.-            clean_up_tokenization_spaces (`bool`, *optional*, defaults to `True`):-                Whether or not to clean up the potential extra spaces in the text output.+            clean_up_tokenization_spaces (`bool`, *optional*):+                Whether or not to clean up the potential extra spaces in the text output. Defaults to the tokenizer's+                `clean_up_tokenization_spaces` setting.             continue_final_message( `bool`, *optional*): This indicates that you want the model to continue the                 last message in the input chat rather than starting a new one, allowing you to "prefill" its response.                 By default this is `True` when the final message in the input chat has the `assistant` role and@@ -388,10 +389,7 @@ def _forward(self, model_inputs, generate_kwargs=None):             model_inputs["input_ids"] if "input_ids" in model_inputs else model_inputs["decoder_input_ids"]         )  # for decoder-only models -        # User-defined `generation_config` passed to the pipeline call take precedence-        if "generation_config" not in generate_kwargs:-            generate_kwargs["generation_config"] = self.generation_config-        generate_kwargs["return_dict_in_generate"] = False+        generate_kwargs = self._prepare_generate_kwargs(generate_kwargs, return_dict_in_generate=False)          generated_sequence = self.model.generate(**model_inputs, **generate_kwargs) 

@@ -331,9 +331,7 @@ def _forward(self, model_inputs, sequential=False, **generate_kwargs):             else:                 outputs = self.batch_inference(**model_inputs)         else:-            # User-defined `generation_config` passed to the pipeline call take precedence-            if "generation_config" not in generate_kwargs:-                generate_kwargs["generation_config"] = self.generation_config+            generate_kwargs = self._prepare_generate_kwargs(generate_kwargs)              outputs = self.model.generate(**model_inputs, **generate_kwargs)         model_outputs = {"model_inputs": model_inputs, "table": table, "outputs": outputs}

@@ -191,8 +191,10 @@ def _sanitize_parameters(          # forward kwargs         if stop_sequence is not None:-            stop_sequence_ids = self.tokenizer.encode(stop_sequence, add_special_tokens=False)-            generate_kwargs["eos_token_id"] = stop_sequence_ids+            if isinstance(stop_sequence, str):+                stop_sequence = [stop_sequence]+            generate_kwargs["stop_strings"] = stop_sequence+            generate_kwargs["tokenizer"] = self.tokenizer         forward_params = generate_kwargs         if self.assistant_model is not None:             forward_params["assistant_model"] = self.assistant_model@@ -264,14 +266,17 @@ def __call__(self, text_inputs, **kwargs):             return_full_text (`bool`, *optional*, defaults to `True`):                 If set to `False` only added text is returned, otherwise the full text is returned. Cannot be                 specified at the same time as `return_text`.-            clean_up_tokenization_spaces (`bool`, *optional*, defaults to `True`):-                Whether or not to clean up the potential extra spaces in the text output.+            clean_up_tokenization_spaces (`bool`, *optional*):+                Whether or not to clean up the potential extra spaces in the text output. Defaults to the tokenizer's+                `clean_up_tokenization_spaces` setting.             continue_final_message( `bool`, *optional*): This indicates that you want the model to continue the                 last message in the input chat rather than starting a new one, allowing you to "prefill" its response.                 By default this is `True` when the final message in the input chat has the `assistant` role and                 `False` otherwise, but you can manually override that behaviour by setting this flag.             prefix (`str`, *optional*):                 Prefix added to prompt.+            stop_sequence (`str` or `list[str]`, *optional*):+                One or more strings that stop generation once generated. The stop string is included in the output.             handle_long_generation (`str`, *optional*):                 By default, this pipelines does not handle long generation (ones that exceed in one form or the other                 the model maximum length). There is no perfect way to address this (more info@@ -346,10 +351,10 @@ def preprocess(          if handle_long_generation == "hole":             cur_len = inputs["input_ids"].shape[-1]-            if "max_new_tokens" in generate_kwargs:-                new_tokens = generate_kwargs["max_new_tokens"]-            else:-                new_tokens = generate_kwargs.get("max_length", self.generation_config.max_length) - cur_len+            generation_params = self._get_set_generation_params(self._prepare_generate_kwargs(generate_kwargs))+            new_tokens = generation_params.get("max_new_tokens")+            if new_tokens is None:+                new_tokens = generation_params["max_length"] - cur_len                 if new_tokens < 0:                     raise ValueError("We cannot infer how many new tokens are expected")             if cur_len + new_tokens > self.tokenizer.model_max_length:@@ -378,27 +383,13 @@ def _forward(self, model_inputs, **generate_kwargs):             in_b = input_ids.shape[0]         prompt_text = model_inputs.pop("prompt_text") -        # If there is a prefix, we may need to adjust the generation length. Do so without permanently modifying-        # generate_kwargs, as some of the parameterization may come from the initialization of the pipeline.         prefix_length = generate_kwargs.pop("prefix_length", 0)-        if prefix_length > 0:-            has_max_new_tokens = "max_new_tokens" in generate_kwargs or (-                "generation_config" in generate_kwargs-                and generate_kwargs["generation_config"].max_new_tokens is not None-            )-            if not has_max_new_tokens:-                generate_kwargs["max_length"] = generate_kwargs.get("max_length") or self.generation_config.max_length-                generate_kwargs["max_length"] += prefix_length-            has_min_new_tokens = "min_new_tokens" in generate_kwargs or (-                "generation_config" in generate_kwargs-                and generate_kwargs["generation_config"].min_new_tokens is not None-            )-            if not has_min_new_tokens and "min_length" in generate_kwargs:-                generate_kwargs["min_length"] += prefix_length--        # User-defined `generation_config` passed to the pipeline call take precedence-        if "generation_config" not in generate_kwargs:-            generate_kwargs["generation_config"] = self.generation_config+        generate_kwargs = self._prepare_generate_kwargs(generate_kwargs)+        if prefix_length:  # a total length (as opposed to a number of new tokens) has to include the prefix+            generation_params = self._get_set_generation_params(generate_kwargs)+            for length, new_tokens in (("max_length", "max_new_tokens"), ("min_length", "min_new_tokens")):+                if length in generation_params and new_tokens not in generation_params:+                    generate_kwargs[length] = generation_params[length] + prefix_length          output = self.model.generate(input_ids=input_ids, attention_mask=attention_mask, **generate_kwargs) @@ -433,7 +424,7 @@ def postprocess(         self,         model_outputs,         return_type=ReturnType.FULL_TEXT,-        clean_up_tokenization_spaces=True,+        clean_up_tokenization_spaces=None,         continue_final_message=None,         skip_special_tokens=None,     ):

@@ -10,7 +10,7 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and-# limitations under the License.from typing import List, Union+# limitations under the License.  import copy from typing import Any, TypedDict, overload@@ -107,12 +107,18 @@ class TextToAudioPipeline(Pipeline):     _default_generation_config = GenerationConfig(max_new_tokens=256)      def __init__(self, *args, vocoder=None, sampling_rate=None, noise_scheduler=None, **kwargs):-        # Some models (e.g., VibeVoice) require noise_scheduler during initialization because `_prepare_generation_config` is called in super().__init__-        if noise_scheduler is not None:-            kwargs["noise_scheduler"] = noise_scheduler         self.noise_scheduler = noise_scheduler          super().__init__(*args, **kwargs)+        # These models pass generation parameters to each of their sub-models, which have their own generation configs:+        # the pipeline doesn't add any+        self._passes_generation_params_to_submodels = self.model.config.model_type in (+            "bark",+            "seamless_m4t",+            "seamless_m4t_v2",+        )+        if self._passes_generation_params_to_submodels:+            self._generation_defaults = {}          self.vocoder = None         if self.model.__class__ in MODEL_FOR_TEXT_TO_SPECTROGRAM_MAPPING.values():@@ -133,7 +139,7 @@ def __init__(self, *args, vocoder=None, sampling_rate=None, noise_scheduler=None         if self.sampling_rate is None:             # get sampling_rate from config and generation config -            config = self.model.config+            config = copy.deepcopy(self.model.config)             gen_config = self.model.__dict__.get("generation_config", None)             if gen_config is not None:                 config.update({k: v for k, v in gen_config.to_dict().items() if v is not None})@@ -159,8 +165,8 @@ def preprocess(self, text, **kwargs):             # bark Tokenizer is called with BarkProcessor which uses those kwargs             # Check if generation_config has semantic_config (BarkGenerationConfig) or use default             max_length = 256-            if hasattr(self.generation_config, "semantic_config"):-                max_length = getattr(self.generation_config.semantic_config, "max_input_semantic_length", 256)+            if hasattr(self.model.generation_config, "semantic_config"):+                max_length = getattr(self.model.generation_config.semantic_config, "max_input_semantic_length", 256)             new_kwargs = {                 "max_length": max_length,                 "add_special_tokens": False,@@ -211,29 +217,9 @@ def _forward(self, model_inputs, **kwargs):             # we expect some kwargs to be additional tensors which need to be on the right device             generate_kwargs = self._ensure_tensor_on_device(generate_kwargs, device=self.device) -            # User-defined `generation_config` passed to the pipeline call take precedence-            if "generation_config" in generate_kwargs:-                generation_config = generate_kwargs.pop("generation_config")-            else:-                generation_config = copy.deepcopy(self.generation_config)--            # ensure dict output to facilitate postprocessing-            generation_config.return_dict_in_generate = True--            # priotiize max_new_tokens to avoid max_length warning-            if generation_config.max_new_tokens is not None:-                generation_config.max_length = None--            # Merge generate_kwargs into generation_config where possible to avoid the-            # deprecation warning about passing generation_config alongside generation params-            model_specific_kwargs = {}-            for k, v in generate_kwargs.items():-                if hasattr(generation_config, k):-                    setattr(generation_config, k, v)-                else:-                    model_specific_kwargs[k] = v-            forward_params["generation_config"] = generation_config-            forward_params.update(model_specific_kwargs)+            # `generate_kwargs` take precedence over `forward_params`. Dict output facilitates postprocessing.+            overrides = {} if self._passes_generation_params_to_submodels else {"return_dict_in_generate": True}+            forward_params = self._prepare_generate_kwargs({**forward_params, **generate_kwargs}, **overrides)              if self.model.config.model_type in ["csm"]:                 # NOTE (ebezzam): CSM does not have the audio tokenizer in the processor therefore `output_audio=True`@@ -289,7 +275,8 @@ def __call__(self, text_inputs, **forward_params):                 The dictionary of ad-hoc parametrization of `generate_config` to be used for the generation call. For a                 complete overview of generate, check the [following                 guide](https://huggingface.co/docs/transformers/en/main_classes/text_generation). `generate_kwargs` are-                only passed to the underlying model if the latter is a generative model.+                only passed to the underlying model if the latter is a generative model, and take precedence over+                `forward_params`.          Return:             `AudioOutput` or a list of `AudioOutput`, which is a `TypedDict` with two keys:@@ -330,6 +317,8 @@ def postprocess(self, audio):         if isinstance(audio, dict):             if "audio" in audio:                 audio = audio["audio"]+            elif "waveform" in audio:  # e.g. SeamlessM4T with `return_intermediate_token_ids=True`+                audio = audio["waveform"]             else:                 needs_decoding = True                 audio = audio["sequences"]

@@ -13,6 +13,7 @@ # limitations under the License.  import unittest+from unittest.mock import patch  import numpy as np import pytest@@ -27,6 +28,9 @@     AutoModelForSpeechSeq2Seq,     AutoProcessor,     AutoTokenizer,+    ParakeetEncoderConfig,+    ParakeetForTDT,+    ParakeetTDTConfig,     Speech2TextForConditionalGeneration,     Wav2Vec2ForCTC,     WhisperForConditionalGeneration,@@ -40,6 +44,7 @@     is_pipeline_test,     is_torch_available,     nested_simplify,+    require_librosa,     require_pyctcdecode,     require_torch,     require_torch_accelerator,@@ -567,6 +572,52 @@ def test_return_timestamps_in_preprocess_longform(self):         )         # fmt: on +    @require_torch+    def test_whisper_generation_parameters_passed_as_kwargs(self):+        # Whisper's `generate()` takes `temperature` as an argument and ignores the one in a `generation_config`+        speech_recognizer = pipeline(task="automatic-speech-recognition", model="openai/whisper-tiny")+        model = speech_recognizer.model+        waveform = np.tile(np.arange(1000, dtype=np.float32), 34)+        with patch.object(model, "generate", wraps=model.generate) as generate:+            _ = speech_recognizer(waveform, generate_kwargs={"temperature": 0.8, "max_new_tokens": 2})+        generate_kwargs = generate.call_args.kwargs+        self.assertNotIn("generation_config", generate_kwargs)+        self.assertEqual(generate_kwargs["temperature"], 0.8)++    @require_torch+    @require_librosa+    def test_transducer_generation_parameters_passed_at_creation(self):+        # Transducers get the generation parameters passed at creation, but not the pipeline's (seq2seq) defaults+        tokenizer = AutoTokenizer.from_pretrained("nvidia/parakeet-tdt-0.6b-v3")+        feature_extractor = AutoFeatureExtractor.from_pretrained("nvidia/parakeet-tdt-0.6b-v3")+        encoder = ParakeetEncoderConfig(+            hidden_size=16,+            num_hidden_layers=1,+            num_attention_heads=2,+            intermediate_size=32,+            subsampling_conv_channels=8,+            num_mel_bins=feature_extractor.feature_size,+        )+        config = ParakeetTDTConfig(+            vocab_size=len(tokenizer) + 1,+            decoder_hidden_size=16,+            encoder_config=encoder.to_dict(),+            blank_token_id=len(tokenizer),+        )+        model = ParakeetForTDT(config)+        model.generation_config.decoder_start_token_id = config.blank_token_id+        speech_recognizer = pipeline(+            task="automatic-speech-recognition",+            model=model,+            tokenizer=tokenizer,+            feature_extractor=feature_extractor,+            max_new_tokens=3,+        )+        with patch.object(model, "generate", wraps=model.generate) as generate:+            _ = speech_recognizer(np.zeros(16000, dtype=np.float32))+        self.assertEqual(generate.call_args.kwargs["max_new_tokens"], 3)+        self.assertNotIn("num_beams", generate.call_args.kwargs)+     @require_torch     def test_return_timestamps_in_init(self):         # segment-level timestamps are accepted

@@ -13,10 +13,16 @@ # limitations under the License.  import unittest+from unittest.mock import patch  from transformers import (     MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING,     AutoTokenizer,+    BartConfig,+    DonutImageProcessor,+    DonutSwinConfig,+    VisionEncoderDecoderConfig,+    VisionEncoderDecoderModel,     is_torch_available,     is_vision_available, )@@ -200,6 +206,38 @@ def test_small_model_pt_donut(self):             nested_simplify(outputs, decimals=4), [{"score": 0.8799, "answer": "2"}, {"score": 0.296, "answer": "1"}]         ) +    @require_torch+    @require_vision+    def test_top_k_is_not_a_generation_parameter(self):+        # With a generative (Donut) model, `top_k` is still the number of answers, not the sampling parameter+        tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bart")+        encoder = DonutSwinConfig(image_size=32, patch_size=4, embed_dim=8, depths=[1], num_heads=[1], window_size=2)+        decoder = BartConfig(+            vocab_size=len(tokenizer),+            d_model=8,+            encoder_layers=1,+            decoder_layers=1,+            encoder_attention_heads=1,+            decoder_attention_heads=1,+            encoder_ffn_dim=8,+            decoder_ffn_dim=8,+        )+        model = VisionEncoderDecoderModel(VisionEncoderDecoderConfig.from_encoder_decoder_configs(encoder, decoder))+        model.generation_config.decoder_start_token_id = tokenizer.bos_token_id+        dqa_pipeline = pipeline(+            "document-question-answering",+            model=model,+            tokenizer=tokenizer,+            image_processor=DonutImageProcessor(size={"height": 32, "width": 32}),+            top_k=2,+            max_new_tokens=3,+        )+        with patch.object(model, "generate", wraps=model.generate) as generate:+            _ = dqa_pipeline(image=Image.new("RGB", (32, 32)), question="What is this?")+        self.assertNotIn("top_k", generate.call_args.kwargs)+        self.assertEqual(generate.call_args.kwargs["max_new_tokens"], 3)+        self.assertEqual(dqa_pipeline._postprocess_params["top_k"], 2)+     @slow     @require_torch     @require_detectron2

@@ -373,6 +373,12 @@ def test_stop_sequence_stopping_criteria(self):         output = text_generator(prompt, stop_sequence=" fe")         self.assertEqual(output, [{"generated_text": "Hello I believe in fe"}]) +        # Multi-token stop sequences only stop generation once the whole sequence is generated+        output = text_generator(prompt, stop_sequence=" fe banana")+        self.assertEqual(output, [{"generated_text": "Hello I believe in fe fe fe fe fe"}])+        output = text_generator(prompt, stop_sequence=[" banana", " fe fe"])+        self.assertEqual(output, [{"generated_text": "Hello I believe in fe fe"}])+     def run_pipeline_test(self, text_generator, _):         model = text_generator.model         tokenizer = text_generator.tokenizer@@ -562,6 +568,77 @@ def test_pipeline_length_setting_warning(self):             _ = text_generator(prompt, max_length=10, max_new_tokens=None)         self.assertNotIn(logger_msg, cl.out) +    def test_pipeline_no_spurious_generation_warnings(self):+        # The pipeline must not warn about generation parameters it sets itself (generation config defaults or+        # parameters passed alongside its generation config)+        text_generator = pipeline("text-generation", model="hf-internal-testing/tiny-random-gpt2")+        logger = logging.get_logger("transformers.generation.utils")+        for kwargs in ({}, {"max_new_tokens": 1}, {"max_length": 10, "min_new_tokens": 1}):+            logger.warning_once.cache_clear()+            with CaptureLogger(logger) as cl:+                _ = text_generator("Hello world", **kwargs)+            self.assertEqual(cl.out, "", f"Unexpected warning with {kwargs}")++    def test_pipeline_length_kwarg_takes_precedence(self):+        # `max_length` passed at call time must override the pipeline's default `max_new_tokens`+        text_generator = pipeline("text-generation", model="hf-internal-testing/tiny-random-gpt2")+        input_length = len(text_generator.tokenizer("Hello world").input_ids)+        out = text_generator("Hello world", max_length=input_length + 2, do_sample=False, return_tensors=True)+        self.assertEqual(len(out[0]["generated_token_ids"]), input_length + 2)++    def test_pipeline_generation_parameters_passed_as_kwargs(self):+        # `generate()` receives the generation parameters as kwargs: those passed at call time on top of those passed+        # at creation, plus the pipeline's defaults for the parameters they don't set+        text_generator = pipeline("text-generation", model="hf-internal-testing/tiny-random-gpt2", max_new_tokens=3)+        model = text_generator.model+        with patch.object(model, "generate", wraps=model.generate) as generate:+            _ = text_generator("Hello world", do_sample=False)+        generate_kwargs = generate.call_args.kwargs+        self.assertNotIn("generation_config", generate_kwargs)+        self.assertEqual(generate_kwargs["max_new_tokens"], 3)+        self.assertFalse(generate_kwargs["do_sample"])+        # The pipeline's default `temperature` only applies to sampling+        self.assertNotIn("temperature", generate_kwargs)++    def test_pipeline_call_generation_config_replaces_creation_parameters(self):+        # A `generation_config` passed at call time replaces the generation parameters passed at creation. Like in+        # `generate()`, passing one together with generation parameters is deprecated (the parameters take precedence).+        text_generator = pipeline("text-generation", model="hf-internal-testing/tiny-random-gpt2", max_new_tokens=3)+        model = text_generator.model+        logger = logging.get_logger("transformers.generation.utils")+        for kwargs, deprecated in (({}, False), ({"max_new_tokens": 2}, True)):+            logger.warning_once.cache_clear()+            with CaptureLogger(logger) as cl, patch.object(model, "generate", wraps=model.generate) as generate:+                _ = text_generator("Hello world", generation_config=GenerationConfig(do_sample=False), **kwargs)+            self.assertEqual("generation-related arguments=({'max_new_tokens'})" in cl.out, deprecated)+            self.assertEqual(generate.call_args.kwargs.get("max_new_tokens"), kwargs.get("max_new_tokens"))+            # The pipeline's default length goes in the `generation_config`, unless a length is passed+            self.assertEqual(generate.call_args.kwargs["generation_config"].max_new_tokens, None if kwargs else 256)++    def test_pipeline_generation_config_with_custom_entries(self):+        # Entries that the model's generation config doesn't have (e.g. Whisper's `no_timestamps_token_id`, when fixing+        # an incomplete config) can only reach `generate()` inside a `generation_config`, at call time or at creation+        generation_config = GenerationConfig(do_sample=False, custom_entry=1)+        call_time = pipeline("text-generation", model="hf-internal-testing/tiny-random-gpt2", max_new_tokens=3)+        creation_time = pipeline(+            "text-generation",+            model=call_time.model,+            tokenizer=call_time.tokenizer,+            generation_config=generation_config,+        )+        logger = logging.get_logger("transformers.generation.utils")+        for text_generator, kwargs in ((call_time, {"generation_config": generation_config}), (creation_time, {})):+            logger.warning_once.cache_clear()+            model = text_generator.model+            with CaptureLogger(logger) as cl, patch.object(model, "generate", wraps=model.generate) as generate:+                _ = text_generator("Hello world", **kwargs)+            self.assertEqual(cl.out, "")+            passed_generation_config = generate.call_args.kwargs["generation_config"]+            self.assertEqual(passed_generation_config.custom_entry, 1)+            self.assertFalse(passed_generation_config.do_sample)+            # The pipeline's default length is added+            self.assertEqual(passed_generation_config.max_new_tokens, 256)+     def test_return_dict_in_generate(self):         text_generator = pipeline("text-generation", model="hf-internal-testing/tiny-random-gpt2", max_new_tokens=2)         out = text_generator(@@ -640,30 +717,36 @@ def test_pipeline_respects_model_generation_config(self):         model = AutoModelForCausalLM.from_pretrained(model_id)         tokenizer = AutoTokenizer.from_pretrained(model_id) +        def generation_param(pipe, key):+            # The value that `generate()` uses+            return pipe._get_set_generation_params(pipe._prepare_generate_kwargs({})).get(key)+         # 1. Modify model.generation_config directly (model_config > pipeline_default)         model.generation_config.max_new_tokens = 500-        model.generation_config.temperature = 0.7+        model.generation_config.temperature = 0.5          # Instantiate pipeline without extra generation kwargs         pipe = pipeline("text-generation", model=model, tokenizer=tokenizer)          # Assert user settings on model.generation_config were respected over pipeline defaults-        self.assertEqual(pipe.generation_config.max_new_tokens, 500)-        self.assertEqual(pipe.generation_config.temperature, 0.7)+        self.assertEqual(generation_param(pipe, "max_new_tokens"), 500)+        self.assertEqual(generation_param(pipe, "temperature"), 0.5)          # 2. Explicit generation_config object overrides model_config and pipeline_default         custom_gc = GenerationConfig(max_new_tokens=250, temperature=0.9)         pipe_gc = pipeline("text-generation", model=model, tokenizer=tokenizer, generation_config=custom_gc)-        self.assertEqual(pipe_gc.generation_config.max_new_tokens, 250)-        self.assertEqual(pipe_gc.generation_config.temperature, 0.9)+        self.assertEqual(generation_param(pipe_gc, "max_new_tokens"), 250)+        self.assertEqual(generation_param(pipe_gc, "temperature"), 0.9) -        # 3. Explicit kwargs override user_generation_config, model_config, and pipeline_default+        # 3. Explicit kwargs override user_generation_config, model_config, and pipeline_default. Like in `generate()`,+        # passing both is deprecated         pipe_kwargs = pipeline(-            "text-generation",-            model=model,-            tokenizer=tokenizer,-            generation_config=custom_gc,-            max_new_tokens=100,+            "text-generation", model=model, tokenizer=tokenizer, generation_config=custom_gc, max_new_tokens=2         )-        self.assertEqual(pipe_kwargs.generation_config.max_new_tokens, 100)-        self.assertEqual(pipe_kwargs.generation_config.temperature, 0.9)+        self.assertEqual(generation_param(pipe_kwargs, "max_new_tokens"), 2)+        self.assertEqual(generation_param(pipe_kwargs, "temperature"), 0.9)+        logger = logging.get_logger("transformers.generation.utils")+        logger.warning_once.cache_clear()+        with CaptureLogger(logger) as cl:+            _ = pipe_kwargs("Hello world")+        self.assertIn("Passing `generation_config` together with generation-related arguments", cl.out)