BUG: astype/construction from float to datetime64 is an unchecked narrowing

@@ -129,6 +129,7 @@ Categorical  Datetimelike ^^^^^^^^^^^^+- Bug in :meth:`DataFrame.astype`, :meth:`Series.astype`, :meth:`Index.astype`, :class:`DatetimeIndex`, :func:`array` and :func:`to_datetime` where ``float`` dtype data outside the ``int64`` range, such as ``inf``, silently became a valid-looking ``datetime64``/``timedelta64`` value or ``NaT``; these now raise ``OutOfBoundsDatetime``/``OutOfBoundsTimedelta`` (:issue:`68926`) - Bug in :meth:`DatetimeIndex.strftime`, :meth:`Series.dt.strftime`, and :meth:`DataFrame.to_csv` with ``date_format`` on Windows silently replacing values that :meth:`Timestamp.strftime` cannot format (e.g. ``%y`` for years before 1900) with their default string representation; these now raise ``ValueError`` (:issue:`58178`)  Timedelta

@@ -60,6 +60,7 @@ from pandas.util._exceptions import find_stack_level from pandas.util._validators import validate_inclusive +from pandas.core.dtypes.astype import raise_if_float_outside_int64 from pandas.core.dtypes.common import (     DT64NS_DTYPE,     INT64_DTYPE,@@ -2643,10 +2644,11 @@ def _sequence_to_dt64(     Raises     ------     TypeError : PeriodDType data is passed+    OutOfBoundsDatetime : float data outside the int64 domain is passed     """      # By this point we are assured to have either a numpy array or Index-    data, copy = maybe_convert_dtype(data, copy, tz=tz)+    data, copy = maybe_convert_dtype(data, copy, tz=tz, out_unit=out_unit)     data_dtype = getattr(data, "dtype", None)      out_dtype = DT64NS_DTYPE@@ -2837,7 +2839,9 @@ def objects_to_datetime64(         raise TypeError(result)  -def maybe_convert_dtype(data, copy: bool, tz: tzinfo | None = None):+def maybe_convert_dtype(+    data, copy: bool, tz: tzinfo | None = None, out_unit: str | None = None+):     """     Convert data based on dtype conventions, issuing     errors where appropriate.@@ -2847,6 +2851,8 @@ def maybe_convert_dtype(data, copy: bool, tz: tzinfo | None = None):     data : np.ndarray or pd.Index     copy : bool     tz : tzinfo or None, default None+    out_unit : str or None, default None+        Resolution the float data is destined for; named in the error message.      Returns     -------@@ -2856,12 +2862,15 @@ def maybe_convert_dtype(data, copy: bool, tz: tzinfo | None = None):     Raises     ------     TypeError : PeriodDType data is passed+    OutOfBoundsDatetime : float data outside the int64 domain is passed     """     if not hasattr(data, "dtype"):         # e.g. collections.deque         return data, copy      if is_float_dtype(data.dtype):+        out_dtype = DT64NS_DTYPE if out_unit is None else np.dtype(f"M8[{out_unit}]")+        raise_if_float_outside_int64(np.asarray(data), out_dtype)         # pre-2.0 we treated these as wall-times, inconsistent with ints         # GH#23675, GH#45573 deprecated to treat symmetrically with integer dtypes.         # Note: data.astype(np.int64) fails ARM tests, see

@@ -34,7 +34,10 @@ ) from pandas.util._exceptions import find_stack_level -from pandas.core.dtypes.astype import astype_is_view+from pandas.core.dtypes.astype import (+    astype_is_view,+    raise_if_float_outside_int64,+) from pandas.core.dtypes.base import ExtensionDtype from pandas.core.dtypes.cast import (     construct_1d_object_array_from_listlike,@@ -782,10 +785,19 @@ def astype(self, dtype: AstypeArg, copy: bool = True) -> ArrayLike:             na_value = np.nan         elif dtype.kind == "M":             unit = np.datetime_data(dtype)[0]-            na_value = np.datetime64("NaT", unit)  # type: ignore[call-overload]+            if unit == "generic":+                # numpy deprecates a unitless NaT; the cast is rejected+                #  downstream, so the sentinel goes unused+                na_value = lib.no_default+            else:+                na_value = np.datetime64("NaT", unit)  # type: ignore[call-overload]         else:             na_value = lib.no_default +        if self.dtype.kind == "f" and dtype.kind in "mM":+            # to_numpy narrows through int64 unchecked (GH#68926)+            raise_if_float_outside_int64(self._data, dtype, mask=self._mask)+         # to_numpy will also raise, but we get somewhat nicer exception messages here         if dtype.kind in "iu" and self._hasna:             raise ValueError("cannot convert NA to integer")

@@ -51,6 +51,7 @@ from pandas.util._decorators import set_module from pandas.util._validators import validate_endpoints +from pandas.core.dtypes.astype import float_outside_int64 from pandas.core.dtypes.common import (     TD64NS_DTYPE,     is_float_dtype,@@ -1315,11 +1316,9 @@ def sequence_to_td64ns(             # On ARM, float-to-int64 overflow saturates to INT64_MAX             # instead of wrapping, which makes the data == int_data             # check pass incorrectly for OOB values like float(2**63).-            # Exclude values outside the int64 domain from the check.-            i64 = np.iinfo(np.int64)-            in_int64_range = (data >= np.float64(i64.min)) & (-                data < np.float64(i64.max)-            )+            # Exclude values outside the int64 domain from the check; NaN is+            # not outside it by that predicate, but mask already covers NaN.+            in_int64_range = ~float_outside_int64(data)             all_round = (mask | (in_int64_range & (data == int_data))).all()             if all_round:                 result = sequence_to_td64ns(

@@ -15,6 +15,11 @@ import numpy as np  from pandas._libs import lib+from pandas._libs.tslibs import (+    OutOfBoundsDatetime,+    OutOfBoundsTimedelta,+    iNaT,+) from pandas.errors import IntCastingNaNError  from pandas.core.dtypes.common import (@@ -74,6 +79,9 @@ def _astype_nansafe(     ------     ValueError         The dtype was a datetime64/timedelta64 dtype, but it had no unit.+    OutOfBoundsDatetime or OutOfBoundsTimedelta+        The dtype was a datetime64/timedelta64 dtype and the float data fell+        outside the int64 domain.     """      # dispatch on extension dtype if needed@@ -134,6 +142,9 @@ def _astype_nansafe(         )         raise ValueError(msg) +    if np.issubdtype(arr.dtype, np.floating) and dtype.kind in "mM":+        raise_if_float_outside_int64(arr, dtype)+     if copy or object in (arr.dtype, dtype):         # Explicit copy, or required since NumPy can't view from / to object.         return arr.astype(dtype, copy=True)@@ -162,6 +173,38 @@ def astype_float_to_int_nansafe(         return values.astype(dtype, copy=copy)  +def float_outside_int64(values: np.ndarray) -> np.ndarray:+    """+    Mask of floats that a narrowing to int64 would alias to NaT or saturate.+    """+    # NaN compares False both ways. float(iNaT) is admitted and round-trips+    #  to NaT, matching cast_from_unit_vectorized.+    return (values >= np.float64(2**63)) | (values < np.float64(iNaT))+++def raise_if_float_outside_int64(+    values: np.ndarray, dtype: np.dtype, mask: np.ndarray | None = None+) -> None:+    """+    Reject floats that a cast to the given datetime64/timedelta64 dtype would+    alias to NaT or saturate to an in-bounds-looking value.++    Entries where ``mask`` is True are ignored.+    """+    unit, step = np.datetime_data(dtype)+    if step != 1 or unit == "generic":+        # unitless and multiplier dtypes are rejected regardless of the values+        #  (multiplier: GH#25611), so the saturated value never surfaces+        return+    oob = float_outside_int64(values)+    if mask is not None:+        oob &= ~mask+    if oob.any():+        bad = values[oob][0]+        err = OutOfBoundsDatetime if dtype.kind == "M" else OutOfBoundsTimedelta+        raise err(f"cannot convert input {bad} with the unit '{unit}'")++ def astype_array(values: ArrayLike, dtype: DtypeObj, copy: bool = False) -> ArrayLike:     """     Cast array (ndarray or ExtensionArray) to the new dtype.

@@ -54,6 +54,7 @@ from pandas.util._decorators import set_module from pandas.util._exceptions import find_stack_level +from pandas.core.dtypes.astype import float_outside_int64 from pandas.core.dtypes.common import (     ensure_object,     is_bool_dtype,@@ -474,6 +475,16 @@ def _convert_listlike_datetimes(             npvalues = np.full(len(arg), np.datetime64("NaT", "ns"))             return DatetimeIndex(npvalues, name=name)         raise+    except OutOfBoundsDatetime:+        if errors == "raise":+            raise+        # GH#68926 coerce just the out-of-range entries; NaN maps to NaT+        oob = float_outside_int64(np.asarray(arg))+        if not oob.any():+            # defensive+            raise+        arg = np.where(oob, np.nan, arg)+        arg, _ = maybe_convert_dtype(arg, copy=False, tz=libtimezones.maybe_get_tz(tz))      arg = ensure_object(arg) 

@@ -7,6 +7,7 @@ ) from functools import partial from operator import attrgetter+import re import zoneinfo  import dateutil@@ -19,7 +20,11 @@     astype_overflowsafe,     timezones, )-from pandas.errors import Pandas4Warning+from pandas.errors import (+    OutOfBoundsDatetime,+    OutOfBoundsTimedelta,+    Pandas4Warning,+)  import pandas as pd import pandas._testing as tm@@ -1366,3 +1371,23 @@ def test_dti_noniso_ambiguous_pytz_raises():         pd.Timestamp(dtstr, tz=tz)     with pytest.raises(ValueError, match="2023-11-05 01:30:00"):         pd.DatetimeIndex([dtstr], tz=tz)+++@pytest.mark.parametrize("value", [np.inf, -np.inf, 1e30, -1e30])+@pytest.mark.parametrize("unit", ["s", "ns"])+def test_dti_constructor_float_out_of_bounds(value, unit):+    # GH#68926 the float is narrowed to int64, where an out-of-range value+    #  saturates to a real-looking timestamp or lands on the NaT sentinel+    arr = np.array([value])+    dtype = f"M8[{unit}]"+    msg = re.escape(f"cannot convert input {value} with the unit '{unit}'")++    with pytest.raises(OutOfBoundsDatetime, match=msg):+        pd.DatetimeIndex(arr, dtype=dtype)+    with pytest.raises(OutOfBoundsDatetime, match=msg):+        pd.Series(arr, dtype=dtype)+    with pytest.raises(OutOfBoundsDatetime, match=msg):+        pd.array(arr, dtype=dtype)+    # the timedelta64 peer already raised; the datetime64 message now matches it+    with pytest.raises(OutOfBoundsTimedelta, match=msg):+        pd.TimedeltaIndex(arr, dtype=f"m8[{unit}]")

@@ -3,14 +3,20 @@     timedelta, ) from importlib import reload+import re import string import sys+import warnings  import numpy as np import pytest  from pandas._libs.tslibs import iNaT-from pandas.errors import Pandas4Warning+from pandas.errors import (+    OutOfBoundsDatetime,+    OutOfBoundsTimedelta,+    Pandas4Warning,+) import pandas.util._test_decorators as td  import pandas as pd@@ -772,3 +778,97 @@ def test_astype_object_to_datetimelike_no_unit(dtype):      with pytest.raises(ValueError, match="dtype with no precision is not allowed"):         ser.astype(dtype)+++@pytest.mark.parametrize("value", [np.inf, -np.inf, 1e30, -1e30, float(2**63)])+@pytest.mark.parametrize("unit", ["s", "ms", "us", "ns"])+def test_astype_float_to_datetimelike_out_of_bounds(value, unit):+    # GH#68926 the cast narrows to int64, where an out-of-range float saturates+    #  to a real-looking timestamp or lands on the NaT sentinel+    ser = pd.Series([value])++    msg = f"cannot convert input {value} with the unit '{unit}'"+    with pytest.raises(OutOfBoundsDatetime, match=re.escape(msg)):+        ser.astype(f"M8[{unit}]")+    with pytest.raises(OutOfBoundsTimedelta, match=re.escape(msg)):+        ser.astype(f"m8[{unit}]")+++@pytest.mark.parametrize("dtype", ["M8[ns]", "m8[ns]", "datetime64[ns, UTC]"])+def test_astype_float_to_datetimelike_in_bounds_unchanged(dtype):+    # GH#68926 in-range floats still truncate toward zero, NaN and the NaT+    #  sentinel still give NaT+    largest = np.nextafter(np.float64(2**63), 0)+    ser = pd.Series([1.5, -1.5, np.nan, float(iNaT), largest])++    result = ser.astype(dtype)++    expected = pd.Series([1, -1, iNaT, iNaT, int(largest)]).astype(dtype)+    tm.assert_series_equal(result, expected)+++@pytest.mark.parametrize("dtype", ["Float64", "Float32"])+@pytest.mark.parametrize("spelling", ["astype", "constructor", "index"])+def test_astype_masked_float_to_datetime64_out_of_bounds(dtype, spelling):+    # GH#68926 masked input goes through the same unchecked int64 cast+    arr = pd.array([np.inf], dtype=dtype)++    with pytest.raises(OutOfBoundsDatetime, match="cannot convert input inf"):+        if spelling == "astype":+            pd.Series(arr).astype("M8[ns]")+        elif spelling == "constructor":+            pd.Series(arr, dtype="M8[ns]")+        else:+            pd.DatetimeIndex(arr)+++def test_astype_masked_float_to_timedelta64_out_of_bounds():+    # GH#68926 masked astype narrows through to_numpy, bypassing _astype_nansafe+    arr = pd.array([np.inf], dtype="Float64")++    with pytest.raises(OutOfBoundsTimedelta, match="cannot convert input inf"):+        pd.Series(arr).astype("m8[ns]")+++def test_astype_masked_float_to_datetime64_in_bounds_unchanged():+    # GH#68926 the guard only looks at the unmasked values, so NA and an+    #  out-of-range float parked behind the mask both still give NaT+    arr = pd.arrays.FloatingArray(+        np.array([1.5, 0.0, np.inf]), np.array([False, True, True])+    )++    result = pd.Series(arr).astype("M8[ns]")++    expected = pd.Series([1, iNaT, iNaT]).astype("M8[ns]")+    tm.assert_series_equal(result, expected)+++@pytest.mark.parametrize("dtype", ["M8", "m8"])+def test_astype_masked_float_to_datetimelike_no_unit(dtype):+    # GH#68926 the out-of-range check must not preempt the no-unit error+    ser = pd.Series(pd.array([np.inf], dtype="Float64"))++    with pytest.raises(TypeError, match="values must have a unit specified"):+        ser.astype(dtype)+++@pytest.mark.parametrize("dtype", ["M8[10s]", "m8[10s]"])+@pytest.mark.parametrize("box", ["numpy", "masked"])+def test_astype_float_to_datetimelike_multiplier_dtype(dtype, box):+    # GH#68926 a multiplier dtype is refused whatever the values, so the+    #  out-of-range guard must not preempt it with the multiplier stripped off+    data = np.array([np.inf]) if box == "numpy" else pd.array([np.inf], dtype="Float64")++    with warnings.catch_warnings():+        # numpy may warn on the saturating cast before pandas rejects the dtype+        warnings.simplefilter("ignore", RuntimeWarning)+        with pytest.raises(ValueError, match="multiplier are not supported"):+            pd.Series(data).astype(dtype)+++def test_astype_float32_to_datetime64_out_of_bounds():+    # GH#68926 every float width narrows to int64, not just float64+    ser = pd.Series(np.array([np.inf], dtype=np.float32))++    with pytest.raises(OutOfBoundsDatetime, match="cannot convert input inf"):+        ser.astype("M8[ns]")

@@ -4986,3 +4986,51 @@ def test_to_datetime_quarterly_string_warns_once_mixed_resolution_tzaware():     ]     assert len(quarter_warnings) == 1     assert result[0] == pd.Timestamp("2014-04-01", tz="UTC")+++@pytest.mark.parametrize("box", [np.array, pd.Series])+def test_to_datetime_float_out_of_bounds_coerce(box):+    # GH#68926 one out-of-range float must not take the whole array down; the+    #  timedelta64 peer already coerces per element+    arg = box([1.0, np.inf, 2.0])++    result = pd.to_datetime(arg, errors="coerce")++    expected = pd.DatetimeIndex(+        ["1970-01-01 00:00:00.000000001", "NaT", "1970-01-01 00:00:00.000000002"]+    )+    if box is pd.Series:+        tm.assert_series_equal(result, pd.Series(expected))+    else:+        tm.assert_index_equal(result, expected)+++def test_to_datetime_float_all_out_of_bounds_coerce():+    # GH#68926 an all-out-of-range input gets the same resolution as all-NaN+    result = pd.to_datetime(np.array([np.inf, -np.inf]), errors="coerce")++    expected = pd.to_datetime(np.array([np.nan, np.nan]), errors="coerce")+    assert expected.dtype == "M8[s]"+    tm.assert_index_equal(result, expected)+++def test_to_datetime_float_out_of_bounds_raises():+    # GH#68926 errors="raise" keeps the message naming the offending value+    msg = re.escape("cannot convert input inf with the unit 'ns'")+    with pytest.raises(OutOfBoundsDatetime, match=msg):+        pd.to_datetime(np.array([1.0, np.inf]))+++def test_to_datetime_float_out_of_bounds_coerce_with_format():+    # GH#68926 the coerced entries must parse like NaN under a format too+    msg = "Parsing integer or float values with a format"+    with tm.assert_produces_warning(Pandas4Warning, match=msg):+        result = pd.to_datetime(+            np.array([2020.0, np.inf]), format="%Y", errors="coerce"+        )+    with tm.assert_produces_warning(Pandas4Warning, match=msg):+        expected = pd.to_datetime(+            np.array([2020.0, np.nan]), format="%Y", errors="coerce"+        )++    tm.assert_index_equal(result, expected)