DEPR: negative min_count in reductions (GH#50022)

@@ -107,6 +107,7 @@ deprecation2 Other deprecations ^^^^^^^^^^^^^^^^^^ - Deprecated ``pandas.to_pickle``, use the :meth:`DataFrame.to_pickle` or :meth:`Series.to_pickle` method instead (:issue:`48402`)+- Deprecated passing a negative ``min_count`` to :meth:`DataFrame.sum`, :meth:`DataFrame.prod`, :meth:`Series.sum`, :meth:`Series.prod`, :meth:`TimedeltaIndex.sum`, the ``sum`` and ``prod`` methods of pandas arrays, and the ``sum``, ``prod``, ``min``, ``max``, ``first``, and ``last`` methods of :class:`.DataFrameGroupBy`, :class:`.SeriesGroupBy`, and :class:`.Resampler`. Pass ``min_count=0`` instead, which behaves the same (:issue:`50022`) - Deprecated the ``cache_dates`` argument in :func:`read_csv`, :func:`read_table`, and :func:`read_fwf` (:issue:`68705`)  .. ---------------------------------------------------------------------------

@@ -49,6 +49,7 @@     set_module, ) from pandas.util._exceptions import find_stack_level+from pandas.util._validators import validate_min_count  from pandas.core.dtypes.cast import (     can_hold_element,@@ -2966,6 +2967,7 @@ def sum(         **kwargs,     ):         nv.validate_sum((), kwargs)+        min_count = validate_min_count(min_count)         return self._reduce(             "sum", skipna=skipna, min_count=min_count, axis=axis, **kwargs         )@@ -3072,6 +3074,7 @@ def prod(         **kwargs,     ):         nv.validate_prod((), kwargs)+        min_count = validate_min_count(min_count)         return self._reduce(             "prod", skipna=skipna, min_count=min_count, axis=axis, **kwargs         )

@@ -33,6 +33,7 @@     Pandas4Warning, ) from pandas.util._exceptions import find_stack_level+from pandas.util._validators import validate_min_count  from pandas.core.dtypes.astype import (     astype_is_view,@@ -1800,6 +1801,7 @@ def sum(         **kwargs,     ):         nv.validate_sum((), kwargs)+        min_count = validate_min_count(min_count)          result = masked_reductions.sum(             self._data,@@ -1821,6 +1823,7 @@ def prod(         **kwargs,     ):         nv.validate_prod((), kwargs)+        min_count = validate_min_count(min_count)          result = masked_reductions.prod(             self._data,

@@ -15,6 +15,7 @@ from pandas.compat.numpy import function as nv from pandas.errors import LossySetitemError from pandas.util._decorators import set_module+from pandas.util._validators import validate_min_count  from pandas.core.dtypes.astype import (     astype_array,@@ -496,6 +497,7 @@ def sum(         **kwargs,     ) -> Scalar:         nv.validate_sum((), kwargs)+        min_count = validate_min_count(min_count)         result = nanops.nansum(             self._ndarray, axis=axis, skipna=skipna, min_count=min_count         )@@ -510,6 +512,7 @@ def prod(         **kwargs,     ) -> Scalar:         nv.validate_prod((), kwargs)+        min_count = validate_min_count(min_count)         result = nanops.nanprod(             self._ndarray, axis=axis, skipna=skipna, min_count=min_count         )

@@ -49,6 +49,7 @@ from pandas.util._exceptions import find_stack_level from pandas.util._validators import (     validate_bool_kwarg,+    validate_min_count, )  from pandas.core.dtypes.astype import astype_array@@ -2269,6 +2270,7 @@ def sum(         """         nv.validate_sum(args, kwargs)         skipna = validate_bool_kwarg(skipna, "skipna")+        min_count = validate_min_count(min_count)          if self.dtype.subtype.kind == "O" and not self._null_fill_value:             # the fill_value * nsparse adjustment below assumes a commutative +,@@ -2361,6 +2363,7 @@ def prod(         """         nv.validate_prod((), kwargs)         skipna = validate_bool_kwarg(skipna, "skipna")+        min_count = validate_min_count(min_count)         return self._dense_reduce("prod", skipna=skipna, min_count=min_count)      def cumsum(self, axis: AxisInt = 0, *args, **kwargs) -> SparseArray:

@@ -34,6 +34,7 @@     set_module, ) from pandas.util._exceptions import find_stack_level+from pandas.util._validators import validate_min_count  from pandas.core.dtypes.base import (     ExtensionDtype,@@ -1107,6 +1108,7 @@ def sum(         **kwargs,     ) -> Scalar:         nv.validate_sum((), kwargs)+        min_count = validate_min_count(min_count)         result = masked_reductions.sum(             values=self._ndarray,             mask=self.isna(),

@@ -49,7 +49,10 @@     OutOfBoundsTimedelta, ) from pandas.util._decorators import set_module-from pandas.util._validators import validate_endpoints+from pandas.util._validators import (+    validate_endpoints,+    validate_min_count,+)  from pandas.core.dtypes.astype import float_outside_int64 from pandas.core.dtypes.common import (@@ -407,6 +410,7 @@ def sum(         nv.validate_sum(             (), {"dtype": dtype, "out": out, "keepdims": keepdims, "initial": initial}         )+        min_count = validate_min_count(min_count)          result = nanops.nansum(             self._ndarray, axis=axis, skipna=skipna, min_count=min_count

@@ -60,6 +60,7 @@     validate_ascending,     validate_bool_kwarg,     validate_inclusive,+    validate_min_count, )  from pandas.core.dtypes.astype import astype_is_view@@ -12073,6 +12074,7 @@ def _min_count_stat_function(                 Pandas4Warning,                 stacklevel=find_stack_level(),             )+        min_count = validate_min_count(min_count)          return self._reduce(             func,

@@ -65,6 +65,7 @@ class providing the base-class of operations. ) from pandas.util._decorators import cache_readonly from pandas.util._exceptions import find_stack_level+from pandas.util._validators import validate_min_count  from pandas.core.dtypes.cast import (     coerce_indexer_dtype,@@ -2839,6 +2840,7 @@ def sum(         1   10   7         2   11  17         """+        min_count = validate_min_count(min_count)         if maybe_use_numba(engine):             from pandas.core._numba.kernels import grouped_sum @@ -2938,6 +2940,7 @@ def prod(         1   16   10         2   30   72         """+        min_count = validate_min_count(min_count)          def prod_compat(obj: NDFrameT):             # GH#18588: see min_compat below@@ -2955,7 +2958,7 @@ def prod_compat(obj: NDFrameT):     def min(         self,         numeric_only: bool = False,-        min_count: int = -1,+        min_count: int = 0,         skipna: bool = True,         engine: Literal["cython", "numba"] | None = None,         engine_kwargs: dict[str, bool] | None = None,@@ -2976,7 +2979,7 @@ def min(                  numeric_only no longer accepts ``None``. -        min_count : int, default -1+        min_count : int, default 0             The required number of valid values to perform the operation. If fewer             than ``min_count`` non-NA values are present the result will be NA. @@ -3048,6 +3051,7 @@ def min(         1   2  2         2   5  8         """+        min_count = validate_min_count(min_count)         if maybe_use_numba(engine):             from pandas.core._numba.kernels import grouped_min_max @@ -3079,7 +3083,7 @@ def min_compat(obj: NDFrameT):     def max(         self,         numeric_only: bool = False,-        min_count: int = -1,+        min_count: int = 0,         skipna: bool = True,         engine: Literal["cython", "numba"] | None = None,         engine_kwargs: dict[str, bool] | None = None,@@ -3100,7 +3104,7 @@ def max(                  numeric_only no longer accepts ``None``. -        min_count : int, default -1+        min_count : int, default 0             The required number of valid values to perform the operation. If fewer             than ``min_count`` non-NA values are present the result will be NA. @@ -3172,6 +3176,7 @@ def max(         1   8  5         2   6  9         """+        min_count = validate_min_count(min_count)         if maybe_use_numba(engine):             from pandas.core._numba.kernels import grouped_min_max @@ -3199,7 +3204,7 @@ def max_compat(obj: NDFrameT):      @final     def first(-        self, numeric_only: bool = False, min_count: int = -1, skipna: bool = True+        self, numeric_only: bool = False, min_count: int = 0, skipna: bool = True     ) -> NDFrameT:         """         Compute the first entry of each column within each group.@@ -3212,7 +3217,7 @@ def first(         ----------         numeric_only : bool, default False             Include only float, int, boolean columns.-        min_count : int, default -1+        min_count : int, default 0             The required number of valid values to perform the operation. If fewer             than ``min_count`` valid values are present the result will be NA.         skipna : bool, default True@@ -3260,6 +3265,7 @@ def first(         1  5.0  1         3  6.0  3         """+        min_count = validate_min_count(min_count)          def first_compat(obj: NDFrameT):             def first(x: Series):@@ -3286,7 +3292,7 @@ def first(x: Series):      @final     def last(-        self, numeric_only: bool = False, min_count: int = -1, skipna: bool = True+        self, numeric_only: bool = False, min_count: int = 0, skipna: bool = True     ) -> NDFrameT:         """         Compute the last entry of each column within each group.@@ -3300,7 +3306,7 @@ def last(         numeric_only : bool, default False             Include only float, int, boolean columns. If None, will attempt to use             everything, then use only numeric data.-        min_count : int, default -1+        min_count : int, default 0             The required number of valid values to perform the operation. If fewer             than ``min_count`` valid values are present the result will be NA.         skipna : bool, default True@@ -3330,6 +3336,7 @@ def last(         1  5.0  2         3  6.0  3         """+        min_count = validate_min_count(min_count)          def last_compat(obj: NDFrameT):             def last(x: Series):

@@ -32,6 +32,7 @@ ) from pandas.util._decorators import set_module from pandas.util._exceptions import find_stack_level+from pandas.util._validators import validate_min_count  from pandas.core.dtypes.dtypes import (     ArrowDtype,@@ -1161,6 +1162,7 @@ def sum(         2023-02-01    7         Freq: MS, dtype: int64         """+        min_count = validate_min_count(min_count)         return self._downsample("sum", numeric_only=numeric_only, min_count=min_count)      @final@@ -1219,6 +1221,7 @@ def prod(         2023-02-01   12         Freq: MS, dtype: int64         """+        min_count = validate_min_count(min_count)         return self._downsample("prod", numeric_only=numeric_only, min_count=min_count)      @final@@ -1276,6 +1279,7 @@ def min(         2023-02-01    3         Freq: MS, dtype: int64         """+        min_count = validate_min_count(min_count)         return self._downsample("min", numeric_only=numeric_only, min_count=min_count)      @final@@ -1333,6 +1337,7 @@ def max(         2023-02-01    4         Freq: MS, dtype: int64         """+        min_count = validate_min_count(min_count)         return self._downsample("max", numeric_only=numeric_only, min_count=min_count)      @final@@ -1387,6 +1392,7 @@ def first(         2023-02-01    3         Freq: MS, dtype: int64         """+        min_count = validate_min_count(min_count)         return self._downsample(             "first", numeric_only=numeric_only, min_count=min_count, skipna=skipna         )@@ -1443,6 +1449,7 @@ def last(         2023-02-01    4         Freq: MS, dtype: int64         """+        min_count = validate_min_count(min_count)         return self._downsample(             "last", numeric_only=numeric_only, min_count=min_count, skipna=skipna         )

@@ -66,7 +66,7 @@ def test_ffill_handles_nan_groups(dropna, method, has_nan_group):     tm.assert_frame_equal(result, expected)  -@pytest.mark.parametrize("min_count, value", [(2, np.nan), (-1, 1.0)])+@pytest.mark.parametrize("min_count, value", [(2, np.nan), (0, 1.0)]) @pytest.mark.parametrize("func", ["first", "last", "max", "min"]) def test_min_count(func, min_count, value):     # GH#37821

@@ -7,6 +7,7 @@ import pytest  from pandas._libs.tslibs import iNaT+from pandas.errors import Pandas4Warning import pandas.util._test_decorators as td  from pandas.core.dtypes.common import pandas_dtype@@ -435,6 +436,22 @@ def test_first_last_skipna(any_real_nullable_dtype, sort, skipna, how):     tm.assert_frame_equal(result, expected)  +@pytest.mark.parametrize("how", ["sum", "prod", "min", "max", "first", "last"])+def test_negative_min_count_deprecated(how):+    # GH#50022+    df = pd.DataFrame({"a": [1, 1, 2], "b": [2.0, np.nan, np.nan]})+    gb = df.groupby("a")+    expected = getattr(gb, how)(min_count=0)+    msg = "Passing a negative value for 'min_count' is deprecated"+    with tm.assert_produces_warning(Pandas4Warning, match=msg):+        result = getattr(gb, how)(min_count=-1)+    tm.assert_frame_equal(result, expected)++    with tm.assert_produces_warning(Pandas4Warning, match=msg):+        result = gb.agg(how, min_count=-1)+    tm.assert_frame_equal(result, expected)++ @pytest.mark.parametrize("how", ["first", "last"]) def test_first_last_ea_min_count(any_real_nullable_dtype, how):     # GH#57591

@@ -8,6 +8,9 @@ import numpy as np import pytest +from pandas.compat import HAS_PYARROW+from pandas.errors import Pandas4Warning+ import pandas as pd import pandas._testing as tm from pandas.core import nanops@@ -1937,3 +1940,72 @@ def test_ea_reduction_method_ddof(any_numeric_ea_and_arrow_dtype, op_name, ddof)      tm.assert_almost_equal(getattr(arr, op_name)(ddof=ddof), expected)     assert pd.isna(getattr(arr, op_name)(skipna=False, ddof=ddof))+++@pytest.mark.parametrize("how", ["sum", "prod"])+@pytest.mark.parametrize(+    "dtype",+    [+        "float64",+        "Float64",+        pytest.param(+            "float64[pyarrow]",+            marks=pytest.mark.skipif(not HAS_PYARROW, reason="requires pyarrow"),+        ),+    ],+)+@pytest.mark.parametrize("min_count", [-1, -1.0])+def test_negative_min_count_deprecated(frame_or_series, how, dtype, min_count):+    # GH#50022; pyarrow used to raise OverflowError+    obj = frame_or_series([2.0, np.nan, 3.0], dtype=dtype)+    expected = getattr(obj, how)(min_count=0)+    msg = "Passing a negative value for 'min_count' is deprecated"+    with tm.assert_produces_warning(Pandas4Warning, match=msg):+        result = getattr(obj, how)(min_count=min_count)+    if frame_or_series is pd.Series:+        assert result == expected+    else:+        tm.assert_series_equal(result, expected)+++@pytest.mark.parametrize(+    "how, dtype",+    [+        ("sum", "Int64"),+        ("prod", "Int64"),+        ("sum", "float64"),+        ("prod", "float64"),+        ("sum", "Sparse[float64]"),+        ("prod", "Sparse[float64]"),+        ("sum", "m8[ns]"),+        ("sum", "string[python]"),+        pytest.param(+            "sum",+            "int64[pyarrow]",+            marks=pytest.mark.skipif(not HAS_PYARROW, reason="requires pyarrow"),+        ),+        pytest.param(+            "prod",+            "int64[pyarrow]",+            marks=pytest.mark.skipif(not HAS_PYARROW, reason="requires pyarrow"),+        ),+    ],+)+def test_negative_min_count_deprecated_array(how, dtype):+    # GH#50022+    data = ["a", None, "b"] if dtype == "string[python]" else [2, None, 3]+    arr = pd.array(data, dtype=dtype)+    expected = getattr(arr, how)(min_count=0)+    msg = "Passing a negative value for 'min_count' is deprecated"+    with tm.assert_produces_warning(Pandas4Warning, match=msg):+        result = getattr(arr, how)(min_count=-1)+    assert result == expected+++def test_negative_min_count_deprecated_timedelta_index():+    # GH#50022+    tdi = pd.to_timedelta([1, 2, 3], unit="D")+    msg = "Passing a negative value for 'min_count' is deprecated"+    with tm.assert_produces_warning(Pandas4Warning, match=msg):+        result = tdi.sum(min_count=-1)+    assert result == tdi.sum(min_count=0)

@@ -544,3 +544,19 @@ def test_first_last_skipna(any_real_nullable_dtype, skipna, how):     expected = getattr(gb, how)(skipna=skipna)     expected.index.freq = "ME"     tm.assert_frame_equal(result, expected)+++@pytest.mark.parametrize("how", ["sum", "prod", "min", "max", "first", "last"])+@pytest.mark.parametrize("periods", [0, 3])+def test_negative_min_count_deprecated(how, periods):+    # GH#50022; periods=0 covers the empty-index path, which skips groupby+    ser = pd.Series(+        [2.0, np.nan, 3.0][:periods],+        index=date_range("2020-01-01", periods=periods, freq="D"),+    )+    rs = ser.resample("2D")+    expected = getattr(rs, how)(min_count=0)+    msg = "Passing a negative value for 'min_count' is deprecated"+    with tm.assert_produces_warning(Pandas4Warning, match=msg):+        result = getattr(rs, how)(min_count=-1)+    tm.assert_series_equal(result, expected)

@@ -15,11 +15,14 @@     TypeVar,     overload, )+import warnings  import numpy as np  from pandas._libs import lib from pandas._libs.missing import NA+from pandas.errors import Pandas4Warning+from pandas.util._exceptions import find_stack_level  from pandas.core.dtypes.common import (     is_bool,@@ -326,6 +329,31 @@ def validate_na_arg(value: object, name: str) -> None:     raise ValueError(f"{name} must be None, pd.NA, np.nan, True, or False; got {value}")  +def validate_min_count(min_count: int) -> int:+    """+    Warn on a negative ``min_count``, which is deprecated (GH#50022).++    Parameters+    ----------+    min_count : int+        The ``min_count`` passed to a reduction.++    Returns+    -------+    int+        ``min_count``, with negative values replaced by 0.+    """+    if (lib.is_integer(min_count) or lib.is_float(min_count)) and min_count < 0:+        warnings.warn(+            "Passing a negative value for 'min_count' is deprecated and will "+            "raise in a future version of pandas. Pass min_count=0 instead.",+            Pandas4Warning,+            stacklevel=find_stack_level(),+        )+        return 0+    return min_count  # pyright: ignore[reportReturnType]++ def validate_percentile(q: float | Iterable[float]) -> np.ndarray:     """     Validate percentiles (used by describe and quantile).