pandas-dev/pandas · #69496

BUG: agg with an empty list or dict raises "No objects to concatenate" (GH#39609)

jbrockmendel · merged Sep 28, 20267 files · 149 + / 4 −
doc/source/whatsnew/v3.1.0.rst1 + / 0 −
@@ -1353,6 +1353,7 @@ Other - Bug in :func:`testing.assert_series_equal`, :func:`testing.assert_frame_equal` and :func:`testing.assert_extension_array_equal` raising ``TypeError`` instead of reporting the difference when comparing unequal :class:`~pandas.api.extensions.ExtensionArray` objects whose scalars define ``__len__``, such as ``pint``'s ``Quantity`` (:issue:`45240`) - Bug in :func:`util.hash_array` and :func:`util.hash_pandas_object` with float and complex dtypes returning different hashes for ``0.0`` and ``-0.0``, and for ``NaN`` values whose sign or payload bits differ (:issue:`28363`) - Bug in :meth:`DataFrame.__setitem__` and :meth:`Series.__setitem__` emitting a spurious :class:`errors.ChainedAssignmentError` warning for a valid assignment made from compiled code, e.g. from a Cython extension module (:issue:`51315`)+- Bug in :meth:`DataFrame.agg`, :meth:`Series.agg`, :meth:`.DataFrameGroupBy.agg` and :meth:`.Resampler.agg` raising ``ValueError`` for an empty list or dict of functions; these now return an empty result, and a groupby aggregation with an empty list has one row per group instead of none (:issue:`39609`) - Bug in :meth:`DataFrame.eval` where a duplicate column name was resolved to a single column (yielding a :class:`Series`), inconsistent with :func:`pandas.eval` using ``resolvers=(df,)`` and with :meth:`DataFrame.__getitem__`, which include every column with that label (:issue:`65588`) - Bug in :meth:`DataFrame.from_dict` where passing a dict of only scalar values raised a :class:`ValueError` telling users to pass an ``index``, even though :meth:`DataFrame.from_dict` has no ``index`` parameter; the message now points to ``orient='index'`` or the :class:`DataFrame` constructor (:issue:`25515`) - Bug in :meth:`DataFrame.query` and :meth:`DataFrame.eval` raising ``UndefinedVariableError`` when accessing an attribute of a class with the ``@`` prefix, e.g. ``df.query("x == @A.a")`` (:issue:`48694`)
pandas/core/apply.py42 + / 1 −
@@ -83,6 +83,7 @@         ExtensionArray,     )     from pandas.core.groupby import GroupBy+    from pandas.core.groupby.groupby import BaseGroupBy     from pandas.core.resample import Resampler     from pandas.core.window.rolling import BaseWindow @@ -526,8 +527,20 @@ def compute_list_like(     def wrap_results_list_like(         self, keys: Iterable[Hashable], results: list[Series | DataFrame]     ):+        from pandas import Index+        from pandas.core.groupby.groupby import BaseGroupBy+         obj = self.obj +        if not results:+            # GH#39609+            if isinstance(obj, ABCSeries):+                return obj._constructor(index=Index([]), name=obj.name)+            elif isinstance(obj, BaseGroupBy) and obj._selected_obj.ndim == 1:+                return obj._selected_obj._constructor_expanddim(+                    index=_groupby_result_index(obj), columns=Index([])+                )+         try:             return concat(results, keys=keys, axis=1, sort=False)         except TypeError as err:@@ -654,9 +667,25 @@ def wrap_results_dict_like(         result_data: list,     ):         from pandas import Index+        from pandas.core.groupby.groupby import BaseGroupBy          obj = self.obj +        if not result_data:+            # GH#39609+            if isinstance(obj, ABCSeries):+                return obj._constructor(index=Index([]), name=obj.name)+            elif isinstance(obj, ABCDataFrame):+                return obj._constructor(index=Index([]), columns=obj.columns[:0])+            elif isinstance(obj, BaseGroupBy):+                index = _groupby_result_index(obj)+                if selected_obj.ndim == 2:+                    frame = cast("DataFrame", selected_obj)+                    return frame._constructor(index=index, columns=frame.columns[:0])+                return selected_obj._constructor_expanddim(+                    index=index, columns=Index([])+                )+         # Avoid making two isinstance calls in all and any below         is_ndframe = [isinstance(r, ABCNDFrame) for r in result_data] @@ -1119,7 +1148,7 @@ def _agg_list_like_frame_reductions(self) -> DataFrame | None:         elif self.kwargs.get("bool_only"):             obj = obj._get_bool_data() -        if obj.columns.empty:+        if obj.columns.empty or not func_names:             return obj._constructor(index=func_names, columns=obj.columns)          # Compute reductions per dtype group to preserve per-column dtypes.@@ -1885,6 +1914,18 @@ def transform(self):         raise NotImplementedError  +def _groupby_result_index(obj: BaseGroupBy) -> Index:+    """+    Index of a groupby aggregation result, used when there is nothing to aggregate.+    """+    from pandas.core.resample import Resampler++    if isinstance(obj, Resampler):+        # a groupby-resample has no single grouper to take the index from+        return obj.size().index+    return obj._grouper.result_index++ def reconstruct_func(     func: AggFuncType | None, allow_skip_normalization: bool = False, /, **kwargs ) -> tuple[bool, AggFuncType, tuple[str, ...] | None, npt.NDArray[np.intp] | None]:
pandas/core/groupby/generic.py3 + / 1 −
@@ -545,7 +545,9 @@ def _aggregate_multiple_funcs(self, arg, *args, **kwargs) -> DataFrame:             return res_df          indexed_output = {key.position: val for key, val in results.items()}-        output = self.obj._constructor_expanddim(indexed_output, index=None)+        # GH#39609 with no funcs, index by the groups+        index = None if results else self._grouper.result_index+        output = self.obj._constructor_expanddim(indexed_output, index=index)         output.columns = Index(key.label for key in results)          return output
pandas/tests/apply/test_frame_apply.py20 + / 0 −
@@ -2017,6 +2017,26 @@ def test_agg_list_like_empty_frame_reduction():     tm.assert_frame_equal(result, expected)  +def test_agg_empty_list():+    # GH#39609+    df = pd.DataFrame({"a": [1, 2], "b": [3, 4]})+    result = df.agg([])+    expected = pd.DataFrame(index=pd.Index([]), columns=df.columns, dtype=object)+    tm.assert_frame_equal(result, expected)++    result = df.agg([], axis=1)+    expected = pd.DataFrame(index=df.index, columns=pd.Index([]))+    tm.assert_frame_equal(result, expected)+++def test_agg_empty_dict():+    # GH#39609+    df = pd.DataFrame({"a": [1, 2], "b": [3, 4]})+    result = df.agg({})+    expected = pd.DataFrame(index=pd.Index([]), columns=df.columns[:0])+    tm.assert_frame_equal(result, expected)++ def test_agg_dist_like_and_nonunique_columns():     # GH#51099     df = pd.DataFrame(
pandas/tests/apply/test_series_apply.py9 + / 0 −
@@ -395,6 +395,15 @@ def test_agg_evaluate_lambdas(string_series):     assert result is pd.Series  +@pytest.mark.parametrize("func", [[], {}])+def test_agg_empty_func(func):+    # GH#39609+    ser = pd.Series([1, 2], name="a")+    result = ser.agg(func)+    expected = pd.Series(index=pd.Index([]), name="a", dtype=object)+    tm.assert_series_equal(result, expected)++ @pytest.mark.parametrize("op_name", ["agg", "apply"]) def test_with_nested_series(datetime_series, op_name):     # GH 2316 & GH52123
pandas/tests/groupby/aggregate/test_aggregate.py29 + / 2 −
@@ -753,7 +753,7 @@ def test_no_args_raises(self):          # but we do allow this         result = gr.agg([])-        expected = pd.DataFrame(columns=[])+        expected = pd.DataFrame(index=np.array([0, 1]), columns=[])         tm.assert_frame_equal(result, expected)      def test_series_named_agg_duplicates_no_raises(self):@@ -1068,7 +1068,34 @@ def test_groupby_aggregate_empty_key_empty_return():     # GH: 32580 Check if everything works, when return is empty     df = pd.DataFrame({"a": [1, 1, 2], "b": [1, 2, 3], "c": [1, 2, 4]})     result = df.groupby("a").agg({"b": []})-    expected = pd.DataFrame(columns=pd.MultiIndex(levels=[["b"], []], codes=[[], []]))+    expected = pd.DataFrame(+        index=pd.Index([1, 2], name="a"),+        columns=pd.MultiIndex(levels=[["b"], []], codes=[[], []]),+    )+    tm.assert_frame_equal(result, expected)+++@pytest.mark.parametrize("func", [[], {}])+def test_groupby_aggregate_empty_func(func):+    # GH#39609+    df = pd.DataFrame({"a": [1, 1, 2], "b": [3, 4, 5]})+    result = df.groupby("a").agg(func)+    if func == []:+        columns = pd.MultiIndex(levels=[["b"], []], codes=[[], []])+    else:+        columns = df.columns[:0]+    expected = pd.DataFrame(index=pd.Index([1, 2], name="a"), columns=columns)+    tm.assert_frame_equal(result, expected)+++@pytest.mark.parametrize("func", [[], {}])+def test_groupby_aggregate_empty_func_as_index_false(func):+    # GH#39609+    df = pd.DataFrame({"a": [1, 1, 2], "b": [3, 4, 5]})+    result = df.groupby("a", as_index=False).agg(func)+    expected = pd.DataFrame({"a": [1, 2]})+    if func == []:+        expected.columns = pd.MultiIndex.from_tuples([("a", "")])     tm.assert_frame_equal(result, expected)  
pandas/tests/resample/test_resample_api.py45 + / 0 −
@@ -323,6 +323,51 @@ def test_apply_without_aggregation2(_test_series):     tm.assert_series_equal(result, _test_series.rename("foo"))  +@pytest.mark.parametrize("func", [[], {}])+def test_agg_empty_func(func):+    # GH#39609+    df = pd.DataFrame({"a": [1, 2, 3]}, index=date_range("2000", periods=3))+    result = df.resample("D").agg(func)+    if func == []:+        columns = pd.MultiIndex.from_product([df.columns, []])+    else:+        columns = df.columns[:0]+    expected = pd.DataFrame(index=df.index, columns=columns)+    tm.assert_frame_equal(result, expected)+++@pytest.mark.parametrize("func", [[], {}])+def test_agg_empty_func_series(func):+    # GH#39609+    ser = pd.Series([1, 2, 3], index=date_range("2000", periods=3), name="a")+    result = ser.resample("D").agg(func)+    expected = pd.DataFrame(index=ser.index, columns=pd.Index([]))+    tm.assert_frame_equal(result, expected)+++@pytest.mark.parametrize("func", [[], {}])+def test_agg_empty_func_groupby(func):+    # GH#39609+    index = pd.DatetimeIndex(["2000-01-01", "2000-01-03", "2000-01-03"])+    df = pd.DataFrame({"a": [1, 2, 3], "g": [1, 1, 2]}, index=index)+    expected_index = pd.MultiIndex.from_arrays(+        [[1, 1, 1, 2], date_range("2000-01-01", periods=3).append(index[-1:])],+        names=["g", None],+    )++    result = df.groupby("g").resample("D").agg(func)+    if func == []:+        columns = pd.MultiIndex.from_product([df.columns, []])+    else:+        columns = df.columns[:0]+    expected = pd.DataFrame(index=expected_index, columns=columns)+    tm.assert_frame_equal(result, expected)++    result = df.groupby("g")["a"].resample("D").agg(func)+    expected = pd.DataFrame(index=expected_index, columns=pd.Index([]))+    tm.assert_frame_equal(result, expected)++ def test_agg_consistency():     # make sure that we are consistent across     # similar aggregations with and w/o selection list