pandas-dev/pandas · #70219

BUG: .loc setitem with an integer slice on a Decimal index

jbrockmendel · merged Oct 4, 20269 files · 35 + / 89 −
doc/source/whatsnew/v3.2.0.rst1 + / 0 −
@@ -166,6 +166,7 @@ Interval Indexing ^^^^^^^^ - Bug in :meth:`DataFrame.loc` and :meth:`DataFrame.iloc` setitem on a 1-column :class:`DataFrame` with an extension dtype raising when the column key is an all-``False`` boolean mask or an empty list-like (:issue:`70232`)+- Bug in :meth:`DataFrame.loc` and :meth:`Series.loc` setitem with an integer slice raising ``TypeError`` where the same ``.loc`` getitem succeeds, e.g. on an index of :class:`decimal.Decimal` values (:issue:`70219`) - Bug in :meth:`Index.difference` dropping elements that are not in ``other`` when ``other`` was a :class:`DatetimeIndex`, :class:`TimedeltaIndex` or :class:`PeriodIndex` and the elements were strings that parse to its values, or when ``other`` was an :class:`IntervalIndex` and the elements were scalars contained in its intervals (:issue:`58971`) - 
pandas/core/indexes/base.py3 + / 49 −
@@ -4269,17 +4269,11 @@ def _convert_slice_indexer(         kind : {'loc', 'getitem'}         """ -        # potentially cast the bounds to integers-        start, stop, step = key.start, key.stop, key.step--        # figure out if this is a positional indexer-        is_index_slice = is_valid_positional_slice(key)-         # TODO(GH#50617): once Series.__[gs]etitem__ is removed we should be able         #  to simplify this.         if kind == "getitem":             # called from the getitem slicers, validate that we are in fact integers-            if is_index_slice:+            if is_valid_positional_slice(key):                 # In this case the _validate_indexer checks below are redundant                 return key             elif self.dtype.kind in "iu":@@ -4289,37 +4283,9 @@ def _convert_slice_indexer(                 self._validate_indexer("slice", key.step, "getitem")                 return key -        # convert the slice to an indexer here; checking that the user didn't-        #  pass a positional slice to loc-        is_positional = is_index_slice and self._should_fallback_to_positional--        # if we are mixed and have integers-        if is_positional:-            try:-                # Validate start & stop-                if start is not None:-                    self.get_loc(start)-                if stop is not None:-                    self.get_loc(stop)-                is_positional = False-            except KeyError:-                pass-         if com.is_null_slice(key):-            # It doesn't matter if we are positional or label based-            indexer = key-        elif is_positional:-            if kind == "loc":-                # GH#16121, GH#24612, GH#31810-                raise TypeError(-                    "Slicing a positional slice with .loc is not allowed, "-                    "Use .loc with labels or .iloc with positions instead.",-                )-            indexer = key-        else:-            indexer = self.slice_indexer(start, stop, step)--        return indexer+            return key+        return self.slice_indexer(key.start, key.stop, key.step)      @final     def _raise_invalid_indexer(@@ -6407,18 +6373,6 @@ def _check_indexing_error(self, key: object) -> None:             # would convert to numpy arrays and raise later any way) - GH29926             raise InvalidIndexError(key) -    @cache_readonly-    def _should_fallback_to_positional(self) -> bool:-        """-        Should an integer key be treated as positional?-        """-        return self.inferred_type not in {-            "integer",-            "mixed-integer",-            "floating",-            "complex",-        }-     def get_indexer_non_unique(         self, target: Axes     ) -> tuple[npt.NDArray[np.intp], npt.NDArray[np.intp]]:
pandas/core/indexes/category.py1 + / 8 −
@@ -11,10 +11,7 @@ import numpy as np  from pandas._libs import index as libindex-from pandas.util._decorators import (-    cache_readonly,-    set_module,-)+from pandas.util._decorators import set_module  from pandas.core.dtypes.common import is_scalar from pandas.core.dtypes.dtypes import CategoricalDtype@@ -155,10 +152,6 @@ class CategoricalIndex(NDArrayBackedExtensionIndex):     def _can_hold_strings(self) -> bool:         return self.categories._can_hold_strings -    @cache_readonly-    def _should_fallback_to_positional(self) -> bool:-        return self.categories._should_fallback_to_positional-     _data: Categorical     _values: Categorical 
pandas/core/indexes/interval.py0 + / 8 −
@@ -1393,14 +1393,6 @@ def _convert_slice_indexer(self, key: slice, kind: Literal["loc", "getitem"]):          return super()._convert_slice_indexer(key, kind) -    @cache_readonly-    def _should_fallback_to_positional(self) -> bool:-        # integer lookups in Series.__getitem__ are unambiguously-        #  positional in this case-        # error: Item "ExtensionDtype"/"dtype[Any]" of "Union[dtype[Any],-        # ExtensionDtype]" has no attribute "subtype"-        return self.dtype.subtype.kind in "mM"  # type: ignore[union-attr]-     def _maybe_cast_slice_bound(self, label, side: str):         return getattr(self, side)._maybe_cast_slice_bound(label, side) 
pandas/core/indexes/multi.py0 + / 8 −
@@ -3383,14 +3383,6 @@ def _check_indexing_error(self, key) -> None:             # We have to explicitly exclude generators, as these are hashable.             raise InvalidIndexError(key) -    @cache_readonly-    def _should_fallback_to_positional(self) -> bool:-        """-        Should integer key(s) be treated as positional?-        """-        # GH#33355-        return self.levels[0]._should_fallback_to_positional-     def _get_indexer_strict(         self, key, axis_name: str     ) -> tuple[Index, npt.NDArray[np.intp]]:
pandas/core/indexes/range.py0 + / 7 −
@@ -596,13 +596,6 @@ def _get_indexer(             locs[valid] = len(self) - 1 - locs[valid]         return ensure_platform_int(locs) -    @cache_readonly-    def _should_fallback_to_positional(self) -> bool:-        """-        Should an integer key be treated as positional?-        """-        return False-     # --------------------------------------------------------------------      def tolist(self) -> list[int]:
pandas/tests/indexing/multiindex/test_partial.py0 + / 2 −
@@ -155,8 +155,6 @@ def test_getitem_intkey_leading_level(             assert mi.levels[0].dtype == np.float64          assert 14 not in mi.levels[0]-        assert not mi.levels[0]._should_fallback_to_positional-        assert not mi._should_fallback_to_positional          with pytest.raises(KeyError, match="14"):             ser[14]
pandas/tests/indexing/test_loc.py30 + / 5 −
@@ -8,6 +8,7 @@     time,     timedelta, )+from decimal import Decimal import re  from dateutil.tz import gettz@@ -3476,7 +3477,8 @@ def test_loc_with_positional_slice_raises():     # GH#31840     ser = pd.Series(range(4), index=["A", "B", "C", "D"]) -    with pytest.raises(TypeError, match="Slicing a positional slice with .loc"):+    msg = r"cannot do slice indexing on Index with these indexers \[3\] of type int"+    with pytest.raises(TypeError, match=msg):         ser.loc[:3] = 2  @@ -3496,18 +3498,41 @@ def test_loc_slice_disallows_positional():         with pytest.raises(TypeError, match=msg):             obj.loc[1:3] -        with pytest.raises(TypeError, match="Slicing a positional slice with .loc"):-            # GH#31840 enforce incorrect behavior+        with pytest.raises(TypeError, match=msg):             obj.loc[1:3] = 1      with pytest.raises(TypeError, match=msg):         df.loc[1:3, 1] -    with pytest.raises(TypeError, match="Slicing a positional slice with .loc"):-        # GH#31840 enforce incorrect behavior+    with pytest.raises(TypeError, match=msg):         df.loc[1:3, 1] = 2  +def test_loc_setitem_int_slice_decimal_index(frame_or_series):+    # GH#70219 integer slice bounds are labels for a Decimal index, as in getitem+    pa = pytest.importorskip("pyarrow")+    values = [Decimal(1), Decimal(3), Decimal(5)]+    index = pd.Index(pd.array(values, dtype=pd.ArrowDtype(pa.decimal128(5, 2))))+    obj = frame_or_series(range(3), index=index)++    expected = frame_or_series([99, 99, 2], index=index)+    tm.assert_equal(obj.loc[0:4], obj.iloc[:2])+    obj.loc[0:4] = 99+    tm.assert_equal(obj, expected)+++def test_loc_setitem_int_slice_decimal_multiindex_level(frame_or_series):+    # GH#70219+    level = pd.Index([Decimal(1), Decimal(3), Decimal(5)], dtype=object)+    index = pd.MultiIndex.from_arrays([level, ["a", "b", "c"]])+    obj = frame_or_series(range(3), index=index)++    expected = frame_or_series([99, 99, 2], index=index)+    tm.assert_equal(obj.loc[0:4], obj.iloc[:2])+    obj.loc[0:4] = 99+    tm.assert_equal(obj, expected)++ def test_loc_datetimelike_mismatched_dtypes():     # GH#32650 dont mix and match datetime/timedelta/period dtypes 
pandas/tests/series/indexing/test_setitem.py0 + / 2 −
@@ -1765,8 +1765,6 @@ def test_setitem_int_not_positional():     # GH#42215 deprecated falling back to positional on __setitem__ with an     #  int not contained in the index; enforced in 2.0     ser = pd.Series([1, 2, 3, 4], index=[1.1, 2.1, 3.0, 4.1])-    assert not ser.index._should_fallback_to_positional-    # assert not ser.index.astype(object)._should_fallback_to_positional      # 3.0 is in our index, so post-enforcement behavior is unchanged     ser[3] = 10