pandas-dev/pandas · #70219
BUG: .loc setitem with an integer slice on a Decimal index
doc/source/whatsnew/v3.2.0.rst1 + / 0 −
@@ -166,6 +166,7 @@ Interval Indexing ^^^^^^^^ - Bug in :meth:`DataFrame.loc` and :meth:`DataFrame.iloc` setitem on a 1-column :class:`DataFrame` with an extension dtype raising when the column key is an all-``False`` boolean mask or an empty list-like (:issue:`70232`)+- Bug in :meth:`DataFrame.loc` and :meth:`Series.loc` setitem with an integer slice raising ``TypeError`` where the same ``.loc`` getitem succeeds, e.g. on an index of :class:`decimal.Decimal` values (:issue:`70219`) - Bug in :meth:`Index.difference` dropping elements that are not in ``other`` when ``other`` was a :class:`DatetimeIndex`, :class:`TimedeltaIndex` or :class:`PeriodIndex` and the elements were strings that parse to its values, or when ``other`` was an :class:`IntervalIndex` and the elements were scalars contained in its intervals (:issue:`58971`) - pandas/core/indexes/base.py3 + / 49 −
@@ -4269,17 +4269,11 @@ def _convert_slice_indexer( kind : {'loc', 'getitem'} """ - # potentially cast the bounds to integers- start, stop, step = key.start, key.stop, key.step-- # figure out if this is a positional indexer- is_index_slice = is_valid_positional_slice(key)- # TODO(GH#50617): once Series.__[gs]etitem__ is removed we should be able # to simplify this. if kind == "getitem": # called from the getitem slicers, validate that we are in fact integers- if is_index_slice:+ if is_valid_positional_slice(key): # In this case the _validate_indexer checks below are redundant return key elif self.dtype.kind in "iu":@@ -4289,37 +4283,9 @@ def _convert_slice_indexer( self._validate_indexer("slice", key.step, "getitem") return key - # convert the slice to an indexer here; checking that the user didn't- # pass a positional slice to loc- is_positional = is_index_slice and self._should_fallback_to_positional-- # if we are mixed and have integers- if is_positional:- try:- # Validate start & stop- if start is not None:- self.get_loc(start)- if stop is not None:- self.get_loc(stop)- is_positional = False- except KeyError:- pass- if com.is_null_slice(key):- # It doesn't matter if we are positional or label based- indexer = key- elif is_positional:- if kind == "loc":- # GH#16121, GH#24612, GH#31810- raise TypeError(- "Slicing a positional slice with .loc is not allowed, "- "Use .loc with labels or .iloc with positions instead.",- )- indexer = key- else:- indexer = self.slice_indexer(start, stop, step)-- return indexer+ return key+ return self.slice_indexer(key.start, key.stop, key.step) @final def _raise_invalid_indexer(@@ -6407,18 +6373,6 @@ def _check_indexing_error(self, key: object) -> None: # would convert to numpy arrays and raise later any way) - GH29926 raise InvalidIndexError(key) - @cache_readonly- def _should_fallback_to_positional(self) -> bool:- """- Should an integer key be treated as positional?- """- return self.inferred_type not in {- "integer",- "mixed-integer",- "floating",- "complex",- }- def get_indexer_non_unique( self, target: Axes ) -> tuple[npt.NDArray[np.intp], npt.NDArray[np.intp]]:pandas/core/indexes/category.py1 + / 8 −
@@ -11,10 +11,7 @@ import numpy as np from pandas._libs import index as libindex-from pandas.util._decorators import (- cache_readonly,- set_module,-)+from pandas.util._decorators import set_module from pandas.core.dtypes.common import is_scalar from pandas.core.dtypes.dtypes import CategoricalDtype@@ -155,10 +152,6 @@ class CategoricalIndex(NDArrayBackedExtensionIndex): def _can_hold_strings(self) -> bool: return self.categories._can_hold_strings - @cache_readonly- def _should_fallback_to_positional(self) -> bool:- return self.categories._should_fallback_to_positional- _data: Categorical _values: Categorical pandas/core/indexes/interval.py0 + / 8 −
@@ -1393,14 +1393,6 @@ def _convert_slice_indexer(self, key: slice, kind: Literal["loc", "getitem"]): return super()._convert_slice_indexer(key, kind) - @cache_readonly- def _should_fallback_to_positional(self) -> bool:- # integer lookups in Series.__getitem__ are unambiguously- # positional in this case- # error: Item "ExtensionDtype"/"dtype[Any]" of "Union[dtype[Any],- # ExtensionDtype]" has no attribute "subtype"- return self.dtype.subtype.kind in "mM" # type: ignore[union-attr]- def _maybe_cast_slice_bound(self, label, side: str): return getattr(self, side)._maybe_cast_slice_bound(label, side) pandas/core/indexes/multi.py0 + / 8 −
@@ -3383,14 +3383,6 @@ def _check_indexing_error(self, key) -> None: # We have to explicitly exclude generators, as these are hashable. raise InvalidIndexError(key) - @cache_readonly- def _should_fallback_to_positional(self) -> bool:- """- Should integer key(s) be treated as positional?- """- # GH#33355- return self.levels[0]._should_fallback_to_positional- def _get_indexer_strict( self, key, axis_name: str ) -> tuple[Index, npt.NDArray[np.intp]]:pandas/core/indexes/range.py0 + / 7 −
@@ -596,13 +596,6 @@ def _get_indexer( locs[valid] = len(self) - 1 - locs[valid] return ensure_platform_int(locs) - @cache_readonly- def _should_fallback_to_positional(self) -> bool:- """- Should an integer key be treated as positional?- """- return False- # -------------------------------------------------------------------- def tolist(self) -> list[int]:pandas/tests/indexing/multiindex/test_partial.py0 + / 2 −
@@ -155,8 +155,6 @@ def test_getitem_intkey_leading_level( assert mi.levels[0].dtype == np.float64 assert 14 not in mi.levels[0]- assert not mi.levels[0]._should_fallback_to_positional- assert not mi._should_fallback_to_positional with pytest.raises(KeyError, match="14"): ser[14]pandas/tests/indexing/test_loc.py30 + / 5 −
@@ -8,6 +8,7 @@ time, timedelta, )+from decimal import Decimal import re from dateutil.tz import gettz@@ -3476,7 +3477,8 @@ def test_loc_with_positional_slice_raises(): # GH#31840 ser = pd.Series(range(4), index=["A", "B", "C", "D"]) - with pytest.raises(TypeError, match="Slicing a positional slice with .loc"):+ msg = r"cannot do slice indexing on Index with these indexers \[3\] of type int"+ with pytest.raises(TypeError, match=msg): ser.loc[:3] = 2 @@ -3496,18 +3498,41 @@ def test_loc_slice_disallows_positional(): with pytest.raises(TypeError, match=msg): obj.loc[1:3] - with pytest.raises(TypeError, match="Slicing a positional slice with .loc"):- # GH#31840 enforce incorrect behavior+ with pytest.raises(TypeError, match=msg): obj.loc[1:3] = 1 with pytest.raises(TypeError, match=msg): df.loc[1:3, 1] - with pytest.raises(TypeError, match="Slicing a positional slice with .loc"):- # GH#31840 enforce incorrect behavior+ with pytest.raises(TypeError, match=msg): df.loc[1:3, 1] = 2 +def test_loc_setitem_int_slice_decimal_index(frame_or_series):+ # GH#70219 integer slice bounds are labels for a Decimal index, as in getitem+ pa = pytest.importorskip("pyarrow")+ values = [Decimal(1), Decimal(3), Decimal(5)]+ index = pd.Index(pd.array(values, dtype=pd.ArrowDtype(pa.decimal128(5, 2))))+ obj = frame_or_series(range(3), index=index)++ expected = frame_or_series([99, 99, 2], index=index)+ tm.assert_equal(obj.loc[0:4], obj.iloc[:2])+ obj.loc[0:4] = 99+ tm.assert_equal(obj, expected)+++def test_loc_setitem_int_slice_decimal_multiindex_level(frame_or_series):+ # GH#70219+ level = pd.Index([Decimal(1), Decimal(3), Decimal(5)], dtype=object)+ index = pd.MultiIndex.from_arrays([level, ["a", "b", "c"]])+ obj = frame_or_series(range(3), index=index)++ expected = frame_or_series([99, 99, 2], index=index)+ tm.assert_equal(obj.loc[0:4], obj.iloc[:2])+ obj.loc[0:4] = 99+ tm.assert_equal(obj, expected)++ def test_loc_datetimelike_mismatched_dtypes(): # GH#32650 dont mix and match datetime/timedelta/period dtypes pandas/tests/series/indexing/test_setitem.py0 + / 2 −
@@ -1765,8 +1765,6 @@ def test_setitem_int_not_positional(): # GH#42215 deprecated falling back to positional on __setitem__ with an # int not contained in the index; enforced in 2.0 ser = pd.Series([1, 2, 3, 4], index=[1.1, 2.1, 3.0, 4.1])- assert not ser.index._should_fallback_to_positional- # assert not ser.index.astype(object)._should_fallback_to_positional # 3.0 is in our index, so post-enforcement behavior is unchanged ser[3] = 10