BUG: handle Unicode thousands and decimal markers in read_csv

@@ -210,6 +210,7 @@ MultiIndex I/O ^^^ - Bug in :class:`DataFrame` repr and :meth:`DataFrame.to_string` where numeric column labels of differing widths were misaligned with their column's values (:issue:`8300`)+- Bug in :func:`read_csv` silently misparsing non-ASCII ``thousands`` and ``decimal`` markers with the C engine; these now fall back to the Python engine with a :class:`errors.ParserWarning`, or raise ``ValueError`` if ``engine="c"`` is explicitly specified (:issue:`71158`) - Bug in :func:`read_csv` with ``engine="c"`` and ``comment`` not skipping a line containing only whitespace before the comment character (:issue:`55916`) - Bug in :func:`read_csv` with ``engine="python"`` and ``index_col`` reading a date like ``"20180101"`` as nanoseconds since the epoch when ``dtype`` is a pyarrow timestamp dtype (:issue:`24542`) - Bug in :func:`read_csv`, :func:`read_excel`, :func:`read_fwf` and :func:`read_html` returning wrong column labels and treating header rows as data when ``header`` was a list not in increasing order, such as ``header=[1, 0]`` (:issue:`47011`)

@@ -581,11 +581,15 @@ cdef class TextReader:          if len(decimal) != 1:             raise ValueError("Only length-1 decimal markers supported")+        if ord(decimal) > 127:+            raise ValueError("Only ASCII decimal markers supported")         self.parser.decimal = <char>ord(decimal)          if thousands is not None:             if len(thousands) != 1:                 raise ValueError("Only length-1 thousands markers supported")+            if ord(thousands) > 127:+                raise ValueError("Only ASCII thousands markers supported")             self.parser.thousands = <char>ord(thousands)          if escapechar is not None:

@@ -589,16 +589,17 @@ def _can_parallelize_csv(filepath_or_buffer, kwds: Mapping[str, Any]) -> bool:     if delimiter is None or (len(delimiter) > 1 and delimiter != r"\s+"):         return False -    # A separator or quotechar wider than one byte forces the python engine-    # too, and warns on the way.  The eligibility check has to catch these+    # A separator, quotechar or numeric marker wider than one byte forces+    # the python engine too, and warns on the way. Catch these options     # here: by the time the name-inference read reports the python engine, it     # has already raised that warning, which the serial read then raises     # again.  GH#66259     if len(delimiter) == 1 and ord(delimiter) > 127:         return False-    quotechar = kwds.get("quotechar")-    if isinstance(quotechar, str) and len(quotechar) == 1 and ord(quotechar) > 127:-        return False+    for option in ("quotechar", "thousands", "decimal"):+        marker = kwds.get(option)+        if isinstance(marker, str) and len(marker) == 1 and ord(marker) > 127:+            return False      # Full-line comments before/inside the preamble shift the header location     # in ways _find_data_start_offset cannot see.@@ -2961,6 +2962,22 @@ def _clean_options(                 )                 engine = "python" +        if engine == "c":+            for option in ("thousands", "decimal"):+                marker = options[option]+                if (+                    isinstance(marker, (str, bytes))+                    and len(marker) == 1+                    and ord(marker) > 127+                ):+                    # The C parser consumes UTF-8 bytes, so even a code point+                    # below 256 cannot be represented by its single char.+                    fallback_reason = (+                        f"the 'c' engine does not support non-ASCII {option} markers"+                    )+                    engine = "python"+                    break+         if fallback_reason and self._engine_specified:             raise ValueError(fallback_reason) 

@@ -25,6 +25,44 @@ skip_pyarrow = pytest.mark.usefixtures("pyarrow_skip")  +@pytest.mark.parametrize("engine", [None, "c", "python"])+@pytest.mark.parametrize(+    "option,marker,value",+    [+        ("thousands", "\u202f", "1/234"),+        ("decimal", "\u066b", "1k5"),+        # Code points below 256 also occupy multiple bytes in UTF-8.+        ("thousands", "\u00a0", "1/234"),+        ("decimal", "\u00a0", "1k5"),+    ],+)+def test_unicode_numeric_marker(engine, option, marker, value):+    # GH#71158: truncating the code point to a byte matched unrelated ASCII.+    data = StringIO(f"number,text\n1{marker}234,{value}\n")+    kwargs = {option: marker, "engine": engine}+    msg = f"the 'c' engine does not support non-ASCII {option} markers"+    if engine == "c":+        with pytest.raises(ValueError, match=msg):+            pd.read_csv(data, **kwargs)+        return++    warning = ParserWarning if engine is None else None+    with tm.assert_produces_warning(warning, match=msg):+        result = pd.read_csv(data, **kwargs)+    expected = pd.DataFrame(+        {"number": [1234 if option == "thousands" else 1.234], "text": [value]}+    )+    tm.assert_frame_equal(result, expected)+++@pytest.mark.parametrize("option", ["thousands", "decimal"])+def test_unicode_numeric_marker_fallback_unsupported_option(option):+    # GH#71158: fallback must not silently discard C-engine-only options.+    msg = f"Falling back.*non-ASCII {option} markers.*low_memory"+    with pytest.raises(ValueError, match=msg):+        pd.read_csv(StringIO("value\n1234\n"), low_memory=False, **{option: "\u202f"})++ def test_bytes_io_input(all_parsers):     encoding = "cp1255"     parser = all_parsers

@@ -2073,9 +2073,12 @@ def test_parallel_fallback_does_not_repeat_python_layer_warning(tmp_path, monkey     ]  -@pytest.mark.parametrize("kwargs", [{"quotechar": "»"}, {"sep": "§"}])-def test_multibyte_sep_or_quotechar_warns_once(tmp_path, monkeypatch, kwargs):-    # Both force TextFileReader onto the python engine, which warns while+@pytest.mark.parametrize(+    "kwargs",+    [{"quotechar": "»"}, {"sep": "§"}, {"thousands": "\u202f"}, {"decimal": "\u066b"}],+)+def test_multibyte_parser_options_warn_once(tmp_path, monkeypatch, kwargs):+    # These force TextFileReader onto the python engine, which warns while     # falling back.  The parallel path has to decline them up front: its     # name-inference read would raise that warning before it can notice the     # engine it got, and the serial read then raises it again (GH#66259).

@@ -33,6 +33,16 @@   class TestTextReader:+    @pytest.mark.parametrize("option", ["thousands", "decimal"])+    @pytest.mark.parametrize("marker", ["\u00a0", "\u202f", "\u066b"])+    def test_unicode_numeric_marker(self, option, marker):+        # GH#71158: reject unsupported markers before casting to char.+        msg = f"Only ASCII {option} markers supported"+        with pytest.raises(ValueError, match=msg):+            TextReader(+                StringIO("value\n1234\n"), **{option: marker}, **_na_value_kwargs+            )+     @pytest.fixture     def csv_path(self, datapath):         return datapath("io", "data", "csv", "test1.csv")