ENH: sniff sep=None with the C engine in read_csv (GH#9645)

@@ -66,11 +66,10 @@ filepath_or_buffer : various   :class:`~python:io.StringIO`). sep : str, defaults to ``','`` for :func:`read_csv`, ``\t`` for :func:`read_table`   Delimiter to use. ``sep=None`` detects the separator from the first valid row-  of the file with Python's builtin sniffer tool, :class:`python:csv.Sniffer`; it-  is supported only by the Python parsing engine, which will be used-  automatically. In addition, separators longer than 1 character-  and different from ``'\s+'`` will be interpreted as regular expressions and-  will force the use of the Python parsing engine. Note that regex+  of the file with Python's builtin sniffer tool, :class:`python:csv.Sniffer`;+  the pyarrow engine does not support it. In addition, separators longer than+  1 character and different from ``'\s+'`` will be interpreted as regular+  expressions and will force the use of the Python parsing engine. Note that regex   delimiters are prone to ignoring quoted data. Regex example: ``'\\r\\t'``. delimiter : str, optional   Alternative argument name for sep.@@ -1433,15 +1432,13 @@ Automatically "sniffing" the delimiter  ``read_csv`` is capable of inferring delimited (not necessarily comma-separated) files, as pandas uses the :class:`python:csv.Sniffer`-class of the csv module. For this, you have to specify ``sep=None``. Passing-``engine='python'`` as well avoids the ``ParserWarning`` raised by the-fallback.+class of the csv module. For this, you have to specify ``sep=None``.  .. ipython:: python     df = pd.DataFrame(np.random.randn(10, 4))    df.to_csv("tmp2.csv", sep=":", index=False)-   pd.read_csv("tmp2.csv", sep=None, engine="python")+   pd.read_csv("tmp2.csv", sep=None)  .. ipython:: python    :suppress:@@ -1515,7 +1512,7 @@ three engines:    "Relative speed","fast","slowest","fastest on large workloads"    "Multithreaded",":ref:`for large files <io.csv.parallel>`","no","yes"    "Regex or multi-character ``sep``","no","yes","no"-   "``sep=None`` (auto-detect the separator)","no","yes","no"+   "``sep=None`` (auto-detect the separator)","yes","yes","no"    "``skipfooter``","no","yes","no"    "``low_memory``","yes","no","no"    "``lineterminator``","yes","no","no"

@@ -28,6 +28,7 @@ enhancement2  Other enhancements ^^^^^^^^^^^^^^^^^^+- :func:`read_csv` and :func:`read_table` with ``sep=None`` now sniff the separator with the C engine instead of falling back to the slower python engine (:issue:`9645`) - :meth:`.DataFrameGroupBy.rank` and :meth:`.SeriesGroupBy.rank` now accept a ``numeric_only`` argument, matching :meth:`DataFrame.rank` (:issue:`44438`) - Display formatting for complex numbers in ``object`` dtype and for ``interval`` dtype values with float endpoints now respects the ``display.precision`` and ``display.float_format`` options (:issue:`25920`). - The :class:`ExtensionArray` base tests in ``pandas.tests.extension.base`` now issue a :class:`errors.PerformanceWarning` when the array uses the default, possibly slow, implementation of ``unique``, ``factorize``, ``argsort``, ``argmin``, ``argmax``, or ``searchsorted`` (:issue:`24433`)

@@ -10,6 +10,7 @@ from collections import (     abc,     defaultdict,+    deque, ) from concurrent.futures import ThreadPoolExecutor import contextlib@@ -18,6 +19,7 @@ import mmap import os import queue+import re import sys import threading from typing import (@@ -108,6 +110,7 @@         Callable,         Hashable,         Iterable,+        Iterator,         Mapping,         Sequence,     )@@ -261,6 +264,10 @@ def _validate_bool_kwargs(kwds: Mapping[str, Any]) -> None: # avoids oversubscribing the machine.  mode.max_threads overrides it in either # direction. _MAX_DEFAULT_WORKERS = 6++# bytes read per call while looking for the row to sniff for sep=None+_SNIFF_CHUNK_SIZE = 64 * 1024+ _pyarrow_unsupported = {     "skipfooter",     "float_precision",@@ -575,9 +582,9 @@ def _can_parallelize_csv(filepath_or_buffer, kwds: Mapping[str, Any]) -> bool:         if codec_name not in ("utf-8", "utf-8-sig"):             return False -    # Separators that force the python-engine fallback inside TextFileReader-    # (sep=None sniffing, or multi-char/regex seps other than r"\s+") cannot-    # use the C-engine buffer-loading fast path.+    # sep=None is only resolved once TextFileReader opens the file, and+    # multi-char/regex seps other than r"\s+" force the python engine, so+    # neither can use the C-engine buffer-loading fast path.     delimiter = kwds.get("delimiter", ",")     if delimiter is None or (len(delimiter) > 1 and delimiter != r"\s+"):         return False@@ -1470,8 +1477,7 @@ def read_csv(     sep : str, default ','         Character or regex pattern to treat as the delimiter. ``sep=None`` detects         the separator from the first valid row of the file with Python's builtin-        sniffer tool, ``csv.Sniffer``; it is supported only by the Python parsing-        engine, which will be used automatically.+        sniffer tool, ``csv.Sniffer``; the pyarrow engine does not support it.         In addition, separators longer than 1 character and different from         ``'\\s+'`` will be interpreted as regular expressions and will force         the use of the Python parsing engine. Note that regex delimiters are prone@@ -2078,8 +2084,7 @@ def read_table(     sep : str, default '\\t' (tab-stop)         Character or regex pattern to treat as the delimiter. ``sep=None`` detects         the separator from the first valid row of the file with Python's builtin-        sniffer tool, ``csv.Sniffer``; it is supported only by the Python parsing-        engine, which will be used automatically.+        sniffer tool, ``csv.Sniffer``; the pyarrow engine does not support it.         In addition, separators longer than 1 character and different from         ``'\\s+'`` will be interpreted as regular expressions and will force         the use of the Python parsing engine. Note that regex delimiters are prone@@ -2904,8 +2909,8 @@ def _clean_options(         sep = options["delimiter"]          if sep is None:-            # sniffing the separator with csv.Sniffer is python-engine only-            if engine in ("c", "pyarrow"):+            # the c engine sniffs the separator in _make_engine+            if engine == "pyarrow":                 fallback_reason = f"the '{engine}' engine does not support sep=None"                 engine = "python"         elif len(sep) > 1:@@ -3092,12 +3097,53 @@ def _make_engine(             raise ValueError(msg)          try:+            if engine == "c" and self.options.get("delimiter", ",") is None:+                assert not isinstance(f, list)  # rejected above for the c engine+                return mapping[engine](self._sniff_delimiter(f), **self.options)             return mapping[engine](f, **self.options)         except Exception:             if self.handles is not None:                 self.handles.close()             raise +    def _sniff_delimiter(self, f: IO[Any]) -> _ReplayHandle:+        """+        Set the delimiter for ``sep=None`` by sniffing the first row that is not+        in ``skiprows``, blank or a full-line comment.++        Returns a handle that replays the data read here and then the rest of ``f``.+        """+        skiprows = self.options["skiprows"]+        skipfunc = skiprows if callable(skiprows) else skiprows.__contains__+        comment = self.options["comment"]+        chunks: list[bytes | str] = []+        delimiter = None+        for pos, raw in enumerate(+            _iter_physical_lines(f, chunks, self.options["lineterminator"])+        ):+            if skipfunc(pos) or not raw:+                continue+            line = raw.decode("utf-8-sig", "replace") if isinstance(raw, bytes) else raw+            line += "\n"+            if comment is not None and comment in line:+                line = line[: line.find(comment)]+                if not line:+                    continue+            delimiter = csv.Sniffer().sniff(line).delimiter+            break++        if delimiter is None:+            # nothing to sniff; the c engine reports the empty file+            delimiter = ","+        elif len(delimiter.encode("utf-8", "surrogatepass")) > 1:+            raise ValueError(+                f"sep=None detected the separator {delimiter!r}, which the 'c' "+                "engine does not support as it is more than one byte in utf-8; "+                "specify engine='python'."+            )+        self.options["delimiter"] = delimiter+        return _ReplayHandle(chunks, f)+     def _failover_to_python(self) -> None:         raise AbstractMethodError(self) @@ -3643,3 +3689,53 @@ def _validate_skipfooter(kwds: dict[str, Any]) -> None:             raise ValueError("'skipfooter' not supported for iteration")         if kwds.get("nrows"):             raise ValueError("'skipfooter' not supported with 'nrows'")+++def _iter_physical_lines(+    handle: IO[Any], chunks: list[bytes | str], lineterminator: str | None+) -> Iterator[bytes | str]:+    """+    Yield each line of ``handle`` without its terminator, appending every chunk+    read to ``chunks``.+    """+    eol: re.Pattern[Any] | None = None+    partial: list[Any] = []+    skip_lf = False+    while True:+        chunk = handle.read(_SNIFF_CHUNK_SIZE)+        if not chunk:+            tail = partial[0][:0].join(partial) if partial else ""+            if tail:+                yield tail+            return+        chunks.append(chunk)+        if eol is None:+            pattern = re.escape(lineterminator) if lineterminator else r"\r\n|\r|\n"+            eol = re.compile(pattern.encode() if isinstance(chunk, bytes) else pattern)+        start = 0+        if skip_lf and chunk[:1] in (b"\n", "\n"):+            # the second half of a "\r\n" split across chunks+            start = 1+        for match in eol.finditer(chunk, start):+            partial.append(chunk[start : match.start()])+            yield chunk[:0].join(partial)+            partial = []+            start = match.end()+        partial.append(chunk[start:])+        skip_lf = not lineterminator and chunk[-1:] in (b"\r", "\r")+++class _ReplayHandle:+    """+    Serve ``chunks`` and then the rest of ``handle`` through ``read``, the only+    method the c engine calls on its source; it accepts reads of any length.+    """++    def __init__(self, chunks: list[bytes | str], handle: IO[Any]) -> None:+        self._chunks = deque(chunks)+        self._handle = handle++    def read(self, size: int) -> bytes | str:+        if self._chunks:+            return self._chunks.popleft()+        return self._handle.read(size)

@@ -386,8 +386,8 @@ def test_open_file(all_parsers, temp_file):      msg = "Could not determine delimiter"     err = csv.Error-    if parser.engine in ("c", "pyarrow"):-        msg = f"the '{parser.engine}' engine does not support sep=None"+    if parser.engine == "pyarrow":+        msg = "the 'pyarrow' engine does not support sep=None"         err = ValueError      file = Path(temp_file)

@@ -24,6 +24,7 @@ from pandas.compat import WASM from pandas.errors import (     DtypeWarning,+    EmptyDataError,     Pandas4Warning,     ParserError,     ParserWarning,@@ -1899,3 +1900,96 @@ def test_unparseable_dtype_names_offending_value(      with pytest.raises(ValueError, match=re.escape(repr(offender))):         parser.read_csv(StringIO(data), sep=";", dtype={"b": dtype}, **kwargs)+++_SNIFF_EXPECTED = pd.DataFrame(+    {"A": [1, 3], "B": [2, 4]}, index=pd.Index(["foo", "bar"], name="index")+)+++@pytest.mark.parametrize("as_bytes", [True, False])+@pytest.mark.parametrize(+    "data,kwargs",+    [+        ("index|A|B\nfoo|1|2\nbar|3|4\n", {}),+        ("index|A|B\r\nfoo|1|2\r\nbar|3|4\r\n", {}),+        ("\nindex|A|B\nfoo|1|2\nbar|3|4\n", {}),+        ("#a,b\nindex|A|B#c,d\n#e\nfoo|1|2\nbar|3|4\n", {"comment": "#"}),+        ("a,b\nc,d\nindex|A|B\nfoo|1|2\nbar|3|4\n", {"skiprows": 2}),+        ("a,b\nindex|A|B\nfoo|1|2\nbar|3|4\n", {"skiprows": lambda x: x == 0}),+    ],+)+def test_sniff_delimiter(c_parser_only, data, kwargs, as_bytes):+    # GH#9645 sniff from the first row not skipped, blank or commented+    parser = c_parser_only+    buf = BytesIO(data.encode()) if as_bytes else StringIO(data)+    result = parser.read_csv(buf, sep=None, index_col=0, **kwargs)+    tm.assert_frame_equal(result, _SNIFF_EXPECTED)+++@pytest.mark.parametrize("chunk_size", [1, 2, 4])+def test_sniff_delimiter_small_reads(c_parser_only, monkeypatch, chunk_size):+    # GH#9645 lines, including "\r\n", split across the reads made while sniffing+    monkeypatch.setattr("pandas.io.parsers.readers._SNIFF_CHUNK_SIZE", chunk_size)+    parser = c_parser_only+    data = "a,b\r\nc,d\r\nindex|A|B\r\nfoo|1|2\r\nbar|3|4\r\n"+    result = parser.read_csv(BytesIO(data.encode()), sep=None, index_col=0, skiprows=2)+    tm.assert_frame_equal(result, _SNIFF_EXPECTED)+++def test_sniff_delimiter_mid_stream(c_parser_only):+    # GH#9645 parsing starts where the handle was, not at the start+    parser = c_parser_only+    buf = BytesIO(b"junk\nindex|A|B\nfoo|1|2\nbar|3|4\n")+    buf.readline()+    result = parser.read_csv(buf, sep=None, index_col=0)+    tm.assert_frame_equal(result, _SNIFF_EXPECTED)+++def test_sniff_delimiter_chunksize(c_parser_only):+    # GH#9645+    parser = c_parser_only+    data = "index|A|B\nfoo|1|2\nbar|3|4\n"+    with parser.read_csv(StringIO(data), sep=None, index_col=0, chunksize=1) as reader:+        result = pd.concat(reader)+    tm.assert_frame_equal(result, _SNIFF_EXPECTED)+++@pytest.mark.parametrize("key", ["sep", "delimiter"])+def test_sniff_delimiter_default_engine(key):+    # GH#9645 the default engine sniffs rather than falling back to python+    with tm.assert_produces_warning(None):+        result = pd.read_csv(StringIO("a;b\n1;2\n"), **{key: None})+    expected = pd.DataFrame({"a": [1], "b": [2]})+    tm.assert_frame_equal(result, expected)+++@pytest.mark.parametrize("data", ["", "a,b\n", "#a,b\n"])+def test_sniff_delimiter_no_rows(c_parser_only, data):+    # GH#9645+    parser = c_parser_only+    with pytest.raises(EmptyDataError, match="No columns to parse from file"):+        parser.read_csv(StringIO(data), sep=None, skiprows=1, comment="#")+++def test_sniff_delimiter_multibyte(c_parser_only):+    # GH#9645 the c engine only supports single-byte separators+    parser = c_parser_only+    data = '"a"\u00a7"b"\n"1"\u00a7"2"\n'+    msg = "sep=None detected the separator '\u00a7'"+    with pytest.raises(ValueError, match=msg):+        parser.read_csv(StringIO(data), sep=None)+++def test_sniff_delimiter_lone_surrogate(c_parser_only):+    # GH#9645+    parser = c_parser_only+    data = b'"a"\x80"b"\n"1"\x80"2"\n'+    msg = r"sep=None detected the separator '\\udc80'"+    with pytest.raises(ValueError, match=msg):+        parser.read_csv(+            BytesIO(data),+            sep=None,+            encoding="ascii",+            encoding_errors="surrogateescape",+        )

@@ -195,8 +195,8 @@ def test_rejects_non_utf8_encoding(self, tmp_path, monkeypatch, encoding):         assert _can_parallelize_csv(path, self._kwds(encoding="utf-8-sig"))      def test_rejects_python_engine_seps(self, tmp_path, monkeypatch):-        # Multi-char/regex seps (other than r"\s+") and sep=None force the-        # python engine inside TextFileReader (GH#64347).+        # Multi-char/regex seps (other than r"\s+") force the python engine+        # and sep=None is sniffed inside TextFileReader (GH#64347).         path = tmp_path / "data.csv"         path.write_text("a,b\n1,2\n", encoding="utf-8")         monkeypatch.setattr(_readers, "_PARALLEL_READ_MIN_BYTES", 1)

@@ -18,7 +18,6 @@     ParserError, ) -import pandas as pd import pandas._testing as tm  from pandas.io.parsers import read_csv@@ -220,30 +219,13 @@ def test_invalid_file_inputs(request, all_parsers):   @pytest.mark.parametrize("key", ["sep", "delimiter"])-def test_sep_none_falls_back_to_python_engine(key):-    # GH#66639, GH#47024 sniffing the separator is python-engine only, so the-    # default engine falls back to it rather than handing sep=None to the C parser-    data = "a;b\n1;2\n"-    expected = pd.DataFrame({"a": [1], "b": [2]})--    with tm.assert_produces_warning(parsers.ParserWarning, match="sep=None"):-        result = read_csv(StringIO(data), **{key: None})-    tm.assert_frame_equal(result, expected)--    with tm.assert_produces_warning(None):-        result = read_csv(StringIO(data), engine="python", **{key: None})-    tm.assert_frame_equal(result, expected)---@pytest.mark.parametrize("key", ["sep", "delimiter"])-@pytest.mark.parametrize("engine", ["c", "pyarrow"])-def test_sep_none_explicit_engine_raises(engine, key):+def test_sep_none_pyarrow_raises(key):     # GH#66639, GH#47024 an explicitly requested engine that cannot sniff the     # separator reports that rather than falling back-    msg = f"the '{engine}' engine does not support sep=None"+    msg = "the 'pyarrow' engine does not support sep=None"      with pytest.raises(ValueError, match=msg):-        read_csv(StringIO("a;b\n1;2\n"), engine=engine, **{key: None})+        read_csv(StringIO("a;b\n1;2\n"), engine="pyarrow", **{key: None})   def test_invalid_dtype_backend(all_parsers):