BUG: Fix warning for extra fields in read_csv with on_bad_lines callable

Author: tisjayy

doc/source/whatsnew/v3.0.0.rst

@@ -814,6 +814,10 @@ I/O
 - Bug in :meth:`set_option` where setting the pandas option ``display.html.use_mathjax`` to ``False`` has no effect (:issue:`59884`)
 - Bug in :meth:`to_csv` where ``quotechar``` is not escaped when ``escapechar`` is not None (:issue:`61407`)
 - Bug in :meth:`to_excel` where :class:`MultiIndex` columns would be merged to a single row when ``merge_cells=False`` is passed (:issue:`60274`)
+- Bug in :func:`read_csv` with ``engine="python"`` and callable ``on_bad_lines`` 
+  where a ``ParserWarning`` for extra fields returned by the callable was only 
+  raised when ``index_col`` was ``None``. Now the warning is consistently raised 
+  regardless of ``index_col`` (:issue:`#61837`)
 
 Period
 ^^^^^^

pandas/io/parsers/base_parser.py

@@ -619,11 +619,11 @@ def _check_data_length(
             # error: No overload variant of "__ror__" of "ndarray" matches
             # argument type "ExtensionArray"
             empty_str_or_na = empty_str | isna(data[-1])  # type: ignore[operator]
-            if len(columns) == len(data) - 1 and np.all(empty_str_or_na):
+            if len(data) > len(columns) :
                 return
             warnings.warn(
-                "Length of header or names does not match length of data. This leads "
-                "to a loss of data with index_col=False.",
+                f"Length of header or names ({len(columns)}) does not match number of "
+                f"fields in line ({len(data)}). Extra field will be dropped.",
                 ParserWarning,
                 stacklevel=find_stack_level(),
             )

pandas/tests/io/parser/test_python_parser_only.py

@@ -322,6 +322,20 @@ def test_malformed_skipfooter(python_parser_only):
         parser.read_csv(StringIO(data), header=1, comment="#", skipfooter=1)
 
 
+def test_on_bad_lines_extra_fields_warns(python_parser_only):
+    parser = python_parser_only
+    data = """id,field_1,field_2
+101,A,B
+102,C,D, E
+103,F,G
+"""
+
+    def line_fixer(_line):
+        return ["1", "2", "3", "4", "5"]
+    for index_col in [None, 0]:
+        with pytest.warns(ParserWarning):
+            parser.read_csv(StringIO(data), on_bad_lines=line_fixer, index_col=index_col)
+
 def test_python_engine_file_no_next(python_parser_only):
     parser = python_parser_only