Merge remote-tracking branch 'upstream/main'

Author: DevastatingRPG

.github/workflows/unit-tests.yml

@@ -30,7 +30,7 @@ jobs:
         env_file: [actions-310.yaml, actions-311.yaml, actions-312.yaml, actions-313.yaml]
         # Prevent the include jobs from overriding other jobs
         pattern: [""]
-        pandas_future_infer_string: ["0"]
+        pandas_future_infer_string: ["1"]
         include:
           - name: "Downstream Compat"
             env_file: actions-311-downstream_compat.yaml
@@ -45,6 +45,10 @@ jobs:
             env_file: actions-313-freethreading.yaml
             pattern: "not slow and not network and not single_cpu"
             platform: ubuntu-24.04
+          - name: "Without PyArrow"
+            env_file: actions-312.yaml
+            pattern: "not slow and not network and not single_cpu"
+            platform: ubuntu-24.04
           - name: "Locale: it_IT"
             env_file: actions-311.yaml
             pattern: "not slow and not network and not single_cpu"
@@ -67,18 +71,9 @@ jobs:
             # It will be temporarily activated during tests with locale.setlocale
             extra_loc: "zh_CN"
             platform: ubuntu-24.04
-          - name: "Future infer strings"
+          - name: "Past no infer strings"
             env_file: actions-312.yaml
-            pandas_future_infer_string: "1"
-            platform: ubuntu-24.04
-          - name: "Future infer strings (without pyarrow)"
-            env_file: actions-311.yaml
-            pandas_future_infer_string: "1"
-            platform: ubuntu-24.04
-          - name: "Pypy"
-            env_file: actions-pypy-39.yaml
-            pattern: "not slow and not network and not single_cpu"
-            test_args: "--max-worker-restart 0"
+            pandas_future_infer_string: "0"
             platform: ubuntu-24.04
           - name: "Numpy Dev"
             env_file: actions-311-numpydev.yaml
@@ -88,7 +83,6 @@ jobs:
           - name: "Pyarrow Nightly"
             env_file: actions-311-pyarrownightly.yaml
             pattern: "not slow and not network and not single_cpu"
-            pandas_future_infer_string: "1"
             platform: ubuntu-24.04
       fail-fast: false
     name: ${{ matrix.name || format('{0} {1}', matrix.platform, matrix.env_file) }}
@@ -103,7 +97,7 @@ jobs:
       PYTEST_TARGET: ${{ matrix.pytest_target || 'pandas' }}
       # Clipboard tests
       QT_QPA_PLATFORM: offscreen
-      REMOVE_PYARROW: ${{ matrix.name == 'Future infer strings (without pyarrow)' && '1' || '0' }}
+      REMOVE_PYARROW: ${{ matrix.name == 'Without PyArrow' && '1' || '0' }}
     concurrency:
       # https://github.community/t/concurrecy-not-work-for-push/183068/7
       group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.extra_apt || '' }}-${{ matrix.pandas_future_infer_string }}-${{ matrix.platform }}
@@ -140,9 +134,6 @@ jobs:
 
       moto:
         image: motoserver/moto:5.0.27
-        env:
-          AWS_ACCESS_KEY_ID: foobar_key
-          AWS_SECRET_ACCESS_KEY: foobar_secret
         ports:
           - 5000:5000
 
@@ -172,12 +163,9 @@ jobs:
       with:
         # xref https://github.com/cython/cython/issues/6870
         werror: ${{ matrix.name != 'Freethreading' }}
-      # TODO: Re-enable once Pypy has Pypy 3.10 on conda-forge
-      if: ${{ matrix.name != 'Pypy' }}
 
     - name: Test (not single_cpu)
       uses: ./.github/actions/run-tests
-      if: ${{ matrix.name != 'Pypy' }}
       env:
         # Set pattern to not single_cpu if not already set
         PATTERN: ${{ env.PATTERN == '' && 'not single_cpu' || matrix.pattern }}

.github/workflows/wheels.yml

@@ -101,7 +101,6 @@ jobs:
         - [macos-14, macosx_arm64]
         - [windows-2022, win_amd64]
         - [windows-11-arm, win_arm64]
-        # TODO: support PyPy?
         python: [["cp310", "3.10"], ["cp311", "3.11"], ["cp312", "3.12"], ["cp313", "3.13"], ["cp313t", "3.13"]]
         include:
         # Build Pyodide wheels and upload them to Anaconda.org

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ ci:
     skip: [pyright, mypy]
 repos:
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.11.12
+    rev: v0.12.2
     hooks:
     -   id: ruff
         args: [--exit-non-zero-on-fix]
@@ -47,7 +47,7 @@ repos:
         types_or: [python, rst, markdown, cython, c]
         additional_dependencies: [tomli]
 -   repo: https://github.com/MarcoGorelli/cython-lint
-    rev: v0.16.6
+    rev: v0.16.7
     hooks:
     -   id: cython-lint
     -   id: double-quote-cython-strings
@@ -95,14 +95,14 @@ repos:
     - id: sphinx-lint
       args: ["--enable", "all", "--disable", "line-too-long"]
 -   repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: v20.1.5
+    rev: v20.1.7
     hooks:
     - id: clang-format
       files: ^pandas/_libs/src|^pandas/_libs/include
       args: [-i]
       types_or: [c, c++]
 -   repo: https://github.com/trim21/pre-commit-mirror-meson
-    rev: v1.8.1
+    rev: v1.8.2
     hooks:
     - id: meson-fmt
       args: ['--inplace']

AUTHORS.md

@@ -7,12 +7,12 @@ About the Copyright Holders
     led by Wes McKinney. AQR released the source under this license in 2009.
 *   Copyright (c) 2011-2012, Lambda Foundry, Inc.
 
-    Wes is now an employee of Lambda Foundry, and remains the pandas project
+    Wes became an employee of Lambda Foundry, and remained the pandas project
     lead.
 *   Copyright (c) 2011-2012, PyData Development Team
 
     The PyData Development Team is the collection of developers of the PyData
-    project. This includes all of the PyData sub-projects, including pandas. The
+    project. This includes all of the PyData sub-projects, such as pandas. The
     core team that coordinates development on GitHub can be found here:
     https://github.com/pydata.
 
@@ -23,11 +23,11 @@ Our Copyright Policy
 
 PyData uses a shared copyright model. Each contributor maintains copyright
 over their contributions to PyData. However, it is important to note that
-these contributions are typically only changes to the repositories. Thus,
+these contributions are typically limited to changes to the repositories. Thus,
 the PyData source code, in its entirety, is not the copyright of any single
 person or institution. Instead, it is the collective copyright of the
 entire PyData Development Team. If individual contributors want to maintain
-a record of what changes/contributions they have specific copyright on,
+a record of the specific changes or contributions they hold copyright to,
 they should indicate their copyright in the commit message of the change
 when they commit the change to one of the PyData repositories.
 
@@ -50,7 +50,7 @@ Other licenses can be found in the LICENSES directory.
 License
 =======
 
-pandas is distributed under a 3-clause ("Simplified" or "New") BSD
+pandas is distributed under the 3-clause ("Simplified" or "New") BSD
 license. Parts of NumPy, SciPy, numpydoc, bottleneck, which all have
-BSD-compatible licenses, are included. Their licenses follow the pandas
+BSD-compatible licenses, are included. Their licenses are compatible with the pandas
 license.

README.md

@@ -175,7 +175,7 @@ All contributions, bug reports, bug fixes, documentation improvements, enhanceme
 
 A detailed overview on how to contribute can be found in the **[contributing guide](https://pandas.pydata.org/docs/dev/development/contributing.html)**.
 
-If you are simply looking to start working with the pandas codebase, navigate to the [GitHub "issues" tab](https://github.com/pandas-dev/pandas/issues) and start looking through interesting issues. There are a number of issues listed under [Docs](https://github.com/pandas-dev/pandas/issues?labels=Docs&sort=updated&state=open) and [good first issue](https://github.com/pandas-dev/pandas/issues?labels=good+first+issue&sort=updated&state=open) where you could start out.
+If you are simply looking to start working with the pandas codebase, navigate to the [GitHub "issues" tab](https://github.com/pandas-dev/pandas/issues) and start looking through interesting issues. There are a number of issues listed under [Docs](https://github.com/pandas-dev/pandas/issues?q=is%3Aissue%20state%3Aopen%20label%3ADocs%20sort%3Aupdated-desc) and [good first issue](https://github.com/pandas-dev/pandas/issues?q=is%3Aissue%20state%3Aopen%20label%3A%22good%20first%20issue%22%20sort%3Aupdated-desc) where you could start out.
 
 You can also triage issues which may include reproducing bug reports, or asking for vital information such as version numbers or reproduction instructions. If you would like to start triaging issues, one easy way to get started is to [subscribe to pandas on CodeTriage](https://www.codetriage.com/pandas-dev/pandas).
 

asv_bench/benchmarks/gil.py

@@ -36,7 +36,7 @@
 from .pandas_vb_common import BaseIO  # isort:skip
 
 
-def test_parallel(num_threads=2, kwargs_list=None):
+def run_parallel(num_threads=2, kwargs_list=None):
     """
     Decorator to run the same function multiple times in parallel.
 
@@ -95,7 +95,7 @@ def setup(self, threads, method):
             {"key": np.random.randint(0, ngroups, size=N), "data": np.random.randn(N)}
         )
 
-        @test_parallel(num_threads=threads)
+        @run_parallel(num_threads=threads)
         def parallel():
             getattr(df.groupby("key")["data"], method)()
 
@@ -123,7 +123,7 @@ def setup(self, threads):
         ngroups = 10**3
         data = Series(np.random.randint(0, ngroups, size=size))
 
-        @test_parallel(num_threads=threads)
+        @run_parallel(num_threads=threads)
         def get_groups():
             data.groupby(data).groups
 
@@ -142,7 +142,7 @@ def setup(self, dtype):
         df = DataFrame({"col": np.arange(N, dtype=dtype)})
         indexer = np.arange(100, len(df) - 100)
 
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def parallel_take1d():
             take_nd(df["col"].values, indexer)
 
@@ -163,7 +163,7 @@ def setup(self):
         k = 5 * 10**5
         kwargs_list = [{"arr": np.random.randn(N)}, {"arr": np.random.randn(N)}]
 
-        @test_parallel(num_threads=2, kwargs_list=kwargs_list)
+        @run_parallel(num_threads=2, kwargs_list=kwargs_list)
         def parallel_kth_smallest(arr):
             algos.kth_smallest(arr, k)
 
@@ -180,42 +180,42 @@ def setup(self):
         self.period = self.dti.to_period("D")
 
     def time_datetime_field_year(self):
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def run(dti):
             dti.year
 
         run(self.dti)
 
     def time_datetime_field_day(self):
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def run(dti):
             dti.day
 
         run(self.dti)
 
     def time_datetime_field_daysinmonth(self):
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def run(dti):
             dti.days_in_month
 
         run(self.dti)
 
     def time_datetime_field_normalize(self):
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def run(dti):
             dti.normalize()
 
         run(self.dti)
 
     def time_datetime_to_period(self):
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def run(dti):
             dti.to_period("s")
 
         run(self.dti)
 
     def time_period_to_datetime(self):
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def run(period):
             period.to_timestamp()
 
@@ -232,7 +232,7 @@ def setup(self, method):
         if hasattr(DataFrame, "rolling"):
             df = DataFrame(arr).rolling(win)
 
-            @test_parallel(num_threads=2)
+            @run_parallel(num_threads=2)
             def parallel_rolling():
                 getattr(df, method)()
 
@@ -249,7 +249,7 @@ def parallel_rolling():
                 "std": rolling_std,
             }
 
-            @test_parallel(num_threads=2)
+            @run_parallel(num_threads=2)
             def parallel_rolling():
                 rolling[method](arr, win)
 
@@ -286,7 +286,7 @@ def setup(self, dtype):
         self.fname = f"__test_{dtype}__.csv"
         df.to_csv(self.fname)
 
-        @test_parallel(num_threads=2)
+        @run_parallel(num_threads=2)
         def parallel_read_csv():
             read_csv(self.fname)
 
@@ -305,7 +305,7 @@ class ParallelFactorize:
     def setup(self, threads):
         strings = Index([f"i-{i}" for i in range(100000)], dtype=object)
 
-        @test_parallel(num_threads=threads)
+        @run_parallel(num_threads=threads)
         def parallel():
             factorize(strings)
 

asv_bench/benchmarks/io/csv.py

@@ -53,6 +53,25 @@ def time_frame(self, kind):
         self.df.to_csv(self.fname)
 
 
+class ToCSVFloatFormatVariants(BaseIO):
+    fname = "__test__.csv"
+
+    def setup(self):
+        self.df = DataFrame(np.random.default_rng(seed=42).random((1000, 1000)))
+
+    def time_old_style_percent_format(self):
+        self.df.to_csv(self.fname, float_format="%.6f")
+
+    def time_new_style_brace_format(self):
+        self.df.to_csv(self.fname, float_format="{:.6f}")
+
+    def time_new_style_thousands_format(self):
+        self.df.to_csv(self.fname, float_format="{:,.2f}")
+
+    def time_callable_format(self):
+        self.df.to_csv(self.fname, float_format=lambda x: f"{x:.6f}")
+
+
 class ToCSVMultiIndexUnusedLevels(BaseIO):
     fname = "__test__.csv"
 

ci/code_checks.sh

@@ -58,7 +58,9 @@ if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then
 
     MSG='Python and Cython Doctests' ; echo "$MSG"
     python -c 'import pandas as pd; pd.test(run_doctests=True)'
-    RET=$(($RET + $?)) ; echo "$MSG" "DONE"
+    # TEMP don't let doctests fail the build until all string dtype changes are fixed
+    # RET=$(($RET + $?)) ; echo "$MSG" "DONE"
+    echo "$MSG" "DONE"
 
 fi
 
@@ -72,6 +74,7 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.Series.dt PR01" `# Accessors are implemented as classes, but we do not document the Parameters section` \
         -i "pandas.Period.freq GL08" \
         -i "pandas.Period.ordinal GL08" \
+        -i "pandas.errors.IncompatibleFrequency SA01,SS06,EX01" \
         -i "pandas.core.groupby.DataFrameGroupBy.plot PR02" \
         -i "pandas.core.groupby.SeriesGroupBy.plot PR02" \
         -i "pandas.core.resample.Resampler.quantile PR01,PR07" \

ci/deps/actions-310-minimum_versions.yaml

@@ -22,7 +22,7 @@ dependencies:
 
   # required dependencies
   - python-dateutil=2.8.2
-  - numpy=1.23.5
+  - numpy=1.26.0
 
   # optional dependencies
   - beautifulsoup4=4.12.3
@@ -41,7 +41,7 @@ dependencies:
   - qtpy=2.3.0
   - openpyxl=3.1.2
   - psycopg2=2.9.6
-  - pyarrow=10.0.1
+  - pyarrow=12.0.1
   - pyiceberg=0.7.1
   - pymysql=1.1.0
   - pyqt=5.15.9
@@ -62,4 +62,4 @@ dependencies:
   - pip:
     - adbc-driver-postgresql==0.10.0
     - adbc-driver-sqlite==0.8.0
-    - tzdata==2022.7
+    - tzdata==2023.3

ci/deps/actions-310.yaml

@@ -39,7 +39,7 @@ dependencies:
   - qtpy>=2.3.0
   - openpyxl>=3.1.2
   - psycopg2>=2.9.6
-  - pyarrow>=10.0.1
+  - pyarrow>=12.0.1
   - pyiceberg>=0.7.1
   - pymysql>=1.1.0
   - pyqt>=5.15.9
@@ -60,4 +60,4 @@ dependencies:
   - pip:
     - adbc-driver-postgresql>=0.10.0
     - adbc-driver-sqlite>=0.8.0
-    - tzdata>=2022.7
+    - tzdata>=2023.3