Merge branch 'main' of https://github.com/pandas-dev/pandas into feature/44764_perf_issue_new_period

Author: Sylvain MARIE

.github/workflows/32-bit-linux.yml

@@ -4,10 +4,12 @@ on:
   push:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
   pull_request:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
     paths-ignore:
       - "doc/**"
@@ -52,3 +54,7 @@ jobs:
           name: Test results
           path: test-data.xml
         if: failure()
+    concurrency:
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-32bit
+      cancel-in-progress: true

.github/workflows/code-checks.yml

@@ -4,10 +4,12 @@ on:
   push:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
   pull_request:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
 
 env:

.github/workflows/docbuild-and-upload.yml

@@ -4,12 +4,14 @@ on:
   push:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
     tags:
       - '*'
   pull_request:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
 
 env:

.github/workflows/macos-windows.yml

@@ -4,10 +4,12 @@ on:
   push:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
   pull_request:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
     paths-ignore:
       - "doc/**"
@@ -16,8 +18,6 @@ env:
   PANDAS_CI: 1
   PYTEST_TARGET: pandas
   PATTERN: "not slow and not db and not network and not single_cpu"
-  ERROR_ON_WARNINGS: "1"
-
 
 permissions:
   contents: read

.github/workflows/package-checks.yml

@@ -4,10 +4,12 @@ on:
   push:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
   pull_request:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
     types: [ labeled, opened, synchronize, reopened ]
 

.github/workflows/sdist.yml

@@ -4,10 +4,12 @@ on:
   push:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
   pull_request:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
     types: [labeled, opened, synchronize, reopened]
     paths-ignore:

.github/workflows/ubuntu.yml

@@ -4,10 +4,12 @@ on:
   push:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
   pull_request:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
     paths-ignore:
       - "doc/**"
@@ -38,7 +40,6 @@ jobs:
           - name: "Minimum Versions"
             env_file: actions-38-minimum_versions.yaml
             pattern: "not slow and not network and not single_cpu"
-            error_on_warnings: "0"
           - name: "Locale: it_IT"
             env_file: actions-38.yaml
             pattern: "not slow and not network and not single_cpu"
@@ -63,12 +64,10 @@ jobs:
             env_file: actions-310.yaml
             pattern: "not slow and not network and not single_cpu"
             pandas_copy_on_write: "1"
-            error_on_warnings: "0"
           - name: "Data Manager"
             env_file: actions-38.yaml
             pattern: "not slow and not network and not single_cpu"
             pandas_data_manager: "array"
-            error_on_warnings: "0"
           - name: "Pypy"
             env_file: actions-pypy-38.yaml
             pattern: "not slow and not network and not single_cpu"
@@ -77,7 +76,6 @@ jobs:
             env_file: actions-310-numpydev.yaml
             pattern: "not slow and not network and not single_cpu"
             test_args: "-W error::DeprecationWarning -W error::FutureWarning"
-            error_on_warnings: "0"
         exclude:
           - env_file: actions-38.yaml
             pyarrow_version: "8"
@@ -97,7 +95,6 @@ jobs:
       ENV_FILE: ci/deps/${{ matrix.env_file }}
       PATTERN: ${{ matrix.pattern }}
       EXTRA_APT: ${{ matrix.extra_apt || '' }}
-      ERROR_ON_WARNINGS: ${{ matrix.error_on_warnings || '1' }}
       LANG: ${{ matrix.lang || '' }}
       LC_ALL: ${{ matrix.lc_all || '' }}
       PANDAS_DATA_MANAGER: ${{ matrix.pandas_data_manager || 'block' }}

.github/workflows/wheels.yml

@@ -173,8 +173,8 @@ jobs:
           pip install hypothesis>=6.34.2 pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17
           cd .. # Not a good idea to test within the src tree
           python -c "import pandas; print(pandas.__version__);
-          pandas.test(extra_args=['-m not clipboard and not single_cpu', '--skip-slow', '--skip-network', '--skip-db', '-n=2']);
-          pandas.test(extra_args=['-m not clipboard and single_cpu', '--skip-slow', '--skip-network', '--skip-db'])"
+          pandas.test(extra_args=['-m not clipboard and not single_cpu and not slow and not network and not db', '-n 2']);
+          pandas.test(extra_args=['-m not clipboard and single_cpu and not slow and not network and not db'])"
       - uses: actions/upload-artifact@v3
         with:
           name: sdist

.pre-commit-config.yaml

@@ -313,7 +313,7 @@ repos:
         entry: python scripts/generate_pip_deps_from_conda.py
         files: ^(environment.yml|requirements-dev.txt)$
         pass_filenames: false
-        additional_dependencies: [pyyaml, toml]
+        additional_dependencies: [tomli, pyyaml]
     -   id: title-capitalization
         name: Validate correct capitalization among titles in documentation
         entry: python scripts/validate_rst_title_capitalization.py
@@ -391,10 +391,11 @@ repos:
         types: [yaml]
     -   id: validate-min-versions-in-sync
         name: Check minimum version of dependencies are aligned
-        entry: python scripts/validate_min_versions_in_sync.py
+        entry: python -m scripts.validate_min_versions_in_sync
         language: python
         files: ^(ci/deps/actions-.*-minimum_versions\.yaml|pandas/compat/_optional\.py)$
-        additional_dependencies: [tomli]
+        additional_dependencies: [tomli, pyyaml]
+        pass_filenames: false
     -   id: validate-errors-locations
         name: Validate errors locations
         description: Validate errors are in appropriate locations.

asv_bench/benchmarks/frame_methods.py

@@ -17,6 +17,21 @@
 from .pandas_vb_common import tm
 
 
+class Clip:
+    params = [
+        ["float64", "Float64", "float64[pyarrow]"],
+    ]
+    param_names = ["dtype"]
+
+    def setup(self, dtype):
+        data = np.random.randn(100_000, 10)
+        df = DataFrame(data, dtype=dtype)
+        self.df = df
+
+    def time_clip(self, dtype):
+        self.df.clip(-1.0, 1.0)
+
+
 class GetNumericData:
     def setup(self):
         self.df = DataFrame(np.random.randn(10000, 25))
@@ -739,4 +754,29 @@ def time_memory_usage_object_dtype(self):
         self.df2.memory_usage(deep=True)
 
 
+class FindValidIndex:
+    param_names = ["dtype"]
+    params = [
+        ["float", "Float64", "float64[pyarrow]"],
+    ]
+
+    def setup(self, dtype):
+        df = DataFrame(
+            np.random.randn(100000, 2),
+            columns=list("AB"),
+            dtype=dtype,
+        )
+        df.iloc[:100, 0] = None
+        df.iloc[:200, 1] = None
+        df.iloc[-100:, 0] = None
+        df.iloc[-200:, 1] = None
+        self.df = df
+
+    def time_first_valid_index(self, dtype):
+        self.df.first_valid_index()
+
+    def time_last_valid_index(self, dtype):
+        self.df.last_valid_index()
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip