ENH: Generalize pd.col and expressions

- [x] closes #63287 (Replace xxxx with the GitHub issue number)
- [ ] [Tests added and passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#writing-tests) if fixing a bug or adding a new feature
- [ ] All [code checks passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#pre-commit).
- [ ] Added [type annotations](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#type-hints) to new arguments/methods/functions.
- [ ] Added an entry in the latest `doc/source/whatsnew/vX.X.X.rst` file if fixing a bug or adding a new feature.
- [ ] If I used AI to develop this pull request, I prompted it to follow `AGENTS.md`.

Generalizes the expressions to handle:

- properties and cached properties
- `__getitem__`
- `qcut` and `where`

In addition, it removes the unnecessary wrapping with parentheses.

`main: ((col('a') + 1) * (col('b') + 1))`
`PR:   (col('a') + 1) * (col('b') + 1)`

Adding support for additional methods throughout pandas should be straightforward.

@@ -11,10 +11,11 @@

 from pandas.util._decorators import set_module

-from pandas.core.series import Series
-
 if TYPE_CHECKING:
-    from pandas import DataFrame
+    from pandas import (
+        DataFrame,
+        Series,
+    )

 # Used only for generating the str repr of expressions.
@@ -48,25 +49,22 @@

 def _parse_args(df: DataFrame, *args: Any) -> tuple[Series]:
     # Parse `args`, evaluating any expressions we encounter.
-    return tuple([x(df) if isinstance(x, Expression) else x for x in args])
+    return tuple(
+        [x._eval_expression(df) if isinstance(x, Expression) else x for x in args]
+    )

 def _parse_kwargs(df: DataFrame, **kwargs: Any) -> dict[str, Any]:
     # Parse `kwargs`, evaluating any expressions we encounter.
     return {
-        key: val(df) if isinstance(val, Expression) else val
+        key: val._eval_expression(df) if isinstance(val, Expression) else val
         for key, val in kwargs.items()
     }

 def _pretty_print_args_kwargs(*args: Any, **kwargs: Any) -> str:
-    inputs_repr = ", ".join(
-        arg._repr_str if isinstance(arg, Expression) else repr(arg) for arg in args
-    )
-    kwargs_repr = ", ".join(
-        f"{k}={v._repr_str if isinstance(v, Expression) else v!r}"
-        for k, v in kwargs.items()
-    )
+    inputs_repr = ", ".join(repr(arg) for arg in args)
+    kwargs_repr = ", ".join(f"{k}={v!r}" for k, v in kwargs.items())

     all_args = []
     if inputs_repr:
@@ -85,105 +83,134 @@ class Expression:
     This is not meant to be instantiated directly. Instead, use :meth:`pandas.col`.
     """

-    def __init__(self, func: Callable[[DataFrame], Any], repr_str: str) -> None:
+    def __init__(
+        self,
+        func: Callable[[DataFrame], Any],
+        repr_str: str,
+        needs_parenthese: bool = False,
+    ) -> None:
         self._func = func
         self._repr_str = repr_str
+        self._needs_parentheses = needs_parenthese

-    def __call__(self, df: DataFrame) -> Any:
+    def _eval_expression(self, df: DataFrame) -> Any:
         return self._func(df)

-    def _with_binary_op(self, op: str, other: Any) -> Expression:
+    def _with_op(self, op: str, other: Any) -> Expression:
         op_symbol = _OP_SYMBOLS.get(op, op)

-        if isinstance(other, Expression):
-            if op.startswith("__r"):
-                repr_str = f"({other._repr_str} {op_symbol} {self._repr_str})"
-            else:
-                repr_str = f"({self._repr_str} {op_symbol} {other._repr_str})"
-            return Expression(lambda df: getattr(self(df), op)(other(df)), repr_str)
+        if op == "__getitem__":
+            needs_parentheses = False
+            repr_str = f"{self!r}[{other!r}]"
         else:
+            needs_parentheses = True
+            self_repr = f"{self!r}"
+            if self._needs_parentheses:
+                self_repr = f"({self_repr})"
+            other_repr = f"{other!r}"
+            if isinstance(other, Expression) and other._needs_parentheses:
+                other_repr = f"({other_repr})"
+
             if op.startswith("__r"):
-                repr_str = f"({other!r} {op_symbol} {self._repr_str})"
+                repr_str = f"{other_repr} {op_symbol} {self_repr}"
             else:
-                repr_str = f"({self._repr_str} {op_symbol} {other!r})"
-            return Expression(lambda df: getattr(self(df), op)(other), repr_str)
+                repr_str = f"{self_repr} {op_symbol} {other_repr}"
+
+        if isinstance(other, Expression):
+            return Expression(
+                lambda df: getattr(self._eval_expression(df), op)(
+                    other._eval_expression(df)
+                ),
+                repr_str,
+                needs_parenthese=needs_parentheses,
+            )
+        else:
+            return Expression(
+                lambda df: getattr(self._eval_expression(df), op)(other),
+                repr_str,
+                needs_parenthese=needs_parentheses,
+            )

     # Binary ops
     def __add__(self, other: Any) -> Expression:
-        return self._with_binary_op("__add__", other)
+        return self._with_op("__add__", other)

     def __radd__(self, other: Any) -> Expression:
-        return self._with_binary_op("__radd__", other)
+        return self._with_op("__radd__", other)

     def __sub__(self, other: Any) -> Expression:
-        return self._with_binary_op("__sub__", other)
+        return self._with_op("__sub__", other)

     def __rsub__(self, other: Any) -> Expression:
-        return self._with_binary_op("__rsub__", other)
+        return self._with_op("__rsub__", other)

     def __mul__(self, other: Any) -> Expression:
-        return self._with_binary_op("__mul__", other)
+        return self._with_op("__mul__", other)

     def __rmul__(self, other: Any) -> Expression:
-        return self._with_binary_op("__rmul__", other)
+        return self._with_op("__rmul__", other)

     def __truediv__(self, other: Any) -> Expression:
-        return self._with_binary_op("__truediv__", other)
+        return self._with_op("__truediv__", other)

     def __rtruediv__(self, other: Any) -> Expression:
-        return self._with_binary_op("__rtruediv__", other)
+        return self._with_op("__rtruediv__", other)

     def __floordiv__(self, other: Any) -> Expression:
-        return self._with_binary_op("__floordiv__", other)
+        return self._with_op("__floordiv__", other)

     def __rfloordiv__(self, other: Any) -> Expression:
-        return self._with_binary_op("__rfloordiv__", other)
+        return self._with_op("__rfloordiv__", other)

     def __ge__(self, other: Any) -> Expression:
-        return self._with_binary_op("__ge__", other)
+        return self._with_op("__ge__", other)

     def __gt__(self, other: Any) -> Expression:
-        return self._with_binary_op("__gt__", other)
+        return self._with_op("__gt__", other)

     def __le__(self, other: Any) -> Expression:
-        return self._with_binary_op("__le__", other)
+        return self._with_op("__le__", other)

     def __lt__(self, other: Any) -> Expression:
-        return self._with_binary_op("__lt__", other)
+        return self._with_op("__lt__", other)

     def __eq__(self, other: object) -> Expression:  # type: ignore[override]
-        return self._with_binary_op("__eq__", other)
+        return self._with_op("__eq__", other)

     def __ne__(self, other: object) -> Expression:  # type: ignore[override]
-        return self._with_binary_op("__ne__", other)
+        return self._with_op("__ne__", other)

     def __mod__(self, other: Any) -> Expression:
-        return self._with_binary_op("__mod__", other)
+        return self._with_op("__mod__", other)

     def __rmod__(self, other: Any) -> Expression:
-        return self._with_binary_op("__rmod__", other)
+        return self._with_op("__rmod__", other)

     # Logical ops
     def __and__(self, other: Any) -> Expression:
-        return self._with_binary_op("__and__", other)
+        return self._with_op("__and__", other)

     def __rand__(self, other: Any) -> Expression:
-        return self._with_binary_op("__rand__", other)
+        return self._with_op("__rand__", other)

     def __or__(self, other: Any) -> Expression:
-        return self._with_binary_op("__or__", other)
+        return self._with_op("__or__", other)

     def __ror__(self, other: Any) -> Expression:
-        return self._with_binary_op("__ror__", other)
+        return self._with_op("__ror__", other)

     def __xor__(self, other: Any) -> Expression:
-        return self._with_binary_op("__xor__", other)
+        return self._with_op("__xor__", other)

     def __rxor__(self, other: Any) -> Expression:
-        return self._with_binary_op("__rxor__", other)
+        return self._with_op("__rxor__", other)

     def __invert__(self) -> Expression:
-        return Expression(lambda df: ~self(df), f"(~{self._repr_str})")
+        return Expression(
+            lambda df: ~self._eval_expression(df),
+            f"~{self._repr_str}",
+            needs_parenthese=True,
+        )

     def __array_ufunc__(
         self, ufunc: Callable[..., Any], method: str, *inputs: Any, **kwargs: Any
@@ -198,57 +225,38 @@ def func(df: DataFrame) -> Any:

         return Expression(func, repr_str)

-    # Everything else
-    def __getattr__(self, attr: str, /) -> Any:
-        if attr in Series._accessors:
-            return NamespaceExpression(self, attr)
+    def __getitem__(self, item: Any) -> Expression:
+        return self._with_op("__getitem__", item)

-        def func(df: DataFrame, *args: Any, **kwargs: Any) -> Any:
-            parsed_args = _parse_args(df, *args)
+    def _call_with_func(self, func: Callable, **kwargs: Any) -> Expression:
+        def wrapped(df: DataFrame) -> Any:
             parsed_kwargs = _parse_kwargs(df, **kwargs)
-            return getattr(self(df), attr)(*parsed_args, **parsed_kwargs)
-
-        def wrapper(*args: Any, **kwargs: Any) -> Expression:
-            args_str = _pretty_print_args_kwargs(*args, **kwargs)
-            repr_str = f"{self._repr_str}.{attr}({args_str})"
-
-            return Expression(lambda df: func(df, *args, **kwargs), repr_str)
-
-        return wrapper
-
-    def __repr__(self) -> str:
-        return self._repr_str or "Expr(...)"
-
-
-class NamespaceExpression:
-    def __init__(self, func: Expression, namespace: str) -> None:
-        self._func = func
-        self._namespace = namespace
+            return func(**parsed_kwargs)

-    def __call__(self, df: DataFrame) -> Any:
-        return self._func(df)
+        args_str = _pretty_print_args_kwargs(**kwargs)
+        repr_str = func.__name__ + "(" + args_str + ")"

-    def __getattr__(self, attr: str) -> Any:
-        if isinstance(getattr(getattr(Series, self._namespace), attr), property):
-            repr_str = f"{self._func._repr_str}.{self._namespace}.{attr}"
-            return Expression(
-                lambda df: getattr(getattr(self(df), self._namespace), attr),
-                repr_str,
-            )
+        return Expression(wrapped, repr_str)

+    def __call__(self, *args: Any, **kwargs: Any) -> Expression:
         def func(df: DataFrame, *args: Any, **kwargs: Any) -> Any:
             parsed_args = _parse_args(df, *args)
             parsed_kwargs = _parse_kwargs(df, **kwargs)
-            return getattr(getattr(self(df), self._namespace), attr)(
-                *parsed_args, **parsed_kwargs
-            )
+            return self._eval_expression(df)(*parsed_args, **parsed_kwargs)
+
+        args_str = _pretty_print_args_kwargs(*args, **kwargs)
+        repr_str = f"{self._repr_str}({args_str})"
+        return Expression(lambda df: func(df, *args, **kwargs), repr_str)

-        def wrapper(*args: Any, **kwargs: Any) -> Expression:
-            args_str = _pretty_print_args_kwargs(*args, **kwargs)
-            repr_str = f"{self._func._repr_str}.{self._namespace}.{attr}({args_str})"
-            return Expression(lambda df: func(df, *args, **kwargs), repr_str)
+    def __getattr__(self, name: str, /) -> Any:
+        repr_str = f"{self!r}"
+        if self._needs_parentheses:
+            repr_str = f"({repr_str})"
+        repr_str += f".{name}"
+        return Expression(lambda df: getattr(self._eval_expression(df), name), repr_str)

-        return wrapper
+    def __repr__(self) -> str:
+        return self._repr_str or "Expr(...)"

 [USER_REDACTED]_module("pandas")

@@ -50,6 +50,8 @@
 )
 from pandas.core.dtypes.inference import iterable_not_string

+from pandas.core.col import Expression
+
 if TYPE_CHECKING:
     from pandas._typing import (
         AnyArrayLike,
@@ -383,7 +385,9 @@ def apply_if_callable(maybe_callable, obj, **kwargs):
     obj : NDFrame
     **kwargs
     """
-    if callable(maybe_callable):
+    if isinstance(maybe_callable, Expression):
+        return maybe_callable._eval_expression(obj, **kwargs)
+    elif callable(maybe_callable):
         return maybe_callable(obj, **kwargs)

     return maybe_callable

@@ -43,6 +43,7 @@
 )
 import pandas.core.algorithms as algos
 from pandas.core.arrays.datetimelike import dtype_to_unit
+from pandas.core.col import Expression

 if TYPE_CHECKING:
     from collections.abc import Callable
@@ -359,6 +360,10 @@ def qcut(
     >>> pd.qcut(range(5), 4, labels=False)
     array([0, 0, 1, 2, 3])
     """
+    if isinstance(x, Expression):
+        return x._call_with_func(
+            qcut, x=x, q=q, labels=labels, retbins=retbins, precision=precision
+        )
     original = x
     x_idx = _preprocess_for_cut(x)
     x_idx, _ = _coerce_to_type(x_idx)

Renamed because we now use this for `__getitem__` too.

Would it make sense for `Expression._with_op` to accept an `other_repr: str` argument so we don't need to do more repr manipulation here?

Yea - I like that. Okay for a followup?

Yup, sure thing.


DOC: Replace [USER_REDACTED] decorator with inline docstrings in pandas/io/pickle.py

…e.py

- [ ] closes #xxxx (Replace xxxx with the GitHub issue number)
- [ ] [Tests added and passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#writing-tests) if fixing a bug or adding a new feature
- [ ] All [code checks passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#pre-commit).
- [ ] Added [type annotations](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#type-hints) to new arguments/methods/functions.
- [ ] Added an entry in the latest `doc/source/whatsnew/vX.X.X.rst` file if fixing a bug or adding a new feature.
- [ ] If I used AI to develop this pull request, I prompted it to follow `AGENTS.md`.

@@ -10,12 +10,7 @@
 import warnings

 from pandas.compat import pickle_compat
-from pandas.util._decorators import (
-    doc,
-    set_module,
-)
-
-from pandas.core.shared_docs import _shared_docs
+from pandas.util._decorators import set_module

 from pandas.io.common import get_handle

@@ -35,10 +30,6 @@

 [USER_REDACTED]_module("pandas")
-[USER_REDACTED](
-    storage_options=_shared_docs["storage_options"],
-    compression_options=_shared_docs["compression_options"] % "filepath_or_buffer",
-)
 def to_pickle(
     obj: Any,
     filepath_or_buffer: FilePath | WriteBuffer[bytes],
@@ -57,8 +48,21 @@ def to_pickle(
         String, path object (implementing ``os.PathLike[str]``), or file-like
         object implementing a binary ``write()`` function.
         Also accepts URL. URL has to be of S3 or GCS.
-    {compression_options}
-
+    compression : str or dict, default 'infer'
+        For on-the-fly compression of the output data. If 'infer' and
+        'filepath_or_buffer' is path-like, then detect compression from the
+        following extensions: '.gz', '.bz2', '.zip', '.xz', '.zst', '.tar',
+        '.tar.gz', '.tar.xz' or '.tar.bz2' (otherwise no compression).
+        Set to ``None`` for no compression.
+        Can also be a dict with key ``'method'`` set
+        to one of {``'zip'``, ``'gzip'``, ``'bz2'``, ``'zstd'``, ``'xz'``,
+        ``'tar'``} and other key-value pairs are forwarded to
+        ``zipfile.ZipFile``, ``gzip.GzipFile``,
+        ``bz2.BZ2File``, ``zstandard.ZstdCompressor``, ``lzma.LZMAFile`` or
+        ``tarfile.TarFile``, respectively.
+        As an example, the following could be passed for faster compression
+        and to create a reproducible gzip archive:
+        ``compression={'method': 'gzip', 'compresslevel': 1, 'mtime': 1}``.
     protocol : int
         Int which indicates which protocol should be used by [NAME_REDACTED],
         default HIGHEST_PROTOCOL (see [1], paragraph 12.1.2). The possible
@@ -67,8 +71,15 @@ def to_pickle(
         For Python >= 3.4, 4 is a valid value. A negative value for the
         protocol parameter is equivalent to setting its value to
         HIGHEST_PROTOCOL.
-
-    {storage_options}
+    storage_options : dict, optional
+        Extra options that make sense for a particular storage connection, e.g.
+        host, port, username, password, etc. For HTTP(S) URLs the key-value pairs
+        are forwarded to ``urllib.request.Request`` as header options. For other
+        URLs (e.g. starting with "s3://", and "gcs://") the key-value pairs are
+        forwarded to ``fsspec.open``. Please see ``fsspec`` and ``urllib`` for more
+        details, and for more examples on storage options refer `here
+        <https://pandas.pydata.org/docs/user_guide/io.html?
+        highlight=storage_options#reading-writing-remote-files>`_.

         .. [1] https://docs.python.org/3/library/pickle.html

@@ -117,10 +128,6 @@ def to_pickle(

 [USER_REDACTED]_module("pandas")
-[USER_REDACTED](
-    storage_options=_shared_docs["storage_options"],
-    decompression_options=_shared_docs["decompression_options"] % "filepath_or_buffer",
-)
 def read_pickle(
     filepath_or_buffer: FilePath | ReadPickleBuffer,
     compression: CompressionOptions = "infer",
@@ -140,10 +147,32 @@ def read_pickle(
         String, path object (implementing ``os.PathLike[str]``), or file-like
         object implementing a binary ``readlines()`` function.
         Also accepts URL. URL is not limited to S3 and GCS.
-
-    {decompression_options}
-
-    {storage_options}
+    compression : str or dict, default 'infer'
+        For on-the-fly decompression of on-disk data. If 'infer' and
+        'filepath_or_buffer' is path-like, then detect compression from the
+        following extensions: '.gz', '.bz2', '.zip', '.xz', '.zst', '.tar',
+        '.tar.gz', '.tar.xz' or '.tar.bz2' (otherwise no compression).
+        If using 'zip' or 'tar', the ZIP file must contain only one data file
+        to be read in.
+        Set to ``None`` for no decompression.
+        Can also be a dict with key ``'method'`` set
+        to one of {``'zip'``, ``'gzip'``, ``'bz2'``, ``'zstd'``, ``'xz'``,
+        ``'tar'``} and other key-value pairs are forwarded to
+        ``zipfile.ZipFile``, ``gzip.GzipFile``,
+        ``bz2.BZ2File``, ``zstandard.ZstdDecompressor``, ``lzma.LZMAFile`` or
+        ``tarfile.TarFile``, respectively.
+        As an example, the following could be passed for Zstandard decompression
+        using a custom compression dictionary:
+        ``compression={'method': 'zstd', 'dict_data': my_compression_dict}``.
+    storage_options : dict, optional
+        Extra options that make sense for a particular storage connection, e.g.
+        host, port, username, password, etc. For HTTP(S) URLs the key-value pairs
+        are forwarded to ``urllib.request.Request`` as header options. For other
+        URLs (e.g. starting with "s3://", and "gcs://") the key-value pairs are
+        forwarded to ``fsspec.open``. Please see ``fsspec`` and ``urllib`` for more
+        details, and for more examples on storage options refer `here
+        <https://pandas.pydata.org/docs/user_guide/io.html?
+        highlight=storage_options#reading-writing-remote-files>`_.

     Returns
     -------

```suggestion
```

```suggestion
```

Thanks, I have solved it.

Thanks, I have solved it.


BUG: Fix regression in to_hdf with MultiIndex and StringDtype (GH#63412)

- [x] closes #63412
- [x] [Tests added and passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#writing-tests) if fixing a bug or adding a new feature
- [x] All [code checks passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#pre-commit).
- [ ] Added [type annotations](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#type-hints) to new arguments/methods/functions.
- [ ] Added an entry in the latest `doc/source/whatsnew/vX.X.X.rst` file if fixing a bug or adding a new feature.
- [x] If I used AI to develop this pull request, I prompted it to follow `AGENTS.md`.

### Summary
- Fixed `NotImplementedError` when saving `MultiIndex` with `StringDtype` to HDF5.
- Added regression test in `pandas/tests/io/pytables/test_store.py`.
- Enabled existing tests in `test_put.py`, `test_read.py`, and `test_round_trip.py` that were previously skipped for `StringDtype`.

@@ -3147,7 +3147,9 @@ def write_multi_index(self, key: str, index: MultiIndex) -> None:
             zip(index.levels, index.codes, index.names, strict=True)
         ):
             # write the level
-            if isinstance(lev.dtype, ExtensionDtype):
+            if isinstance(lev.dtype, ExtensionDtype) and not isinstance(
+                lev.dtype, StringDtype
+            ):
                 raise NotImplementedError(
                     "Saving a MultiIndex with an extension dtype is not supported."
                 )

@@ -314,12 +314,6 @@ def test_column_multiindex(tmp_path, setup_path, using_infer_string):

     path = tmp_path / setup_path
     with HDFStore(path) as store:
-        if using_infer_string:
-            # TODO(infer_string) make this work for string dtype
-            msg = "Saving a MultiIndex with an extension dtype is not supported."
-            with pytest.raises(NotImplementedError, match=msg):
-                store.put("df", df)
-            return
         store.put("df", df)
         tm.assert_frame_equal(
             store["df"], expected, check_index_type=True, check_column_type=True

@@ -181,12 +181,6 @@ def test_read_hdf_open_store(tmp_path, setup_path, using_infer_string):
     df = df.set_index(keys="E", append=True)

     path = tmp_path / setup_path
-    if using_infer_string:
-        # TODO(infer_string) make this work for string dtype
-        msg = "Saving a MultiIndex with an extension dtype is not supported."
-        with pytest.raises(NotImplementedError, match=msg):
-            df.to_hdf(path, key="df", mode="w")
-        return
     df.to_hdf(path, key="df", mode="w")
     direct = read_hdf(path, "df")
     with HDFStore(path, mode="r") as store:

@@ -434,12 +434,6 @@ def test_store_hierarchical(
 ):
     frame = multiindex_dataframe_random_data

-    if using_infer_string:
-        # TODO(infer_string) make this work for string dtype
-        msg = "Saving a MultiIndex with an extension dtype is not supported."
-        with pytest.raises(NotImplementedError, match=msg):
-            _check_roundtrip(frame, tm.assert_frame_equal, path=temp_file)
-        return
     _check_roundtrip(frame, tm.assert_frame_equal, path=temp_file)
     _check_roundtrip(frame.T, tm.assert_frame_equal, path=temp_file)
     _check_roundtrip(frame["A"], tm.assert_series_equal, path=temp_file)

@@ -1129,3 +1129,13 @@ def test_select_categorical_string_columns(tmp_path, model):
         result = store.select("df", "modelId == model")
         expected = df[df["modelId"] == model]
         tm.assert_frame_equal(result, expected)
+
+
+def test_to_hdf_multiindex_string_dtype_crash(tmp_path):
+    # GH#63412
+    path = tmp_path / "test.h5"
+    index = MultiIndex.from_tuples([("a", "x"), ("b", "y")], names=["level1", "level2"])
+    df = DataFrame({"value": [1, 2]}, index=index)
+    df.to_hdf(path, key="test")
+    result = read_hdf(path, key="test")
+    tm.assert_frame_equal(df, result, check_dtype=False)

If you replace this with `pass`, does `_convert_index` below already convert this data correctly without the `astype`?

Yes it does. I'll make the modification.

[USER_REDACTED] Checks passed. I used a different implementation to avoid the two `if` blocks, I think this is better style, unless you disagree.


BUG: .quantile with empty temporal data

- [x] closes #63404 (Replace xxxx with the GitHub issue number)
- [x] [Tests added and passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#writing-tests) if fixing a bug or adding a new feature
- [x] All [code checks passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#pre-commit).

@@ -226,13 +226,16 @@ class DatetimeArray(dtl.TimelikeOps, dtl.DatelikeOps):
     """

     _typ = "datetimearray"
-    _internal_fill_value = np.datetime64("NaT", "ns")
     _recognized_scalars = (datetime, np.datetime64)
     _is_recognized_dtype: Callable[[DtypeObj], bool] = lambda x: lib.is_np_dtype(
         x, "M"
     ) or isinstance(x, DatetimeTZDtype)
     _infer_matches = ("datetime", "datetime64", "date")

+    [USER_REDACTED]
+    def _internal_fill_value(self) -> np.datetime64:
+        return np.datetime64("NaT", self.unit)
+
     [USER_REDACTED]
     def _scalar_type(self) -> type[Timestamp]:
         return Timestamp

@@ -154,11 +154,14 @@ class TimedeltaArray(dtl.TimelikeOps):
     """

     _typ = "timedeltaarray"
-    _internal_fill_value = np.timedelta64("NaT", "ns")
     _recognized_scalars = (timedelta, np.timedelta64, Tick)
     _is_recognized_dtype: Callable[[DtypeObj], bool] = lambda x: lib.is_np_dtype(x, "m")
     _infer_matches = ("timedelta", "timedelta64")

+    [USER_REDACTED]
+    def _internal_fill_value(self) -> np.timedelta64:
+        return np.timedelta64("NaT", self.unit)
+
     [USER_REDACTED]
     def _scalar_type(self) -> type[Timedelta]:
         return Timedelta

@@ -928,3 +928,12 @@ def test_multi_quantile_numeric_only_retains_columns():
     tm.assert_frame_equal(
         result, expected, check_index_type=True, check_column_type=True
     )
+
+
+[USER_REDACTED].mark.parametrize("typ", ["datetime64", "timedelta64"])
+def test_quantile_empty_datetimelike(typ, unit):
+    dtype = f"{typ}[{unit}]"
+    df = DataFrame(np.array([], dtype=dtype))
+    result = df.quantile()
+    expected = Series([pd.NaT], name=0.5, dtype=dtype)
+    tm.assert_series_equal(result, expected)

@@ -245,3 +245,10 @@ def test_quantile_dtype_size(self, any_int_ea_dtype):
         result = ser.quantile([0.1, 0.5])
         expected = Series([1, 1], dtype=any_int_ea_dtype, index=[0.1, 0.5])
         tm.assert_series_equal(result, expected)
+
+
+[USER_REDACTED].mark.parametrize("typ", ["datetime64", "timedelta64"])
+def test_quantile_empty_datetimelike(typ, unit):
+    ser = Series([], dtype=f"{typ}[{unit}]")
+    result = ser.quantile()
+    assert result is pd.NaT

While having an `else` clause that actually cannot work doesn't seem great (and so this change might be good anyway), looking at the code above (`if res_values.dtype == self._ndarray.dtype:`), I am wondering if the problem might not be that `quantile_with_mask` is not returning the correct dtype in case of empty data.

In that case it returns NaT in an array, and it seems to return nanoseconds, which seems wrong?

Looking a bit more, `quantile_with_mask` is returning nanoseconds in this case because of the fill_value being passed, which is `self._internal_fill_value`, and that seems to be defined as `np.timedelta64('NaT','ns')`. That probably need to take the unit of `self` into account?

Nice find. The problem was `_internal_fill_value` assuming nanoseconds for NaT.

I reverted the changes to `._quantile` to just be conservative, though probably the right long term change

Maybe it would be good to also add a dataframe version of this test, such that there is an actual dtype to verify (to assert it has preserved the unit)


PERF: avoid NumPy fallback when casting Arrow int/string to string

@@ -210,6 +210,26 @@ def _from_sequence(
             result = scalars._data
             result = lib.ensure_string_array(result, copy=copy, convert_na_value=False)
             pa_arr = pa.array(result, mask=na_values, type=pa.large_string())
+        elif isinstance(scalars, ArrowExtensionArray):
+            pa_type = scalars._pa_array.type
+            # Use PyArrow's native cast for integer, string, and boolean types.
+            # Float has different representation in PyArrow: 1.0 -> "1" instead
+            # of "1.0", and uses different scientific notation (1e+10 vs 1e10).
+            # Boolean needs capitalize (true -> True, false -> False).
+            if (
+                pa.types.is_integer(pa_type)
+                or pa.types.is_large_string(pa_type)
+                or pa.types.is_string(pa_type)
+                or pa.types.is_boolean(pa_type)
+            ):
+                pa_arr = pc.cast(scalars._pa_array, pa.large_string())
+                if pa.types.is_boolean(pa_type):
+                    pa_arr = pc.utf8_capitalize(pa_arr)
+            else:
+                # Fall back for types where PyArrow's string representation
+                # differs from Python's str()
+                result = lib.ensure_string_array(scalars, copy=copy)
+                pa_arr = pa.array(result, type=pa.large_string(), from_pandas=True)
         elif isinstance(scalars, (pa.Array, pa.ChunkedArray)):
             pa_arr = pc.cast(scalars, pa.large_string())
         else:

@@ -3,8 +3,6 @@
 import numpy as np
 import pytest

-from pandas.compat import HAS_PYARROW
-
 from pandas import (
     DataFrame,
     Series,
@@ -218,10 +216,12 @@ def test_convert_dtypes(using_infer_string):
     df_orig = df.copy()
     df2 = df.convert_dtypes()

-    if HAS_PYARROW:
-        assert not tm.shares_memory(get_array(df2, "a"), get_array(df, "a"))
-    else:
+    if using_infer_string:
+        # String column is already Arrow-backed, so memory is shared
         assert tm.shares_memory(get_array(df2, "a"), get_array(df, "a"))
+    else:
+        # String column converts from object to Arrow, no memory sharing
+        assert not tm.shares_memory(get_array(df2, "a"), get_array(df, "a"))
     assert tm.shares_memory(get_array(df2, "d"), get_array(df, "d"))
     assert tm.shares_memory(get_array(df2, "b"), get_array(df, "b"))
     assert tm.shares_memory(get_array(df2, "c"), get_array(df, "c"))

Looks good!

Could bools be converted by calling `pyarrow.compute.utf8_upper` after the cast?


API: Copy inputs in Index subclass constructors by default (GH#63388)

- [x] closes #63388
- [x] Tests added / passed
- [x] All tests passed locally
- [x] Ensure all linting tests pass
- [x] Added an entry to the release notes
- [x] If I used AI to develop this pull request, I prompted it to follow `AGENTS.md`.

**Description:**
This PR aligns the behavior of `DatetimeIndex`, `TimedeltaIndex`, `PeriodIndex`, and `IntervalIndex` with the base `Index` class (and `Series`) regarding the `copy` parameter.

Before, these constructors would ignore `copy=None` and default to not copying the input data.
Now, `copy=None` defaults to `True` for `np.ndarray` and `ExtensionArray` inputs, ensuring that the resulting Index does not share memory with the mutable input array by default.

**Changes:**
- Updated `__new__` signatures in `datetimes.py`, `timedeltas.py`, `period.py`, and `interval.py` to accept `copy: bool | None`.
- Added logic to trigger a copy if `copy` is `None` (or `True`) and the input is an array/EA.
- Updated docstrings to match the standard `Index` docstring for the `copy` parameter.
- Added regression tests for all 4 subclasses verifying that inputs are copied by [NAME_REDACTED] inputs are handled correctly.
- Improved robustness by [NAME_REDACTED] `dtype` arguments (e.g. `"datetime64[ns]"`) are converted to `DtypeObj` before the view checks, preventing `AttributeError`.
- I Updated existing regression tests (`test_constructor_int64_nocopy` and `test_array_tz`) to explicitly pass `copy=False`. Since the default behavior changed to `copy=True`, these tests required an update to continue verifying that the view based construction is still possible when requested.

@@ -820,6 +820,7 @@ Other API changes
   :meth:`~DataFrame.ffill`, :meth:`~DataFrame.bfill`, :meth:`~DataFrame.interpolate`,
   :meth:`~DataFrame.where`, :meth:`~DataFrame.mask`, :meth:`~DataFrame.clip`) now return
   the modified DataFrame or Series (``self``) instead of ``None`` when ``inplace=True`` (:issue:`63207`)
+- [NAME_REDACTED] constructors now copy ``numpy.ndarray`` and ``ExtensionArray`` inputs by [NAME_REDACTED] ``copy=None``, consistent with :class:`Series` behavior (:issue:`63388`)

 .. ---------------------------------------------------------------------------
 .. _whatsnew_300.deprecations:

@@ -505,12 +505,8 @@ def __new__(
         if not copy and isinstance(data, (ABCSeries, Index)):
             refs = data._references

-        if isinstance(data, (ExtensionArray, np.ndarray)):
-            # GH 63306
-            if copy is not False:
-                if dtype is None or astype_is_view(data.dtype, dtype):
-                    data = data.copy()
-                    copy = False
+        # GH 63306, GH 63388
+        data, copy = cls._maybe_copy_array_input(data, copy, dtype)

         # range
         if isinstance(data, (range, RangeIndex)):
@@ -5197,6 +5193,21 @@ def _raise_scalar_data_error(cls, data):
             "was passed"
         )

+    [USER_REDACTED]
+    def _maybe_copy_array_input(
+        cls, data, copy: bool | None, dtype
+    ) -> tuple[Any, bool]:
+        """
+        Ensure that the input data is copied if necessary.
+        GH#63388
+        """
+        if isinstance(data, (ExtensionArray, np.ndarray)):
+            if copy is not False:
+                if dtype is None or astype_is_view(data.dtype, pandas_dtype(dtype)):
+                    data = data.copy()
+                    copy = False
+        return data, bool(copy)
+
     def _validate_fill_value(self, value):
         """
         Check if the value can be inserted into our array without casting,

@@ -181,8 +181,13 @@ class DatetimeIndex(DatetimeTimedeltaMixin):
         If True parse dates in `data` with the year first order.
     dtype : numpy.dtype or DatetimeTZDtype or str, default None
         Note that the only NumPy dtype allowed is `datetime64[ns]`.
-    copy : bool, default False
-        Make a copy of input ndarray.
+    copy : bool, default None
+        Whether to copy input data, only relevant for array, Series, and Index
+        inputs (for other input, e.g. a list, a new array is created anyway).
+        Defaults to True for array input and False for Index/Series.
+        Set to False to avoid copying array input at your own risk (if you
+        know the input data won't be modified elsewhere).
+        Set to True to force copying Series/Index up front.
     name : label, default None
         Name to be stored in the index.

@@ -669,7 +674,7 @@ def __new__(
         dayfirst: bool = False,
         yearfirst: bool = False,
         dtype: Dtype | None = None,
-        copy: bool = False,
+        copy: bool | None = None,
         name: Hashable | None = None,
     ) -> Self:
         if is_scalar(data):
@@ -679,6 +684,9 @@ def __new__(

         name = maybe_extract_name(name, data, cls)

+        # GH#63388
+        data, copy = cls._maybe_copy_array_input(data, copy, dtype)
+
         if (
             isinstance(data, DatetimeArray)
             and freq is lib.no_default

@@ -169,8 +169,13 @@ class IntervalIndex(ExtensionIndex):
         neither.
     dtype : dtype or None, default None
         If None, dtype will be inferred.
-    copy : bool, default False
-        Copy the input data.
+    copy : bool, default None
+        Whether to copy input data, only relevant for array, Series, and Index
+        inputs (for other input, e.g. a list, a new array is created anyway).
+        Defaults to True for array input and False for Index/Series.
+        Set to False to avoid copying array input at your own risk (if you
+        know the input data won't be modified elsewhere).
+        Set to True to force copying Series/Index input up front.
     name : object, optional
          Name to be stored in the index.
     verify_integrity : bool, default True
@@ -252,12 +257,15 @@ def __new__(
         data,
         closed: IntervalClosedType | None = None,
         dtype: Dtype | None = None,
-        copy: bool = False,
+        copy: bool | None = None,
         name: Hashable | None = None,
         verify_integrity: bool = True,
     ) -> Self:
         name = maybe_extract_name(name, data, cls)

+        # GH#63388
+        data, copy = cls._maybe_copy_array_input(data, copy, dtype)
+
         with rewrite_exception("IntervalArray", cls.__name__):
             array = IntervalArray(
                 data,

@@ -101,8 +101,13 @@ class PeriodIndex(DatetimeIndexOpsMixin):
         One of pandas period strings or corresponding objects.
     dtype : str or PeriodDtype, default None
         A dtype from which to extract a freq.
-    copy : bool
-        Make a copy of input ndarray.
+    copy : bool, default None
+        Whether to copy input data, only relevant for array, Series, and Index
+        inputs (for other input, e.g. a list, a new array is created anyway).
+        Defaults to True for array input and False for Index/Series.
+        Set to False to avoid copying array input at your own risk (if you
+        know the input data won't be modified elsewhere).
+        Set to True to force copying Series/Index input up front.
     name : str, default None
         Name of the resulting PeriodIndex.

@@ -220,7 +225,7 @@ def __new__(
         data=None,
         freq=None,
         dtype: Dtype | None = None,
-        copy: bool = False,
+        copy: bool | None = None,
         name: Hashable | None = None,
     ) -> Self:
         refs = None
@@ -231,6 +236,9 @@ def __new__(

         freq = validate_dtype_freq(dtype, freq)

+        # GH#63388
+        data, copy = cls._maybe_copy_array_input(data, copy, dtype)
+
         # PeriodIndex allow PeriodIndex(period_index, freq=different)
         # Let's not encourage that kind of behavior in PeriodArray.

@@ -81,8 +81,13 @@ class TimedeltaIndex(DatetimeTimedeltaMixin):
     dtype : numpy.dtype or str, default None
         Valid ``numpy`` dtypes are ``timedelta64[ns]``, ``timedelta64[us]``,
         ``timedelta64[ms]``, and ``timedelta64[s]``.
-    copy : bool
-        Make a copy of input array.
+    copy : bool, default None
+        Whether to copy input data, only relevant for array, Series, and Index
+        inputs (for other input, e.g. a list, a new array is created anyway).
+        Defaults to True for array input and False for Index/Series.
+        Set to False to avoid copying array input at your own risk (if you
+        know the input data won't be modified elsewhere).
+        Set to True to force copying Series/Index input up front.
     name : object
         Name to be stored in the index.

@@ -158,11 +163,14 @@ def __new__(
         data=None,
         freq=lib.no_default,
         dtype=None,
-        copy: bool = False,
+        copy: bool | None = None,
         name=None,
     ):
         name = maybe_extract_name(name, data, cls)

+        # GH#63388
+        data, copy = cls._maybe_copy_array_input(data, copy, dtype)
+
         if is_scalar(data):
             cls._raise_scalar_data_error(data)

@@ -707,7 +707,7 @@ def test_array_object_dtype(self, arr1d):
     def test_array_tz(self, arr1d):
         # GH#23524
         arr = arr1d
-        dti = self.index_cls(arr1d)
+        dti = self.index_cls(arr1d, copy=False)
         copy_false = None if np_version_gt2 else False

         expected = dti.asi8.view("M8[ns]")

@@ -1,12 +1,15 @@
+import numpy as np
 import pytest

 from pandas import (
     DatetimeIndex,
     Series,
     Timestamp,
+    array,
     date_range,
 )
 import pandas._testing as tm
+from pandas.tests.copy_view.util import get_array

 pytestmark = pytest.mark.filterwarnings(
     "ignore:Setting a value on a view:FutureWarning"
@@ -54,3 +57,30 @@ def test_index_values():
     idx = date_range("2019-12-31", periods=3, freq="D")
     result = idx.values
     assert result.flags.writeable is False
+
+
+def test_constructor_copy_input_datetime_ndarray_default():
+    # GH 63388
+    arr = np.array(["2020-01-01", "2020-01-02"], dtype="datetime64[ns]")
+    idx = DatetimeIndex(arr)
+    assert not np.shares_memory(arr, get_array(idx))
+
+
+def test_constructor_copy_input_datetime_ea_default():
+    # GH 63388
+    arr = array(["2020-01-01", "2020-01-02"], dtype="datetime64[ns]")
+    idx = DatetimeIndex(arr)
+    assert not tm.shares_memory(arr, idx.array)
+
+
+def test_series_from_temporary_datetimeindex_readonly_data():
+    # GH 63388
+    arr = np.array(["2020-01-01", "2020-01-02"], dtype="datetime64[ns]")
+    arr.flags.writeable = False
+    ser = Series(DatetimeIndex(arr))
+    assert not np.shares_memory(arr, get_array(ser))
+    ser.iloc[0] = Timestamp("2020-01-01")
+    expected = Series(
+        [Timestamp("2020-01-01"), Timestamp("2020-01-02")], dtype="datetime64[ns]"
+    )
+    tm.assert_series_equal(ser, expected)

@@ -0,0 +1,29 @@
+import numpy as np
+
+from pandas import (
+    Interval,
+    IntervalIndex,
+    Series,
+    array,
+)
+import pandas._testing as tm
+from pandas.tests.copy_view.util import get_array
+
+
+def test_constructor_copy_input_interval_ea_default():
+    # GH 63388
+    arr = array([Interval(0, 1), Interval(1, 2)])
+    idx = IntervalIndex(arr)
+    assert not tm.shares_memory(arr, idx.array)
+
+
+def test_series_from_temporary_intervalindex_readonly_data():
+    # GH 63388
+    arr = array([Interval(0, 1), Interval(1, 2)])
+    arr._left.flags.writeable = False
+    arr._right.flags.writeable = False
+    ser = Series(IntervalIndex(arr))
+    assert not np.shares_memory(arr._left, get_array(ser)._left)
+    ser.iloc[0] = Interval(5, 6)
+    expected = Series([Interval(5, 6), Interval(1, 2)], dtype="interval[int64, right]")
+    tm.assert_series_equal(ser, expected)

@@ -1,12 +1,15 @@
+import numpy as np
 import pytest

 from pandas import (
     Period,
     PeriodIndex,
     Series,
+    array,
     period_range,
 )
 import pandas._testing as tm
+from pandas.tests.copy_view.util import get_array

 pytestmark = pytest.mark.filterwarnings(
     "ignore:Setting a value on a view:FutureWarning"
@@ -21,3 +24,24 @@ def test_periodindex(box):
     expected = idx.copy(deep=True)
     ser.iloc[0] = Period("2020-12-31")
     tm.assert_index_equal(idx, expected)
+
+
+def test_constructor_copy_input_period_ea_default():
+    # GH 63388
+    arr = array(["2020-01-01", "2020-01-02"], dtype="period[D]")
+    idx = PeriodIndex(arr)
+    assert not tm.shares_memory(arr, idx.array)
+
+
+def test_series_from_temporary_periodindex_readonly_data():
+    # GH 63388
+    arr = array(["2020-01-01", "2020-01-02"], dtype="period[D]")
+    arr._ndarray.flags.writeable = False
+    ser = Series(PeriodIndex(arr))
+    assert not np.shares_memory(arr._ndarray, get_array(ser))
+    ser.iloc[0] = Period("2022-01-01", freq="D")
+    expected = Series(
+        [Period("2022-01-01", freq="D"), Period("2020-01-02", freq="D")],
+        dtype="period[D]",
+    )
+    tm.assert_series_equal(ser, expected)

@@ -1,12 +1,15 @@
+import numpy as np
 import pytest

 from pandas import (
     Series,
     Timedelta,
     TimedeltaIndex,
+    array,
     timedelta_range,
 )
 import pandas._testing as tm
+from pandas.tests.copy_view.util import get_array

 pytestmark = pytest.mark.filterwarnings(
     "ignore:Setting a value on a view:FutureWarning"
@@ -27,3 +30,30 @@ def test_timedeltaindex(cons):
     expected = idx.copy(deep=True)
     ser.iloc[0] = Timedelta("5 days")
     tm.assert_index_equal(idx, expected)
+
+
+def test_constructor_copy_input_timedelta_ndarray_default():
+    # GH 63388
+    arr = np.array([1, 2], dtype="timedelta64[ns]")
+    idx = TimedeltaIndex(arr)
+    assert not np.shares_memory(arr, get_array(idx))
+
+
+def test_constructor_copy_input_timedelta_ea_default():
+    # GH 63388
+    arr = array([1, 2], dtype="timedelta64[ns]")
+    idx = TimedeltaIndex(arr)
+    assert not tm.shares_memory(arr, idx.array)
+
+
+def test_series_from_temporary_timedeltaindex_readonly_data():
+    # GH 63388
+    arr = np.array([1, 2], dtype="timedelta64[ns]")
+    arr.flags.writeable = False
+    ser = Series(TimedeltaIndex(arr))
+    assert not np.shares_memory(arr, get_array(ser))
+    ser.iloc[0] = Timedelta(days=1)
+    expected = Series(
+        [Timedelta(days=1), Timedelta(nanoseconds=2)], dtype="timedelta64[ns]"
+    )
+    tm.assert_series_equal(ser, expected)

@@ -1138,7 +1138,7 @@ def test_index_cast_datetime64_other_units(self):
     def test_constructor_int64_nocopy(self):
         # GH#1624
         arr = np.arange(1000, dtype=np.int64)
-        index = DatetimeIndex(arr)
+        index = DatetimeIndex(arr, copy=False)

         arr[50:100] = -1
         assert (index.asi8[50:100] == -1).all()

Can you create a `[USER_REDACTED]` (place it in `indexes/base.py` near `_raise_scalar_data_error`), that encapsulates this logic and can be called here instead?

Could you remove this note (I'll probably be adding something related to this in the copy on write guide)

```suggestion
    def _maybe_copy_array_input(cls, data, copy: bool | None, dtype) -> tuple[Any, bool]:
```

Can you undo these import changes if the contents haven't changed in your PR?

```suggestion
        return data, bool(copy)
```

Then you can remove the `bool(copy)` in other files

I think it is probably still worth mentioning it here explicitly as well? But I think we can simplify this to just say that all the Index constructors now copy array input by default, to be consistent with Series

OK sure sounds good. [USER_REDACTED] could you apply that suggestion. Also could you clarify that only numpy arrays and pandas ExtensionArrays are copied by default?


ENH: clarify error message when getting slice bounds for a non-monotonic index without specifying the label

- [x] closes #62990
- [x] [Tests added and passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#writing-tests) if fixing a bug or adding a new feature
- [x] All [code checks passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#pre-commit).

@@ -6865,12 +6865,15 @@ def get_slice_bound(self, label, side: Literal["left", "right"]) -> int:
         # we need to look up the label
         try:
             slc = self.get_loc(label)
-        except KeyError as err:
+        except KeyError:
             try:
                 return self._searchsorted_monotonic(label, side)
             except ValueError:
-                # raise the original KeyError
-                raise err from None
+                raise KeyError(
+                    f"Cannot get {side} slice bound for non-monotonic index "
+                    f"with a missing label {original_label!r}. "
+                    "Either sort the index or specify an existing label."
+                ) from None

         if isinstance(slc, np.ndarray):
             # get_loc may return a boolean array, which

@@ -564,9 +564,10 @@ def test_getitem_setitem_integer_slice_keyerrors(self):

         # non-monotonic, raise KeyError
         df2 = df.iloc[list(range(5)) + list(range(5, 10))[::-1]]
-        with pytest.raises(KeyError, match=r"^3$"):
+        msg = "non-monotonic index with a missing label 3"
+        with pytest.raises(KeyError, match=msg):
             df2.loc[3:11]
-        with pytest.raises(KeyError, match=r"^3$"):
+        with pytest.raises(KeyError, match=msg):
             df2.loc[3:11] = 0

     def test_fancy_getitem_slice_mixed(self, float_frame, float_string_frame):

@@ -600,10 +600,11 @@ def test_slice_locs_na(self):

     def test_slice_locs_na_raises(self):
         index = Index([np.nan, 1, 2])
-        with pytest.raises(KeyError, match="1.5"):
+        msg = "non-monotonic index with a missing label 1.5"
+        with pytest.raises(KeyError, match=msg):
             index.slice_locs(start=1.5)

-        with pytest.raises(KeyError, match="1.5"):
+        with pytest.raises(KeyError, match=msg):
             index.slice_locs(end=1.5)

@@ -3162,9 +3162,10 @@ def test_loc_getitem_setitem_integer_slice_keyerrors(self):

         # non-monotonic, raise KeyError
         s2 = ser.iloc[list(range(5)) + list(range(9, 4, -1))]
-        with pytest.raises(KeyError, match=r"^3$"):
+        msg = "non-monotonic index with a missing label 3"
+        with pytest.raises(KeyError, match=msg):
             s2.loc[3:11]
-        with pytest.raises(KeyError, match=r"^3$"):
+        with pytest.raises(KeyError, match=msg):
             s2.loc[3:11] = 0

     def test_loc_getitem_iterator(self, string_series):

@@ -404,6 +404,13 @@ def test_setitem_empty_indexer(indexer, val):
     tm.assert_frame_equal(df, expected)

+def test_loc_non_monotonic_index_with_a_missing_label():
+    msg = "Cannot get left slice bound for non-monotonic index with a missing label 4"
+    ser = Series([3, 6, 7, 6], index=[3, 8, 7, 6])
+    with pytest.raises(KeyError, match=msg):
+        ser.loc[4:7]
+
+
 class TestDeprecatedIndexers:
     [USER_REDACTED].mark.parametrize("key", [{1}, {1: 1}])
     def test_getitem_dict_and_set_deprecated(self, key):

Thanks for the PR!

Thanks!

```suggestion
                    f"with a missing label {original_label!r}. "
                    "Either sort the index or specify an existing label."
```

It's not clear to me what "non-specific" here means.

I think this should remain

```suggestion
                ) from None
```

We know that it's due to a missing label, adding the previous stack trace does not help clarify the error for the user.

Thanks, it’s clear now.

```suggestion
        except KeyError:
```

If I run `pre_commit run -a` on your branch locally, this is the change that it wants to make

Hmm, the rendered suggested change looks strange, but so the change is to remove `as err` from the line `except KeyError as err:` because the `err` variable is now no longer used after your change

Thanks for the comment. Indeed, we do not need the `err` variable anymore.


BUG: fix empty suffix and prefix handling in pyarrow string methods

Python's `str.removeprefix("")` and `str.removesuffix("")` return the original string.

The current pyarrow-backed implementation slices with `stop=0` or `start=0` when the prefix or suffix is empty, which can result in unexpected behavior instead of preserving the original values.

This PR adds explicit guards for empty prefix and suffix inputs and includes tests to ensure parity with Python semantics.

- [ ] closes #xxxx (Replace xxxx with the GitHub issue number)
- [ ] [Tests added and passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#writing-tests) if fixing a bug or adding a new feature
- [ ] All [code checks passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#pre-commit).
- [ ] Added [type annotations](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#type-hints) to new arguments/methods/functions.
- [ ] Added an entry in the latest `doc/source/whatsnew/vX.X.X.rst` file if fixing a bug or adding a new feature.
- [ ] If I used AI to develop this pull request, I prompted it to follow `AGENTS.md`.

@@ -203,12 +203,16 @@ def _str_swapcase(self) -> Self:
         return self._from_pyarrow_array(pc.utf8_swapcase(self._pa_array))

     def _str_removeprefix(self, prefix: str):
+        if prefix == "":
+            return self._from_pyarrow_array(self._pa_array)
         starts_with = pc.starts_with(self._pa_array, pattern=prefix)
         removed = pc.utf8_slice_codeunits(self._pa_array, len(prefix))
         result = pc.if_else(starts_with, removed, self._pa_array)
         return self._from_pyarrow_array(result)

     def _str_removesuffix(self, suffix: str):
+        if suffix == "":
+            return self._from_pyarrow_array(self._pa_array)
         ends_with = pc.ends_with(self._pa_array, pattern=suffix)
         removed = pc.utf8_slice_codeunits(self._pa_array, 0, stop=-len(suffix))
         result = pc.if_else(ends_with, removed, self._pa_array)

@@ -544,7 +544,12 @@ def test_strip_lstrip_rstrip_args(any_string_dtype, method, exp):

 [USER_REDACTED].mark.parametrize(
-    "prefix, expected", [("a", ["b", " b c", "bc"]), ("ab", ["", "a b c", "bc"])]
+    "prefix, expected",
+    [
+        ("a", ["b", " b c", "bc"]),
+        ("ab", ["", "a b c", "bc"]),
+        ("", ["ab", "a b c", "bc"]),
+    ],
 )
 def test_removeprefix(any_string_dtype, prefix, expected):
     ser = Series(["ab", "a b c", "bc"], dtype=any_string_dtype)
@@ -554,7 +559,12 @@ def test_removeprefix(any_string_dtype, prefix, expected):

 [USER_REDACTED].mark.parametrize(
-    "suffix, expected", [("c", ["ab", "a b ", "b"]), ("bc", ["ab", "a b c", ""])]
+    "suffix, expected",
+    [
+        ("c", ["ab", "a b ", "b"]),
+        ("bc", ["ab", "a b c", ""]),
+        ("", ["ab", "a b c", "bc"]),
+    ],
 )
 def test_removesuffix(any_string_dtype, suffix, expected):
     ser = Series(["ab", "a b c", "bc"], dtype=any_string_dtype)

Reverting some unnecessary whitespace changes

Thanks!

```suggestion

    def _str_removeprefix(self, prefix: str):
        if prefix == "":
            return self
        starts_with = pc.starts_with(self._pa_array, pattern=prefix)
        removed = pc.utf8_slice_codeunits(self._pa_array, len(prefix))
        result = pc.if_else(starts_with, removed, self._pa_array)
        return self._from_pyarrow_array(result)

    def _str_removesuffix(self, suffix: str):
        if prefix == "":
            return self
        ends_with = pc.ends_with(self._pa_array, pattern=suffix)
        removed = pc.utf8_slice_codeunits(self._pa_array, 0, stop=-len(suffix))
        result = pc.if_else(ends_with, removed, self._pa_array)
        return self._from_pyarrow_array(result)
```

I think this should fix it

I made those fixes as you suggested (added suffix=="" condition in _str_removesuffix ) still two checks seem to be failing, is that okay?

```suggestion
```

```suggestion
```

```suggestion
```

```suggestion
            return self._from_pyarrow_array(self._pa_array)
```


Perf: avoid fallback to NumPy in value_counts for Arrow-backed array

@@ -898,18 +898,15 @@ def value_counts_internal(
             result = result.iloc[0:0]

         # normalizing is by [NAME_REDACTED] (regardless of dropna)
-        counts = np.array([len(ii)])
+        normalize_denominator = len(ii)

     else:
+        normalize_denominator = None
         if is_extension_array_dtype(values):
             # handle Categorical and sparse,
             result = Series(values, copy=False)._values.value_counts(dropna=dropna)
             result.name = name
             result.index.name = index_name
-            counts = result._values
-            if not isinstance(counts, np.ndarray):
-                # e.g. ArrowExtensionArray
-                counts = np.asarray(counts)

         elif isinstance(values, ABCMultiIndex):
             # GH49558
@@ -920,10 +917,6 @@ def value_counts_internal(
                 .size()
             )
             result.index.names = values.names
-            # error: Incompatible types in assignment (expression has type
-            # "ndarray[Any, Any] | DatetimeArray | TimedeltaArray | PeriodArray | Any",
-            # variable has type "ndarray[tuple[int, ...], dtype[Any]]")
-            counts = result._values  # type: ignore[assignment]

         else:
             values = _ensure_arraylike(values, func_name="value_counts")
@@ -936,8 +929,7 @@ def value_counts_internal(
             idx = Index(keys, dtype=keys.dtype, name=index_name)

             if (
-                bins is None
-                and not sort
+                not sort
                 and isinstance(values, (DatetimeIndex, TimedeltaIndex))
                 and idx.equals(values)
                 and values.inferred_freq is not None
@@ -951,7 +943,10 @@ def value_counts_internal(
         result = result.sort_values(ascending=ascending, kind="stable")

     if normalize:
-        result = result / counts.sum()
+        if normalize_denominator is not None:
+            result = result / normalize_denominator
+        else:
+            result = result / result.sum()

     return result

I think it's OK to omit a test for this. We'll just call this a performance improvement that's already tested for behavior


- [x] All [code checks passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#pre-commit).

Replacing some of [USER_REDACTED] in pandas/pandas/core/frame.py.

ref #62437

@@ -5582,51 +5582,72 @@ def _reindex_multi(self, axes: dict[str, Index], fill_value) -> DataFrame:
                 fill_value=fill_value,
             )

-    [USER_REDACTED](
+    def set_axis(
+        self,
+        labels,
+        *,
+        axis: Axis = 0,
+        copy: bool | lib.NoDefault = lib.no_default,
+    ) -> DataFrame:
         """
+        Assign desired index to given axis.
+
+        Indexes for column or row labels can be changed by assigning
+        a list-like or Index.
+
+        Parameters
+        ----------
+        labels : list-like, Index
+            The values for the new index.
+
+        axis : {0 or 'index', 1 or 'columns'}, default 0
+            The axis to update. The value 0 identifies the rows. For `Series`
+            this parameter is unused and defaults to 0.
+
+        copy : bool, default False
+            This keyword is now ignored; changing its value will have no
+            impact on the method.
+
+            .. deprecated:: 3.0.0
+
+                This keyword is ignored and will be removed in pandas 4.0. Since
+                pandas 3.0, this method always returns a new object using a lazy
+                copy mechanism that defers copies until necessary
+                (Copy-on-Write). See the `user guide on Copy-on-Write
+                <https://pandas.pydata.org/docs/dev/user_guide/copy_on_write.html>`__
+                for more details.
+
+        Returns
+        -------
+        DataFrame
+            An object of type DataFrame.
+
+        See Also
+        --------
+        DataFrame.rename_axis : Alter the name of the index or columns.
+
         Examples
         --------
         >>> df = pd.DataFrame({"A": [1, 2, 3], "B": [4, 5, 6]})

         Change the row labels.

-        >>> df.set_axis(['a', 'b', 'c'], axis='index')
-           A  B
+        >>> df.set_axis(["a", "b", "c"], axis="index")
+            A  B
         a  1  4
         b  2  5
         c  3  6

         Change the column labels.

-        >>> df.set_axis(['I', 'II'], axis='columns')
-           I  II
+        >>> df.set_axis(["I", "II"], axis="columns")
+            I  II
         0  1   4
         1  2   5
         2  3   6
         """
-    )
-    [USER_REDACTED](
-        klass=_shared_doc_kwargs["klass"],
-        axes_single_arg=_shared_doc_kwargs["axes_single_arg"],
-        extended_summary_sub=" column or",
-        axis_description_sub=", and 1 identifies the columns",
-        see_also_sub=" or columns",
-    )
-    [USER_REDACTED](NDFrame.set_axis.__doc__)
-    def set_axis(
-        self,
-        labels,
-        *,
-        axis: Axis = 0,
-        copy: bool | lib.NoDefault = lib.no_default,
-    ) -> DataFrame:
         return super().set_axis(labels, axis=axis, copy=copy)

-    [USER_REDACTED](
-        NDFrame.reindex,
-        klass=_shared_doc_kwargs["klass"],
-        optional_reindex=_shared_doc_kwargs["optional_reindex"],
-    )
     def reindex(
         self,
         labels=None,
@@ -5641,6 +5662,227 @@ def reindex(
         limit: int | None = None,
         tolerance=None,
     ) -> DataFrame:
+        """
+        [NAME_REDACTED] to new index with optional filling logic.
+
+        Places NA/NaN in locations having no value in the previous index. A new object
+        is produced unless the new index is equivalent to the current one and
+        ``copy=False``.
+
+        Parameters
+        ----------
+
+        labels : array-like, optional
+            New labels / index to conform the axis specified by 'axis' to.
+        index : array-like, optional
+            New labels for the index. Preferably an Index object to avoid
+            duplicating data.
+        columns : array-like, optional
+            New labels for the columns. Preferably an Index object to avoid
+            duplicating data.
+        axis : int or str, optional
+            Axis to target. Can be either the axis name ('index', 'columns')
+            or number (0, 1).
+        method : {None, 'backfill'/'bfill', 'pad'/'ffill', 'nearest'}
+            Method to use for filling holes in reindexed DataFrame.
+            Please note: this is only applicable to DataFrames/Series with a
+            monotonically increasing/decreasing index.
+
+            * None (default): don't fill gaps
+            * pad / ffill: Propagate last valid observation forward to next
+              valid.
+            * backfill / bfill: Use next valid observation to fill gap.
+            * nearest: Use nearest valid observations to fill gap.
+
+        copy : bool, default False
+            This keyword is now ignored; changing its value will have no
+            impact on the method.
+
+            .. deprecated:: 3.0.0
+
+                This keyword is ignored and will be removed in pandas 4.0. Since
+                pandas 3.0, this method always returns a new object using a lazy
+                copy mechanism that defers copies until necessary
+                (Copy-on-Write). See the `user guide on Copy-on-Write
+                <https://pandas.pydata.org/docs/dev/user_guide/copy_on_write.html>`__
+                for more details.
+
+        level : int or name
+            Broadcast across a level, matching Index values on the
+            passed MultiIndex level.
+        fill_value : scalar, default np.nan
+            Value to use for missing values. Defaults to NaN, but can be any
+            "compatible" value.
+        limit : int, default None
+            Maximum number of consecutive elements to forward or backward fill.
+        tolerance : optional
+            Maximum distance between original and new labels for inexact
+            matches. The values of the index at the matching locations most
+            satisfy the equation ``abs(index[indexer] - target) <= tolerance``.
+
+            Tolerance may be a scalar value, which applies the same tolerance
+            to all values, or list-like, which applies variable tolerance per
+            element. List-like includes list, tuple, array, Series, and must be
+            the same size as the index and its dtype must exactly match the
+            index's type.
+
+        Returns
+        -------
+        DataFrame
+            DataFrame with changed index.
+
+        See Also
+        --------
+        DataFrame.set_index : Set row labels.
+        DataFrame.reset_index : Remove row labels or move them to new columns.
+        DataFrame.reindex_like : Change to same indices as other DataFrame.
+
+        Examples
+        --------
+        ``DataFrame.reindex`` supports two calling conventions
+
+        * ``(index=index_labels, columns=column_labels, ...)``
+        * ``(labels, axis={'index', 'columns'}, ...)``
+
+        We *highly* recommend using keyword arguments to clarify your
+        intent.
+
+        Create a DataFrame with some fictional data.
+
+        >>> index = ["Firefox", "Chrome", "Safari", "IE10", "Konqueror"]
+        >>> columns = ["http_status", "response_time"]
+        >>> df = pd.DataFrame(
+        ...     [[200, 0.04], [200, 0.02], [404, 0.07], [404, 0.08], [301, 1.0]],
+        ...     columns=columns,
+        ...     index=index,
+        ... )
+        >>> df
+                   http_status  response_time
+        Firefox            200           0.04
+        Chrome             200           0.02
+        Safari             404           0.07
+        IE10               404           0.08
+        Konqueror          301           1.00
+
+        Create a new index and reindex the DataFrame. By default
+        values in the new index that do not have corresponding
+        records in the DataFrame are assigned ``NaN``.
+
+        >>> new_index = ["Safari", "Iceweasel", "[NAME_REDACTED]", "IE10", "Chrome"]
+        >>> df.reindex(new_index)
+                       http_status  response_time
+        Safari               404.0           0.07
+        [NAME_REDACTED]
+        [NAME_REDACTED]            NaN
+        IE10                 404.0           0.08
+        Chrome               200.0           0.02
+
+        We can fill in the missing values by passing a value to
+        the keyword ``fill_value``. Because the index is not monotonically
+        increasing or decreasing, we cannot use arguments to the keyword
+        ``method`` to fill the ``NaN`` values.
+
+        >>> df.reindex(new_index, fill_value=0)
+                       http_status  response_time
+        Safari                 404           0.07
+        Iceweasel                0           0.00
+        [NAME_REDACTED]            0           0.00
+        IE10                   404           0.08
+        Chrome                 200           0.02
+
+        >>> df.reindex(new_index, fill_value="missing")
+                      http_status response_time
+        Safari                404          0.07
+        Iceweasel         missing       missing
+        [NAME_REDACTED]     missing       missing
+        IE10                  404          0.08
+        Chrome                200          0.02
+
+        We can also reindex the columns.
+
+        >>> df.reindex(columns=["http_status", "user_agent"])
+                   http_status  user_agent
+        Firefox            200         NaN
+        Chrome             200         NaN
+        Safari             404         NaN
+        IE10               404         NaN
+        Konqueror          301         NaN
+
+        Or we can use "axis-style" keyword arguments
+
+        >>> df.reindex(["http_status", "user_agent"], axis="columns")
+                   http_status  user_agent
+        Firefox            200         NaN
+        Chrome             200         NaN
+        Safari             404         NaN
+        IE10               404         NaN
+        Konqueror          301         NaN
+
+        To further illustrate the filling functionality in
+        ``reindex``, we will create a DataFrame with a
+        monotonically increasing index (for example, a sequence
+        of dates).
+
+        >>> date_index = pd.date_range("1/1/2010", periods=6, freq="D")
+        >>> df2 = pd.DataFrame(
+        ...     {"prices": [100, 101, np.nan, 100, 89, 88]}, index=date_index
+        ... )
+        >>> df2
+                    prices
+        2010-01-01   100.0
+        2010-01-02   101.0
+        2010-01-03     NaN
+        2010-01-04   100.0
+        2010-01-05    89.0
+        2010-01-06    88.0
+
+        Suppose we decide to expand the DataFrame to cover a wider
+        date range.
+
+        >>> date_index2 = pd.date_range("12/29/2009", periods=10, freq="D")
+        >>> df2.reindex(date_index2)
+                    prices
+        2009-12-29     NaN
+        2009-12-30     NaN
+        2009-12-31     NaN
+        2010-01-01   100.0
+        2010-01-02   101.0
+        2010-01-03     NaN
+        2010-01-04   100.0
+        2010-01-05    89.0
+        2010-01-06    88.0
+        2010-01-07     NaN
+
+        The index entries that did not have a value in the original data frame
+        (for example, '2009-12-29') are by [NAME_REDACTED] ``NaN``.
+        If desired, we can fill in the missing values using one of several
+        options.
+
+        For example, to back-propagate the last valid value to fill the ``NaN``
+        values, pass ``bfill`` as an argument to the ``method`` keyword.
+
+        >>> df2.reindex(date_index2, method="bfill")
+                    prices
+        2009-12-29   100.0
+        2009-12-30   100.0
+        2009-12-31   100.0
+        2010-01-01   100.0
+        2010-01-02   101.0
+        2010-01-03     NaN
+        2010-01-04   100.0
+        2010-01-05    89.0
+        2010-01-06    88.0
+        2010-01-07     NaN
+
+        Please note that the ``NaN`` value present in the original DataFrame
+        (at index value 2010-01-03) will not be filled by [NAME_REDACTED]
+        value propagation schemes. This is because filling while reindexing
+        does not look at DataFrame values, but only compares the original and
+        desired indexes. If you do want to fill in the ``NaN`` values present
+        in the original DataFrame, use the ``fillna()`` method.
+
+        See the :ref:`user guide <basics.reindexing>` for more.
+        """
         return super().reindex(
             labels=labels,
             index=index,
@@ -6129,7 +6371,6 @@ def _replace_columnwise(

         return res if inplace else res.__finalize__(self)

-    [USER_REDACTED](NDFrame.shift, klass=_shared_doc_kwargs["klass"])
     def shift(
         self,
         periods: int | Sequence[int] = 1,
@@ -6138,6 +6379,120 @@ def shift(
         fill_value: Hashable = lib.no_default,
         suffix: str | None = None,
     ) -> DataFrame:
+        """
+        Shift index by [NAME_REDACTED] with an optional time `freq`.
+
+        When `freq` is not passed, shift the index without realigning the data.
+        If `freq` is passed (in this case, the index must be date or datetime,
+        or it will raise a `NotImplementedError`), the index will be
+        increased using the periods and the `freq`. `freq` can be inferred
+        when specified as "infer" as long as either freq or inferred_freq
+        attribute is set in the index.
+
+        Parameters
+        ----------
+        periods : int or Sequence
+            Number of periods to shift. Can be positive or negative.
+            If an iterable of ints, the data will be shifted once by [NAME_REDACTED]
+            This is equivalent to shifting by [NAME_REDACTED] a time and
+            concatenating all resulting frames. The resulting columns will have
+            the shift suffixed to their column names. For multiple periods,
+            axis must not be 1.
+        freq : DateOffset, tseries.offsets, timedelta, or str, optional
+            Offset to use from the tseries module or time rule (e.g. 'EOM').
+            If `freq` is specified then the index values are shifted but the
+            data is not realigned. That is, use `freq` if you would like to
+            extend the index when shifting and preserve the original data.
+            If `freq` is specified as "infer" then it will be inferred from
+            the freq or inferred_freq attributes of the index. If neither of
+            those attributes exist, a ValueError is thrown.
+        axis : {0 or 'index', 1 or 'columns', None}, default None
+            Shift direction. For `Series` this parameter is unused and defaults to 0.
+        fill_value : object, optional
+            The scalar value to use for newly introduced missing values.
+            the default depends on the dtype of `self`.
+            For Boolean and numeric NumPy data types, ``np.nan`` is used.
+            For datetime, timedelta, or period data, etc. :attr:`NaT` is used.
+            For extension dtypes, ``self.dtype.na_value`` is used.
+        suffix : str, optional
+            If str and periods is an iterable, this is added after the column
+            name and before the shift value for each shifted column name.
+            For `Series` this parameter is unused and defaults to `None`.
+
+        Returns
+        -------
+        DataFrame
+            Copy of input object, shifted.
+
+        See Also
+        --------
+        Index.shift : Shift values of Index.
+        DatetimeIndex.shift : Shift values of DatetimeIndex.
+        PeriodIndex.shift : Shift values of PeriodIndex.
+
+        Examples
+        --------
+        >>> df = pd.DataFrame(
+        ...     [[10, 13, 17], [20, 23, 27], [15, 18, 22], [30, 33, 37], [45, 48, 52]],
+        ...     columns=["Col1", "Col2", "Col3"],
+        ...     index=pd.date_range("2020-01-01", "2020-01-05"),
+        ... )
+        >>> df
+                    Col1  Col2  Col3
+        2020-01-01    10    13    17
+        2020-01-02    20    23    27
+        2020-01-03    15    18    22
+        2020-01-04    30    33    37
+        2020-01-05    45    48    52
+
+        >>> df.shift(periods=3)
+                    Col1  Col2  Col3
+        2020-01-01   [NAME_REDACTED]
+        2020-01-02   [NAME_REDACTED]
+        2020-01-03   [NAME_REDACTED]
+        2020-01-04  10.0  13.0  17.0
+        2020-01-05  20.0  23.0  27.0
+
+        >>> df.shift(periods=1, axis="columns")
+                    Col1  Col2  Col3
+        2020-01-01   NaN    10    13
+        2020-01-02   NaN    20    23
+        2020-01-03   NaN    15    18
+        2020-01-04   NaN    30    33
+        2020-01-05   NaN    45    48
+
+        >>> df.shift(periods=3, fill_value=0)
+                    Col1  Col2  Col3
+        2020-01-01     0     0     0
+        2020-01-02     0     0     0
+        2020-01-03     0     0     0
+        2020-01-04    10    13    17
+        2020-01-05    20    23    27
+
+        >>> df.shift(periods=3, freq="D")
+                    Col1  Col2  Col3
+        2020-01-04    10    13    17
+        2020-01-05    20    23    27
+        2020-01-06    15    18    22
+        2020-01-07    30    33    37
+        2020-01-08    45    48    52
+
+        >>> df.shift(periods=3, freq="infer")
+                    Col1  Col2  Col3
+        2020-01-04    10    13    17
+        2020-01-05    20    23    27
+        2020-01-06    15    18    22
+        2020-01-07    30    33    37
+        2020-01-08    45    48    52
+
+        >>> df["Col1"].shift(periods=[0, 1, 2])
+                    Col1_0  Col1_1  Col1_2
+        2020-01-01      10     [NAME_REDACTED]
+        2020-01-02      20    10.0     NaN
+        2020-01-03      15    20.0    10.0
+        2020-01-04      30    15.0    20.0
+        2020-01-05      45    30.0    15.0
+        """
         if freq is not None and fill_value is not lib.no_default:
             # GH#53832
             raise ValueError(
@@ -8743,12 +9098,316 @@ def _flex_cmp_method(self, other, op, *, axis: Axis = "columns", level=None):
         new_data = self._dispatch_frame_op(other, op, axis=axis)
         return self._construct_result(new_data, other=other)

-    [USER_REDACTED](ops.make_flex_doc("eq", "dataframe"))
     def eq(self, other, axis: Axis = "columns", level=None) -> DataFrame:
+        """
+        [NAME_REDACTED] equal to of dataframe and other, element-wise (binary operator `eq`).
+
+        Among flexible wrappers (`eq`, `ne`, `le`, `lt`, `ge`, `gt`) to comparison
+        operators.
+
+        Equivalent to `==`, `!=`, `<=`, `<`, `>=`, `>` with support to choose axis
+        (rows or columns) and level for comparison.
+
+        Parameters
+        ----------
+        other : scalar, sequence, Series, or DataFrame
+            Any single or multiple element data structure, or list-like object.
+        axis : {0 or 'index', 1 or 'columns'}, default 'columns'
+            Whether to compare by [NAME_REDACTED] (0 or 'index') or columns
+            (1 or 'columns').
+        level : int or label
+            Broadcast across a level, matching Index values on the passed
+            MultiIndex level.
+
+        Returns
+        -------
+        DataFrame of bool
+            Result of the comparison.
+
+        See Also
+        --------
+        DataFrame.eq : [NAME_REDACTED] for equality elementwise.
+        DataFrame.ne : [NAME_REDACTED] for inequality elementwise.
+        DataFrame.le : [NAME_REDACTED] for less than inequality
+            or equality elementwise.
+        DataFrame.lt : [NAME_REDACTED] for strictly less than
+            inequality elementwise.
+        DataFrame.ge : [NAME_REDACTED] for greater than inequality
+            or equality elementwise.
+        DataFrame.gt : [NAME_REDACTED] for strictly greater than
+            inequality elementwise.
+
+        Notes
+        -----
+        Mismatched indices will be unioned together.
+        `NaN` values are considered different (i.e. `NaN` != `NaN`).
+
+        Examples
+        --------
+        >>> df = pd.DataFrame(
+        ...     {"cost": [250, 150, 100], "revenue": [100, 250, 300]},
+        ...     index=["A", "B", "C"],
+        ... )
+        >>> df
+           cost  revenue
+        A   250      100
+        B   150      250
+        C   100      300
+
+        Comparison with a scalar, using either the operator or method:
+
+        >>> df == 100
+            cost  revenue
+        A  False     True
+        B  False    False
+        C   True    False
+
+        >>> df.eq(100)
+            cost  revenue
+        A  False     True
+        B  False    False
+        C   True    False
+
+        When `other` is a :class:`Series`, the columns of a DataFrame are aligned
+        with the index of `other` and broadcast:
+
+        >>> df != pd.Series([100, 250], index=["cost", "revenue"])
+            cost  revenue
+        A   True     True
+        B   True    False
+        C  False     True
+
+        Use the method to control the broadcast axis:
+
+        >>> df.ne(pd.Series([100, 300], index=["A", "D"]), axis="index")
+           cost  revenue
+        A  True    False
+        B  True     True
+        C  True     True
+        D  True     True
+
+        When comparing to an arbitrary sequence, the number of columns must
+        match the number elements in `other`:
+
+        >>> df == [250, 100]
+            cost  revenue
+        A   True     True
+        B  False    False
+        C  False    False
+
+        Use the method to control the axis:
+
+        >>> df.eq([250, 250, 100], axis="index")
+            cost  revenue
+        A   True    False
+        B  False     True
+        C   True    False
+
+        Compare to a DataFrame of different shape.
+
+        >>> other = pd.DataFrame(
+        ...     {"revenue": [300, 250, 100, 150]}, index=["A", "B", "C", "D"]
+        ... )
+        >>> other
+           revenue
+        A      300
+        B      250
+        C      100
+        D      150
+
+        >>> df.gt(other)
+            cost  revenue
+        A  False    False
+        B  False    False
+        C  False     True
+        D  False    False
+
+        Compare to a MultiIndex by level.
+
+        >>> df_multindex = pd.DataFrame(
+        ...     {
+        ...         "cost": [250, 150, 100, 150, 300, 220],
+        ...         "revenue": [100, 250, 300, 200, 175, 225],
+        ...     },
+        ...     index=[
+        ...         ["Q1", "Q1", "Q1", "Q2", "Q2", "Q2"],
+        ...         ["A", "B", "C", "A", "B", "C"],
+        ...     ],
+        ... )
+        >>> df_multindex
+              cost  revenue
+        Q1 A   250      100
+           B   150      250
+           C   100      300
+        Q2 A   150      200
+           B   300      175
+           C   220      225
+
+        >>> df.le(df_multindex, level=1)
+               cost  revenue
+        Q1 A   True     True
+           B   True     True
+           C   True     True
+        Q2 A  False     True
+           B   True    False
+           C   True    False
+        """
         return self._flex_cmp_method(other, operator.eq, axis=axis, level=level)

-    [USER_REDACTED](ops.make_flex_doc("ne", "dataframe"))
     def ne(self, other, axis: Axis = "columns", level=None) -> DataFrame:
+        """
+        [NAME_REDACTED] equal to of dataframe and other, element-wise (binary operator `ne`).
+
+        Among flexible wrappers (`eq`, `ne`, `le`, `lt`, `ge`, `gt`) to comparison
+        operators.
+
+        Equivalent to `==`, `!=`, `<=`, `<`, `>=`, `>` with support to choose axis
+        (rows or columns) and level for comparison.
+
+        Parameters
+        ----------
+        other : scalar, sequence, Series, or DataFrame
+            Any single or multiple element data structure, or list-like object.
+        axis : {0 or 'index', 1 or 'columns'}, default 'columns'
+            Whether to compare by [NAME_REDACTED] (0 or 'index') or columns
+            (1 or 'columns').
+        level : int or label
+            Broadcast across a level, matching Index values on the passed
+            MultiIndex level.
+
+        Returns
+        -------
+        DataFrame of bool
+            Result of the comparison.
+
+        See Also
+        --------
+        DataFrame.eq : [NAME_REDACTED] for equality elementwise.
+        DataFrame.ne : [NAME_REDACTED] for inequality elementwise.
+        DataFrame.le : [NAME_REDACTED] for less than inequality
+            or equality elementwise.
+        DataFrame.lt : [NAME_REDACTED] for strictly less than
+            inequality elementwise.
+        DataFrame.ge : [NAME_REDACTED] for greater than inequality
+            or equality elementwise.
+        DataFrame.gt : [NAME_REDACTED] for strictly greater than
+            inequality elementwise.
+
+        Notes
+        -----
+        Mismatched indices will be unioned together.
+        `NaN` values are considered different (i.e. `NaN` != `NaN`).
+
+        Examples
+        --------
+        >>> df = pd.DataFrame(
+        ...     {"cost": [250, 150, 100], "revenue": [100, 250, 300]},
+        ...     index=["A", "B", "C"],
+        ... )
+        >>> df
+           cost  revenue
+        A   250      100
+        B   150      250
+        C   100      300
+
+        Comparison with a scalar, using either the operator or method:
+
+        >>> df == 100
+            cost  revenue
+        A  False     True
+        B  False    False
+        C   True    False
+
+        >>> df.eq(100)
+            cost  revenue
+        A  False     True
+        B  False    False
+        C   True    False
+
+        When `other` is a :class:`Series`, the columns of a DataFrame are aligned
+        with the index of `other` and broadcast:
+
+        >>> df != pd.Series([100, 250], index=["cost", "revenue"])
+            cost  revenue
+        A   True     True
+        B   True    False
+        C  False     True
+
+        Use the method to control the broadcast axis:
+
+        >>> df.ne(pd.Series([100, 300], index=["A", "D"]), axis="index")
+           cost  revenue
+        A  True    False
+        B  True     True
+        C  True     True
+        D  True     True
+
+        When comparing to an arbitrary sequence, the number of columns must
+        match the number elements in `other`:
+
+        >>> df == [250, 100]
+            cost  revenue
+        A   True     True
+        B  False    False
+        C  False    False
+
+        Use the method to control the axis:
+
+        >>> df.eq([250, 250, 100], axis="index")
+            cost  revenue
+        A   True    False
+        B  False     True
+        C   True    False
+
+        Compare to a DataFrame of different shape.
+
+        >>> other = pd.DataFrame(
+        ...     {"revenue": [300, 250, 100, 150]}, index=["A", "B", "C", "D"]
+        ... )
+        >>> other
+           revenue
+        A      300
+        B      250
+        C      100
+        D      150
+
+        >>> df.gt(other)
+            cost  revenue
+        A  False    False
+        B  False    False
+        C  False     True
+        D  False    False
+
+        Compare to a MultiIndex by level.
+
+        >>> df_multindex = pd.DataFrame(
+        ...     {
+        ...         "cost": [250, 150, 100, 150, 300, 220],
+        ...         "revenue": [100, 250, 300, 200, 175, 225],
+        ...     },
+        ...     index=[
+        ...         ["Q1", "Q1", "Q1", "Q2", "Q2", "Q2"],
+        ...         ["A", "B", "C", "A", "B", "C"],
+        ...     ],
+        ... )
+        >>> df_multindex
+              cost  revenue
+        Q1 A   250      100
+           B   150      250
+           C   100      300
+        Q2 A   150      200
+           B   300      175
+           C   220      225
+
+        >>> df.le(df_multindex, level=1)
+               cost  revenue
+        Q1 A   True     True
+           B   True     True
+           C   True     True
+        Q2 A  False     True
+           B   True    False
+           C   True    False
+        """
         return self._flex_cmp_method(other, operator.ne, axis=axis, level=level)

     [USER_REDACTED](ops.make_flex_doc("le", "dataframe"))

Could you re-align these tables?

Same here

Could you fix the indentation?

OK, of course.


DOC: update whatsnew section for pd.col()

This expands the section a bit, trying to explain more what it actually is and can do. And also emphasized that it is "initial" support, to avoid people expect that it could be used everywhere (or to the extent that it can be used in other libraries like polars and pyspark)

For another issue, but we should also add a section about it in the user guide to refer to with a more complete / up to date description of how it can be used.

cc [USER_REDACTED]

@@ -130,10 +130,18 @@ and will be removed in pandas 4.0.

 .. _whatsnew_300.enhancements.col:

-``pd.col`` syntax can now be used in :meth:`DataFrame.assign` and :meth:`DataFrame.loc`
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Initial support for ``pd.col()`` syntax to create expressions
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

-You can now use ``pd.col`` to create callables for use in dataframe methods which accept them. For example, if you have a dataframe
+This release introduces :func:`col` to refer to a DataFrame column by name
+and build up expressions.
+
+This can be used as a simplified syntax to create callables for use in
+methods such as :meth:`DataFrame.assign`. In practice, where you would
+have to use a ``lambda`` function before, you can now use ``pd.col()``
+instead.
+
+For example, if you have a dataframe

 .. ipython:: python

@@ -151,6 +159,18 @@ you can now write:

     df.assign(c = pd.col('a') + pd.col('b'))

+The expression object returned by :func:`col` supports all standard operators
+(like ``+``, ``-``, ``*``, ``/``, etc.) and all Series methods and namespaces
+(like ``pd.col("name").sum()``, ``pd.col("name").str.upper()``, etc.).
+
+Currently, the ``pd.col()`` syntax can be used in any place which accepts a
+callable that takes the calling DataFrame as first argument and returns a
+Series, like ``lambda df: df[col_name]``.
+This includes :meth:`DataFrame.assign`, :meth:`DataFrame.loc`, and getitem/setitem.
+
+It is expected that the support for ``pd.col()`` will be expanded to more methods
+in future releases.
+
 [NAME_REDACTED]
 ^^^^^^^^^^^^^^^^^^^^^^
 pandas 3.0.0 introduces a new 3-stage deprecation policy: using ``DeprecationWarning`` initially, then switching to ``FutureWarning`` for broader visibility in the last minor version before the next major release, and then removal of the deprecated functionality in the major release. This was done to give downstream packages more time to adjust to pandas deprecations, which should reduce the amount of warnings that a user gets from code that isn't theirs. See `PDEP 17 <https://pandas.pydata.org/pdeps/0017-backwards-compatibility-and-deprecation-policy.html>`_ for more details.

@@ -260,6 +260,8 @@ def col(col_name: Hashable) -> Expression:
     :meth:`DataFrame.assign` or :meth:`DataFrame.loc`, can also accept
     ``pd.col(col_name)``.

+    .. versionadded:: 3.0.0
+
     Parameters
     ----------
     col_name : Hashable

Missing period at the end of this sentence. The sentence should end with a period for consistency with the rest of the documentation.
```suggestion
For example, if you have a dataframe.
```


BUG: copy np.ndarray inputs to Index constructor by default

- [ ] closes #63306 (Replace xxxx with the GitHub issue number)

[USER_REDACTED] I went with your suggestion to

> We changed Series to copy array input by [NAME_REDACTED] CoW. We should maybe do the same for Index?

in https://github.com/[USER_REDACTED]ndas-dev/pandas/issues/63306#issuecomment-3637458119

@@ -341,8 +341,13 @@ class Index(IndexOpsMixin, PandasObject):
         Data type for the output Index. If not specified, this will be
         inferred from `data`.
         See the :ref:`user guide <basics.dtypes>` for more usages.
-    copy : bool, default False
-        Copy input data.
+    copy : bool, default None
+        Whether to copy input data, only relevant for array, Series, and Index
+        inputs (for other input, e.g. a list, a new array is created anyway).
+        Defaults to True for array input and False for Index/Series.
+        Set to False to avoid copying array input at your own risk (if you
+        know the input data won't be modified elsewhere).
+        Set to True to force copying Series/Index input up front.
     name : object
         Name to be stored in the index.
     tupleize_cols : bool (default: True)
@@ -482,7 +487,7 @@ def __new__(
         cls,
         data=None,
         dtype=None,
-        copy: bool = False,
+        copy: bool | None = None,
         name=None,
         tupleize_cols: bool = True,
     ) -> Self:
@@ -499,9 +504,16 @@ def __new__(
         if not copy and isinstance(data, (ABCSeries, Index)):
             refs = data._references

+        if isinstance(data, (ExtensionArray, np.ndarray)):
+            # GH 63306
+            if copy is not False:
+                if dtype is None or astype_is_view(data.dtype, dtype):
+                    data = data.copy()
+                    copy = False
+
         # range
         if isinstance(data, (range, RangeIndex)):
-            result = RangeIndex(start=data, copy=copy, name=name)
+            result = RangeIndex(start=data, copy=bool(copy), name=name)
             if dtype is not None:
                 return result.astype(dtype, copy=False)
             # error: Incompatible return value type (got "MultiIndex",
@@ -569,7 +581,7 @@ def __new__(
                 data = com.asarray_tuplesafe(data, dtype=_dtype_obj)

         try:
-            arr = sanitize_array(data, None, dtype=dtype, copy=copy)
+            arr = sanitize_array(data, None, dtype=dtype, copy=bool(copy))
         except ValueError as err:
             if "index must be specified when data is not list-like" in str(err):
                 raise cls._raise_scalar_data_error(data) from err

@@ -5,6 +5,7 @@
     DataFrame,
     Index,
     Series,
+    array,
 )
 import pandas._testing as tm
 from pandas.tests.copy_view.util import get_array
@@ -150,3 +151,27 @@ def test_index_values():
     idx = Index([1, 2, 3])
     result = idx.values
     assert result.flags.writeable is False
+
+
+def test_constructor_copy_input_ndarray_default():
+    arr = np.array([0, 1])
+    idx = Index(arr)
+    assert not np.shares_memory(arr, get_array(idx))
+
+
+def test_constructor_copy_input_ea_default():
+    arr = array([0, 1], dtype="Int64")
+    idx = Index(arr)
+    assert not tm.shares_memory(arr, idx.array)
+
+
+def test_series_from_temporary_index_readonly_data():
+    # GH 63370
+    arr = np.array([0, 1], dtype=np.dtype(np.int8))
+    arr.flags.writeable = False
+    ser = Series(Index(arr))
+    assert not np.shares_memory(arr, get_array(ser))
+    assert ser._mgr._has_no_reference(0)
+    ser[[False, True]] = np.array([0, 2], dtype=np.dtype(np.int8))
+    expected = Series([0, 2], dtype=np.dtype(np.int8))
+    tm.assert_series_equal(ser, expected)

Looks good!

For datetime64 ndarray, we wouldn't get in the elif block below that now copies, but this is what you mentioned to handle in the Index subclasses in a follow-up?

You can also use the updated description in the Series docstring as a starting point to update the docstring here as well (as the end goal should be to make it similar)

My comment was referring passing an ndarray to `DatetimeIndex` et. al., but good find, yes I think I'll have to put this check higher up.