API: Period.to_timestamp default to microsecond unit

- [ ] closes #xxxx (Replace xxxx with the GitHub issue number)
- [x] [Tests added and passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#writing-tests) if fixing a bug or adding a new feature
- [x] All [code checks passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#pre-commit).
- [ ] Added [type annotations](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#type-hints) to new arguments/methods/functions.
- [x] Added an entry in the latest `doc/source/whatsnew/vX.X.X.rst` file if fixing a bug or adding a new feature.
- [ ] If I used AI to develop this pull request, I prompted it to follow `AGENTS.md`.

cc [USER_REDACTED] do we want to get this in under the wire?

@@ -800,6 +800,7 @@ Other API changes
 ^^^^^^^^^^^^^^^^^
 - 3rd party ``py.path`` objects are no longer explicitly supported in IO methods. Use :py:class:`pathlib.Path` objects instead (:issue:`57091`)
 - :func:`read_table`'s ``parse_dates`` argument defaults to ``None`` to improve consistency with :func:`read_csv` (:issue:`57476`)
+- :meth:`Period.to_timestamp` and :meth:`PeriodIndex.to_timestamp` now give microsecond-unit objects when possible, and nanosecond-unit objects in other cases. This affects the actual value of :meth:`Period.end_time` and :meth:`PeriodIndex.end_time` (:issue:`56164`)
 - All classes inheriting from builtin ``tuple`` (including types created with :func:`collections.namedtuple`) are now hashed and compared as builtin ``tuple`` during indexing operations (:issue:`57922`)
 - Made ``dtype`` a required argument in :meth:`ExtensionArray._from_sequence_of_strings` (:issue:`56519`)
 - Passing a :class:`Series` input to :func:`json_normalize` will now retain the :class:`Series` :class:`Index`, previously output had a new :class:`RangeIndex` (:issue:`51452`)

@@ -38,7 +38,10 @@ from libc.time cimport (
     tm,
 )

-from pandas._libs.tslibs.dtypes cimport c_OFFSET_TO_PERIOD_FREQSTR
+from pandas._libs.tslibs.dtypes cimport (
+    PeriodDtypeCode,
+    c_OFFSET_TO_PERIOD_FREQSTR,
+)

 from pandas._libs.tslibs.np_datetime import OutOfBoundsDatetime

@@ -69,13 +72,6 @@ from pandas._libs.tslibs.ccalendar cimport (
     get_week_of_year,
     is_leapyear,
 )
-from pandas._libs.tslibs.timedeltas cimport (
-    delta_to_nanoseconds,
-    is_any_td_scalar,
-)
-
-from pandas._libs.tslibs.conversion import DT64NS_DTYPE
-
 from pandas._libs.tslibs.dtypes cimport (
     FR_ANN,
     FR_BUS,
@@ -95,6 +91,10 @@ from pandas._libs.tslibs.dtypes cimport (
     freq_group_code_to_npy_unit,
 )
 from pandas._libs.tslibs.parsing cimport quarter_to_myear
+from pandas._libs.tslibs.timedeltas cimport (
+    delta_to_nanoseconds,
+    is_any_td_scalar,
+)

 from pandas._libs.tslibs.parsing import parse_datetime_string_with_reso

@@ -973,13 +973,13 @@ def periodarr_to_dt64arr(const int64_t[:] periodarr, int freq):
         for i in range(N):
             out[i] = period_ordinal_to_dt64(periodarr[i], freq)

-        return out.base  # .base to access underlying np.ndarray
+        return out.base.view("M8[us]")  # .base to access underlying np.ndarray

     else:
         # Short-circuit for performance
         if freq == FR_NS:
             # TODO: copy?
-            return periodarr.base
+            return periodarr.base.view("M8[ns]")

         if freq == FR_US:
             dta = periodarr.base.view("M8[us]")
@@ -993,7 +993,8 @@ def periodarr_to_dt64arr(const int64_t[:] periodarr, int freq):
             dta = periodarr.base.view("M8[h]")
         elif freq == FR_DAY:
             dta = periodarr.base.view("M8[D]")
-        return astype_overflowsafe(dta, dtype=DT64NS_DTYPE)
+        # GH#63760 give microseconds for everything other than freq="ns"
+        return astype_overflowsafe(dta, dtype=np.dtype("M8[us]"))

 cdef void get_asfreq_info(int from_freq, int to_freq,
@@ -1161,13 +1162,19 @@ cdef int64_t period_ordinal_to_dt64(int64_t ordinal, int freq) except? -1:
     if ordinal == NPY_NAT:
         return NPY_NAT

+    if freq == PeriodDtypeCode.N:
+        # We have to return nanosecond unit, but this is a no-op
+        return ordinal
+
     get_date_info(ordinal, freq, &dts)

     try:
-        result = npy_datetimestruct_to_datetime(NPY_DATETIMEUNIT.NPY_FR_ns, &dts)
+        result = npy_datetimestruct_to_datetime(NPY_DATETIMEUNIT.NPY_FR_us, &dts)
     except OverflowError as err:
         fmt = dts_to_iso_string(&dts)
-        raise OutOfBoundsDatetime(f"Out of bounds nanosecond timestamp: {fmt}") from err
+        raise OutOfBoundsDatetime(
+            f"Out of bounds microsecond timestamp: {fmt}"
+        ) from err

     return result

@@ -1681,7 +1688,7 @@ cdef class PeriodMixin:
         Timestamp('2012-01-01 00:00:00')

         >>> period.end_time
-        Timestamp('2012-01-01 23:59:59.999999999')
+        Timestamp('2012-01-01 23:59:59.999999')
         """
         return self.to_timestamp(how="start")

@@ -1706,7 +1713,7 @@ cdef class PeriodMixin:
         For Period:

         >>> pd.Period('2020-01', 'D').end_time
-        Timestamp('2020-01-01 23:59:59.999999999')
+        Timestamp('2020-01-01 23:59:59.999999')

         For Series:

@@ -1718,19 +1725,19 @@ cdef class PeriodMixin:
         2   2020-03
         dtype: period[M]
         >>> s.dt.end_time
-        0   2020-01-31 23:59:59.999999999
-        1   2020-02-29 23:59:59.999999999
-        2   2020-03-31 23:59:59.999999999
-        dtype: datetime64[ns]
+        0   2020-01-31 23:59:59.999999
+        1   2020-02-29 23:59:59.999999
+        2   2020-03-31 23:59:59.999999
+        dtype: datetime64[us]

         For PeriodIndex:

         >>> idx = pd.PeriodIndex(["2023-01", "2023-02", "2023-03"], freq="M")
         >>> idx.end_time
-        DatetimeIndex(['2023-01-31 23:59:59.999999999',
-                       '2023-02-28 23:59:59.999999999',
-                       '2023-03-31 23:59:59.999999999'],
-                       dtype='datetime64[ns]', freq=None)
+        DatetimeIndex(['2023-01-31 23:59:59.999999',
+                       '2023-02-28 23:59:59.999999',
+                       '2023-03-31 23:59:59.999999'],
+                       dtype='datetime64[us]', freq=None)
         """
         return self.to_timestamp(how="end")

@@ -2061,6 +2068,9 @@ cdef class _Period(PeriodMixin):
         Uses the target frequency specified at the part of the period specified
         by `how`, which is either `Start` or `Finish`.

+        If possible, gives microsecond-unit Timestamp. Otherwise gives nanosecond
+        unit.
+
         Parameters
         ----------
         freq : str or DateOffset
@@ -2089,14 +2099,19 @@ cdef class _Period(PeriodMixin):
         """
         how = validate_end_alias(how)

+        if self._dtype._dtype_code == PeriodDtypeCode.N or freq == "ns":
+            unit = "ns"
+        else:
+            unit = "us"
+
         end = how == "E"
         if end:
             if freq == "B" or self._freq == "B":
                 # roll forward to ensure we land on B date
-                adjust = np.timedelta64(1, "D") - np.timedelta64(1, "ns")
+                adjust = np.timedelta64(1, "D") - np.timedelta64(1, unit)
                 return self.to_timestamp(how="start") + adjust
             endpoint = (self + self._freq).to_timestamp(how="start")
-            return endpoint - np.timedelta64(1, "ns")
+            return endpoint - np.timedelta64(1, unit)

         if freq is None:
             freq_code = self._dtype._get_to_timestamp_base()
@@ -2110,7 +2125,7 @@ cdef class _Period(PeriodMixin):
         val = self.asfreq(freq, how)

         dt64 = period_ordinal_to_dt64(val.ordinal, base)
-        return Timestamp(dt64)
+        return Timestamp(dt64, unit=unit)

     [USER_REDACTED]
     def year(self) -> int:

@@ -759,6 +759,9 @@ def to_timestamp(self, freq=None, how: str = "start") -> DatetimeArray:
         """
         Cast to DatetimeArray/Index.

+        If possible, gives microsecond-unit DatetimeArray/Index. Otherwise
+        gives nanosecond unit.
+
         Parameters
         ----------
         freq : str or DateOffset, optional
@@ -789,34 +792,39 @@ def to_timestamp(self, freq=None, how: str = "start") -> DatetimeArray:
         >>> idx = pd.PeriodIndex(["2023-01", "2023-02", "2023-03"], freq="M")
         >>> idx.to_timestamp()
         DatetimeIndex(['2023-01-01', '2023-02-01', '2023-03-01'],
-        dtype='datetime64[ns]', freq='MS')
+        dtype='datetime64[us]', freq='MS')

         The frequency will not be inferred if the index contains less than
         three elements, or if the values of index are not strictly monotonic:

         >>> idx = pd.PeriodIndex(["2023-01", "2023-02"], freq="M")
         >>> idx.to_timestamp()
-        DatetimeIndex(['2023-01-01', '2023-02-01'], dtype='datetime64[ns]', freq=None)
+        DatetimeIndex(['2023-01-01', '2023-02-01'], dtype='datetime64[us]', freq=None)

         >>> idx = pd.PeriodIndex(
         ...     ["2023-01", "2023-02", "2023-02", "2023-03"], freq="2M"
         ... )
         >>> idx.to_timestamp()
         DatetimeIndex(['2023-01-01', '2023-02-01', '2023-02-01', '2023-03-01'],
-        dtype='datetime64[ns]', freq=None)
+        dtype='datetime64[us]', freq=None)
         """
         from pandas.core.arrays import DatetimeArray

         how = libperiod.validate_end_alias(how)

+        if self.freq.base == "ns" or freq == "ns":
+            unit = "ns"
+        else:
+            unit = "us"
+
         end = how == "E"
         if end:
             if freq == "B" or self.freq == "B":
                 # roll forward to ensure we land on B date
-                adjust = Timedelta(1, "D") - Timedelta(1, "ns")
+                adjust = Timedelta(1, unit="D") - Timedelta(1, unit=unit)
                 return self.to_timestamp(how="start") + adjust
             else:
-                adjust = Timedelta(1, "ns")
+                adjust = Timedelta(1, unit=unit)
                 return (self + self.freq).to_timestamp(how="start") - adjust

         if freq is None:
@@ -831,7 +839,8 @@ def to_timestamp(self, freq=None, how: str = "start") -> DatetimeArray:
         new_parr = self.asfreq(freq, how=how)

         new_data = libperiod.periodarr_to_dt64arr(new_parr.asi8, base)
-        dta = DatetimeArray._from_sequence(new_data, dtype=np.dtype("M8[ns]"))
+        dta = DatetimeArray._from_sequence(new_data, dtype=new_data.dtype)
+        assert dta.unit == unit

         if self.freq.name == "B":
             # See if we can retain BDay instead of Day in cases where

@@ -16256,7 +16256,7 @@ def to_timestamp(
         2023-01-01     1      3
         2024-01-01     2      4
         >>> df1.index
-        DatetimeIndex(['2023-01-01', '2024-01-01'], dtype='datetime64[ns]', freq=None)
+        DatetimeIndex(['2023-01-01', '2024-01-01'], dtype='datetime64[us]', freq=None)

         Using `freq` which is the offset that the Timestamps will have

@@ -16267,7 +16267,7 @@ def to_timestamp(
         2023-01-31     1      3
         2024-01-31     2      4
         >>> df2.index
-        DatetimeIndex(['2023-01-31', '2024-01-31'], dtype='datetime64[ns]', freq=None)
+        DatetimeIndex(['2023-01-31', '2024-01-31'], dtype='datetime64[us]', freq=None)
         """
         self._check_copy_deprecation(copy)
         new_obj = self.copy(deep=False)

@@ -896,7 +896,10 @@ def _parsed_string_to_bounds(
         """
         freq = OFFSET_TO_PERIOD_FREQSTR.get(reso.attr_abbrev, reso.attr_abbrev)
         per = Period(parsed, freq=freq)
-        start, end = per.start_time, per.end_time
+        start = per.start_time
+        # Can't use end_time here bc that will subtract a microsecond
+        #  instead of a nanosecond
+        end = (per + 1).start_time - np.timedelta64(1, "ns")
         start = start.as_unit(self.unit)
         end = end.as_unit(self.unit)

@@ -8,7 +8,6 @@

 from pandas._libs import (
     NaT,
-    OutOfBoundsDatetime,
     Timestamp,
 )
 from pandas._libs.tslibs import to_offset
@@ -1110,28 +1109,25 @@ def test_to_timestamp_roundtrip_bday(self):
         parr = dta.to_period()
         result = parr.to_timestamp()
         assert result.freq == "B"
-        tm.assert_extension_array_equal(result, dta)
+        tm.assert_extension_array_equal(result, dta.as_unit("us"))

         dta2 = dta[::2]
         parr2 = dta2.to_period()
         result2 = parr2.to_timestamp()
         assert result2.freq == "2B"
-        tm.assert_extension_array_equal(result2, dta2)
+        tm.assert_extension_array_equal(result2, dta2.as_unit("us"))

         parr3 = dta.to_period("2B")
         result3 = parr3.to_timestamp()
         assert result3.freq == "B"
-        tm.assert_extension_array_equal(result3, dta)
+        tm.assert_extension_array_equal(result3, dta.as_unit("us"))

     def test_to_timestamp_out_of_bounds(self):
         # GH#19643 previously overflowed silently
         pi = pd.period_range("1500", freq="Y", periods=3)
-        msg = "Out of bounds nanosecond timestamp: 1500-01-01 00:00:00"
-        with pytest.raises(OutOfBoundsDatetime, match=msg):
-            pi.to_timestamp()
-
-        with pytest.raises(OutOfBoundsDatetime, match=msg):
-            pi._data.to_timestamp()
+        pi.to_timestamp()
+        dta = pi._data.to_timestamp()
+        assert dta[0] == Timestamp(1500, 1, 1)

     [USER_REDACTED].mark.parametrize("propname", PeriodArray._bool_ops)
     def test_bool_properties(self, arr1d, propname):

@@ -393,11 +393,11 @@ def test_to_csv_idx_types(self, nrows, r_idx_type, c_idx_type, ncols, temp_file)
         if r_idx_type == "dt":
             expected.index = expected.index.astype("M8[us]")
         elif r_idx_type == "p":
-            expected.index = expected.index.astype("M8[ns]")
+            expected.index = expected.index.astype("M8[us]")
         if c_idx_type == "dt":
             expected.columns = expected.columns.astype("M8[us]")
         elif c_idx_type == "p":
-            expected.columns = expected.columns.astype("M8[ns]")
+            expected.columns = expected.columns.astype("M8[us]")
         tm.assert_frame_equal(result, expected, check_names=False)

     [USER_REDACTED].mark.slow

@@ -21,7 +21,6 @@ def _get_with_delta(delta, freq="YE-DEC"):
         to_datetime("1/1/2001") + delta,
         to_datetime("12/31/2009") + delta,
         freq=freq,
-        unit="ns",
     )

@@ -37,15 +36,15 @@ def test_to_timestamp(self, frame_or_series):
         obj["mix"] = "a"
         obj = tm.get_obj(obj, frame_or_series)

-        exp_index = date_range("1/1/2001", end="12/31/2009", freq="YE-DEC", unit="ns")
-        exp_index = exp_index + Timedelta(1, "D") - Timedelta(1, "ns")
+        exp_index = date_range("1/1/2001", end="12/31/2009", freq="YE-DEC")
+        exp_index = exp_index + Timedelta(1, "D") - Timedelta(1, "us")
         result = obj.to_timestamp("D", "end")
         tm.assert_index_equal(result.index, exp_index)
         tm.assert_numpy_array_equal(result.values, obj.values)
         if frame_or_series is Series:
             assert result.name == "A"

-        exp_index = date_range("1/1/2001", end="1/1/2009", freq="YS-JAN", unit="ns")
+        exp_index = date_range("1/1/2001", end="1/1/2009", freq="YS-JAN")
         result = obj.to_timestamp("D", "start")
         tm.assert_index_equal(result.index, exp_index)

@@ -55,19 +54,19 @@ def test_to_timestamp(self, frame_or_series):
         delta = timedelta(hours=23)
         result = obj.to_timestamp("H", "end")
         exp_index = _get_with_delta(delta)
-        exp_index = exp_index + Timedelta(1, "h") - Timedelta(1, "ns")
+        exp_index = exp_index + Timedelta(1, "h") - Timedelta(1, "us")
         tm.assert_index_equal(result.index, exp_index)

         delta = timedelta(hours=23, minutes=59)
         result = obj.to_timestamp("T", "end")
         exp_index = _get_with_delta(delta)
-        exp_index = exp_index + Timedelta(1, "m") - Timedelta(1, "ns")
+        exp_index = exp_index + Timedelta(1, "m") - Timedelta(1, "us")
         tm.assert_index_equal(result.index, exp_index)

         result = obj.to_timestamp("S", "end")
         delta = timedelta(hours=23, minutes=59, seconds=59)
         exp_index = _get_with_delta(delta)
-        exp_index = exp_index + Timedelta(1, "s") - Timedelta(1, "ns")
+        exp_index = exp_index + Timedelta(1, "s") - Timedelta(1, "us")
         tm.assert_index_equal(result.index, exp_index)

     def test_to_timestamp_columns(self):
@@ -83,37 +82,37 @@ def test_to_timestamp_columns(self):
         # columns
         df = df.T

-        exp_index = date_range("1/1/2001", end="12/31/2009", freq="YE-DEC", unit="ns")
-        exp_index = exp_index + Timedelta(1, "D") - Timedelta(1, "ns")
+        exp_index = date_range("1/1/2001", end="12/31/2009", freq="YE-DEC")
+        exp_index = exp_index + Timedelta(1, "D") - Timedelta(1, "us")
         result = df.to_timestamp("D", "end", axis=1)
         tm.assert_index_equal(result.columns, exp_index)
         tm.assert_numpy_array_equal(result.values, df.values)

-        exp_index = date_range("1/1/2001", end="1/1/2009", freq="YS-JAN", unit="ns")
+        exp_index = date_range("1/1/2001", end="1/1/2009", freq="YS-JAN")
         result = df.to_timestamp("D", "start", axis=1)
         tm.assert_index_equal(result.columns, exp_index)

         delta = timedelta(hours=23)
         result = df.to_timestamp("H", "end", axis=1)
         exp_index = _get_with_delta(delta)
-        exp_index = exp_index + Timedelta(1, "h") - Timedelta(1, "ns")
+        exp_index = exp_index + Timedelta(1, "h") - Timedelta(1, "us")
         tm.assert_index_equal(result.columns, exp_index)

         delta = timedelta(hours=23, minutes=59)
         result = df.to_timestamp("min", "end", axis=1)
         exp_index = _get_with_delta(delta)
-        exp_index = exp_index + Timedelta(1, "m") - Timedelta(1, "ns")
+        exp_index = exp_index + Timedelta(1, "m") - Timedelta(1, "us")
         tm.assert_index_equal(result.columns, exp_index)

         result = df.to_timestamp("S", "end", axis=1)
         delta = timedelta(hours=23, minutes=59, seconds=59)
         exp_index = _get_with_delta(delta)
-        exp_index = exp_index + Timedelta(1, "s") - Timedelta(1, "ns")
+        exp_index = exp_index + Timedelta(1, "s") - Timedelta(1, "us")
         tm.assert_index_equal(result.columns, exp_index)

         result1 = df.to_timestamp("5min", axis=1)
         result2 = df.to_timestamp("min", axis=1)
-        expected = date_range("2001-01-01", "2009-01-01", freq="YS", unit="ns")
+        expected = date_range("2001-01-01", "2009-01-01", freq="YS")
         assert isinstance(result1.columns, DatetimeIndex)
         assert isinstance(result2.columns, DatetimeIndex)
         tm.assert_numpy_array_equal(result1.columns.asi8, expected.asi8)
@@ -138,11 +137,9 @@ def test_to_timestamp_hourly(self, frame_or_series):
         if frame_or_series is not Series:
             obj = obj.to_frame()

-        exp_index = date_range(
-            "1/1/2001 00:59:59", end="1/2/2001 00:59:59", freq="h", unit="ns"
-        )
+        exp_index = date_range("1/1/2001 00:59:59", end="1/2/2001 00:59:59", freq="h")
         result = obj.to_timestamp(how="end")
-        exp_index = exp_index + Timedelta(1, "s") - Timedelta(1, "ns")
+        exp_index = exp_index + Timedelta(1, "s") - Timedelta(1, "us")
         tm.assert_index_equal(result.index, exp_index)
         if frame_or_series is Series:
             assert result.name == "foo"

@@ -120,11 +120,11 @@ def test_to_period_infer(self):
     def test_period_dt64_round_trip(self):
         dti = date_range("1/1/2000", "1/7/2002", freq="B", unit="ns")
         pi = dti.to_period()
-        tm.assert_index_equal(pi.to_timestamp(), dti)
+        tm.assert_index_equal(pi.to_timestamp(), dti.as_unit("us"))

         dti = date_range("1/1/2000", "1/7/2002", freq="B", unit="ns")
         pi = dti.to_period(freq="h")
-        tm.assert_index_equal(pi.to_timestamp(), dti)
+        tm.assert_index_equal(pi.to_timestamp(), dti.as_unit("us"))

     def test_to_period_millisecond(self):
         index = DatetimeIndex(

@@ -22,34 +22,34 @@ def test_to_timestamp_non_contiguous(self):
         pi = dti.to_period()

         result = pi[::2].to_timestamp()
-        expected = dti[::2]
+        expected = dti[::2].as_unit("us")
         tm.assert_index_equal(result, expected)

         result = pi._data[::2].to_timestamp()
-        expected = dti._data[::2]
+        expected = dti._data[::2].as_unit("us")
         # TODO: can we get the freq to round-trip?
         tm.assert_datetime_array_equal(result, expected, check_freq=False)

         result = pi[::-1].to_timestamp()
-        expected = dti[::-1]
+        expected = dti[::-1].as_unit("us")
         tm.assert_index_equal(result, expected)

         result = pi._data[::-1].to_timestamp()
-        expected = dti._data[::-1]
+        expected = dti._data[::-1].as_unit("us")
         tm.assert_datetime_array_equal(result, expected, check_freq=False)

         result = pi[::2][::-1].to_timestamp()
-        expected = dti[::2][::-1]
+        expected = dti[::2][::-1].as_unit("us")
         tm.assert_index_equal(result, expected)

         result = pi._data[::2][::-1].to_timestamp()
-        expected = dti._data[::2][::-1]
+        expected = dti._data[::2][::-1].as_unit("us")
         tm.assert_datetime_array_equal(result, expected, check_freq=False)

     def test_to_timestamp_freq(self):
         idx = period_range("2017", periods=12, freq="Y-DEC")
         result = idx.to_timestamp()
-        expected = date_range("2017", periods=12, freq="YS-JAN", unit="ns")
+        expected = date_range("2017", periods=12, freq="YS-JAN")
         tm.assert_index_equal(result, expected)

     def test_to_timestamp_pi_nat(self):
@@ -59,7 +59,7 @@ def test_to_timestamp_pi_nat(self):
         result = index.to_timestamp("D")
         expected = DatetimeIndex(
             [NaT, datetime(2011, 1, 1), datetime(2011, 2, 1)],
-            dtype="M8[ns]",
+            dtype="M8[us]",
             name="idx",
         )
         tm.assert_index_equal(result, expected)
@@ -101,38 +101,38 @@ def test_to_timestamp_pi_mult(self):

         result = idx.to_timestamp()
         expected = DatetimeIndex(
-            ["2011-01-01", "NaT", "2011-02-01"], dtype="M8[ns]", name="idx"
+            ["2011-01-01", "NaT", "2011-02-01"], dtype="M8[us]", name="idx"
         )
         tm.assert_index_equal(result, expected)

         result = idx.to_timestamp(how="E")
         expected = DatetimeIndex(
-            ["2011-02-28", "NaT", "2011-03-31"], dtype="M8[ns]", name="idx"
+            ["2011-02-28", "NaT", "2011-03-31"], dtype="M8[us]", name="idx"
         )
-        expected = expected + Timedelta(1, "D") - Timedelta(1, "ns")
+        expected = expected + Timedelta(1, "D") - Timedelta(1, "us")
         tm.assert_index_equal(result, expected)

     def test_to_timestamp_pi_combined(self):
         idx = period_range(start="2011", periods=2, freq="1D1h", name="idx")

         result = idx.to_timestamp()
         expected = DatetimeIndex(
-            ["2011-01-01 00:00", "2011-01-02 01:00"], dtype="M8[ns]", name="idx"
+            ["2011-01-01 00:00", "2011-01-02 01:00"], dtype="M8[us]", name="idx"
         )
         tm.assert_index_equal(result, expected)

         result = idx.to_timestamp(how="E")
         expected = DatetimeIndex(
-            ["2011-01-02 00:59:59", "2011-01-03 01:59:59"], name="idx", dtype="M8[ns]"
+            ["2011-01-02 00:59:59", "2011-01-03 01:59:59"], name="idx", dtype="M8[us]"
         )
-        expected = expected + Timedelta(1, "s") - Timedelta(1, "ns")
+        expected = expected + Timedelta(1, "s") - Timedelta(1, "us")
         tm.assert_index_equal(result, expected)

         result = idx.to_timestamp(how="E", freq="h")
         expected = DatetimeIndex(
-            ["2011-01-02 00:00", "2011-01-03 01:00"], dtype="M8[ns]", name="idx"
+            ["2011-01-02 00:00", "2011-01-03 01:00"], dtype="M8[us]", name="idx"
         )
-        expected = expected + Timedelta(1, "h") - Timedelta(1, "ns")
+        expected = expected + Timedelta(1, "h") - Timedelta(1, "us")
         tm.assert_index_equal(result, expected)

     def test_to_timestamp_1703(self):

@@ -14,16 +14,14 @@ class TestPeriodIndexOps:
     def test_start_time(self):
         # GH#17157
         index = period_range(freq="M", start="2016-01-01", end="2016-05-31")
-        expected_index = date_range(
-            "2016-01-01", end="2016-05-31", freq="MS", unit="ns"
-        )
+        expected_index = date_range("2016-01-01", end="2016-05-31", freq="MS")
         tm.assert_index_equal(index.start_time, expected_index)

     def test_end_time(self):
         # GH#17157
         index = period_range(freq="M", start="2016-01-01", end="2016-05-31")
         expected_index = date_range("2016-01-01", end="2016-05-31", freq="ME")
-        expected_index += Timedelta(1, "D") - Timedelta(1, "ns")
+        expected_index += Timedelta(1, "D") - Timedelta(1, "us")
         tm.assert_index_equal(index.end_time, expected_index)

     [USER_REDACTED].mark.filterwarnings(r"ignore:PeriodDtype\[B\] is deprecated:FutureWarning")
@@ -36,5 +34,5 @@ def test_end_time_business_friday(self):
         result = pi.end_time

         dti = date_range("1990-01-05", freq="D", periods=1)._with_freq(None)
-        expected = dti + Timedelta(days=1, nanoseconds=-1)
+        expected = dti + Timedelta(1, "D") - Timedelta(1, "us")
         tm.assert_index_equal(result, expected)

@@ -682,6 +682,7 @@ def test_secondary_y_ts(self):
         axes = fig.get_axes()
         line = ax.get_lines()[0]
         xp = Series(line.get_ydata(), line.get_xdata()).to_timestamp()
+        xp.index = xp.index.as_unit("ns")
         tm.assert_series_equal(ser, xp)
         assert ax.get_yaxis().get_ticks_position() == "right"
         assert not axes[0].get_yaxis().get_visible()

@@ -1112,7 +1112,7 @@ def test_resample_anchored_intraday(unit):
     result = df.resample("ME").mean()
     expected = df.resample("ME").mean().to_period()
     expected = expected.to_timestamp(how="end")
-    expected.index += Timedelta(1, "ns") - Timedelta(1, "D")
+    expected.index += Timedelta(1, unit="us") - Timedelta(1, unit="D")
     expected.index = expected.index.as_unit(unit)._with_freq("infer")
     assert expected.index.freq == "ME"
     tm.assert_frame_equal(result, expected)
@@ -1121,7 +1121,7 @@ def test_resample_anchored_intraday(unit):
     exp = df.shift(1, freq="D").resample("ME").mean().to_period()
     exp = exp.to_timestamp(how="end")

-    exp.index = exp.index + Timedelta(1, "ns") - Timedelta(1, "D")
+    exp.index = exp.index + Timedelta(1, unit="us") - Timedelta(1, unit="D")
     exp.index = exp.index.as_unit(unit)._with_freq("infer")
     assert exp.index.freq == "ME"
     tm.assert_frame_equal(result, exp)
@@ -1134,7 +1134,7 @@ def test_resample_anchored_intraday2(unit):
     result = df.resample("QE").mean()
     expected = df.resample("QE").mean().to_period()
     expected = expected.to_timestamp(how="end")
-    expected.index += Timedelta(1, "ns") - Timedelta(1, "D")
+    expected.index += Timedelta(1, unit="us") - Timedelta(1, unit="D")
     expected.index._data.freq = "QE"
     expected.index._freq = lib.no_default
     expected.index = expected.index.as_unit(unit)
@@ -1144,7 +1144,7 @@ def test_resample_anchored_intraday2(unit):
     expected = df.shift(1, freq="D").resample("QE").mean()
     expected = expected.to_period()
     expected = expected.to_timestamp(how="end")
-    expected.index += Timedelta(1, "ns") - Timedelta(1, "D")
+    expected.index += Timedelta(1, unit="us") - Timedelta(1, unit="D")
     expected.index._data.freq = "QE"
     expected.index._freq = lib.no_default
     expected.index = expected.index.as_unit(unit)

@@ -1,7 +1,6 @@
 import pytest

 from pandas._libs.tslibs.period import INVALID_FREQ_ERR_MSG
-from pandas.errors import OutOfBoundsDatetime

 from pandas import (
     Period,
@@ -42,10 +41,8 @@ def test_to_timestamp_out_of_bounds(self):
         # GH#19643, used to incorrectly give Timestamp in 1754
         with tm.assert_produces_warning(FutureWarning, match=bday_msg):
             per = Period("0001-01-01", freq="B")
-        msg = "Out of bounds nanosecond timestamp"
-        with pytest.raises(OutOfBoundsDatetime, match=msg):
-            with tm.assert_produces_warning(FutureWarning, match=bday_msg):
-                per.to_timestamp()
+        with tm.assert_produces_warning(FutureWarning, match=bday_msg):
+            per.to_timestamp()

     def test_asfreq_corner(self):
         val = Period(freq="Y", year=2007)
@@ -694,14 +691,7 @@ def test_conv_microsecond(self):
         start = per.start_time
         expected = Timestamp("2020-01-30 15:57:27.576166")
         assert start == expected
-        assert start._value == per.ordinal * 1000
-
-        per2 = Period("2300-01-01", "us")
-        msg = "2300-01-01"
-        with pytest.raises(OutOfBoundsDatetime, match=msg):
-            per2.start_time
-        with pytest.raises(OutOfBoundsDatetime, match=msg):
-            per2.end_time
+        assert start._value == per.ordinal

     def test_asfreq_mult(self):
         # normal freq to mult freq

@@ -8,16 +8,21 @@
 import numpy as np
 import pytest

-from pandas._libs.tslibs import iNaT
+from pandas._libs.tslibs import (
+    iNaT,
+    to_offset,
+)
 from pandas._libs.tslibs.ccalendar import (
     DAYS,
     MONTHS,
 )
-from pandas._libs.tslibs.np_datetime import OutOfBoundsDatetime
 from pandas._libs.tslibs.parsing import DateParseError
 from pandas._libs.tslibs.period import INVALID_FREQ_ERR_MSG
 from pandas.compat import PY314
-from pandas.errors import Pandas4Warning
+from pandas.errors import (
+    OutOfBoundsDatetime,
+    Pandas4Warning,
+)

 from pandas import (
     NaT,
@@ -682,12 +687,12 @@ def test_hash(self):
     def test_to_timestamp_mult(self):
         p = Period("2011-01", freq="M")
         assert p.to_timestamp(how="S") == Timestamp("2011-01-01")
-        expected = Timestamp("2011-02-01") - Timedelta(1, "ns")
+        expected = Timestamp("2011-02-01") - Timedelta(1, unit="us")
         assert p.to_timestamp(how="E") == expected

         p = Period("2011-01", freq="3M")
         assert p.to_timestamp(how="S") == Timestamp("2011-01-01")
-        expected = Timestamp("2011-04-01") - Timedelta(1, "ns")
+        expected = Timestamp("2011-04-01") - Timedelta(1, unit="us")
         assert p.to_timestamp(how="E") == expected

     [USER_REDACTED].mark.filterwarnings(
@@ -712,8 +717,8 @@ def test_to_timestamp(self):

         def _ex(p):
             if p.freq == "B":
-                return p.start_time + Timedelta(days=1, nanoseconds=-1)
-            return Timestamp((p + p.freq).start_time._value - 1)
+                return p.start_time + Timedelta(days=1) - Timedelta(microseconds=1)
+            return Timestamp((p + p.freq).start_time._value - 1, unit="us")

         for fcode in from_lst:
             p = Period("1982", freq=fcode)
@@ -729,19 +734,19 @@ def _ex(p):
         p = Period("1985", freq="Y")

         result = p.to_timestamp("h", how="end")
-        expected = Timestamp(1986, 1, 1) - Timedelta(1, "ns")
+        expected = Timestamp(1986, 1, 1) - Timedelta(1, unit="us")
         assert result == expected
         result = p.to_timestamp("3h", how="end")
         assert result == expected

         result = p.to_timestamp("min", how="end")
-        expected = Timestamp(1986, 1, 1) - Timedelta(1, "ns")
+        expected = Timestamp(1986, 1, 1) - Timedelta(1, unit="us")
         assert result == expected
         result = p.to_timestamp("2min", how="end")
         assert result == expected

         result = p.to_timestamp(how="end")
-        expected = Timestamp(1986, 1, 1) - Timedelta(1, "ns")
+        expected = Timestamp(1986, 1, 1) - Timedelta(1, unit="us")
         assert result == expected

         expected = datetime(1985, 1, 1)
@@ -761,7 +766,7 @@ def test_to_timestamp_business_end(self):
             per = Period("1990-01-05", "B")  # Friday
             result = per.to_timestamp("B", how="E")

-        expected = Timestamp("1990-01-06") - Timedelta(nanoseconds=1)
+        expected = Timestamp("1990-01-06") - Timedelta(1, unit="us")
         assert result == expected

     [USER_REDACTED].mark.parametrize(
@@ -920,8 +925,21 @@ def _period_constructor(bound, offset):
     def test_outer_bounds_start_and_end_time(self, bound, offset, period_property):
         # GH #13346
         period = TestPeriodProperties._period_constructor(bound, offset)
-        with pytest.raises(OutOfBoundsDatetime, match="Out of bounds nanosecond"):
-            getattr(period, period_property)
+        # post-GH#63760 this no longer raises OutOfBoundsDatetime
+        getattr(period, period_property)
+
+        per = Period._from_ordinal(bound.value, freq=to_offset("ms"))
+
+        if period_property == "end_time" and offset == 1:
+            err = OverflowError
+            msg = "value too large"
+        else:
+            err = OutOfBoundsDatetime
+            msg = "Out of bounds microsecond"
+        # But if we do the same thing with a much-more-remote date, we still
+        #  raise OutOfBoundsDatetime (as opposed to silently giving a wrong result)
+        with pytest.raises(err, match=msg):
+            getattr(per, period_property)

     [USER_REDACTED].mark.parametrize("bound, offset", [(Timestamp.min, -1), (Timestamp.max, 1)])
     [USER_REDACTED].mark.parametrize("period_property", ["start_time", "end_time"])
@@ -946,66 +964,60 @@ def test_start_time(self):
     def test_end_time(self):
         p = Period("2012", freq="Y")

-        def _ex(*args):
-            return Timestamp(Timestamp(datetime(*args)).as_unit("ns")._value - 1)
-
-        xp = _ex(2013, 1, 1)
-        assert xp == p.end_time
+        exp = Timestamp(2012, 12, 31, 23, 59, 59, 999999)
+        assert p.end_time == exp

         p = Period("2012", freq="Q")
-        xp = _ex(2012, 4, 1)
-        assert xp == p.end_time
+        exp = Timestamp(2012, 3, 31, 23, 59, 59, 999999)
+        assert p.end_time == exp

         p = Period("2012", freq="M")
-        xp = _ex(2012, 2, 1)
-        assert xp == p.end_time
+        exp = Timestamp(2012, 1, 31, 23, 59, 59, 999999)
+        assert p.end_time == exp

         p = Period("2012", freq="D")
-        xp = _ex(2012, 1, 2)
-        assert xp == p.end_time
+        exp = Timestamp(2012, 1, 1, 23, 59, 59, 999999)
+        assert p.end_time == exp

         p = Period("2012", freq="h")
-        xp = _ex(2012, 1, 1, 1)
-        assert xp == p.end_time
+        exp = Timestamp(2012, 1, 1, 0, 59, 59, 999999)
+        assert p.end_time == exp

         with tm.assert_produces_warning(FutureWarning, match=bday_msg):
             p = Period("2012", freq="B")
-            xp = _ex(2012, 1, 3)
-            assert xp == p.end_time
+            exp = Timestamp(2012, 1, 2, 23, 59, 59, 999999)
+            assert p.end_time == exp

         p = Period("2012", freq="W")
-        xp = _ex(2012, 1, 2)
-        assert xp == p.end_time
+        exp = Timestamp(2012, 1, 1, 23, 59, 59, 999999)
+        assert p.end_time == exp

         # Test for GH 11738
         p = Period("2012", freq="15D")
-        xp = _ex(2012, 1, 16)
-        assert xp == p.end_time
+        exp = Timestamp(2012, 1, 15, 23, 59, 59, 999999)
+        assert p.end_time == exp

         p = Period("2012", freq="1D1h")
-        xp = _ex(2012, 1, 2, 1)
-        assert xp == p.end_time
+        exp = Timestamp(2012, 1, 2, 0, 59, 59, 999999)
+        assert p.end_time == exp

         p = Period("2012", freq="1h1D")
-        xp = _ex(2012, 1, 2, 1)
-        assert xp == p.end_time
+        exp = Timestamp(2012, 1, 2, 0, 59, 59, 999999)
+        assert p.end_time == exp

     def test_end_time_business_friday(self):
         # GH#34449
         with tm.assert_produces_warning(FutureWarning, match=bday_msg):
             per = Period("1990-01-05", "B")
             result = per.end_time

-        expected = Timestamp("1990-01-06") - Timedelta(nanoseconds=1)
+        expected = Timestamp("1990-01-06") - Timedelta(1, unit="us")
         assert result == expected

     def test_anchor_week_end_time(self):
-        def _ex(*args):
-            return Timestamp(Timestamp(datetime(*args)).as_unit("ns")._value - 1)
-
         p = Period("2013-1-1", "W-SAT")
-        xp = _ex(2013, 1, 6)
-        assert p.end_time == xp
+        exp = Timestamp(2013, 1, 5, 23, 59, 59, 999999)
+        assert p.end_time == exp

     def test_properties_annually(self):
         # Test properties on Periods with annually frequency.

Yes, I think that would be good!

```suggestion
        # give microseconds for everything other than freq="ns"
```

```suggestion
                adjust = Timedelta(1, unit=unit)
```

Is that still needed now? (I thought we follow the input unit for integers)

Is there a specific reason you are switching here from Timedelta to np.timedelta64 ?

If you would change the freq to use ns, then it would keep raising the out of bounds?

i think you're right, will update

will update with the GH ref

i think i started this branch before we changed the Timedelta behavior.  will revert this bit

do you mean the freq we pass to the Period constructor in this example?  that would raise at construction.

Ah, yes, that's what I meant, but indeed not helping to test this

(the same change is used in some other files as well)


TST: add regression test for subtraction with mismatched index name/type (GH#57524)

Adds a regression test to ensure subtraction works correctly
when Series have indices with different names/types (GH#57524).

The test passes on the current main branch.

closes #57524

@@ -399,6 +399,28 @@ def test_add_list_to_masked_array_boolean(self):
         result = [True, None, True] + ser  # noqa: RUF005
         tm.assert_series_equal(result, expected)

+    def test_subtraction_index_name_type_mismatch_regression(self):
+        # GH#57524
+        s1 = Series(
+            [23, 22, 21],
+            index=Index(["a", "b", "c"], name="index a"),
+            dtype="Int64",
+        )
+        s2 = Series(
+            [21, 22, 23],
+            index=Index(
+                ["a", "b", "c"],
+                name="index b",
+                dtype="string",
+            ),
+            dtype="Int64",
+        )
+
+        result = s1 - s2
+        expected = Series([2, 0, -2], index=s1.index, dtype="Int64")
+
+        tm.assert_series_equal(result, expected)
+

 # ------------------------------------------------------------------
 # Comparisons

These series do not have the same `dtype` as the original issue

Thanks for catching that — you're right.

I've updated the regression test to match the original issue more closely:
- use nullable Int64 dtype for the Series
- use a string-dtyped Index with differing name/type

The test now reproduces the original scenario and passes locally.


API: rename read_iceberg selected_fields keyword to columns

Briefly brought this up during the dev meeting as well, and there seemed general agreement

Currently `read_iceberg` essentially just passes through (except for translating the case of "all columns") to pyiceberg, and thus also using their naming for all keyword. But when providing this IO method in pandas, we could also give a bit more consistent interface to pandas users (what this PR is doing).

Specifically `selected_fields` uses a term that is otherwise never used in pandas (for a concept that does exist).
We are not actually fully consistent about it ourselves in the other IO methods, but generally use `usecols` (for read_csv (and read_table, read_fwf), read_excel, read_spss) or `columns` (for read_parquet (and read_feather, read_orc), read_hdf, read_sql_table, read_stata). My feeling is that `columns` is the more "modern" one, so went with that.

@@ -14,8 +14,8 @@ def read_iceberg(
     catalog_name: str | None = None,
     *,
     catalog_properties: dict[str, Any] | None = None,
+    columns: list[str] | None = None,
     row_filter: str | None = None,
-    selected_fields: tuple[str] | None = None,
     case_sensitive: bool = True,
     snapshot_id: int | None = None,
     limit: int | None = None,
@@ -38,11 +38,11 @@ def read_iceberg(
         The name of the catalog.
     catalog_properties : dict of {str: str}, optional
         The properties that are used next to the catalog configuration.
+    columns : list of str, optional
+        A list of strings representing the column names to return in the output
+        dataframe.
     row_filter : str, optional
         A string that describes the desired rows.
-    selected_fields : tuple of str, optional
-        A tuple of strings representing the column names to return in the output
-        dataframe.
     case_sensitive : bool, default True
         If True column matching is case sensitive.
     snapshot_id : int, optional
@@ -71,7 +71,7 @@ def read_iceberg(
     ...     catalog_name="my_catalog",
     ...     catalog_properties={"s3.secret-access-key": "my-secret"},
     ...     row_filter="trip_distance >= 10.0",
-    ...     selected_fields=("VendorID", "tpep_pickup_datetime"),
+    ...     columns=["VendorID", "tpep_pickup_datetime"],
     ... )  # doctest: +SKIP
     """
     pyiceberg_catalog = import_optional_dependency("pyiceberg.catalog")
@@ -82,8 +82,10 @@ def read_iceberg(
     table = catalog.load_table(table_identifier)
     if row_filter is None:
         row_filter = pyiceberg_expressions.AlwaysTrue()
-    if selected_fields is None:
+    if columns is None:
         selected_fields = ("*",)
+    else:
+        selected_fields = tuple(columns)  # type: ignore[assignment]
     if scan_properties is None:
         scan_properties = {}
     result = table.scan(

@@ -115,7 +115,7 @@ def test_read_with_case_sensitive(self, catalog):
         result = read_iceberg(
             "ns.my_table",
             catalog_properties={"uri": catalog.uri},
-            selected_fields=["a"],
+            columns=["a"],
             case_sensitive=False,
         )
         tm.assert_frame_equal(result, expected)
@@ -124,7 +124,7 @@ def test_read_with_case_sensitive(self, catalog):
             read_iceberg(
                 "ns.my_table",
                 catalog_properties={"uri": catalog.uri},
-                selected_fields=["a"],
+                columns=["a"],
                 case_sensitive=True,
             )

```suggestion
        selected_fields = ["*"]
```


TST: add regression test for xs/loc mismatch on MultiIndex timeIndex (#19451)

- [x] closes #19451
- [x] [Tests added and passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#writing-tests) if fixing a bug or adding a new feature
- [x] All [code checks passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#pre-commit).
- [ ] Added [type annotations](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#type-hints) to new arguments/methods/functions.
- [ ] Added an entry in the latest `doc/source/whatsnew/vX.X.X.rst` file if fixing a bug or adding a new feature.
- [ ] If I used AI to develop this pull request, I prompted it to follow `AGENTS.md`.

@@ -14,6 +14,7 @@
     Timedelta,
     Timestamp,
     date_range,
+    to_datetime,
 )
 import pandas._testing as tm

@@ -382,6 +383,30 @@ def test_partial_slicing_with_multiindex_series(self):
         result = df2.loc[Timestamp("2000-1-4")]
         tm.assert_frame_equal(result, expected)

+    def test_xs_matches_loc_after_sort_multiindex(self):
+        # GH 19451
+        # after sort_index, .xs should
+        # return the same result as .loc
+        df = DataFrame(
+            [
+                ["2017-01-01", "000001", "a", 4, 5],
+                ["2017-01-02", "000001", "b", 4, 5],
+                ["2017-01-01", "000002", "b", 4, 5],
+                ["2017-01-02", "000002", "b", 4, 5],
+                ["2017-02-02", "000001", "b", 4, 5],
+                ["2017-02-03", "000002", "b", 4, 5],
+            ],
+            columns=["date", "code", "c", "d", "e"],
+        )
+        df = (
+            df.assign(date=to_datetime(df.date))
+            .set_index(["date", "code"])
+            .sort_index()
+        )
+        expected = df.loc["2017-01", slice(None)]
+        result = df.xs("2017-01", level=0)
+        tm.assert_frame_equal(result, expected)
+
     def test_partial_slice_requires_monotonicity(self):
         # Disallowed since 2.0 (GH 37819)
         ser = Series(np.arange(10), date_range("2014-01-01", periods=10))

Can you make this a dedicated test function?

Nit: Could you use the smaller, input DataFrame like in https://github.com/[USER_REDACTED]ndas-dev/pandas/issues/19451#issuecomment-361738163?


PERF: Skip normalization of named aggregations in groupby.agg when possible

- [x] closes #63742
- [x] [Tests added and passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#writing-tests) if fixing a bug or adding a new feature
- [x] All [code checks passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#pre-commit).
- [x] Added [type annotations](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#type-hints) to new arguments/methods/functions.
- [x] Added an entry in the latest `doc/source/whatsnew/vX.X.X.rst` file if fixing a bug or adding a new feature.
- [ ] If I used AI to develop this pull request, I prompted it to follow `AGENTS.md`.

This improves the named aggregation performance to be similar to un named (around 3X faster)

```python
In [1]: import numpy as np
   ...: import pandas as pd
   ...:
   ...:
   ...: def generate_data(n_rows: int) -> pd.DataFrame:
   ...:     # Random date range within 2024
   ...:     date_range = pd.date_range(start="2024-01-01", end="2024-12-31", freq="D")
   ...:     dates = np.random.choice(date_range, size=n_rows)
   ...:
   ...:     # Random number of users and values
   ...:     num_users = np.random.randint(1, 100, size=n_rows)
   ...:     values = np.random.uniform(1.0, 1000.0, size=n_rows)
   ...:
   ...:     # [NAME_REDACTED]
   ...:     df = pd.DataFrame({"date": dates, "num_users": num_users, "value": values})
   ...:     return df
   ...:
   ...:
   ...: def named(df: pd.DataFrame):
   ...:     return df.groupby("date").agg(
   ...:         **{"num_users": ("num_users", "sum"), "value": ("value", "sum")}
   ...:     )
   ...:
   ...:
   ...: def unnamed(df: pd.DataFrame):
   ...:     return df.groupby("date").agg({"num_users": "sum", "value": "sum"})
   ...:

In [2]: df = generate_data(10**3)

In [3]: %timeit named(df)
359 μs ± 4.67 μs per loop (mean ± std. dev. of 7 runs, 1,000 loops each)

In [4]: %timeit unnamed(df)
362 μs ± 5.03 μs per loop (mean ± std. dev. of 7 runs, 1,000 loops each)
```

@@ -1770,6 +1770,7 @@ def reconstruct_func(
             raise TypeError("Must provide 'func' or tuples of '(column, aggfunc).")

     if relabeling:
+        normalization_needed = False
         # error: Incompatible types in assignment (expression has type
         # "MutableMapping[Hashable, list[Callable[..., Any] | str]]", variable has type
         # "Callable[..., Any] | str | list[Callable[..., Any] | str] |
@@ -1778,18 +1779,32 @@ def reconstruct_func(
         converted_kwargs = {}
         for key, val in kwargs.items():
             if isinstance(val, NamedAgg):
+                column = val.column
                 aggfunc = val.aggfunc
                 if val.args or val.kwargs:
                     aggfunc = lambda x, func=aggfunc, a=val.args, kw=val.kwargs: func(
                         x, *a, **kw
                     )
-                converted_kwargs[key] = (val.column, aggfunc)
             else:
-                converted_kwargs[key] = val
+                column, aggfunc = val

-        func, columns, order = normalize_keyword_aggregation(  # type: ignore[assignment]
-            converted_kwargs
-        )
+            if column != key:
+                normalization_needed = True
+            converted_kwargs[key] = column, aggfunc
+
+        if normalization_needed:
+            func, columns, order = normalize_keyword_aggregation(  # type: ignore[assignment]
+                converted_kwargs
+            )
+        else:
+            # When names match, convert to dict format {column: aggfunc}
+            # and disable relabeling since no reordering is needed
+            func = {
+                column: aggfunc for key, (column, aggfunc) in converted_kwargs.items()
+            }
+            # Normalization is skipped if all kwargs keys are the same as their
+            # corresponding column name.
+            relabeling = False

     assert func is not None

@@ -1982,3 +1982,32 @@ def func(x):
     result = df.groupby(["Group"], as_index=False)["Data"].agg(func)
     expected = DataFrame(columns=["Group", "Data"])
     tm.assert_frame_equal(result, expected)
+
+
+def test_agg_relabel_with_name_match():
+    # GH-63742
+    df = DataFrame(
+        {
+            "group": ["a", "a", "b", "b"],
+            "value": [1, 2, 3, 4],
+            "count": [10, 20, 30, 40],
+        }
+    )
+
+    # Test with tuple format where output names match column names
+    result = df.groupby("group").agg(value=("value", "sum"), count=("count", "max"))
+
+    # Should produce same result as dict format
+    expected = df.groupby("group").agg({"value": "sum", "count": "max"})
+
+    tm.assert_frame_equal(result, expected)
+
+
+def test_agg_relabel_with_name_match_and_namedagg():
+    # GH-63742
+    df = DataFrame({"A": [0, 0, 1, 1], "B": [1, 2, 3, 4]})
+
+    result = df.groupby("A").agg(B=pd.NamedAgg("B", "sum"))
+
+    expected = DataFrame({"B": [3, 7]}, index=Index([0, 1], name="A"))
+    tm.assert_frame_equal(result, expected)

Looks good!

lgtm

Probably would be nicer to change how relabeling is calculated initially, but i wanted to create a simple POC and validate that this kind of fix would be acceptable

Agreed this could be cleaned up with some refactoring. But I think this is good as-is and refactor could be a separate PR if you're interested. Can you add a comment above this line, something like

    # Normalization is skipped if all kwargs keys are the same as their corresponding column name.

Cool,
Thanks!


CI: Re-enable `-Werror` in Linux and MacOS

- [x] xref #63701 (Replace xxxx with the GitHub issue number)
- [ ] [Tests added and passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#writing-tests) if fixing a bug or adding a new feature
- [x] All [code checks passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#pre-commit).
- [ ] Added [type annotations](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#type-hints) to new arguments/methods/functions.
- [ ] Added an entry in the latest `doc/source/whatsnew/vX.X.X.rst` file if fixing a bug or adding a new feature.
- [ ] If I used AI to develop this pull request, I prompted it to follow `AGENTS.md`.

---

This patch also fixes the warning of unused function on linux by [NAME_REDACTED] of some functions.

@@ -6,8 +6,7 @@ inputs:
     default: true
   werror:
     description: Enable werror flag for build
-    # TMP disable werror until we fix all warnings with more recent meson
-    default: false
+    default: true
 runs:
   using: composite
   steps:

@@ -166,10 +166,11 @@ jobs:
     - name: [NAME_REDACTED]
       id: build
       uses: ./.github/actions/build_pandas
-      # TMP disable werror until we fix all warnings with more recent meson
-      # with:
-      #   # xref https://github.com/[USER_REDACTED]n/cython/issues/6870
-      #   werror: ${{ matrix.name != 'Freethreading' }}
+      with:
+        # python3.14t/cpython/pyatomic_gcc.h:63:10:
+        #   error: ‘__atomic_fetch_add_8’
+        #   writing 8 bytes into a region of size 0 overflows the destination [-Werror=stringop-overflow=]
+        werror: ${{ matrix.name != 'Freethreading' }}

     - name: Test (not single_cpu)
       uses: ./.github/actions/run-tests
@@ -216,6 +217,11 @@ jobs:

       - name: [NAME_REDACTED]
         uses: ./.github/actions/build_pandas
+        with:
+          # Windows contains a lot of warnings regarding unsafe type conversion.
+          # It's possible to replicate some of them in clang with the `-Wconversion` flag.
+          # Tracking issue: https://github.com/[USER_REDACTED]ndas-dev/pandas/issues/63701
+          werror: ${{ runner.os != 'Windows' }}

       - name: Test
         uses: ./.github/actions/run-tests

@@ -134,7 +134,7 @@ except ImportError:

 [USER_REDACTED].wraparound(False)
 [USER_REDACTED].boundscheck(False)
-def memory_usage_of_objects(arr: object[:]) -> int64_t:
+def memory_usage_of_objects(ndarray[object, ndim=1] arr) -> int64_t:
     """
     Return the memory usage of an object array in bytes.

@@ -3190,7 +3190,9 @@ def to_object_array_tuples(rows: object) -> np.ndarray:

 [USER_REDACTED].wraparound(False)
 [USER_REDACTED].boundscheck(False)
-def fast_multiget(dict mapping, object[:] keys, default=np.nan) -> "ArrayLike":
+def fast_multiget(
+    dict mapping, ndarray[object, ndim=1] keys, default=np.nan
+) -> "ArrayLike":
     cdef:
         Py_ssize_t i, n = len(keys)
         object val

@@ -60,7 +60,7 @@ def unstack(const numeric_object_t[:, :] values, const uint8_t[:] mask,

 [USER_REDACTED].wraparound(False)
 [USER_REDACTED].boundscheck(False)
-def explode(object[:] values):
+def explode(ndarray[object, ndim=1] values):
     """
     transform array list-likes to long form
     preserve non-list entries

Is it worth reporting to Cython that using object memoryview generats C code that raises compiler warnings?

I am going to raise an issue.


DOC: Fix GL08 for pandas.tseries.offsets.QuarterBegin.startingMonth

- [ ] related to #60365

Fixes

```
pandas.tseries.offsets.BQuarterBegin.startingMonth GL08
pandas.tseries.offsets.BQuarterEnd.startingMonth GL08
pandas.tseries.offsets.QuarterBegin.startingMonth GL08
pandas.tseries.offsets.QuarterEnd.startingMonth GL08
```

@@ -73,8 +73,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.Series.dt PR01" `# Accessors are implemented as classes, but we do not document the Parameters section` \
         -i "pandas.Period.ordinal GL08" \
         -i "pandas.tseries.offsets.BDay PR02,SA01" \
-        -i "pandas.tseries.offsets.BQuarterBegin.startingMonth GL08" \
-        -i "pandas.tseries.offsets.BQuarterEnd.startingMonth GL08" \
         -i "pandas.tseries.offsets.BusinessDay PR02,SA01" \
         -i "pandas.tseries.offsets.BusinessDay.is_on_offset GL08" \
         -i "pandas.tseries.offsets.BusinessDay.weekmask GL08" \
@@ -127,8 +125,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.tseries.offsets.MonthBegin.is_on_offset GL08" \
         -i "pandas.tseries.offsets.MonthEnd.is_on_offset GL08" \
         -i "pandas.tseries.offsets.Nano.is_on_offset GL08" \
-        -i "pandas.tseries.offsets.QuarterBegin.startingMonth GL08" \
-        -i "pandas.tseries.offsets.QuarterEnd.startingMonth GL08" \
         -i "pandas.tseries.offsets.Second.is_on_offset GL08" \
         -i "pandas.tseries.offsets.SemiMonthBegin.day_of_month GL08" \
         -i "pandas.tseries.offsets.SemiMonthBegin.is_on_offset GL08" \

@@ -3157,20 +3157,47 @@ cdef class QuarterOffset(SingleConstructorOffset):
     _from_name_starting_month: ClassVar[int]

     cdef readonly:
-        int startingMonth
+        int _startingMonth

     def __init__(self, n=1, normalize=False, startingMonth=None):
         BaseOffset.__init__(self, n, normalize)

         if startingMonth is None:
             startingMonth = self._default_starting_month
-        self.startingMonth = startingMonth
+        self._startingMonth = startingMonth

     cpdef __setstate__(self, state):
-        self.startingMonth = state.pop("startingMonth")
+        self._startingMonth = state.pop("startingMonth")
         self._n = state.pop("n")
         self._normalize = state.pop("normalize")

+    [USER_REDACTED]
+    def startingMonth(self) -> int:
+        """
+        Return the month of the year from which quarters start.
+
+        This value determines which month marks the beginning of a quarterly period.
+        For example, with startingMonth=1, quarters start in January, April, July,
+        and October.
+
+        See Also
+        --------
+        QuarterOffset.rule_code : Return the rule code for the quarter offset.
+        HalfYearOffset.startingMonth : Similar property for half-year-based offsets.
+
+        Examples
+        --------
+        >>> pd.offsets.BQuarterBegin().startingMonth
+        3
+
+        >>> pd.offsets.QuarterEnd().startingMonth
+        3
+
+        >>> pd.offsets.QuarterBegin(startingMonth=1).startingMonth
+        1
+        """
+        return self._startingMonth
+
     [USER_REDACTED]
     def _from_name(cls, suffix=None):
         kwargs = {}

Shouldn't this new method be after these two lines?

looks like my rebase from the main had some issue. fixed.


TST: test concat_compat for datetime-EA with non-ns

- [x] closes #33331

The reproducible example works correctly on main. Added a test to cover it.

@@ -286,6 +286,16 @@ def test_concat_NaT_series_dataframe_all_NaT(self, tz1, tz2):
         result = concat([first, second])
         tm.assert_frame_equal(result, expected)

+    def test_concat_compat_on_non_ns_datetime_EA(self):
+        # GH#33331
+        first = Series(np.array([datetime(2010, 1, 1)], dtype="datetime64[D]"))
+        second = Series(pd.array(["a", "b"], dtype="category"))
+
+        expected = Series([Timestamp("2010-01-01 00:00:00"), "a", "b"])
+
+        result = concat([first, second], ignore_index=True)
+        tm.assert_series_equal(result, expected)
+

 class TestTimezoneConcat:
     def test_concat_tz_series(self):

Is it possible to test this without going through a private API like concat_compat?

```suggestion
import datetime
```

IMO it's a lot clearer just to not shorten this

Yes,I think we can test this using the public function `concat`. I moved the test to `pandas/tests/reshape/concat/test_datetimes.py`.

Thanks, I won’t shorten it.


BUG: .str methods failing on PyArrow using regex with \Z

Closes #63385

This is only for match and needs to be generalized, if we think this approach is workable

The RE2 engine only supports `\z` and not `\Z`, and Python 3.14 actually also just added `\z` and kept `\Z` as an alias for compat. So for python users, up to recently, you could only use `\Z`, but pyarrow engine requires `\z`. As a user changing manually to use `\z` only works with Python 3.14+, so I think it would be good to handle this under the hood for the user.

@@ -1211,7 +1211,7 @@ Strings
 - Bug in :meth:`Series.str.match` failing to raise when given a compiled ``re.Pattern`` object and conflicting ``case`` or ``flags`` arguments (:issue:`62240`)
 - Bug in :meth:`Series.str.zfill` raising ``AttributeError`` for :class:`ArrowDtype` (:issue:`61485`)
 - Bug in :meth:`Series.value_counts` would not respect ``sort=False`` for series having ``string`` dtype (:issue:`55224`)
-- Bug in :ref:`Series.str methods <api.series.str>` would fail when using regex with lookaheads or lookbehinds on data backed by PyArrow (:issue:`60833`)
+- Bug in :ref:`Series.str methods <api.series.str>` would fail when using regex with lookaheads or lookbehinds, backreferences, or ``\Z`` on data backed by PyArrow (:issue:`60833`, :issue:`63683`)
 - Bug in multiplication with a :class:`StringDtype` incorrectly allowing multiplying by bools; explicitly cast to integers instead (:issue:`62595`)

 Interval

@@ -376,7 +376,9 @@ def _str_match(
     ):
         if not pat.startswith("^"):
             pat = f"^({pat})"
-        return self._str_contains(pat, case, flags, na, regex=True)
+        return ArrowStringArrayMixin._str_contains(
+            self, pat, case, flags, na, regex=True
+        )

     def _str_fullmatch(
         self,
@@ -391,7 +393,7 @@ def _str_fullmatch(
             pat = f"^({pat[1:]})$"
         elif not pat.startswith("^"):
             pat = f"^({pat[0:-1]})$"
-        return self._str_match(pat, case, flags, na)
+        return ArrowStringArrayMixin._str_match(self, pat, case, flags, na)

     def _str_find(self, sub: str, start: int = 0, end: int | None = None):
         if (start == 0 or start is None) and end is None:

@@ -372,16 +372,31 @@ def _is_re_pattern_with_flags(pat: str | re.Pattern) -> bool:
         )

     [USER_REDACTED]
-    def _preprocess_re_pattern(pat: re.Pattern, case: bool) -> tuple[str, bool, int]:
-        pattern = pat.pattern
-        flags = pat.flags
-        # flags is not supported by pyarrow, but `case` is -> extract and remove
-        if flags & re.IGNORECASE:
-            case = False
-            flags = flags & ~re.IGNORECASE
-        # when creating a pattern with re.compile and a string, it automatically
-        # gets a UNICODE flag, while pyarrow assumes unicode for strings anyway
-        flags = flags & ~re.UNICODE
+    def _preprocess_re_pattern(
+        pat: str | re.Pattern, case: bool, flags: int
+    ) -> tuple[str, bool, int]:
+        if isinstance(pat, re.Pattern):
+            pattern = pat.pattern
+            # TODO flags passed separately by [NAME_REDACTED]
+            flags = pat.flags
+            # flags is not supported by pyarrow, but `case` is -> extract and remove
+            if flags & re.IGNORECASE:
+                case = False
+                flags = flags & ~re.IGNORECASE
+            # when creating a pattern with re.compile and a string, it automatically
+            # gets a UNICODE flag, while pyarrow assumes unicode for strings anyway
+            flags = flags & ~re.UNICODE
+        else:
+            pattern = pat
+
+        if (
+            pattern.endswith("\\Z")
+            # Second condition counts the number of `\` that patterns ends with
+            # prior to Z -> needs to be odd to end with an unescaped \Z
+            and (len(pattern) - len(pattern[:-1].rstrip("\\")) + 1) % 2 == 1
+        ):
+            pattern = pattern[:-2] + "\\z"
+
         return pattern, case, flags

     def _str_contains(
@@ -398,10 +413,8 @@ def _str_contains(
             or (regex and self._has_unsupported_regex(pat))
         ):
             return super()._str_contains(pat, case, flags, na, regex)
-        if isinstance(pat, re.Pattern):
-            # TODO flags passed separately by [NAME_REDACTED]
-            pat, case, flags = self._preprocess_re_pattern(pat, case)

+        pat, case, flags = self._preprocess_re_pattern(pat, case, flags)
         return ArrowStringArrayMixin._str_contains(self, pat, case, flags, na, regex)

     def _str_match(
@@ -417,9 +430,8 @@ def _str_match(
             or self._has_unsupported_regex(pat)
         ):
             return super()._str_match(pat, case, flags, na)
-        if isinstance(pat, re.Pattern):
-            pat, case, flags = self._preprocess_re_pattern(pat, case)

+        pat, case, flags = self._preprocess_re_pattern(pat, case, flags)
         return ArrowStringArrayMixin._str_match(self, pat, case, flags, na)

     def _str_fullmatch(
@@ -429,11 +441,14 @@ def _str_fullmatch(
         flags: int = 0,
         na: Scalar | lib.NoDefault = lib.no_default,
     ):
-        if flags or self._is_re_pattern_with_flags(pat):
+        if (
+            flags
+            or self._is_re_pattern_with_flags(pat)
+            or self._has_unsupported_regex(pat)
+        ):
             return super()._str_fullmatch(pat, case, flags, na)
-        if isinstance(pat, re.Pattern):
-            pat, case, flags = self._preprocess_re_pattern(pat, case)

+        pat, case, flags = self._preprocess_re_pattern(pat, case, flags)
         return ArrowStringArrayMixin._str_fullmatch(self, pat, case, flags, na)

     def _str_replace(
@@ -458,6 +473,9 @@ def _str_replace(
         ):
             return super()._str_replace(pat, repl, n, case, flags, regex)

+        if regex:
+            pat, case, flags = self._preprocess_re_pattern(pat, case, flags)
+
         return ArrowStringArrayMixin._str_replace(
             self, pat, repl, n, case, flags, regex
         )
@@ -471,6 +489,8 @@ def _str_repeat(self, repeats: int | Sequence[int]):
     def _str_count(self, pat: str, flags: int = 0):
         if flags or self._has_unsupported_regex(pat):
             return super()._str_count(pat, flags)
+
+        pat, _, _ = self._preprocess_re_pattern(pat, True, 0)
         result = pc.count_substring_regex(self._pa_array, pat)
         return self._convert_int_result(result)

@@ -761,3 +761,24 @@ def test_extractall_lookarounds(any_string_dtype, pat, expected_data):
     expected = Series(expected_data, dtype=any_string_dtype).to_frame()
     expected.index.names = [None, "match"]
     tm.assert_frame_equal(result, expected)
+
+
+def test_extract_end_of_string(any_string_dtype):
+    # https://github.com/[USER_REDACTED]ndas-dev/pandas/pull/63613
+    ser = Series(["aa", "abc", "bb\n"], dtype=any_string_dtype)
+
+    # with dollar sign
+    result = ser.str.extract("([ab]+)$")
+    expected = Series(["aa", np.nan, "bb"], dtype=any_string_dtype).to_frame()
+    tm.assert_frame_equal(result, expected)
+
+    # with \Z (ensure this is translated to \z for pyarrow)
+    result = ser.str.extract(r"([ab]+)\Z")
+    expected = Series(["aa", np.nan, np.nan], dtype=any_string_dtype).to_frame()
+    tm.assert_frame_equal(result, expected)
+
+    # ensure finding a literal \Z still works
+    ser = Series([r"aa\Z", "abc", "bb\\Z\n"], dtype=any_string_dtype)
+    result = ser.str.extract(r"([ab]+)\\Z")
+    expected = Series(["aa", np.nan, "bb"], dtype=any_string_dtype).to_frame()
+    tm.assert_frame_equal(result, expected)

@@ -372,6 +372,48 @@ def test_contains_lookarounds(any_string_dtype, pat, expected_data, na):
     tm.assert_series_equal(result, expected)

+def test_contains_end_of_string(any_string_dtype):
+    # https://github.com/[USER_REDACTED]ndas-dev/pandas/pull/63613
+    expected_dtype = (
+        np.bool_ if is_object_or_nan_string_dtype(any_string_dtype) else "boolean"
+    )
+
+    ser = Series(["baz", "bar", "bars", "bar\n"], dtype=any_string_dtype)
+
+    # with dollar sign
+    result = ser.str.contains("bar$")
+    if any_string_dtype == "string" and any_string_dtype.storage == "pyarrow":
+        # pyarrow (RE2) only matches $ at the very end of the line
+        expected = Series([False, True, False, False], dtype=expected_dtype)
+    else:
+        # python matches $ before or after an ending newline
+        expected = Series([False, True, False, True], dtype=expected_dtype)
+    tm.assert_series_equal(result, expected)
+
+    # with \Z (ensure this is translated to \z for pyarrow)
+    result = ser.str.contains(r"bar\Z")
+    expected = Series([False, True, False, False], dtype=expected_dtype)
+    tm.assert_series_equal(result, expected)
+
+    # ensure finding a literal \Z still works
+    ser = Series(
+        ["bar", r"bar{}".format("\\"), r"bar\Z", r"bar\\Z", "bars", "bar\n"],
+        dtype=any_string_dtype,
+    )
+
+    result = ser.str.contains(r"bar\\Z")
+    expected = Series([False, False, True, False, False, False], dtype=expected_dtype)
+    tm.assert_series_equal(result, expected)
+
+    result = ser.str.contains(r"bar\\\Z")
+    expected = Series([False, True, False, False, False, False], dtype=expected_dtype)
+    tm.assert_series_equal(result, expected)
+
+    result = ser.str.contains(r"bar\\\\Z")
+    expected = Series([False, False, False, True, False, False], dtype=expected_dtype)
+    tm.assert_series_equal(result, expected)
+
+
 # --------------------------------------------------------------------------------------
 # str.startswith
 # --------------------------------------------------------------------------------------
@@ -961,6 +1003,32 @@ def test_replace_lookarounds(any_string_dtype, pat, expected_data):
     tm.assert_series_equal(result, expected)

+def test_replace_end_of_string(any_string_dtype):
+    # https://github.com/[USER_REDACTED]ndas-dev/pandas/pull/63613
+    ser = Series(["baz", "bar", "bars", "bar\n"], dtype=any_string_dtype)
+
+    # with dollar sign
+    result = ser.str.replace("bar$", "x", regex=True)
+    if any_string_dtype == "string" and any_string_dtype.storage == "pyarrow":
+        # pyarrow (RE2) only matches $ at the very end of the line
+        expected = Series(["baz", "x", "bars", "bar\n"], dtype=any_string_dtype)
+    else:
+        # python matches $ before or after an ending newline
+        expected = Series(["baz", "x", "bars", "x\n"], dtype=any_string_dtype)
+    tm.assert_series_equal(result, expected)
+
+    # with \Z (ensure this is translated to \z for pyarrow)
+    result = ser.str.replace(r"bar\Z", "x", regex=True)
+    expected = Series(["baz", "x", "bars", "bar\n"], dtype=any_string_dtype)
+    tm.assert_series_equal(result, expected)
+
+    # ensure finding a literal \Z still works
+    ser = Series([r"bar\Z", "bar", "bars", "bar\n"], dtype=any_string_dtype)
+    result = ser.str.replace(r"bar\\Z", "x", regex=True)
+    expected = Series(["x", "bar", "bars", "bar\n"], dtype=any_string_dtype)
+    tm.assert_series_equal(result, expected)
+
+
 # --------------------------------------------------------------------------------------
 # str.match
 # --------------------------------------------------------------------------------------
@@ -1145,6 +1213,36 @@ def test_match_lookarounds(any_string_dtype, pat, expected_data):
     tm.assert_series_equal(result, expected)

+def test_match_end_of_string(any_string_dtype):
+    # https://github.com/[USER_REDACTED]ndas-dev/pandas/pull/63613
+    expected_dtype = (
+        np.bool_ if is_object_or_nan_string_dtype(any_string_dtype) else "boolean"
+    )
+
+    ser = Series(["baz", "bar", "bars", "bar\n"], dtype=any_string_dtype)
+
+    # with dollar sign
+    result = ser.str.match("bar$")
+    if any_string_dtype == "string" and any_string_dtype.storage == "pyarrow":
+        # pyarrow (RE2) only matches $ at the very end of the line
+        expected = Series([False, True, False, False], dtype=expected_dtype)
+    else:
+        # python matches $ before or after an ending newline
+        expected = Series([False, True, False, True], dtype=expected_dtype)
+    tm.assert_series_equal(result, expected)
+
+    # with \Z (ensure this is translated to \z for pyarrow)
+    result = ser.str.match(r"bar\Z")
+    expected = Series([False, True, False, False], dtype=expected_dtype)
+    tm.assert_series_equal(result, expected)
+
+    # ensure finding a literal \Z still works
+    ser = Series([r"bar\Z", "bar", "bars", "bar\n"], dtype=any_string_dtype)
+    result = ser.str.match(r"bar\\Z")
+    expected = Series([True, False, False, False], dtype=expected_dtype)
+    tm.assert_series_equal(result, expected)
+
+
 # --------------------------------------------------------------------------------------
 # str.fullmatch
 # --------------------------------------------------------------------------------------
@@ -1308,6 +1406,30 @@ def test_fullmatch_lookarounds(any_string_dtype, pat):
     tm.assert_series_equal(result, expected)

+def test_fullmatch_end_of_string(any_string_dtype):
+    # https://github.com/[USER_REDACTED]ndas-dev/pandas/pull/63613
+    expected_dtype = (
+        np.bool_ if is_object_or_nan_string_dtype(any_string_dtype) else "boolean"
+    )
+
+    ser = Series(["baz", "bar", "bars", "bar\n"], dtype=any_string_dtype)
+
+    # with dollar sign (for fullmatch, no difference between python and pyarrow)
+    result = ser.str.fullmatch("bar$")
+    expected = Series([False, True, False, False], dtype=expected_dtype)
+    tm.assert_series_equal(result, expected)
+
+    # with \Z (ensure this is translated to \z for pyarrow)
+    result = ser.str.fullmatch(r"bar\Z")
+    tm.assert_series_equal(result, expected)
+
+    # ensure finding a literal \Z still works
+    ser = Series([r"bar\Z", "bar", "bars", "bar\n"], dtype=any_string_dtype)
+    result = ser.str.fullmatch(r"bar\\Z")
+    expected = Series([True, False, False, False], dtype=expected_dtype)
+    tm.assert_series_equal(result, expected)
+
+
 # --------------------------------------------------------------------------------------
 # str.findall
 # --------------------------------------------------------------------------------------
@@ -1380,6 +1502,27 @@ def test_findall_lookarounds(any_string_dtype, pat, expected_data):
     tm.assert_series_equal(result, expected)

+def test_findall_end_of_string(any_string_dtype):
+    # https://github.com/[USER_REDACTED]ndas-dev/pandas/pull/63613
+    ser = Series(["baz", "bar", "bars", "bar\n"], dtype=any_string_dtype)
+
+    # with dollar sign
+    result = ser.str.findall("bar$")
+    expected = Series([[], ["bar"], [], ["bar"]], dtype=object)
+    tm.assert_series_equal(result, expected)
+
+    # with \Z (ensure this is translated to \z for pyarrow)
+    result = ser.str.findall(r"bar\Z")
+    expected = Series([[], ["bar"], [], []], dtype=object)
+    tm.assert_series_equal(result, expected)
+
+    # ensure finding a literal \Z still works
+    ser = Series([r"bar\Z", "bar", "bars", "bar\n"], dtype=any_string_dtype)
+    result = ser.str.findall(r"bar\\Z")
+    expected = Series([["bar\\Z"], [], [], []], dtype=object)
+    tm.assert_series_equal(result, expected)
+
+
 # --------------------------------------------------------------------------------------
 # str.find
 # --------------------------------------------------------------------------------------

@@ -332,6 +332,27 @@ def test_split_lookarounds(any_string_dtype, pat, expected_data):
     tm.assert_series_equal(result, expected)

+def test_split_regex_end_of_string(any_string_dtype):
+    # https://github.com/[USER_REDACTED]ndas-dev/pandas/pull/63613
+    ser = Series(["baz", "bar", "bars", "bar\n"], dtype=any_string_dtype)
+
+    # with dollar sign
+    result = ser.str.split("r$", regex=True)
+    expected = Series([["baz"], ["ba", ""], ["bars"], ["ba", "\n"]], dtype=object)
+    tm.assert_series_equal(result, expected)
+
+    # with \Z (ensure this is translated to \z for pyarrow)
+    result = ser.str.split(r"r\Z", regex=True)
+    expected = Series([["baz"], ["ba", ""], ["bars"], ["bar\n"]], dtype=object)
+    tm.assert_series_equal(result, expected)
+
+    # ensure finding a literal \Z still works
+    ser = Series([r"bar\Z", "bar", r"bar\Zs", "bar\n"], dtype=any_string_dtype)
+    result = ser.str.split(r"r\\Z", regex=True)
+    expected = Series([["ba", ""], ["bar"], ["ba", "s"], ["bar\n"]], dtype=object)
+    tm.assert_series_equal(result, expected)
+
+
 def test_rsplit_to_dataframe_expand_no_splits(any_string_dtype):
     s = Series(["nosplit", "alsonosplit"], dtype=any_string_dtype)
     result = s.str.rsplit("_", expand=True)

@@ -89,6 +89,36 @@ def test_count_lookarounds(any_string_dtype, pat, expected_data):
     tm.assert_series_equal(result, expected)

+def test_count_end_of_string(any_string_dtype):
+    # https://github.com/[USER_REDACTED]ndas-dev/pandas/pull/63613
+    expected_dtype = (
+        "int64" if is_object_or_nan_string_dtype(any_string_dtype) else "Int64"
+    )
+
+    ser = Series(["baz", "bar", "bars", "bar\n"], dtype=any_string_dtype)
+
+    # with dollar sign
+    result = ser.str.count("bar$")
+    if any_string_dtype == "string" and any_string_dtype.storage == "pyarrow":
+        # pyarrow (RE2) only matches $ at the very end of the line
+        expected = Series([0, 1, 0, 0], dtype=expected_dtype)
+    else:
+        # python matches $ before or after an ending newline
+        expected = Series([0, 1, 0, 1], dtype=expected_dtype)
+    tm.assert_series_equal(result, expected)
+
+    # with \Z (ensure this is translated to \z for pyarrow)
+    result = ser.str.count(r"bar\Z")
+    expected = Series([0, 1, 0, 0], dtype=expected_dtype)
+    tm.assert_series_equal(result, expected)
+
+    # ensure finding a literal \Z still works
+    ser = Series([r"bar\Z", "bar", "bars", "bar\n"], dtype=any_string_dtype)
+    result = ser.str.count(r"bar\\Z")
+    expected = Series([1, 0, 0, 0], dtype=expected_dtype)
+    tm.assert_series_equal(result, expected)
+
+
 def test_repeat(any_string_dtype):
     ser = Series(["a", "b", np.nan, "c", np.nan, "d"], dtype=any_string_dtype)

lgtm; good to merge this as-is without handling higher number of `\` as discussed above.

This correctly handles what I think would be the vast majority of real cases and as far as I can tell has no false positives. However it does have false negatives. The following should give true:

- `r"\\\Z"` (in general, an odd number of `\`)
- `r"text(\Z)`
- `r"\Z text"`

Only the first seems like it could maybe occur in practice, the other two are likely erroneous. And while I could see this being used with lookarounds, we'll already be falling back to object dtype with those so we don't need to be worried about them here.

[USER_REDACTED] I changed this and added a `self._has_regex_lookaround(pat)` to ArrowStringArray._str_fullmatch, to ensure the method here certainly uses the mixin version of `_str_match`, and not the ArrowStringArray, which would otherwise call the validation methods a second time (which can fail after replacing \Z to \z for older python, and generally we can also avoid the overhead of validating twice)

Thanks for the analysis!

> `r"\\\Z"` (in general, an odd number of `\`)

Should we count the number of trailing `\`? (and so only get here if the number of `\` is not even). Something like:

```
if pat.endswith("\\Z") and not ((len(pat[:-1]) - len(pat[:-1].rstrip("\\"))) % 2 == 0):
```

We can remove one of the `pat[:-1]` (which I think will give a copy of the entire string). Also removed the `not` but I'm fine if that remains as the original.

```python
# Second condition counts the number of `\` that pat ends with prior to Z
if pat.endswith("\\Z") and (len(pat) - len(pat[:-1].rstrip("\\")) + 1) % 2 == 1:
```

Added that and added some test cases to one of the tests


BLD: unpin meson(-python) in environment.yml, bump minimum meson-python in build-system requires

Partial redo of https://github.com/[USER_REDACTED]ndas-dev/pandas/pull/62086 (and https://github.com/[USER_REDACTED]ndas-dev/pandas/pull/60681), seeing how this goes now without the additional changes (although I assume we will have to add some of those build config updates to get the builds passing)

While the temporary suppressing of `-Werror` is not ideal (for new warnings), in practice we already have all those warnings anyway (there are present in the wheels builds which already use newer meson, and just don't bubble up in most CI with the older versions).

Closes #63686

@@ -6,7 +6,8 @@ inputs:
     default: true
   werror:
     description: Enable werror flag for build
-    default: true
+    # TMP disable werror until we fix all warnings with more recent meson
+    default: false
 runs:
   using: composite
   steps:

@@ -166,9 +166,10 @@ jobs:
     - name: [NAME_REDACTED]
       id: build
       uses: ./.github/actions/build_pandas
-      with:
-        # xref https://github.com/[USER_REDACTED]n/cython/issues/6870
-        werror: ${{ matrix.name != 'Freethreading' }}
+      # TMP disable werror until we fix all warnings with more recent meson
+      # with:
+      #   # xref https://github.com/[USER_REDACTED]n/cython/issues/6870
+      #   werror: ${{ matrix.name != 'Freethreading' }}

     - name: Test (not single_cpu)
       uses: ./.github/actions/run-tests

@@ -10,7 +10,7 @@ dependencies:
   - versioneer
   - cython<4.0.0a0
   - meson=1.2.1
-  - meson-python=0.13.1
+  - meson-python=0.17.1

   # test dependencies
   - pytest>=8.3.4

@@ -7,8 +7,8 @@ dependencies:
   # build dependencies
   - versioneer
   - cython<4.0.0a0
-  - meson=1.2.1
-  - meson-python=0.13.1
+  - meson=1.10.0
+  - meson-python=0.18.0

   # test dependencies
   - pytest>=8.3.4

@@ -8,8 +8,8 @@ dependencies:
   # build dependencies
   - versioneer
   - cython<4.0.0a0
-  - meson=1.2.1
-  - meson-python=0.13.1
+  - meson=1.10.0
+  - meson-python=0.18.0

   # test dependencies
   - pytest>=8.3.4

@@ -8,7 +8,7 @@ dependencies:
   - setuptools
   - versioneer
   - cython<4.0.0a0
-  - meson=1.8.0
+  - meson=1.10.0
   - meson-python=0.18.0

   # test dependencies

@@ -6,8 +6,8 @@ dependencies:

   # build dependencies
   - versioneer
-  - meson=1.2.1
-  - meson-python=0.13.1
+  - meson=1.10.0
+  - meson-python=0.18.0
   - cython<4.0.0a0

   # test dependencies

@@ -6,9 +6,9 @@ dependencies:

   # build dependencies
   - versioneer
-  - meson=1.2.1
   - cython<4.0.0a0
-  - meson-python=0.13.1
+  - meson=1.10.0
+  - meson-python=0.18.0

   # test dependencies
   - pytest>=8.3.4

@@ -9,8 +9,8 @@ dependencies:
   # build dependencies
   - versioneer
   - cython<4.0.0a0
-  - meson=1.2.1
-  - meson-python=0.13.1
+  - meson>=1.2.1,<2
+  - meson-python>=0.17.1,<1

   # test dependencies
   - pytest>=8.3.4

@@ -2,13 +2,13 @@
 # Minimum requirements for the build system to execute.
 # See https://github.com/scipy/scipy/pull/12940 for the AIX issue.
 requires = [
-    "meson-python>=0.13.1",
+    "meson-python>=0.17.1,<1",
     "meson>=1.2.1,<2",
     "wheel",
     "Cython<4.0.0a0",  # Note: sync with setup.py, environment.yml and asv.conf.json
-    # Force numpy higher than 2.0rc1, so that built wheels are compatible
+    # Force numpy higher than 2.0, so that built wheels are compatible
     # with both numpy 1 and 2
-    "numpy>=2.0.0rc1",
+    "numpy>=2.0.0",
     "versioneer[toml]"
 ]

@@ -4,8 +4,8 @@
 pip
 versioneer[toml]
 cython<4.0.0a0
-meson[ninja]==1.2.1
-meson-python==0.13.1
+meson[ninja]>=1.2.1,<2
+meson-python>=0.17.1,<1
 pytest>=8.3.4
 pytest-cov
 pytest-xdist>=3.6.1

LGTM. Bumping the build tools is necessary. Disabling `werror` temporarily is a pragmatic move to get this landed.

Could you create a follow up issue about re-enabling werror?


DOC: Fix GL08 for pandas.tseries.offsets.BusinessDay.calendar

- [ ] related to #60365

Fixes

```
pandas.tseries.offsets.BusinessDay.calendar GL08
pandas.tseries.offsets.BusinessHour.calendar GL08
pandas.tseries.offsets.CustomBusinessDay.calendar GL08
pandas.tseries.offsets.CustomBusinessHour.calendar GL08
pandas.tseries.offsets.CustomBusinessMonthBegin.calendar GL08
pandas.tseries.offsets.CustomBusinessMonthEnd.calendar GL08
```

@@ -82,11 +82,9 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.tseries.offsets.BYearBegin.is_on_offset GL08" \
         -i "pandas.tseries.offsets.BYearEnd.is_on_offset GL08" \
         -i "pandas.tseries.offsets.BusinessDay PR02,SA01" \
-        -i "pandas.tseries.offsets.BusinessDay.calendar GL08" \
         -i "pandas.tseries.offsets.BusinessDay.is_on_offset GL08" \
         -i "pandas.tseries.offsets.BusinessDay.weekmask GL08" \
         -i "pandas.tseries.offsets.BusinessHour PR02,SA01" \
-        -i "pandas.tseries.offsets.BusinessHour.calendar GL08" \
         -i "pandas.tseries.offsets.BusinessHour.is_on_offset GL08" \
         -i "pandas.tseries.offsets.BusinessHour.weekmask GL08" \
         -i "pandas.tseries.offsets.BusinessMonthBegin.is_on_offset GL08" \
@@ -95,19 +93,15 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.tseries.offsets.CBMonthEnd PR02" \
         -i "pandas.tseries.offsets.CDay PR02,SA01" \
         -i "pandas.tseries.offsets.CustomBusinessDay PR02,SA01" \
-        -i "pandas.tseries.offsets.CustomBusinessDay.calendar GL08" \
         -i "pandas.tseries.offsets.CustomBusinessDay.is_on_offset GL08" \
         -i "pandas.tseries.offsets.CustomBusinessDay.weekmask GL08" \
         -i "pandas.tseries.offsets.CustomBusinessHour PR02,SA01" \
-        -i "pandas.tseries.offsets.CustomBusinessHour.calendar GL08" \
         -i "pandas.tseries.offsets.CustomBusinessHour.is_on_offset GL08" \
         -i "pandas.tseries.offsets.CustomBusinessHour.weekmask GL08" \
         -i "pandas.tseries.offsets.CustomBusinessMonthBegin PR02" \
-        -i "pandas.tseries.offsets.CustomBusinessMonthBegin.calendar GL08" \
         -i "pandas.tseries.offsets.CustomBusinessMonthBegin.m_offset GL08" \
         -i "pandas.tseries.offsets.CustomBusinessMonthBegin.weekmask GL08" \
         -i "pandas.tseries.offsets.CustomBusinessMonthEnd PR02" \
-        -i "pandas.tseries.offsets.CustomBusinessMonthEnd.calendar GL08" \
         -i "pandas.tseries.offsets.CustomBusinessMonthEnd.m_offset GL08" \
         -i "pandas.tseries.offsets.CustomBusinessMonthEnd.weekmask GL08" \
         -i "pandas.tseries.offsets.DateOffset.is_on_offset GL08" \

@@ -1905,7 +1905,7 @@ cdef class BusinessMixin(SingleConstructorOffset):
     cdef readonly:
         timedelta _offset
         # [NAME_REDACTED] subclasses use weekmask, holiday, calendar
-        object weekmask, _holidays, calendar
+        object weekmask, _holidays, _calendar

     def __init__(self, n=1, normalize=False, offset=timedelta(0)):
         BaseOffset.__init__(self, n, normalize)
@@ -1923,7 +1923,7 @@ cdef class BusinessMixin(SingleConstructorOffset):
         # holidays, weekmask
         self.weekmask = weekmask
         self._holidays = holidays
-        self.calendar = calendar
+        self._calendar = calendar

     [USER_REDACTED]
     def offset(self):
@@ -2011,6 +2011,50 @@ cdef class BusinessMixin(SingleConstructorOffset):
         """
         return self._holidays

+    [USER_REDACTED]
+    def calendar(self):
+        """
+        Return the calendar used for business day calculations.
+
+        This property returns the numpy busdaycalendar object used for
+        determining valid business days. For standard business day offsets
+        (e.g., BusinessDay, BusinessHour), this returns None. For custom
+        business day offsets, this returns the calendar that was either
+        passed directly or constructed from weekmask and holidays.
+
+        Returns
+        -------
+        np.busdaycalendar or None
+            The business day calendar used for calculations, or None if
+            using default business day rules.
+
+        See Also
+        --------
+        BusinessDay.holidays : Holidays for standard business day offset.
+        CustomBusinessDay.holidays : Holidays for custom business day offset.
+        CustomBusinessDay.weekmask : Weekmask for custom business day offset.
+
+        Examples
+        --------
+        For standard business offsets, calendar is None:
+
+        >>> bd = pd.offsets.BusinessDay()
+        >>> bd.calendar is None
+        True
+
+        >>> bh = pd.offsets.BusinessHour()
+        >>> bh.calendar is None
+        True
+
+        For custom business day with explicit holidays:
+
+        >>> holidays = [pd.Timestamp("2023-12-25"), pd.Timestamp("2024-01-01")]
+        >>> cbd = pd.offsets.CustomBusinessDay(holidays=holidays)
+        >>> isinstance(cbd.calendar, np.busdaycalendar)
+        True
+        """
+        return self._calendar
+
     def _repr_attrs(self) -> str:
         if self.offset:
             attrs = [f"offset={repr(self.offset)}"]
@@ -2037,7 +2081,7 @@ cdef class BusinessMixin(SingleConstructorOffset):
                                                holidays=holidays,
                                                calendar=None)
             self.weekmask = weekmask
-            self.calendar = calendar
+            self._calendar = calendar
             self._holidays = holidays

         BaseOffset.__setstate__(self, state)

Why does this need a skip?

forgot to remove that. as I was testing them one by one. changing it.


TST: ensure categorical index levels remain categorical in merge

- [x] closes #37480

The reproducible example works correctly on main. Added a test to cover it.

@@ -2214,6 +2214,37 @@ def test_merge_on_int_array(self):
         )
         tm.assert_frame_equal(result, expected)

+    def test_merge_category_index_levels_stay_category(self):
+        # GH#37480
+        df1 = DataFrame(
+            {
+                "x": list(range(1, 10)),
+                "y": list(range(1, 10)),
+                "z": list(range(1, 10)),
+                "d": list(range(1, 10)),
+            }
+        )
+
+        df2 = df1.astype({"x": "category", "y": "category", "z": "category"})
+
+        df3 = df2.iloc[:4, :].groupby(["z", "x"], observed=True).agg({"d": "sum"})
+        df4 = df2.iloc[7:, :].groupby(["z", "x", "y"], observed=True).agg({"d": "sum"})
+        result = merge(df3, df4, left_index=True, right_index=True, how="outer")
+
+        cats = list(range(1, 10))
+        z = Categorical(list(range(1, 5)) + list(range(8, 10)), categories=cats)
+        x = Categorical(list(range(1, 5)) + list(range(8, 10)), categories=cats)
+        y = Categorical([np.nan] * 4 + list(range(8, 10)), categories=cats)
+        idx = MultiIndex.from_arrays([z, x, y], names=["z", "x", "y"])
+        expected = DataFrame(
+            {
+                "d_x": list(map(float, range(1, 5))) + [np.nan] * 2,
+                "d_y": [np.nan] * 4 + list(map(float, range(8, 10))),
+            },
+            index=idx,
+        )
+        tm.assert_frame_equal(result, expected)
+

 class TestMergeOnIndexes:
     [USER_REDACTED].mark.parametrize(

1. Could you just assign `result = merge(df3, df4, left_index=True, right_index=True, how="outer")`
2. Could you construct an `expected = ` DataFrame an use `tm.assert_frame_equal`?

Sorry one last comment. Could we make the data sizes smaller i.e. use 10 instead of 100?

[USER_REDACTED], thank you for reviewing this PR. I reduced the data sizes to 10.