source/app.tex

\section{Additional figures}
\label{sec:fifig}

\begin{figure}[htb]
    \centering
    \begin{subfigure}{0.32\textwidth}
        \includegraphics[width=\linewidth]{fig/dropout/7c.pdf}
        \caption{Housing ($n=4000$)}
        \label{fig:housingdropout}
    \end{subfigure}
    \hfill
    \begin{subfigure}{0.32\textwidth}
        \includegraphics[width=\linewidth]{fig/dropout/7b.pdf}
        \caption{LSAC ($n=4000$)}
        \label{fig:LSACmanydropout}
    \end{subfigure}
    \hfill
    \begin{subfigure}{0.32\textwidth}
        \includegraphics[width=\linewidth]{fig/dropout/7a.pdf}
        \caption{LSAC ($n=100$)}
        \label{fig:LSACfewdropout}
    \end{subfigure}
     \caption{Same experiments as in \Cref{fig:real}, using dropout regularization instead of ridge. Although dropout induces a more complex form of regularization (even for linear regression~\cite{dropoutversusl2}), the findings are similar. For the Housing dataset in the plot (a), dropout provides benefits only for large performative effects, which can be explained by the small dimension $d=8$. For LSAC in the plots (b)-(c), the observed effects are the same as for ridge regularization. Finally, for LSAC in the plot (c), being in the proportional setting with large noise, the optimal dropout rate increases with the strength of the performative effect and the optimal risk get smaller, as it was the case for ridge regularization.
     }
    \label{fig:dropout}
\end{figure}

\begin{figure}[htb]
    \centering
    \begin{subfigure}{0.32\textwidth}
        \includegraphics[width=\linewidth]{fig/lasso/risk_vs_lambda_all_b_5c.pdf}
        \caption{Housing ($n=4000$)}
        \label{fig:housinglasso}
    \end{subfigure}
    \hfill
    \begin{subfigure}{0.32\textwidth}
        \includegraphics[width=\linewidth]{fig/lasso/risk_vs_lambda_all_b_5b.pdf}
        \caption{LSAC ($n=4000$)}
        \label{fig:LSACmanylasso}
    \end{subfigure}
    \hfill
    \begin{subfigure}{0.32\textwidth}
        \includegraphics[width=\linewidth]{fig/lasso/risk_vs_lambda_all_b_5a.pdf}
        \caption{LSAC ($n=100$)}
        \label{fig:LSACfewlasso}
    \end{subfigure}
       \caption{Same experiments as in \Cref{fig:real}, using Lasso regularization instead of ridge. Similar conclusions hold: the performative effect worsens performance in the population regime and helps in the proportional regime; the optimal regularizer continues to be non-decreasing with the strength of the performative effect in the population regime. The optimal regularizer seems close to constant in the proportional regime, potentially due to the number of features being too small to observe a dependency on the Lasso regularization. Indeed, with $22$ features, the support of $\theta$ cannot change smoothly as the regularization increases.}
    \label{fig:lasso}
\end{figure}
\begin{figure}[htb]
    \centering
    \begin{subfigure}{0.32\textwidth}
        \includegraphics[width=\linewidth]{fig/elasticnet/6c.pdf}
        \caption{Housing ($n=4000$)}
        \label{fig:housingelasticnet}
    \end{subfigure}
    \hfill
    \begin{subfigure}{0.32\textwidth}
        \includegraphics[width=\linewidth]{fig/elasticnet/6b.pdf}
        \caption{LSAC ($n=4000$)}
        \label{fig:LSACmanyelasticnet}
    \end{subfigure}
    \hfill
    \begin{subfigure}{0.32\textwidth}
        \includegraphics[width=\linewidth]{fig/elasticnet/6a.pdf}
        \caption{LSAC ($n=100$)}
        \label{fig:LSACfewelasticnet}
    \end{subfigure}
 \caption{Same experiments as in \Cref{fig:real}, using an elastic net regularization with an equal ratio between $\ell_1$ and $\ell_2$ penalties instead of ridge. As intuition suggests, the results lie between those obtained with ridge and Lasso regularization.}
    \label{fig:elasticnet}
\end{figure}


\newpage

\section{Additional proofs for \Cref{sec:pop}}
\label{app:pop}



This appendix contains the missing proofs for \Cref{sec:pop}. %
We start with the convergence at exponential rate to the fixed point $\theta^{\infty}$ in (\ref{eq:fppop}). Then, we prove Theorem \ref{thm:pop} giving the first-order approximation of the risk, as well as the expression in (\ref{eq:pophigh}) giving the higher-order approximation of the risk. Finally, we prove the upper bound on $\|F\|_{\mathrm{op}}$ in (\ref{eq:opnF}).

\begin{lemma}\label{lemma:cr}
    The sequence $(\theta_k)_k$ converges to the fixed point \[\theta^{\infty} = (I_p +\lambda \Sigma^{-1} - D)^{-1}\thetapop.\]
  Moreover, for any \(\varepsilon \in (0, 1)\), if we start at \(\theta_0 = 0\), after at most 
     \[k_{\varepsilon} = \left\lceil \frac{\ln \left(1/\varepsilon\right)}{\ln\left(1/\left(\frac{\|\Sigma\|_{\mathrm{op}}}{\|\Sigma\|_{\mathrm{op}} + \lambda} \max\left\{\|b \|_\infty, \|c \|_{\infty}\right\}\right)\right)}\right\rceil\] iterations, the relative error $
        \frac{\|\theta^{k_{\varepsilon}} - \theta^{\infty}\|_2}{\|\theta^{\infty}\|_2}$
        is smaller than $\varepsilon$.

\end{lemma}
\begin{proof}
Denoting $T = (\Sigma + \lambda I_p)^{-1} \Sigma D$, the recurrence relation is 
\[\theta^k = T \theta^{k-1} + (\Sigma + \lambda I_p)^{-1} \Sigma \thetapop.  \]
When going to the limit, $\sum_i T^i \rightarrow (I_p - T)^{-1}$. The convergence requires the matrix $T$ to have smaller eigenvalues than one, which is guaranteed by  $\|b\|_{\infty}$ and $\|c\|_{\infty}$ being smaller than one. Thus, we have 
\[\theta^{\infty} = (I_p - T)^{-1}(\Sigma + \lambda I_p)^{-1} \Sigma \thetapop. \]
Noticing that $I_p = (\Sigma + \lambda I_p)^{-1}(\Sigma + \lambda I_p)$ and using the definition of $T$ gives the expression of $\theta_{\infty}$.

Let $e_k=\theta^{k}-\theta^\infty$. Using $\theta^\infty=T\theta^\infty+(\Sigma+\lambda I_p)^{-1}\Sigma\,\thetapop$, we have
\[
e_k
= \theta^{k}-\theta^\infty
= T\theta^{k-1}+(\Sigma+\lambda I_p)^{-1}\Sigma \thetapop - \theta^\infty
= T(\theta^{k-1}-\theta^\infty)
= T e_{k-1}.
\]
Thus,
\[
e_k=T^k e_0 = -T^{k} \theta^\infty \implies \|e_k\|_2 \le \|T\|^k_{\mathrm{op}} \| \theta^\infty \|_2 \implies \frac{\|e_k\|_2}{\|\theta^\infty\|_2} =  \frac{\|\theta^{k}-\theta^\infty\|_2}{\|\theta^\infty\|_2} \le  \|T\|_{\mathrm{op}}^k. 
\]
Consequently, $\|T\|^k\le\varepsilon$ suffices, i.e.,
$k \ge \ln(1/\varepsilon)/\ln\left(1/\|T\|_{\mathrm{op}}\right)$. We finally note that
\begin{equation*}
    \begin{split}
        \|T\|_{\mathrm{op}} &\le \|(\Sigma+\lambda I_p)^{-1}\Sigma\|_{\mathrm{op}}\|D\|_{\mathrm{op}}
= \frac{\|\Sigma\|_{\mathrm{op}}}{\|\Sigma\|_{\mathrm{op}} + \lambda} \max\left\{\|b \|_\infty, \|c \|_{\infty}\right\}.
    \end{split}
\end{equation*}
Combining the last two inequalities gives the wanted convergence rate.
\end{proof}

\begin{proof}[Proof of Theorem \ref{thm:pop} and of the higher-order approximation in (\ref{eq:pophigh})]
    We start by computing the Taylor expansion:
    \begin{align*}
        A = (\Sigma + \lambda I_p - \Sigma D)^{-1} \Sigma - I_p
        = (I_p - (D-\lambda \Sigma^{-1}))^{-1} - I_p 
        = \sum_{i = 1}^{\infty} (D- \lambda\Sigma^{-1})^i
        = \sum_{i = 1}^{\infty} F^i.
    \end{align*}
    Let us define 
    \(A^{(k)} = \sum_{i=1}^{k} (D- \lambda\Sigma^{-1})^i \). For the two first orders, we have:
    \[ A^{(1) \, \top} \Sigma A^{(1)} = (D- \lambda\Sigma^{-1}) \Sigma (D- \lambda\Sigma^{-1}) = D\Sigma D - 2\lambda D + \lambda^2 \Sigma^{-1}.
    \]
    This is independent of $c$ and gives the simple formula
    \[R^{(1)}(\lambda) = \frac{1}{d} \tr(\di(b^2)\Sigma_1) - 2 \lambda \bar b +\frac{1}{d} \lambda^2 \tr(S_1), \]
    where $\bar{b} := \frac{1}{d}\tr[\di(b)] =
\frac{1}{d}\sum_{i=1}^d b_i$, $b^2:=[b_1^2, \ldots, b_d^2]\in\mathbb R^d$ and $S_1 = (\Sigma_1 - \Sigma_{12}\Sigma_2^{-1} \Sigma_{21})^{-1}$ denotes the Schur complement of $\Sigma$.
    We go further in the expansion to recover (\ref{eq:pophigh}):
    \begin{align*}
        A^{(2) \,\top}\Sigma A^{(2)}
        & = 
         A^{(1) \, \top} \Sigma A^{(1)} + A^{(1)\,\top} \Sigma \left(A^{(2)} - A^{(1)} \right) + \left(A^{(2)} - A^{(1)} \right)^\top \Sigma A^{(1)}  \\&\quad + \left(A^{(2)} - A^{(1)} \right)^\top \Sigma \left(A^{(2)} - A^{(1)} \right)
        \\& =  D \Sigma D + D^2 \Sigma D + D \Sigma D^2  - \lambda\left[ D\Sigma D \Sigma^{-1}
        + \Sigma^{-1} D \Sigma D + 2D + 4D^2\right] \\
        &\quad + \lambda^2\left[\Sigma^{-1}
        + 3(\Sigma^{-1} D + D \Sigma^{-1}) \right] -2 \lambda^3 \Sigma^{-2} + O\left(\|F\|_{\mathrm{op}}^4\right).
    \end{align*}
The final formula results from taking the trace of the first block. We write the matrix product block per block to prove that $$\tr\bigl[(D\Sigma D \Sigma^{-1}
        + \Sigma^{-1} D \Sigma D)_1\bigr] = 2\tr\!\bigl[\di(b)\Sigma_1\di(b)S_1\bigr]
         + 2\tr\!\bigl[\di(b)\Sigma_{12}\di(c)S_{21}\bigr],$$
         where $S_{21}^\top= -(\Sigma_1 - \Sigma_{12}\Sigma_2^{-1} \Sigma_{21})^{-1}\Sigma_{12} \Sigma_2^{-1}$. This concludes the proof.
\end{proof}

\begin{lemma}\label{lemma:weyl}
Let $F=D-\lambda\,\Sigma^{-1}$. Then, we have that
\begin{equation}\label{eq:fopn1}
\|F\|_{\mathrm{op}} \le 
\max\left(
\left| \max_{1\le i \le d}\{b_i, c_i\} - \frac{\lambda}{\lambda_{\max}(\Sigma)} \right|,
\left| \min_{1\le i \le d}\{b_i, c_i\} - \frac{\lambda}{\lambda_{\min}(\Sigma)} \right|
\right).
\end{equation}
\end{lemma}
\begin{proof}
By Weyl's inequalities for Hermitian matrices,
\begin{align*}
\lambda_{\max}(F) &\le \lambda_{\max}(D) - \lambda \lambda_{\min}(\Sigma^{-1}) 
= \max_{1\le i \le d}\{b_i, c_i\} - \frac{\lambda}{\lambda_{\max}(\Sigma)},\\
\lambda_{\min}(F) &\ge \lambda_{\min}(D) - \lambda \lambda_{\max}(\Sigma^{-1})
= \min_{1\le i \le d}\{b_i, c_i\} - \frac{\lambda}{\lambda_{\min}(\Sigma)}.
\end{align*}
Therefore, we have
\[
\begin{aligned}
\|F\|_{\mathrm{op}} &= \max\left\{ |\lambda_{\max}(F)|,|\lambda_{\min}(F)| \right\}
\\&\le \max\left(
\left| \max_{1\le i \le d}\{b_i, c_i\} - \frac{\lambda}{\lambda_{\max}(\Sigma)} \right|,
\left| \min_{1\le i \le d}\{b_i, c_i\} - \frac{\lambda}{\lambda_{\min}(\Sigma)} \right|
\right),
\end{aligned}
\]
and the equality happens if and only if \(\Sigma\) and \(D\) are simultaneously diagonalizable.
\end{proof}
Finally, we can rewrite this result as 
\begin{equation}   \label{eq:opnF} 
\|F\|_{\mathrm{op}} \le 
\max\left(
\begin{split}
\left| \max_{1\le i \le d}\{b_i, c_i\} - \frac{\lambda}{\|\Sigma\|_{\mathrm{op}}} \right|,\\
\left| \min_{1\le i \le d}\{b_i, c_i\} - \frac{\lambda}{\lambda_{\min}(\Sigma)} \right|
\end{split}
\right),
\end{equation}
where $\lambda_{\min}(\Sigma)$ is the smallest eigenvalue of $\Sigma$, since $\lambda_{\max}(\Sigma)=\|\Sigma\|_{\mathrm{op}}$, due to $\Sigma$ being a covariance matrix and, hence, positive semidefinite.

\section{Proof of Theorem \ref{thm:over}}\label{app:pf}



\paragraph{Deterministic equivalent for $\mathcal R_{1}(\Sigma, \theta_{1}, \thetapop)$.}

Let $\mathcal R_k(\Sigma, \theta_k, \thetapop)$ be the excess risk of the estimator $\theta_k$ given by \eqref{eq:thetak}, i.e.,
\[
    \mathcal R_k(\Sigma, \theta_k, \thetapop) = \left\|\theta_k - \thetapop \right\|_\Sigma^2. 
\]
Having fixed the initialization $\theta_0$, the only randomness in $\mathcal R_{1}(\Sigma, \theta_{1}, \thetapop)$ comes from $(X^{(0)}, y^{(0)})$. This corresponds to the setting in which one trains from the (deterministic) vector of regression coefficients $\thetapop+D\theta_0$. The following lemma gives %
a deterministic equivalent for $\mathcal R_{1}(\Sigma, \theta_{1}, \thetapop)$, conditional on $\theta_0$.

\begin{lemma}\label{lemma:1step}
\revised{Let Assumption \ref{assum:model} hold.} Let $R>0$ be a constant such that $\thetapop, \theta_0\in B_p(R)$. Assume that $\kappa, 
\sigma, \lambda \in (1/M, M)$ and $\|\Sigma\|_{\mathrm{op}},\ \|\Sigma^{-1}\|_{\mathrm{op}} \le M$ for some constant $M>1$. Then, there exists a constant $C=C\left(M, R\right)$ such that, for any $\delta \in (0, 1/2]$, the following holds %
\begin{equation}\label{eq:det1}    
\sup_{\thetapop, \theta_0\in B_p(R)}
\Pr\left(\left|\mathcal R_{1}(\Sigma, \theta_{1}, \thetapop)-\fixedriskeq^{(1)}\left(\Sigma, \theta_0, \thetapop\right)\right|\ge \delta\right)
\le Cpe^{-p\delta^{4}/C},
\end{equation}
\revised{with probability at least $1-Cpe^{-p\delta^{4}/C}$,} where
\begin{equation} \label{eq:R1eq}   
\begin{aligned}
\fixedriskeq^{(1)}\left(\Sigma, \theta_0, \thetapop\right)
&= \left\|\left( \Sigma + \tau I_p \right)^{-1} \Sigma (\thetapop+D\theta_0) - \thetapop  \right\|_\Sigma^2 \\&\quad + \kappa \tr\left[ \Sigma^2 \left( \Sigma + \tau I_p \right)^{-2} \right]\frac{   \sigma^2 + \tau^2 \left\| \left( \Sigma + \tau I_p \right)^{-1} (\thetapop+D\theta_0) \right\|_\Sigma^2  }{ p - \kappa \tr\left[ \Sigma^2 \left( \Sigma + \tau I_p \right)^{-2} \right] },
\end{aligned}
\end{equation}
and $\tau$ is the unique solution of \eqref{eq:tau}. 
\end{lemma}

\begin{proof}
Note that we are generating labels using $\thetaperfok:=\thetapop+D\theta_0$ as a vector of regression coefficients. Thus, we can apply Theorem 3 by \cite{ildizhigh} (which utilizes the non-asymptotic characterization of the minimum norm interpolator by \cite{han2023distribution}), replacing $\beta^s$ with $\thetaperfok$ in that statement. This gives that \eqref{eq:det1} holds with $\fixedriskeq^{(1)}\left(\Sigma, \theta_0, \thetapop\right)$ replaced by $\tfixedriskeq^{(1)}\left(\Sigma, \thetapop, \thetaperfok\right)$ defined as 
\begin{equation}\label{eq:tildeR1}    
\tfixedriskeq^{(1)}\left(\Sigma, \thetapop, \thetaperfok\right)= \E_{g^{(1)}}\left[\left\|X^{(1)}\left(\Sigma, \thetaperfok, g^{(1)}\right)-\thetapop\right\|_\Sigma^2 \right] ,
\end{equation}
where
\begin{align}
X^{(1)}\left(\Sigma, \thetaperfok, g^{(1)}\right)
&=  (\Sigma + \tau I_p)^{-1} \Sigma \left[\thetaperfok + \frac{\Sigma^{-1/2} \gamma^{(1)}(\thetaperfok)g^{(1)}}{\sqrt{p}}\right] ,\label{eq:X1} \\
\left(\gamma^{(1)}(\thetaperfok)\right)^2
&= \kappa\left(\sigma^2 + \tfixedriskeq^{(1)}\left(\Sigma, \thetaperfok, \thetaperfok\right)\right),\label{eq:gamma1}
\end{align}
$\tau$ is the unique solution of \eqref{eq:tau} and $g^{(1)}\sim \mathcal N(0, I_p)$. By plugging \eqref{eq:X1} into \eqref{eq:tildeR1} and computing the expectation with respect to $g^{(1)}$, we get
\begin{equation}\label{eq:tildeR2}
    \tfixedriskeq^{(1)}\left(\Sigma, \thetapop, \thetaperfok\right)=\left\| \left(\Sigma + \tau I_p \right)^{-1} \Sigma \thetaperfok - \thetapop  \right\|_\Sigma^2 + \frac{\left(\gamma^{(1)}(\thetaperfok)\right)^2}{p} \tr \left[ \Sigma^2 \left(\Sigma + \tau I_p \right)^{-2} \right].
\end{equation}
Next, we solve the fixed point equation in $\gamma^{(1)}(\thetaperfok)$: 
\[
\begin{aligned}
    \left(\gamma^{(1)}(\thetaperfok)\right)^2
&= \kappa\left(\sigma^2 + \tfixedriskeq^{(1)}\left(\Sigma, \thetaperfok, \thetaperfok\right)\right)  \\&= \kappa\left(\sigma^2 + \left\| \left(\left(\Sigma + \tau I_p \right)^{-1} \Sigma - I_p\right) \thetaperfok  \right\|_\Sigma^2 + \frac{\left(\gamma^{(1)}(\thetaperfok)\right)^2}{p} \tr \left[ \Sigma^2 \left(\Sigma + \tau I_p \right)^{-2} \right]\right) \\
&= \kappa\left(\sigma^2 + \tau^2\left\| \left(\Sigma + \tau I_p \right)^{-1} \thetaperfok  \right\|_\Sigma^2 + \frac{\left(\gamma^{(1)}(\thetaperfok)\right)^2}{p} \tr \left[ \Sigma^2 \left(\Sigma + \tau I_p \right)^{-2} \right]\right).
\end{aligned}
\]
The last equality comes from
\[
I_p - \left(\Sigma + \tau I_p \right)^{-1} \Sigma  =  \left(\Sigma + \tau I_p \right)^{-1} \left( \Sigma + \tau I_p \right) - \left(\Sigma + \tau I_p \right)^{-1}\Sigma = \tau \left(\Sigma + \tau I_p \right)^{-1}.
\]
Rearranging gives that
\begin{equation}\label{eq:gamma1ex}
\left(\gamma^{(1)} (\thetaperfok)\right)^2 = \kappa\frac{ \sigma^2 + \tau^2 \left\| \left( \Sigma + \tau I_p \right)^{-1} \thetaperfok  \right\|_\Sigma^2  }{ 1 - \frac{1}{n} \tr\left[ \Sigma^2 \left( \Sigma + \tau I_p \right)^{-2} \right] },
\end{equation}
which plugged into \eqref{eq:tildeR2} gives the desired result.
\end{proof}

We note that the expression in \eqref{eq:R1eq} depends on $\theta_0$ and, in fact, it keeps depending on it even after neglecting terms of order $O(\|D\|_{\mathrm{op}}^2)$.

\paragraph{Deterministic equivalent for $\mathcal R_{2}(\Sigma, \theta_{2}, \thetapop)$.}  Next, by %
iterating twice the strategy of Lemma \ref{lemma:1step}, we derive
a deterministic equivalent for $\mathcal R_{2}(\Sigma, \theta_{2}, \thetapop)$. %

\begin{lemma}\label{lemma:2step}
\revised{Let Assumption \ref{assum:model} hold.} Let $R>0$ be a constant such that $\thetapop, \theta_0\in B_p(R)$. Assume that $\kappa, 
\sigma, \lambda \in (1/M, M)$ and $\|\Sigma\|_{\mathrm{op}},\ \|\Sigma^{-1}\|_{\mathrm{op}} \le M$ for some constant $M>1$. Then, there exists a constant $C=C\left(M, R\right)$ such that, for any $\delta \in (0, 1/2]$, the following holds %
\begin{equation}\label{eq:det2}    
\sup_{\thetapop, \theta_0\in B_p(R)}
\Pr\left(\left|\mathcal R_{2}(\Sigma, \theta_{2}, \thetapop)-\fixedriskeq^{(2)}\left(\Sigma, \theta_0, \thetapop\right)\right|\ge \delta\right)
\le Cpe^{-p\delta^{4}/C},
\end{equation}
\revised{with probability at least $1-Cpe^{-p\delta^{4}/C}$,} where
\begin{equation}  \label{eq:fixedriskeq2}  
\begin{aligned}
&\fixedriskeq^{(2)}\left(\Sigma, \theta_0, \thetapop\right)
= \left\| \left( \Sigma + \tau I_p \right)^{-1} \Sigma D \left( \Sigma + \tau I_p \right)^{-1} \Sigma (\thetapop+D\theta_0) - \tau(\Sigma+\tau I_p)^{-1}\thetapop \right\|_\Sigma^2  \\
&\quad+ \kappa \tr\left[ \Sigma \left( \Sigma + \tau I_p \right)^{-2} D  \Sigma^3 \left( \Sigma + \tau I_p \right)^{-2} D \right]
\frac{ \sigma^2 + \tau^2 \big\| \left( \Sigma + \tau I_p \right)^{-1} (\thetapop+D\theta_0) \big\|_\Sigma^2 }{p - \kappa \tr\left[ \Sigma^2 \left( \Sigma + \tau I_p \right)^{-2} \right] } \\
&\quad+ \kappa \tr\left[ \Sigma^2 \left( \Sigma + \tau I_p \right)^{-2} \right]
\frac{ \sigma^2 + \tau^2 \big\| \left( \Sigma + \tau I_p \right)^{-1} \left(\thetapop + D \left( \Sigma + \tau I_p \right)^{-1} \Sigma (\thetapop+D\theta_0) \right) \big\|_\Sigma^2 }{p - \kappa \tr\left[ \Sigma^2 \left( \Sigma + \tau I_p \right)^{-2} \right] } \\
&\quad+ \kappa^2 \tau^2  \tr\left[ \Sigma^2 \left( \Sigma + \tau I_p \right)^{-2} \right] 
\tr\left[ \Sigma \left( \Sigma + \tau I_p \right)^{-2} D  \Sigma \left( \Sigma + \tau I_p \right)^{-2} D \right]
\\
&\hspace{15em}\cdot\frac{ \sigma^2 + \tau^2 \big\| \left( \Sigma + \tau I_p \right)^{-1} (\thetapop+D\theta_0) \big\|_\Sigma^2 }
{\big( p - \kappa \tr\left[ \Sigma^2 \left( \Sigma + \tau I_p \right)^{-2} \right] \big)^2 },
\end{aligned}
\end{equation}
and $\tau$ is the unique solution of \eqref{eq:tau}. 
\end{lemma}

\begin{proof}
The proof extends the argument of Theorem 2 by \citep{ildizhigh} to the ridge regression case, and it applies the distributional characterization of the minimum norm interpolator by \cite{han2023distribution} twice. First, note that $\|\theta_{1}\|_2$ is bounded by a constant $C_1=C_1(R, M)$ independent of $n, p$, with probability at least $C_2 e^{-p/C_2}$, where $C_2=C_2(R, M)$ is a constant independent of $n, p$. This follows from a direct adaptation of Proposition 11 by \cite{ildizhigh}. Define $R':=\max(C_1, \|\thetapop\|_2)$. Then, upon conditioning on $\theta_{1}$, we can apply Lemma \ref{lemma:1step} (after re-defining $R$ to be $R'$), which gives that, for some constant $C_3=C_3(R, M)$,
\begin{equation}\label{eq:det3}    
\sup_{\thetapop, \theta_{1}\in B_p(R')}
\Pr\left(\left|\mathcal R_{2}(\Sigma, \theta_{2}, \thetapop)-\fixedriskeq^{(1)}\left(\Sigma, \theta_{1}, \thetapop\right)\right|\ge \delta\right)
\le C_3pe^{-p\delta^{4}/C_3},
\end{equation}
where $\fixedriskeq^{(1)}\left(\Sigma, \theta_{1}, \thetapop\right)$ is defined in \eqref{eq:R1eq}
and $\tau$ is the unique solution of \eqref{eq:tau}. 

We now evaluate the first term in the expression for $\fixedriskeq^{(1)}\left(\Sigma, \theta_{1}, \thetapop\right)$:
\begin{equation*}
\begin{split}
    \fixedriskeqa^{(1)}\left(\Sigma, \theta_{1}, \thetapop\right):&=\left\|\left( \Sigma + \tau I_p \right)^{-1} \Sigma (\thetapop+D\theta_{1}) - \thetapop  \right\|_\Sigma^2\\
    &=\left\|\left( \Sigma + \tau I_p \right)^{-1} \Sigma D\theta_{1} - \tau (\Sigma+\tau I_p)^{-1}\thetapop  \right\|_\Sigma^2.
    \end{split}
\end{equation*}
Let $M_1=\Sigma^{1/2}$, $M_2=(\Sigma+\tau I_p)^{-1}\Sigma D$ and $a=\tau(\Sigma+\tau I_p)^{-1}\thetapop$. Then, the function $\theta_{1}\mapsto \fixedriskeqa^{(1)}\left(\Sigma, \theta_{1}, \thetapop\right)$ can be expressed as
$$
f(\theta_{1})=\|M_1(M_2 \theta_{1}-a)\|_2^2,
$$
which has gradient
$$
\nabla f(\theta_{1})=2M_2^\top M_1^\top M_1(M_2 \theta_{1}-a).
$$
As $\|\theta_{1}\|_2\le C_1$, $f$ is Lipschitz and its Lipschitz constant is $2\|M_1\|_{\mathrm{op}}^2\|M_2\|_{\mathrm{op}}(\|M_1\|_{\mathrm{op}} C_1+\|a\|_2)$. As $\|M_1\|_{\mathrm{op}}, \|M_2\|_{\mathrm{op}}, C_1, \|a\|_2$ are all upper bounded by constants dependent only on $R, M$, the Lipschitz constant of $f$ is also upper bounded by a constant dependent only on $R, M$. Thus, 
an application of the distributional characterization by \cite{han2023distribution} (restated as Theorem 4 in \citep{ildizhigh}) gives that, for some constant $C_4=C_4(R, M)$,
\begin{equation}\label{eq:det4}    
\sup_{\thetapop, \theta_{1}\in B_p(R')}
\Pr\left(\left|\fixedriskeqa^{(1)}\left(\Sigma, \theta_{1}, \thetapop\right)-\tfixedriskeqa^{(1)}\left(\Sigma, \thetapop, \thetaperfok\right)\right|\ge \delta\right)
\le C_4pe^{-p\delta^{4}/C_4},
\end{equation}
where
\begin{equation}\label{eq:tfixedriskeqa}
    \tfixedriskeqa^{(1)}\left(\Sigma, \thetapop, \thetaperfok\right)= \E_{g^{(1)}}\left[\left\|(\Sigma+\tau I_p)^{-1}\Sigma D X^{(1)}\left(\Sigma, \thetaperfok, g^{(1)}\right)-\tau(\Sigma+\tau I_p)^{-1}\thetapop\right\|_\Sigma^2 \right].
\end{equation}
We recall from Lemma \ref{lemma:1step} that $\thetaperfok=\thetapop+D\theta_0$, $g^{(1)}\sim \mathcal N(0, I_p)$ and $X^{(1)}\left(\Sigma, \thetaperfok, g^{(1)}\right)$ is given by \eqref{eq:X1}. By plugging \eqref{eq:X1} into the RHS of \eqref{eq:tfixedriskeqa} and computing the expectation with respect to $g^{(1)}$, we have 
\begin{equation}\label{eq:tfixedriskeqa2}
    \begin{split}
\tfixedriskeqa^{(1)}\left(\Sigma, \thetapop, \thetaperfok\right)&= \E_{g^{(1)}}\Biggl[\biggl\|(\Sigma+\tau I_p)^{-1}\Sigma D(\Sigma+\tau I_p)^{-1}\Sigma\thetaperfok-\tau(\Sigma+\tau I_p)^{-1}\thetapop\\
&\hspace{5em}+(\Sigma+\tau I_p)^{-1}\Sigma D(\Sigma+\tau I_p)^{-1}\Sigma^{1/2}\frac{ \gamma^{(1)}(\thetaperfok)g^{(1)}}{\sqrt{p}}\biggr\|_\Sigma^2 \Biggr]\\
&=\biggl\|(\Sigma+\tau I_p)^{-1}\Sigma D(\Sigma+\tau I_p)^{-1}\Sigma\thetaperfok-\tau(\Sigma+\tau I_p)^{-1}\thetapop\biggr\|_\Sigma^2 \\
&\hspace{5em}+\frac{\left(\gamma^{(1)}(\thetaperfok)\right)^2}{p}\tr\left[ \Sigma \left( \Sigma + \tau I_p \right)^{-2} D  \Sigma^3 \left( \Sigma + \tau I_p \right)^{-2} D \right],
    \end{split}
\end{equation}
where in the last step we have used the circulant property of the trace. By using the expression for $\gamma^{(1)}(\thetaperfok)$ in \eqref{eq:gamma1ex} and recalling that $\thetaperfok=\thetapop+D\theta_0$, one readily obtains that the RHS of \eqref{eq:tfixedriskeqa2} coincides with the first two lines of the RHS of \eqref{eq:fixedriskeq2}.

Finally, we evaluate the second term in the expression for $\fixedriskeq^{(1)}\left(\Sigma, \theta_{1}, \thetapop\right)$:
\begin{equation*}
\begin{split}
    \fixedriskeqb^{(1)}\left(\Sigma, \theta_{1}, \thetapop\right):&=\kappa \tr\left[ \Sigma^2 \left( \Sigma + \tau I_p \right)^{-2} \right]\frac{   \sigma^2 + \tau^2 \left\| \left( \Sigma + \tau I_p \right)^{-1} (\thetapop+D\theta_{1}) \right\|_\Sigma^2  }{ p - \kappa \tr\left[ \Sigma^2 \left( \Sigma + \tau I_p \right)^{-2} \right] }.
    \end{split}
\end{equation*}
Let $M_1=\Sigma^{1/2}$, $M_2=(\Sigma+\tau I_p)^{-1}D$ and $a=(\Sigma+\tau I_p)^{-1}\thetapop$. Then, the function $\theta_{1}\mapsto \left\| \left( \Sigma + \tau I_p \right)^{-1} (\thetapop+D\theta_{1}) \right\|_\Sigma^2$ can be expressed as
$$
f(\theta_{1})=\|M_1(M_2 \theta_{1}+a)\|_2^2,
$$
which has gradient
$$
\nabla f(\theta_{1})=2M_2^\top M_1^\top M_1(M_2 \theta_{1}+a).
$$
As $\|\theta_{1}\|_2\le C_1$, $f$ is Lipschitz and its Lipschitz constant is $2\|M_1\|_{\mathrm{op}}^2\|M_2\|_{\mathrm{op}}(\|M_1\|_{\mathrm{op}} C_1+\|a\|_2)$. As $\|M_1\|_{\mathrm{op}}, \|M_2\|_{\mathrm{op}}, C_1, \|a\|_2$ are all upper bounded by constants dependent only on $R, M$, the Lipschitz constant of $f$ is also upper bounded by a constant dependent only on $R, M$. Note that the quantity $|p-\kappa \tr\left[\Sigma^2(\Sigma+\tau I_p)^{-2}\right]|$ is lower bounded by a constant dependent only on $R, M$, as a consequence of Proposition 2.1 in \cite{han2023distribution}. Thus, we have that the function $\theta_{1}\mapsto\fixedriskeqb^{(1)}\left(\Sigma, \theta_{1}, \thetapop\right)$ is Lipschitz and its Lipschitz constant is $C_5=C_5(R, M)$. Hence,
another application of the distributional characterization by \cite{han2023distribution} (cf.\ Theorem 4 in \citep{ildizhigh}) gives that, for some constant $C_6=C_6(R, M)$,
\begin{equation}\label{eq:det5}    
\sup_{\thetapop, \theta_{1}\in B_p(R')}
\Pr\left(\left|\fixedriskeqb^{(1)}\left(\Sigma, \theta_{1}, \thetapop\right)-\tfixedriskeqb^{(1)}\left(\Sigma, \thetapop, \thetaperfok\right)\right|\ge \delta\right)
\le C_6pe^{-p\delta^{4}/C_6},
\end{equation}
where
\begin{equation}\label{eq:tfixedriskeqb}
    \begin{split}
\tfixedriskeqb^{(1)}&\left(\Sigma, \thetapop, \thetaperfok\right)= \kappa \tr\left[ \Sigma^2 \left( \Sigma + \tau I_p \right)^{-2} \right]\\
  &  \cdot \frac{   \sigma^2 + \tau^2 \mathbb E_{g^{(1)}}\left[\left\| \left( \Sigma + \tau I_p \right)^{-1} \left(\thetapop+DX^{(1)}\left(\Sigma, \thetaperfok, g^{(1)}\right)\right) \right\|_\Sigma^2 \right] }{ p - \kappa \tr\left[ \Sigma^2 \left( \Sigma + \tau I_p \right)^{-2} \right] }.
  \end{split}
\end{equation}
By using \eqref{eq:X1} and computing the expectation with respect to $g^{(1)}$, we have 

\begin{equation}\label{eq:tfixedriskeqb2}
\begin{split}    
\mathbb E_{g^{(1)}}&\left[    \left\| \left( \Sigma + \tau I_p \right)^{-1} \left(\thetapop+DX^{(1)}\left(\Sigma, \thetaperfok, g^{(1)}\right)\right) \right\|_\Sigma^2\right]\\&=    \mathbb E_{g^{(1)}}\Biggl[\biggl\| \left( \Sigma + \tau I_p \right)^{-1} \thetapop+\left( \Sigma + \tau I_p \right)^{-1}D\left( \Sigma + \tau I_p \right)^{-1}\Sigma \thetaperfok\\
&\qquad\qquad\qquad  +\left( \Sigma + \tau I_p \right)^{-1} D\left( \Sigma + \tau I_p \right)^{-1}\Sigma^{1/2}\frac{ \gamma^{(1)}(\thetaperfok)g^{(1)}}{\sqrt{p}} \biggr\|_\Sigma^2\Biggr]\\
&=\left\| \left( \Sigma + \tau I_p \right)^{-1} \thetapop+\left( \Sigma + \tau I_p \right)^{-1}D\left( \Sigma + \tau I_p \right)^{-1}\Sigma \thetaperfok\right\|_\Sigma^2\\
&\qquad\qquad\qquad+\frac{\left(\gamma^{(1)}(\thetaperfok)\right)^2}{p}\tr\left[ \Sigma \left( \Sigma + \tau I_p \right)^{-2} D  \Sigma \left( \Sigma + \tau I_p \right)^{-2} D \right],
\end{split}
\end{equation}
where in the last step we have used the circulant property of the trace. By plugging \eqref{eq:tfixedriskeqb2} into \eqref{eq:tfixedriskeqb}, using the expression for $\gamma^{(1)}(\thetaperfok)$ in \eqref{eq:gamma1ex} and recalling that $\thetaperfok=\thetapop+D\theta_0$, one readily obtains that the RHS of \eqref{eq:tfixedriskeqb} coincides with the last two lines of the RHS of \eqref{eq:fixedriskeq2}. As $\fixedriskeq^{(1)}\left(\Sigma, \theta_{1}, \thetapop\right)=\fixedriskeqa^{(1)}\left(\Sigma, \theta_{1}, \thetapop\right)+\fixedriskeqb^{(1)}\left(\Sigma, \theta_{1}, \thetapop\right)$, the desired result readily follows by combining 
\eqref{eq:det3}, \eqref{eq:det4} and \eqref{eq:det5}.
\end{proof}

\paragraph{Concluding the argument.}
Note that
\begin{equation*}
    \begin{split}
 \tr\big[\Sigma (\Sigma+\tau I_p)^{-2} D \Sigma^{3} (\Sigma+\tau I_p)^{-2} D\big]&= O(\|D\|_{\mathrm{op}}^{2}),\\
\tr\big[\Sigma (\Sigma+\tau I_p)^{-2} D \Sigma (\Sigma+\tau I_p)^{-2} D\big] &= O(\|D\|_{\mathrm{op}}^{2}).
    \end{split}
\end{equation*}
Furthermore, we have
\begin{equation*}
    \begin{split}
        &\left\| \left( \Sigma + \tau I_p \right)^{-1} \Sigma D \left( \Sigma + \tau I_p \right)^{-1} \Sigma (\thetapop+D\theta_0) - \tau(\Sigma+\tau I_p)^{-1}\thetapop \right\|_\Sigma^2\\
        & =\left\| \left( \Sigma + \tau I_p \right)^{-1} \Sigma D \left( \Sigma + \tau I_p \right)^{-1} \Sigma \thetapop - \tau(\Sigma+\tau I_p)^{-1}\thetapop \right\|_\Sigma^2+O(\|D\|_{\mathrm{op}}^{2})\\
        & =\left\|\tau(\Sigma+\tau I_p)^{-1}\thetapop\right\|_\Sigma^2\\
        &\qquad\qquad -2\tau\langle (\Sigma+\tau I_p)^{-1}\thetapop, \Sigma \left( \Sigma + \tau I_p \right)^{-1} \Sigma D \left( \Sigma + \tau I_p \right)^{-1} \Sigma \thetapop\rangle+O(\|D\|_{\mathrm{op}}^{2})\\
        &=\tau
\langle \thetapop, (\Sigma+\tau I_p )^{-1}\left(\tau I_p
-2
( \Sigma+\tau I_p )^{-1}
\Sigma^2 D
\right)\Sigma( \Sigma+\tau I_p )^{-1}
\thetapop\rangle+O(\|D\|_{\mathrm{op}}^{2}). 
    \end{split}
\end{equation*}
Similarly, we have
\begin{equation*}
    \begin{split}
&\left\| \left( \Sigma + \tau I_p \right)^{-1} \left(\thetapop + D \left( \Sigma + \tau I_p \right)^{-1} \Sigma (\thetapop+D\theta_0) \right) \right\|_\Sigma^2  \\
&=\left\| \left( \Sigma + \tau I_p \right)^{-1} \left(\thetapop + D \left( \Sigma + \tau I_p \right)^{-1} \Sigma \thetapop \right) \right\|_\Sigma^2+O(\|D\|_{\mathrm{op}}^{2})\\
&=\left\| \left( \Sigma + \tau I_p \right)^{-1} \thetapop  \right\|_\Sigma^2+2\langle \left( \Sigma + \tau I_p \right)^{-1}\thetapop, \Sigma\left( \Sigma + \tau I_p \right)^{-1}D\left( \Sigma + \tau I_p \right)^{-1}\Sigma\thetapop \rangle +O(\|D\|_{\mathrm{op}}^{2})\\
&=\langle \thetapop,\left( \Sigma+\tau I_p \right)^{-1}
\left(I_p+2\left( \Sigma+\tau I_p \right)^{-1}
\Sigma D\right)\Sigma\left( \Sigma+\tau I_p \right)^{-1}\thetapop\rangle+O(\|D\|_{\mathrm{op}}^{2}).
    \end{split}
\end{equation*}
Recalling the definitions \eqref{eq:defdet} and \eqref{eq:fixedriskeq2}, we conclude that  
\begin{equation}\label{eq:equalityc}    
\fixedriskeq^{(2)}\left(\Sigma, \theta_0, \thetapop\right)=\fixedriskeq\left(\Sigma, \thetapop, D, \lambda\right)+O(\|D\|_{\mathrm{op}}^{2}).
\end{equation}
Thus, the desired result follows from \eqref{eq:equalityc} and Lemma \ref{lemma:2step}. 

\section{\revised{Extension to sub-Gaussian data}}\label{app:extension}

\revised{Throughout this appendix, we relax Assumption \ref{assum:model} as follows.}

\revised{\begin{assumption}[Regression performative model -- relaxed assumption]
For $\theta \in \R^p$, samples from $\D(\theta)$ are taken i.i.d.\ with features $x$ drawn independently of $\theta$ and such that $\Sigma^{-1/2}x$ has independent, zero mean, unit variance and uniformly sub-Gaussian entries.  The label $y$ is given by
\vspace{-.3em}
\begin{equation}\label{eq:data-bis}    
y = x^\top \thetapop + x^\top D \theta + w, \quad w \sim \N(0, \sigma^2).
\end{equation}
We assume $p = 2d$, $(\thetapop)^{\top} = (a^\top, 0)$ with $a$ having zero mean and covariance $I_d/d$, and $D = \di(b, c)$ where $b, c \in \R^d$ with $\|b\|_{\infty}, \|c\|_{\infty} < 1$. We further assume that $a\sqrt{d}$ has sub-Gaussian norm upper bounded by a universal constant (independent of $d$).
\label{assum:model-bis}
\end{assumption}}

\revised{\begin{theorem}[Excess risk -- over-parameterized, relaxed assumptions]\label{thm:over-rel}
Let Assumption \ref{assum:model-bis} hold. Let $R>0$ be a constant s.t.\ $\thetapop\in B_p(R)$ and let $\theta_0$ be sampled uniformly on the unit sphere. Assume that $\kappa, \sigma, \lambda\in (1/M, M)$ and $\|\Sigma\|_{\mathrm{op}},\ \|\Sigma^{-1}\|_{\mathrm{op}} \le M$ for some constant $M>1$. Then, there exists a constant $C=C\left(M, R\right)$ such that for any $\delta \in (0,1/2]$, with probability at least $1-C\delta^{-7}p^{-1/8}$, %
\begin{equation}    
\left|\mathcal{R}(\Sigma, \theta_{2}, \thetapop)-\fixedriskeq\left(\Sigma, \thetapop, D, \lambda\right)\right|\le \delta+O(\|D\|_{\mathrm{op}}^2),
\end{equation}
where $\fixedriskeq\left(\Sigma, \thetapop, D, \lambda\right)$ is given by (\ref{eq:defdet}). 
\end{theorem}}



\revised{\begin{lemma}[Norm control]\label{lemma:norm}
In the setting of Theorem \ref{thm:over-rel}, we have that
    \begin{align}
\|\theta_1\|_2&\le C,\label{eq:condnorm1}\\
\|\theta_2\|_2&\le C,\label{eq:condnorm1-bis}\\
\|\theta_0\|_\infty&\le C\frac{\log p}{\sqrt{p}},\label{eq:condnorm2}\\
\|\thetapop\|_\infty&\le C\frac{\log p}{\sqrt{p}},\label{eq:condnorm3}\\
\|\theta_1\|_\infty&\le C\frac{\log p}{\sqrt{p}},\label{eq:condnorm4}
    \end{align}    
    with probability at least $1-Ce^{-\log^2 p/C}$, where $C=C(R, M)$ is a constant depending only on $R, M$ (and not on $n, p$).
\end{lemma}}

\begin{proof}
\revised{  We start by proving (\ref{eq:condnorm1}). The claim follows by extending the argument of Proposition 11 in \cite{ildizhigh} and we repeat it here for completeness. Recall that
  $$
  \theta_{1} = \frac{1}{p} \left(\frac{1}{p} X^{0  \top} X^{0} + \lambda I_p\right)^{-1} X^{0 \top} X^0(\thetapop+D\theta_0)+\frac{1}{p} \left(\frac{1}{p} X^{0  \top} X^{0} + \lambda I_p\right)^{-1} X^{0 \top}w. 
  $$
 Note that 
$$
 \left\|\frac{1}{p} \left(\frac{1}{p} X^{0  \top} X^{0} + \lambda I_p\right)^{-1} X^{0 \top} X^0\right\|_{\mathrm{op}}\le 1,
 $$
 which implies that 
 $$
\left\| \frac{1}{p} \left(\frac{1}{p} X^{0  \top} X^{0} + \lambda I_p\right)^{-1} X^{0 \top} X^0(\thetapop+D\theta_0)\right\|_2\le C_1,
 $$
 for some constant $C_1=C_1(R, M)$. Next, we can write
 \begin{equation*}
 \begin{split}
\left\|\frac{1}{p} \left(\frac{1}{p} X^{0  \top} X^{0} + \lambda I_p\right)^{-1} X^{0 \top}w\right\|_2^2 &= \frac{w^\top X^0}{p}\left(\frac{1}{p} X^{0  \top} X^{0} + \lambda I_p\right)^{-2}\frac{X^{0 \top}w}{p}\\
&\le \frac{w^\top w}{p}\left\|\frac{1}{p}X^0\left(\frac{1}{p} X^{0  \top} X^{0} + \lambda I_p\right)^{-2}X^{0 \top}\right\|_{\mathrm{op}}.
\end{split}
\end{equation*}
Using Bernstein's inequality, we have that $w^\top w/p$ is upper bounded by $C_2=C_2(R, M)$ with probability at least $1-e^{-p/C_2}$. Furthermore, $
\left\|\frac{1}{p}X^0\left(\frac{1}{p} X^{0  \top} X^{0} + \lambda I_p\right)^{-2}X^{0 \top}\right\|_{\mathrm{op}}\le \left\|\frac{1}{p}X^0\left(\frac{1}{p} X^{0  \top} X^{0} + \lambda I_p\right)^{-2}X^{0 \top}\right\|_{\mathrm{op}}
$ is also upper bounded by a universal constant. Thus, an application of the triangle inequality gives (\ref{eq:condnorm1}). Repeating the same argument with $\theta_1$ in place of $\theta_0$ and $X^1$ in place of $X^0$ readily gives (\ref{eq:condnorm1-bis}).}

\revised{Let $v\in \mathbb R^p$ be a vector such that $v\sqrt{p}$ has sub-Gaussian norm upper bounded by a universal constant (independent of $p$). We will now show that
\begin{equation}\label{eq:inftyn}
    \|v\|_\infty\le C\frac{\log p}{\sqrt{p}},
\end{equation}
with probability at least $1-e^{-\log^2 p}$. To see this, it suffices to note that the $j$-th coordinate $v_j\sqrt{p}$ is sub-Gaussian with sub-Gaussian norm upper bounded by a universal constant. Thus, 
$$
\mathbb P(|v_j\sqrt{p}|>t)\le 2e^{-t^2/C_3},
$$
for some universal constant $C_3$. Taking $t=C\log p$ and doing a union bound over $j\in \{1, \ldots, p\}$ gives (\ref{eq:inftyn}).}

\revised{Since $\theta_0$ is sampled uniformly on the sphere, (\ref{eq:condnorm2}) is implied by (\ref{eq:inftyn}). Therefore, $\theta_0\sqrt{p}$ has sub-Gaussian norm upper bounded by a universal constant (independent of $p$). Furthermore,  (\ref{eq:condnorm3}) is implied by (\ref{eq:inftyn}) since $\thetapop$ satisfies Assumption \ref{assum:model-bis}. Finally, letting $\|\cdot\|_{\psi_2}$ denote the sub-Gaussian norm of a vector, we have
\begin{equation}
\begin{split}
    \|\theta_1\sqrt{p}\|_{\psi_2}&\le \left\|\frac{1}{p} \left(\frac{1}{p} X^{0  \top} X^{0} + \lambda I_p\right)^{-1} X^{0 \top} X^0\right\|_{\mathrm{op}} (\|\thetapop\sqrt{p}\|_{\psi_2}+\|\theta_0\sqrt{p}\|_{\psi_2})\\
    &\hspace{10em}+\left\|\frac{1}{\sqrt{p}} \left(\frac{1}{p} X^{0  \top} X^{0} + \lambda I_p\right)^{-1} X^{0 \top} \right\|_{\mathrm{op}}\left\|w\right\|_{\psi_2}\\
    &\le \|\thetapop\sqrt{p}\|_{\psi_2}+\|\theta_0\sqrt{p}\|_{\psi_2}+\left\|w\right\|_{\psi_2},
\end{split}
\end{equation}
which is upper bounded by a universal constant. 
Thus, (\ref{eq:condnorm4}) is also implied by (\ref{eq:inftyn}) and the proof is complete.}
\end{proof}

\revised{\begin{lemma}\label{lemma:1step-bis}
Let Assumption \ref{assum:model-bis} hold. Let $R>0$ be a constant such that $\thetapop\in B_p(R)$ and let $\theta_0$ be sampled uniformly on the unit sphere. Assume that $\kappa, 
\sigma, \lambda \in (1/M, M)$ and $\|\Sigma\|_{\mathrm{op}},\ \|\Sigma^{-1}\|_{\mathrm{op}} \le M$ for some constant $M>1$. Then, there exists a constant $C=C\left(M, R\right)$ such that, for any $\delta \in (0, 1/2]$, the following holds %
\begin{equation}\label{eq:det1-ter}    
\sup_{\thetapop, \theta_0\in B_p(R)}
\Pr\left(\left|\mathcal R_{1}(\Sigma, \theta_{1}, \thetapop)-\fixedriskeq^{(1)}\left(\Sigma, \theta_0, \thetapop\right)\right|\ge \delta\right)
\le Cpe^{-p\delta^{4}/C},
\end{equation}
with probability at least $1-C\delta^{-7}p^{-1/8}$,
where $\fixedriskeq^{(1)}$ is given by (\ref{eq:R1eq}).
\end{lemma}}

\begin{proof}
\revised{By Lemma \ref{lemma:norm}, we have that $\thetapop+D\theta_0$ satisfies the delocalization condition of Proposition 10.3 by \cite{han2023distribution}. This implies that the hypotheses of Theorem 2.4 by \cite{han2023distribution} are satisfied when we train using $\thetapop+D\theta_0$. Thus, we can now follow the same steps as in Lemma \ref{lemma:1step} which invokes Theorem 3 by \cite{ildizhigh}. In particular, Theorem 3 by \cite{ildizhigh} uses Theorem 4 therein plus the bound on $\|\theta_1\|_2$ given by Lemma \ref{lemma:norm}. Thus, it suffices to replace the application of Theorem 4 by \cite{ildizhigh} with the application of Theorem 2.4 by \cite{han2023distribution}, and the desired result readily holds.} 
\end{proof}

\revised{
\begin{proof}[Proof of Theorem \ref{thm:over-rel}]
    By Lemma \ref{lemma:norm}, we have that $\thetapop+D\theta_0$ and $\thetapop+D\theta_1$ satisfy the delocalization condition of Proposition 10.3 by \cite{han2023distribution}. This implies that the hypotheses of Theorem 2.4 by \cite{han2023distribution} are satisfied when we train using either $\thetapop+D\theta_0$ or $\thetapop+D\theta_1$ as vector of regression coefficients. Consequently, the desired result is obtained by following the same steps as in the proof of Theorem \ref{thm:over}, the only differences being that \emph{(i)} we apply Lemma \ref{lemma:1step-bis} in place of Lemma \ref{lemma:1step}, and \emph{(ii)} we apply Theorem 2.4 by \cite{han2023distribution} in place of Theorem 4 by \cite{ildizhigh}. This requires an upper bound on $\|\theta_1\|_2, \|\theta_2\|_2$ which is provided by Lemma \ref{lemma:norm}.
\end{proof}}

\section{Proof of Theorem \ref{thm:equiv}}\label{app:pfequiv}

We start by computing explicitly $\mathbb E_{\thetapop}\fixedriskeq\left(\Sigma, \thetapop, D, \lambda\right)$.

\begin{lemma}\label{lemma:explicit}
Consider the setting of Theorem \ref{thm:over}, assume that $a$ has covariance $I_d/d$, and let $\Sigma=\begin{bmatrix}I_d&\rho I_d\\ \rho I_d&I_d\end{bmatrix}$. Then, we have that
\begin{equation*}
\begin{split}    
\mathbb E_{\thetapop}\fixedriskeq\left(\Sigma, \thetapop, D, \lambda\right)&=\widetilde{\mathcal R}(D, \lambda, \rho)+ O(\bar b\rho^2+\rho^4),\\
 \widetilde{\mathcal R}(D, \lambda, \rho)&:=\mathcal R_0(\lambda, \rho) + \bar b  A_1(\lambda)   + \bar c \rho^2 A_2(\lambda),
\end{split}
\end{equation*}
where $\bar b=\tr[\di(b)]/d, \bar c=\tr[\di(c)]/d$ and the auxiliary functions 
 $\mathcal R_0(\lambda, \rho)$, $A_1(\lambda)$, and $A_2(\lambda)$ %
 are given by 
\begin{equation}   \label{eq:explexpr} 
\begin{aligned}
\mathcal R_0(\lambda, \rho) &= \frac{\tau^{2}}{(1+\tau)^{2}}
+  \frac{\kappa}{(1+\tau)^{2}-\kappa} \left( \sigma^{2}+\frac{\tau^{2}}{(1+\tau)^{2}} \right)\\&\hspace{-1em} + \rho^2 \left( \frac{\tau^{2} (1-2\tau)}{(1+\tau)^{4}}
+ \frac{\kappa \tau^{2} (1-2\tau)}{(1+\tau)^{4} \left((1+\tau)^{2}-\kappa \right)}
+ \frac{\kappa\tau(\tau-2)}{\left((1+\tau)^{2}-\kappa\right)^{2}}\left(\sigma^{2}+\frac{\tau^{2}}{(1+\tau)^{2}}\right) \right), \\
A_1(\lambda) &= - \frac{2\tau}{(1+\tau)^{3}}
+ \frac{2\kappa\tau^{2}}{(1+\tau)^{3}\left((1+\tau)^{2}-\kappa\right)}, \\
A_2(\lambda) &= - \frac{4\tau^{3}}{(1+\tau)^{5}} + \frac{2 \kappa \tau^{3} (\tau^{2}-1)}{(1+\tau)^{6}\left((1+\tau)^{2}-\kappa\right)}.
\end{aligned}
\end{equation}
\end{lemma}


\begin{proof}
Given a $p\times p$ matrix $M$, let us denote by $(M)_1$ its top-left $d\times d$ block. For any $M\in \mathbb R^{p\times p}$, we have 
\begin{equation*}
\begin{split}
        \mathbb E_{\thetapop}\left[\langle \thetapop, M\thetapop\rangle\right]&=    \mathbb E_{\thetapop}\left[ (\thetapop)^\top M\thetapop\right]=    \mathbb E_{\thetapop}\left[\tr\left[ (\thetapop)^\top M\thetapop\right]\right]\\
        &=    \mathbb E_{\thetapop}\left[\tr\left[M  \thetapop(\thetapop)^\top\right]\right]=    \tr\left[(M)_1\right]/d,
        \end{split}
\end{equation*}
where the third equality uses the circulant property of the trace and the last one that $(\thetapop)^{\top} = (a^\top, 0)$ with $a$ having covariance $I_d/d$. Thus, from \eqref{eq:defdet}, we have
\begin{equation}  \label{eq:defdetexp}  
\begin{aligned}
\mathbb E_{\thetapop}\fixedriskeq\left(\Sigma, \thetapop, D, \lambda\right)
&= 
\frac{\tau^{2}}{d}
\tr\left[
\left(
\Sigma
\left( \Sigma+\tau I_p \right)^{-2}
\right)_{1}
\right]
\\
&\quad
+ \kappa
\tr\left[\Sigma^{2}\left( \Sigma+\tau I_p \right)^{-2}\right]
\frac{
\sigma^{2}
+ 
\frac{\tau^{2}}{d}
\tr\left[
\left(\Sigma
\left( \Sigma+\tau I_p \right)^{-2}
\right)_{1}
\right]
}{
p - \kappa
\tr\left[\Sigma^{2}\left( \Sigma+\tau I_p \right)^{-2}\right]
}
\\
&\quad
- 
\frac{2\tau}{d}
\tr\left[
\left(
\left( \Sigma+\tau I_p \right)^{-2}
\Sigma^{2} D
\left( \Sigma+\tau I_p \right)^{-1}
\Sigma
\right)_{1}
\right]
\\
&\quad
+ \frac{2\kappa\tau^{2}}{d}
\tr\left[\Sigma^{2}\left( \Sigma+\tau I_p \right)^{-2}\right]
\frac{
\tr\left[
\left(
\left( \Sigma+\tau I_p \right)^{-2}
\Sigma D
\left( \Sigma+\tau I_p \right)^{-1}
\Sigma
\right)_{1}
\right]
}{
p - \kappa
\tr\left[\Sigma^{2}\left( \Sigma+\tau I_p \right)^{-2}\right]
}.
\end{aligned}
\end{equation}
Note that
\(
\Sigma +\tau I_p=
\begin{bmatrix}1+\tau&\rho\\ \rho&1+\tau\end{bmatrix}\otimes I_d
\)
has inverse
\(
(\Sigma+\tau I_p)^{-1}=
\frac{1}{(1+\tau)^2-\rho^2}
\begin{bmatrix}1+\tau&-\rho\\ -\rho&1+\tau\end{bmatrix}\otimes I_d.
\)
Furthermore,
\begin{equation*}
\begin{split}
    (\Sigma+\tau I_p)^{-2}
&=\frac{1}{\left((1+\tau)^2-\rho^2\right)^2}
\begin{bmatrix}(1+\tau)^2+\rho^2&-2\rho(1+\tau)\\ -2\rho(1+\tau)&(1+\tau)^2+\rho^2\end{bmatrix}\otimes I_d,\\
\Sigma^2&=
\begin{bmatrix}1+\rho^2&2\rho\\ 2\rho&1+\rho^2\end{bmatrix}\otimes I_d.
\end{split}    
\end{equation*}
A direct block multiplication gives
\begin{align*}
\tr\left[\left(\Sigma(\Sigma+\tau I_p)^{-2}\right)_1\right]
&= d\ \frac{(1+\tau)^2-\rho^2(1+2\tau)}{\left((1+\tau)^2-\rho^2\right)^2}, \\
\tr\left[\Sigma^{2}(\Sigma+\tau I_p)^{-2}\right]
&= 2d\ \frac{(1+\tau-\rho^2)^2+\rho^2\tau^2}{\left((1+\tau)^2-\rho^2\right)^2},\\
\tr\left[\left((\Sigma+\tau I_p)^{-2}\Sigma^{2}D(\Sigma+\tau I_p)^{-1}\Sigma\right)_1\right]
&=\frac{(1+\tau-\rho^2)\left((1+\tau-\rho^2)^2+\rho^2\tau^2\right)}{\left((1+\tau)^2-\rho^2\right)^3}\tr[\di(b)]\\
&\qquad
 +\frac{2(1+\tau-\rho^2)\rho^2\tau^2}{\left((1+\tau)^2-\rho^2\right)^3}\tr[\di(c)],\\
\tr\left[\left((\Sigma+\tau I_p)^{-2}\Sigma D(\Sigma+\tau I_p)^{-1}\Sigma\right)_1\right]
&=\frac{(1+\tau-\rho^2)\left((1+\tau)(1+\tau-\rho^2)-\rho^2\tau\right)}{\left((1+\tau)^2-\rho^2\right)^3}\tr[\di(b)]\\
&\qquad
+\frac{\rho^2\tau\left(\rho^2+\tau^2-1\right)}{\left((1+\tau)^2-\rho^2\right)^3}\tr[\di(c)].
\end{align*}
Expanding each rational function at $\rho=0$ using
\begin{equation*}
    \begin{split}        
\frac{1}{( (1+\tau)^2-\rho^2 )^2}
&=\frac{1}{(1+\tau)^4}\left(1+\frac{2\rho^2}{(1+\tau)^2}\right)+O(\rho^4), \\
 \frac{1}{( (1+\tau)^2-\rho^2 )^3}
&=\frac{1}{(1+\tau)^6}\left(1+\frac{3\rho^2}{(1+\tau)^2}\right)+O(\rho^4),
    \end{split}
\end{equation*}
yields, to order $\rho^2$,
\begin{align*}
\frac{1}{d}\tr\left[\left(\Sigma(\Sigma+\tau I_p)^{-2}\right)_1\right]
&= \left(\frac{1}{(1+\tau)^2}+\rho^2\frac{1-2\tau}{(1+\tau)^4}\right)+O(\rho^4),\\
\frac{1}{d}\tr\left[\Sigma^{2}(\Sigma+\tau I_p)^{-2}\right]
&= \frac{2}{(1+\tau)^2}+2\rho^2\frac{\tau^2-2\tau}{(1+\tau)^4}+O(\rho^4),\\
\frac{1}{d}\tr\left[\left((\Sigma+\tau I_p)^{-2}\Sigma^{2}D(\Sigma+\tau I_p)^{-1}\Sigma\right)_1\right]
&=\frac{\bar b}{(1+\tau)^{3}}
+ \rho^{2}\left(\frac{\tau^{2}-3\tau}{(1+\tau)^{5}}\bar b+\frac{2\tau^{2}}{(1+\tau)^{5}}\bar c\right)
+O(\rho^4),\\
\frac{1}{d}\tr\left[\left((\Sigma+\tau I_p)^{-2}\Sigma D(\Sigma+\tau I_p)^{-1}\Sigma\right)_1\right]
&=\frac{\bar b}{(1+\tau)^{3}}
+ \rho^{2}\left(\frac{1-3\tau}{(1+\tau)^{5}}\bar b+\frac{\tau(\tau^{2}-1)}{(1+\tau)^{6}}\bar c\right)
+O(\rho^4).
\end{align*}
Moreover, we have that
\[
\frac{\kappa\operatorname{tr}\left[\Sigma^{2}(\Sigma+\tau I_p)^{-2}\right]}
{p-\kappa\operatorname{tr}\left[\Sigma^{2}(\Sigma+\tau I_p)^{-2}\right]}
= \frac{\kappa}{(1+\tau)^{2}-\kappa}
+ \rho^{2}\frac{\kappa\tau(\tau-2)}{\left((1+\tau)^{2}-\kappa\right)^{2}}
+ O(\rho^{4}).
\]
Plugging these into \eqref{eq:defdetexp} gives the claimed result. 
\end{proof}

Let us further define
\begin{equation}
\tau^*(D, \rho):=\arg\min_{\tau\ge 0}\widetilde{\mathcal R}(D, \lambda, \rho),\qquad \tau_0^*(\rho):=\arg\min_{\tau\ge 0}\mathcal R_0(\lambda, \rho), \qquad  \tau_0:=\tau_0^*(0).
\end{equation}
Then, the following result proves an expression for $\tau^*(D, \rho)$, up to order $\rho^2$. %

\begin{lemma}\label{lemma:taustar}
In the setting of Theorem \ref{thm:equiv}, we have that
\begin{align*}
\tau^*(D, \rho)
&= \tau^{*}_{0}(\rho) + \bar b \left( B_3(\sigma, \kappa) + O(\rho^2)\right) + \bar c\left( \rho^2 C_3(\sigma, \kappa) + O(\rho^4) \right) + O(\bar b^2+\bar c^2),
\end{align*}
where
\begin{equation}\label{eq:tau0}
    \begin{aligned}
\tau_0 &= \frac{1 + \kappa + \kappa \sigma^2 + \sqrt{(1 + \kappa + \kappa \sigma^2)^2 - 4\kappa}}{2} - 1,\\
    \tau^{*}_{0}(\rho) &= \tau_{0} -\rho^{2}
\frac{\kappa\tau_{0}^{2}}
{(1+\tau_{0})\left((1+\tau_{0})^{2}-\kappa\right)} + O(\rho^4), \\
B_3(\sigma, \kappa) &= -
\frac{2(1+\tau_{0})^{4}-3(\kappa+1)(1+\tau_{0})^{3}
+4\kappa(1+\tau_{0})^{2}+\kappa(\kappa+1)(1+\tau_{0})-2\kappa^{2}}
{(1+\tau_{0})^{2}\left((1+\tau_{0})^{2}-\kappa\right)}, \\
C_3(\sigma, \kappa) &= - \frac{\tau_{0}^{2} \left(
 4\tau_{0}^{4}+(6-3\kappa)\tau_{0}^{3}-(6+3\kappa)\tau_{0}^{2}
+(\kappa^{2}+9\kappa-14)\tau_{0}-3\kappa^{2}+9\kappa-6 \right)}{(1+\tau_{0})^{4}\left((1+\tau_{0})^{2}-\kappa\right)}.
\end{aligned}
\end{equation}
\end{lemma}

\begin{proof}
A direct differentiation gives
\[
\begin{aligned}
\frac{\mathrm{d}}{\mathrm{d}\tau}\widetilde{\mathcal R}(D,\lambda,\rho)
&= \frac{2}{1+\tau}\left(
\frac{\tau}{(1+\tau)^{2}-\kappa}
-\frac{\kappa\left(\sigma^{2}(1+\tau)^{2}+\tau^{2}\right)}{\left((1+\tau)^{2}-\kappa\right)^{2}}
\right) \\
&\quad+\rho^2 \left( \frac{2\tau\left(\tau^{2}-4\tau+1\right)}{(1+\tau)^{5}} +\frac{2\kappa\tau\left((1+\tau)^{2}\left(3\tau^{2}-5\tau+1\right)-\kappa\left(\tau^{2}-4\tau+1\right)\right)}{(1+\tau)^{5}\left((1+\tau)^{2}-\kappa\right)^{2}} \right.\\ &\hspace{10em} -\frac{2\kappa^2\left(\kappa\sigma^{2}(\tau-1)(1+\tau)^{3}+\kappa\tau^{2}(\tau^{2}+\tau-3) \right)}{(1+\tau)^{3}\left((1+\tau)^{2}-\kappa\right)^{3}} \\& \left. \hspace{10em} -\frac{2\kappa \left(\sigma^{2}(1+\tau)^{4}(\tau^{2}-4\tau+1)+\tau^{2}(1+\tau)^{2}(\tau^{2}-5\tau+3)\right)}{(1+\tau)^{3}\left((1+\tau)^{2}-\kappa\right)^{3}}   \right)
\\
&\quad+\bar b \left(
\frac{4\tau-2}{(1+\tau)^{4}}+\frac{\kappa\tau(4-6\tau)}{(1+\tau)^{4}\left((1+\tau)^{2}-\kappa \right)}
-\frac{4\kappa^{2}\tau^{2}}{(1+\tau)^{4} \left((1+\tau)^{2}-\kappa\right)^{2}}
\right)\\
&\quad+\bar c \rho^2 \left( \frac{4\tau^{2}(2\tau-3)}{(1+\tau)^{6}} + \frac{2\kappa\tau^{2}(1+\tau)\left(\kappa(\tau^{2}-6\tau+3)-(1+\tau)^{2}(3\tau^{2}-8\tau+3)\right)}{(1+\tau)^{7}\left((1+\tau)^{2}-\kappa\right)^{2}}\right).
\end{aligned}
\]
With this explicit derivatives, the stationarity equation $\frac{\mathrm{d}}{\mathrm{d}\tau}\widetilde{\mathcal R}(D, \lambda, \rho)=0$ is equivalent to
\(
\frac{F\left(\tau,\bar b,\bar c,\rho^{2}\right)}{(1 + \tau)^7 ((1+\tau)^2 - \kappa)^3} =0,
\)
where
\[
F\left(\tau,\bar b,\bar c,\rho^{2}\right)
= F_{0}(\tau)
\rho^2 F_{\rho}(\tau) + \bar b F_{b}(\tau)
 + \bar c \rho^2 F_{\rho c}(\tau),
\]
\[
F_{0}(\tau)
= 2(1+\tau)^{7}\left(\kappa-(1+\tau)^{2}\right)
\left(\kappa\sigma^{2}\tau+\kappa\sigma^{2}+\kappa\tau-\tau^{2}-\tau\right),
\]
\[
F_{b}(\tau)
= 2(1+\tau)^{4}\left((1+\tau)^{2}-\kappa\right)
\left(
(\tau-1)\kappa^{2}
+(\tau+1)(2-2\tau-3\tau^{2})\kappa
+(1+\tau)^{3}(2\tau-1)
\right),
\]
\[
\begin{aligned}
F_{\rho}(\tau)
= -2(1+\tau)^{5}\left(
\right.&
\kappa^{2}\left(\tau(\tau^{2}+\tau-1)+\sigma^{2}(1+\tau)^{2}(\tau-1)\right)\\
&\quad
+\kappa(1+\tau)^{2}\left(\tau(\tau^{2}-6\tau+2)+\sigma^{2}(1+\tau)(\tau^{2}-4\tau+1)\right)\\
&\left.\quad
-\tau(1+\tau)^{3}(\tau^{2}-4\tau+1)
\right),
\end{aligned}
\]
\[
F_{\rho c}(\tau)
= 2\tau^{2}(1+\tau)^{2}\left((1+\tau)^{2}-\kappa\right)
\left(
(\tau-3)\kappa^{2}
-3(\tau+1)(\tau^{2}-3)\kappa
+2(\tau+1)^{3}(2\tau-3)
\right).
\]


Setting $\bar b=\bar c=\rho^{2}=0$ yields
\[
(1+\tau)^{2}-(1+\kappa+\kappa\sigma^{2})(1+\tau)+\kappa=0,
\]
and the desired minimum corresponds to its largest solution, which is given by $\tau_{0}$ as expressed in the statement. It is easy to see that 
\[
\begin{aligned}
\partial_{\tau}F\left(\tau_{0},0,0,0\right))
&= -2(1+\tau_0)^7\left((1+\tau_0)^2-\kappa\right)\left(2(1+\tau_0)-(1+\kappa+\kappa\sigma^2)\right) \\&= -2(1+\tau_0)^7\left((1+\tau_0)^2-\kappa\right)\sqrt{(1+\kappa+\kappa\sigma^2)^2-4\kappa}\neq 0.
\end{aligned}
\]
Therefore, the implicit function theorem gives a smooth map
\(
\tau^{*}(\bar b,\bar c,\rho^{2})
\)
with $\tau^{*}(0,0,0)=\tau_{0}$ and $F(\tau^{*},\cdot)=0$.
Differentiating $F=0$ at $(\tau_{0},0,0,0)$ in each small parameter and dividing by $\partial_{\tau}F(\tau_{0},0,0,0)$ yields the linear expansion for $\tau^{*}-\tau_{0}$. The coefficient for $\rho^2$ in $\tau_0^*(\rho)$ is given by
\[
 \partial_{\rho^2}\tau^*(0,0,0) = - \frac{\partial_{\rho^2}F}{\partial_{\tau}F} \Bigg|_{(\tau_0,0,0,0)} = - \frac{F_{\rho}(\tau_0)}{\partial_{\tau}F_0(\tau_0,0,0,0)}.
\]
The $\bar b$ coefficient, $B_3$, is
\[
 B_3(\sigma, \kappa) = \partial_{\bar b}\tau^*(0,0,0) = - \frac{\partial_{\bar b}F}{\partial_{\tau}F} \Bigg|_{(\tau_0,0,0,0)} = - \frac{F_b(\tau_0)}{\partial_{\tau}F_0(\tau_0,0,0,0)}.
\]
The $\bar c \rho^2$ coefficient, $C_3$, is found from the mixed partial derivative:
\[
 C_3(\sigma, \kappa) = \partial_{\rho^2}\partial_{\bar c}\tau^*(0,0,0) = - \frac{\partial_{\rho^2}\partial_{\bar c}F}{\partial_{\tau}F} \Bigg|_{(\tau_0,0,0,0)} = - \frac{F_{\rho c}(\tau_0)}{\partial_{\tau}F_0(\tau_0,0,0,0)}.
\]
Substituting the expressions for $\partial_{\tau}F_0(\tau_0)$, $F_{\rho}(\tau_0)$, $F_b(\tau_0)$, and $F_{\rho c}(\tau_0)$ and cancelling common factors gives the coefficients as stated in \eqref{eq:tau0}.
\end{proof}

As $\lambda$ and $\tau$ are linked by the fixed point equation \eqref{eq:tau}, an application of Lemma \ref{lemma:taustar} readily gives that 
\begin{equation}
\begin{split}    
\lambdaeqs(D, \rho)&= \lambdaeqsz(\rho)+\bar b (B_1(\sigma, \kappa)+O(\rho^2)) +\bar c \rho^2( C_1(\sigma, \kappa)+O(\rho^2))+O(\bar b^2+\bar c^2),
\end{split}
\end{equation}
where
\begin{equation}\label{eq:lfor}
\begin{aligned}
    \lambdaeqsz(\rho) &= \tau_{0}\left(\kappa^{-1}-\frac{1}{1+\tau_{0}}\right)  + \rho^{2}\left(\tau_{0}\left(\frac{1}{(1+\tau_{0})^{2}}-\frac{1}{(1+\tau_{0})^{3}}\right)\right.\\
&\hspace{10em}\left.-\left(\kappa^{-1}-\frac{1}{(1+\tau_{0})^{2}}\right)
\frac{\kappa\tau_{0}^{2}}{(1+\tau_{0})\left((1+\tau_{0})^{2}-\kappa\right)}
\right) + O(\rho^4), \\
 B_1(\sigma, \kappa) &= -
\frac{2(1+\tau_{0})^{4}-3(\kappa+1)(1+\tau_{0})^{3}
+4\kappa(1+\tau_{0})^{2}+\kappa(\kappa+1)(1+\tau_{0})-2\kappa^{2}}
{\kappa(1+\tau_{0})^{4}},\\
C_1(\sigma, \kappa) &= - 
\frac{\tau_{0}^{2} \left(
4\tau_{0}^{4}+(6-3\kappa)\tau_{0}^{3}-(6+3\kappa)\tau_{0}^{2}
+(\kappa^{2}+9\kappa-14)\tau_{0}-3\kappa^{2}+9\kappa-6 \right)}{\kappa(1+\tau_{0})^{6}}.
\end{aligned}
\end{equation}
This proves \eqref{eq:thmequivl}. %
Next, the corollary below proves %
\eqref{eq:thmequivR}.

\begin{corollary}
Consider the setting of Theorem \ref{thm:equiv} and let $\tau_0$ be given by \eqref{eq:tau0}. Then, we have that %
\begin{equation}
    \fixedriskeqs(D, \rho)= \fixedriskeqs(\rho)+\bar b (B_2(\sigma, \kappa)+O(\rho^2))+\bar c \rho^2( C_2(\sigma, \kappa)+O(\rho^2))+O(\bar b^2+\bar c^2),
\end{equation}
where
\begin{equation}\label{eq:Rfor}    
\begin{aligned}
    \fixedriskeqs(\rho) &= \frac{\tau_{0}^{2}}{(1+\tau_{0})^{2}}
+\frac{\kappa}{(1+\tau_{0})^{2}-\kappa}\left(\sigma^{2}+\frac{\tau_{0}^{2}}{(1+\tau_{0})^{2}}\right) \\&\quad +\rho^{2}\left(
\frac{\tau_{0}^{2}(1-2\tau_{0})}{(1+\tau_{0})^{4}}
+ \frac{\kappa \tau_{0}^{2}(1-2\tau_{0})}{(1+\tau_{0})^{4} \left( (1+\tau_{0})^{2}-\kappa \right)}
\right.\\
&\hspace{10em}\left.+ \frac{\kappa\tau_{0}(\tau_{0}-2)}{\left((1+\tau_{0})^{2}-\kappa\right)^{2}}\left(\sigma^{2}+\frac{\tau_{0}^{2}}{(1+\tau_{0})^{2}}\right) \right) + O(\rho^4), \\
B_2(\sigma, \kappa) &= -\frac{2\tau_{0}}{(1+\tau_{0})^{3}}
+\frac{2\kappa\tau_{0}^{2}}{(1+\tau_{0})^{3}\left((1+\tau_{0})^{2}-\kappa\right)}, \\
C_2(\sigma, \kappa) &= - \frac{4\tau_{0}^{3}}{(1+\tau_{0})^{5}}
+ \frac{2 \kappa \tau_{0}^{3} (\tau_{0}^{2}-1)}{(1+\tau_{0})^{6} \left((1+\tau_{0})^{2}-\kappa \right)}.
\end{aligned}
\end{equation}
\end{corollary}

\begin{proof}
Let us re-define $\widetilde{\mathcal R}(D, \lambda, \rho)$ given in  \eqref{eq:explexpr} as $\widetilde{R}(\tau,\bar b,\bar c,\rho^{2})$ to emphasize its dependence on $\tau,\bar b,\bar c$. %
By definition of $\tau_{0}$, we have $\partial_{\tau}\widetilde{R}(\tau_{0},0,0,0)=0$. Furthermore, from Lemma \ref{lemma:taustar}, we have
\[
\tau^{*}(D, \rho)=\tau_{0}
+O(\bar b + (1 + \bar c) \rho^2 ).
\]
A first–order Taylor expansion of $\widetilde{R}(\tau^{*}(D, \rho), \bar b, \bar c,\rho^{2})$ around $(\tau;\bar b,\bar c,\rho^{2})=(\tau_{0};0,0,0)$ gives
\[
\begin{aligned}
\widetilde{R}(\tau^{*}(D, \rho), \bar b, \bar c,\rho^{2})
&=
\widetilde{R}(\tau_{0},\bar b, \bar c,\rho^{2})
+\partial_{\tau}\widetilde{R}(\tau_{0},0,0,0)(\tau^{*}(D, \rho)-\tau_{0})
\\
&\quad
+~O\left((\tau^{*}(D, \rho)-\tau_{0})\bar b\right)
+O\left((\tau^{*}(D, \rho)-\tau_{0})\rho^{2}\right)
+O(\bar b^2+\bar c^2+\rho^4).
\end{aligned}
\]
As $\partial_{\tau}\widetilde{R}(\tau_{0},0,0,0)=0$, we conclude that %
\[
\widetilde{R}(\tau^{*},\bar b, \bar c, \rho^{2})
=
\widetilde{R}(\tau_{0}, \bar b, \bar c, \rho^2)
+O(\bar b^2+\bar c^2+\rho^4),
\]
and substituting $\tau=\tau_{0}$ in \eqref{eq:explexpr} gives the claimed expansion.
\end{proof}

We now move to the proof of \eqref{eq:relations1}, which follows from the lemma below.

\begin{lemma}\label{lem:transition-kappa-3}
Let $B_1(\sigma, \kappa)$ be given by \eqref{eq:lfor}. Then, for any $\kappa>1$, 
$B_1(\kappa,\cdot)$ has exactly one zero $\sigma_{B_1}(\kappa)>0$, with
\[
B_1(\kappa,\sigma)\ge 0 \ \text{for } 0\le \sigma\le \sigma_{B_1}(\kappa),
\qquad
B_1(\kappa,\sigma)\le 0 \ \text{for } \sigma\ge \sigma_{B_1}(\kappa).
\]
Moreover, as $\kappa\to\infty$,
\[
\sigma_{B_1}^{2}(\kappa)
=\frac{1}{2}
-\frac{7}{18}\kappa^{-1}
+O\left(\kappa^{-2}\right).
\]
\end{lemma}

\begin{proof}
Let us define the shorthands
\begin{equation}
    s(\sigma):=1+\tau_0,\qquad N_{B_1}(s,\kappa):=2s^{4}-3(\kappa+1)s^{3}+4\kappa s^{2}+\kappa(\kappa+1)s-2\kappa^{2},
\end{equation}
with $\tau_0$ given by \eqref{eq:tau0}. Note that 
\[
s(\sigma) = \frac{1 + \kappa + \kappa \sigma^2 + \sqrt{(1 + \kappa + \kappa \sigma^2)^2-4\kappa}}{2} \ge \frac{1 + \kappa + \sqrt{(1 + \kappa)^2-4\kappa}}{2} = \kappa.
\]
Now let us also define
\(
\Phi(s):=-N_{B_1}(s,\kappa)/\left(\kappa s^{4}\right)
\)
for $s\ge \kappa$.
A direct calculation gives the factorization
\[
\frac{\mathrm d}{\mathrm ds}\Phi(s)
=\frac{-N_{B_1}'(s)s+4N_{B_1}(s)}{\kappa s^{5}}
=\frac{-(s^{2}-\kappa)\left(3(\kappa+1)s-8\kappa\right)}{\kappa s^{5}}.
\]
For $s\ge \kappa$ we have $s^{2}-\kappa>0$, hence $\Phi'(s)$ changes sign only once at
\(
s_{*}:=\frac{8\kappa}{3(\kappa+1)}
\).
If $\kappa\ge 5/3$, then $s_{*}\le \kappa$ and $\Phi$ is strictly decreasing on $[\kappa,\infty)$.
If $1<\kappa<5/3$, then $\kappa<s_{*}$ and $\Phi$ is increasing on $[\kappa,s_{*})$ and strictly decreasing on $(s_{*},\infty)$.

Note that $B_1(\kappa,\sigma)=-N_{B_1}\left(s(\sigma),\kappa\right)/(\kappa s(\sigma)^{4})=\Phi\left(s(\sigma)\right),
\)
$s(\sigma)$ is strictly increasing in $\sigma$, and
\(
\Phi\left(s(\sigma)\right)\to -2/\kappa
\)
as $\sigma\to\infty$ (since $s(\sigma)\to \kappa\sigma^{2}$ and $N_{B_1}(s,\kappa)\to 2s^{4}$).
Furthermore, $s(0)=\kappa$ and
\[
N_{B_1}(\kappa,\kappa)=-\kappa^{2}(\kappa-1)^{2}<0
\implies
B_1(\kappa,0)=-\frac{N_{B_1}(\kappa,\kappa)}{\kappa^{5}}>0 .
\]
Therefore, $B_1(\kappa,\sigma)$ is strictly decreasing on $[0,\infty)$ if $\kappa\ge 5/3$, and for $1<\kappa<5/3$ it increases for small $\sigma$ and then strictly decreases; in either case, since $B_1(\kappa,0)>0$ and $\lim_{\sigma\to\infty}B_1(\kappa,\sigma)=-2/\kappa<0$, it crosses $0$ exactly once, which proves the existence and uniqueness of $\sigma_{B_1}(\kappa)$.
At the crossing $B_1(\kappa,\sigma_{B_1})=0$, hence $N_{B_1}\left(s(\sigma_{B_1}),\kappa\right)=0$.

Now
let $\varepsilon:=\kappa^{-1}$ and write $s=\kappa c$.
Dividing $N_{B_1}(\kappa c,\kappa)=0$ by $\kappa^{4}$ yields the analytic equation
\[
F(\varepsilon,c)=0,
\qquad
F(\varepsilon,c):=2c^{4}-3(1+\varepsilon)c^{3}+4\varepsilon c^{2}+(\varepsilon+\varepsilon^{2})c-2\varepsilon^{2}.
\]
At $\varepsilon=0$,
\(
F(0,c)=2c^{4}-3c^{3}
\)
has the positive root
\(
c_{0}=\tfrac32,
\)
and
\(
\partial_{c}F(0,c_{0})
=8c_{0}^{3}-9c_{0}^{2}
=\tfrac{27}{4}\neq 0.
\)
By the implicit function theorem there exists a unique analytic branch $c(\varepsilon)$ with $c(0)=\tfrac32$, having the expansion
\(
c(\varepsilon)=\frac{3}{2}+c_{1}\varepsilon+O(\varepsilon^{2}).
\)
Substituting into $F(\varepsilon,c)=0$ gives that, up to first order,
\[
\frac{27}{4}c_{1}+\frac{3}{8}=0
\implies
c_{1}=-\frac{1}{18}.
\]
Thus,
\[
\sigma_{B_1}^{2}
=\frac{(s_{c}-1)(s_{c}-\kappa)}{\kappa s_{c}}
=\left(1-\frac{1}{\kappa c(\varepsilon)}\right)\left(c(\varepsilon)-1\right),
\]
with
\(
s_{c}=\kappa c(\varepsilon)
=\frac{3}{2}\kappa-\frac{1}{18}+O(\kappa^{-1}).
\)
Substituting $c(\varepsilon)=\tfrac32-\tfrac{1}{18}\varepsilon+O(\varepsilon^{2})$ and expanding yields
\[
\sigma_{B_1}^{2}
=\frac{1}{2}
-\frac{7}{18}\kappa^{-1}
+O(\kappa^{-2}).
\]
The analyticity of $c(\varepsilon)$ implies the remainder $O(\varepsilon^{2})$ in $c$ and, consequently, the remainder $O(\kappa^{-2})$ in the displayed expansion.
\end{proof}

Next, we move to the proof of \eqref{eq:relations2}, which follows from the lemma below. 

\begin{lemma}\label{lemma:rel2}
Let $C_1(\sigma, \kappa)$ be given by \eqref{eq:lfor}.
Then, for every $\kappa\ge2$ and all $\sigma\ge0$,
\(
C_1(\kappa,\sigma)\le0.
\)
\end{lemma}

\begin{proof}
Let $s(\sigma) =1+\tau_{0}$, with $\tau_0$ given by \eqref{eq:tau0}, and note that \(\tau_0^2 / (\kappa s(\sigma)^6) > 0\).
Thus, the sign of $C_1$ is the opposite of the sign of $N_{C_1}(s(\sigma)-1,\kappa)$, where
\[
N_{C_1}(t,\kappa)=4t^{4}+(6-3\kappa)t^{3}-(6+3\kappa)t^{2}
+(\kappa^{2}+9\kappa-14)t-3\kappa^{2}+9\kappa-6.
\]
At $\sigma=0$, one has $s(0)-1=\kappa-1$, and a direct substitution yields
\[
N_{C_1}(\kappa-1,\kappa):=\kappa^{4}-3\kappa^{3}+2\kappa^{2}
=\kappa^{2}(\kappa-1)(\kappa-2).
\]
Moreover, differentiating in $t$ gives
\[
N_{C_1}''(t,\kappa)=6(-3\kappa t-\kappa+8t^{2}+6t-2)>0,
\]
for all $t\ge \kappa-1$ and $\kappa\ge2$, so $N_{C_1}'(\cdot,\kappa)$ is increasing on $[\kappa-1,\infty)$. As
\[
N_{C_1}'(\kappa-1,\kappa)=\kappa(\kappa-2)(7\kappa-3)\ge0,
\]
for $\kappa\ge2$, it follows that $N_{C_1}(\cdot,\kappa)$ is increasing on $[\kappa-1,\infty)$. Therefore, for every $\sigma\ge0$,
\[
N_{C_1}(s(\sigma)-1,\kappa)\ \ge\ N_{C_1}(\kappa-1,\kappa)=\kappa^{2}(\kappa-1)(\kappa-2)\ge0
\quad\text{for }\kappa\ge2.
\]
Since the prefactor is positive, $C_1(\kappa,\sigma)\le0$ for all $\sigma$ as soon as $\kappa\ge2$.
\end{proof}


\begin{lemma}\label{lemma:rel3}
Let $B_2(\sigma, \kappa)$ be given by \eqref{eq:Rfor}.
Then, for every $\kappa > 1$ and all $\sigma\ge0$,
\(
B_2(\kappa,\sigma)\le0.
\)
\end{lemma}

\begin{proof}
We have
\[
\begin{aligned}
B_2(\sigma,\kappa)
&=-\frac{2\tau_0}{(1+\tau_0)^3}
+\frac{2\kappa\tau_0^2}{(1+\tau_0)^3\bigl((1+\tau_0)^2-\kappa\bigr)}\\
&=\frac{2\tau_0}{(1+\tau_0)^3}
\left[
-1+\frac{\kappa\tau_0}{(1+\tau_0)^2-\kappa}
\right]\\
&=\frac{2\tau_0}{(1+\tau_0)^3}
\frac{(1+\tau_0)\bigl(\kappa-(1+\tau_0)\bigr)}{(1+\tau_0)^2-\kappa}\\
&=\frac{2\tau_0}{(1+\tau_0)^2}
\frac{\kappa-(1+\tau_0)}{(1+\tau_0)^2-\kappa}.
\end{aligned}
\]

Since \((1 + \tau_0)^2 > (1 + \tau_0) \ge \kappa\), we have $B_2(\sigma, \kappa)\le 0$.
\end{proof}


\begin{lemma}\label{lemma:rel4}
Let $C_2(\sigma, \kappa)$ be given by \eqref{eq:Rfor}.
Then, for every $\kappa > 1$ and all $\sigma\ge0$,
\(
C_2(\kappa,\sigma)\le0.
\)
\end{lemma}

\begin{proof}
Let us again write $s(\sigma)=1+\tau_{0}>0$ and note $\tau_{0}^{2}-1=(s(\sigma)-1)^{2}-1=s(\sigma)^{2}-2s(\sigma)$. Then, substituting and simplifying, we have
\[
C_2(\kappa,\sigma)
=-\frac{4\tau_{0}^{3}}{s(\sigma)^{5}}
+\frac{2\tau_{0}^{3}}{s(\sigma)^{6}}
\frac{\kappa(s(\sigma)^{2}-2s(\sigma))}{s(\sigma)^{2}-\kappa}
=\frac{2\tau_{0}^{3}}{s(\sigma)^{6}}\left(
\frac{\kappa s(\sigma)^{2}-2s(\sigma)^{3}}{s(\sigma)^{2}-\kappa}\right)
=\frac{2\tau_{0}^{3}}{s(\sigma)^{4}}
\frac{\kappa-2s(\sigma)}{s(\sigma)^{2}-\kappa}.
\]
Note that \(s(\sigma) > \kappa\), and therefore $s(\sigma)^{2}-\kappa>0$ and $\kappa-2s(\sigma)\le \kappa-2\kappa=-\kappa<0$. Because $\tau_{0}>0$ for $\kappa>1$, the prefactor $2\tau_{0}^{3}/s(\sigma)^{4}>0$. Therefore $C_2(\kappa,\sigma)\le0$.
\end{proof}

 Combining the results from Lemmas~\ref{lem:transition-kappa-3},~\ref{lemma:rel2}, ~\ref{lemma:rel3} and ~\ref{lemma:rel4}  concludes the proof of Theorem \ref{thm:equiv}.


\section{\revised{
Test risk evaluated on $\mathcal D(\theta)$ in the over-parameterized setting}}\label{app:test-bis}
\revised{Let $\overline{\mathcal R}_k(\Sigma, \theta_k, \thetapop)$ be the excess risk of the estimator $\theta_k$ given by \eqref{eq:thetak} evaluated on $\mathcal D(\theta)$, i.e.,
\[
    \overline{\mathcal R}_k(\Sigma, \theta_k, \thetapop) = \left\|\theta_k - (\thetapop+D\theta_{k-1}) \right\|_\Sigma^2. 
\]
}
\revised{\begin{theorem}[Excess risk on $\mathcal D(\theta)$-- over-parameterized]\label{thm:over2}
Let $R>0$ be a constant s.t.\ $\thetapop, \theta_0\in B_p(R)$. Assume that $\kappa, \sigma, \lambda\in (1/M, M)$ and $\|\Sigma\|_{\mathrm{op}},\ \|\Sigma^{-1}\|_{\mathrm{op}} \le M$ for some constant $M>1$. Then, there exists a constant $C=C\left(M, R\right)$ such that for any $\delta \in (0,1/2]$, with probability at least $1-Cpe^{-p\delta^{4}/C}$, %
\begin{equation}    
\left|\overline{\mathcal{R}}_2(\Sigma, \theta_{2}, \thetapop)-\ofixedriskeq\left(\Sigma, \thetapop, D, \lambda\right)\right|\le \delta+O(\|D\|_{\mathrm{op}}^2),
\end{equation}
where
\begin{equation}   \label{eq:defdet2} 
\begin{aligned}
&\ofixedriskeq\left(\Sigma, \thetapop, D, \lambda\right)
\\
&
\hspace{-.2em}=\hspace{-.2em}\frac{
\sigma^2 \kappa
\tr\left[\Sigma^{2}\left( \Sigma+\tau I_p \right)^{-2}\right]\hspace{-.2em}+\hspace{-.2em} p
\tau^{2}
\langle \thetapop,\hspace{-.2em}\left( \Sigma+\tau I_p \right)^{-1}\hspace{-.2em}
\left(\hspace{-.1em}I_p\hspace{-.1em}+\hspace{-.1em}2\left( \Sigma+\tau I_p \right)^{-1}
\Sigma D\hspace{-.1em}\right)\hspace{-.1em}\Sigma\left( \Sigma+\tau I_p \right)^{-1}\hspace{-.1em}\thetapop\rangle
}{
p - \kappa
\tr\left[\Sigma^{2}\left( \Sigma+\tau I_p \right)^{-2}\right]
},
\end{aligned}
\end{equation}
and $\tau$ is the unique solution of \eqref{eq:tau}.
\end{theorem}}

\revised{\begin{proof}
The argument is analogous to that used to prove Theorem \ref{thm:over}, and we only report differences. Using the same approach as Lemma \ref{lemma:1step}, we have
\begin{equation}\label{eq:det1-bis}  
\sup_{\thetapop, \theta_0\in B_p(R)}
\Pr\left(\left|\overline{\mathcal R}_{1}(\Sigma, \theta_{1}, \thetapop)-\ofixedriskeq^{(1)}\left(\Sigma, \theta_0, \thetapop\right)\right|\ge \delta\right)
\le Cpe^{-p\delta^{4}/C},
\end{equation}
where
\begin{equation} \label{eq:R1eq-bis} 
\begin{aligned}
\ofixedriskeq^{(1)}\left(\Sigma, \theta_0, \thetapop\right)
&= %
\frac{   \sigma^2\kappa \tr\left[ \Sigma^2 \left( \Sigma + \tau I_p \right)^{-2} \right] + p\tau^2 \left\| \left( \Sigma + \tau I_p \right)^{-1} (\thetapop+D\theta_0) \right\|_\Sigma^2  }{ p - \kappa \tr\left[ \Sigma^2 \left( \Sigma + \tau I_p \right)^{-2} \right] }.
\end{aligned}
\end{equation}
Next, using the same approach\footnote{In fact, the derivation is simpler since the term corresponding to $\fixedriskeqa^{(1)}\left(\Sigma, \theta_{1}, \thetapop\right)$ here is absent.} as Lemma \ref{lemma:2step}, we have
\begin{equation}\label{eq:det2-bis} 
\sup_{\thetapop, \theta_0\in B_p(R)}
\Pr\left(\left|\overline{\mathcal R}_{2}(\Sigma, \theta_{2}, \thetapop)-\ofixedriskeq^{(2)}\left(\Sigma, \theta_0, \thetapop\right)\right|\ge \delta\right)
\le Cpe^{-p\delta^{4}/C},
\end{equation}
where
\begin{equation}  \label{eq:fixedriskeq2-bis}  
\begin{aligned}
&\ofixedriskeq^{(2)}\left(\Sigma, \theta_0, \thetapop\right)
\\
&=  
\frac{ \sigma^2\kappa \tr\left[ \Sigma^2 \left( \Sigma + \tau I_p \right)^{-2} \right] + p\tau^2 \big\| \left( \Sigma + \tau I_p \right)^{-1} \left(\thetapop + D \left( \Sigma + \tau I_p \right)^{-1} \Sigma (\thetapop+D\theta_0) \right) \big\|_\Sigma^2 }{p - \kappa \tr\left[ \Sigma^2 \left( \Sigma + \tau I_p \right)^{-2} \right] } \\
&+ p\kappa \tau^2   
\tr\left[ \Sigma \left( \Sigma + \tau I_p \right)^{-2} D  \Sigma \left( \Sigma + \tau I_p \right)^{-2} D \right]
\cdot\frac{ \sigma^2 + \tau^2 \big\| \left( \Sigma + \tau I_p \right)^{-1} (\thetapop+D\theta_0) \big\|_\Sigma^2 }
{\big( p - \kappa \tr\left[ \Sigma^2 \left( \Sigma + \tau I_p \right)^{-2} \right] \big)^2 }.
\end{aligned}
\end{equation}
Noting that the quantity in the second line is $O(\|D\|_{\mathrm{op}}^2)$ and that
\begin{equation}
\begin{split}    
    \big\| &\left( \Sigma + \tau I_p \right)^{-1} \left(\thetapop + D \left( \Sigma + \tau I_p \right)^{-1} \Sigma (\thetapop+D\theta_0) \right) \big\|_\Sigma^2\\
    &=\langle \thetapop,\hspace{-.2em}\left( \Sigma+\tau I_p \right)^{-1}\hspace{-.2em}
\left(\hspace{-.1em}I_p\hspace{-.1em}+\hspace{-.1em}2\left( \Sigma+\tau I_p \right)^{-1}
\Sigma D\hspace{-.1em}\right)\hspace{-.1em}\Sigma\left( \Sigma+\tau I_p \right)^{-1}\hspace{-.1em}\thetapop\rangle
+O(\|D\|_{\mathrm{op}}^2)
\end{split}
\end{equation}
concludes the argument. 
\end{proof}}

\revised{\begin{lemma}\label{lemma:explicit-bis}
Consider the setting of Theorem \ref{thm:over2}, assume that $a$ has covariance $I_d/d$, and let $\Sigma=\begin{bmatrix}I_d&\rho I_d\\ \rho I_d&I_d\end{bmatrix}$. Then, we have that
\begin{equation*}
\begin{split}
\mathbb E_{\thetapop}\ofixedriskeq\left(\Sigma, \thetapop, D, \lambda\right)&=\overline{\mathcal R}(D, \lambda, \rho)+ O(\bar b\rho^2+\rho^4),\\
\overline{\mathcal R}(D, \lambda, \rho)&:=\mathcal R_0(\lambda, \rho) + \bar b \overline{A}_1(\lambda) + \bar c \rho^2 \overline{A}_2(\lambda),
\end{split}
\end{equation*}
where $\bar b=\tr[\di(b)]/d, \bar c=\tr[\di(c)]/d$, the auxiliary function $\mathcal R_0(\lambda, \rho)$ is given by \eqref{eq:explexpr}, and the new auxiliary functions $\overline{A}_1(\lambda)$ and $\overline{A}_2(\lambda)$ are given by
\begin{equation} \label{eq:explexpr-bis}
\begin{aligned}
\overline{A}_1(\lambda) &= \frac{2\tau^2}{(1+\tau)((1+\tau)^{2}-\kappa)}, \\
\overline{A}_2(\lambda) &= \frac{2\tau^3(\tau^{2}-1)}{(1+\tau)^{4}((1+\tau)^{2}-\kappa)}.
\end{aligned}
\end{equation}
\end{lemma}}

\revised{\begin{proof}
Using $\mathbb E_{\thetapop}\left[\langle \thetapop, M\thetapop\rangle\right]= \tr\left[(M)_1\right]/d$ from the proof of Lemma \ref{lemma:explicit}, we take the expectation of \eqref{eq:defdet2}:
\begin{equation} \label{eq:defdetexp-bis}
\begin{aligned}
&\mathbb E_{\thetapop}\ofixedriskeq\left(\Sigma, \thetapop, D, \lambda\right)
\\
&
=
\frac{
\sigma^2 \kappa
\tr\left[\Sigma^{2}\left( \Sigma+\tau I_p \right)^{-2}\right]
}{
p - \kappa
\tr\left[\Sigma^{2}\left( \Sigma+\tau I_p \right)^{-2}\right]
}
\\
&\quad
+
\frac{p \tau^2}{p - \kappa
\tr\left[\Sigma^{2}\left( \Sigma+\tau I_p \right)^{-2}\right]}
\frac{1}{d}
\tr\left[
\left(
\Sigma
\left( \Sigma+\tau I_p \right)^{-2}
\right)_{1}
\right]
\\
&\quad
+
\frac{p \tau^2}{p - \kappa
\tr\left[\Sigma^{2}\left( \Sigma+\tau I_p \right)^{-2}\right]}
\frac{2}{d}
\tr\left[
\left(
\left( \Sigma+\tau I_p \right)^{-2}
\Sigma D
\left( \Sigma+\tau I_p \right)^{-1}
\Sigma
\right)_{1}
\right].
\end{aligned}
\end{equation}
The first two terms correspond to the risk with $D=0$ (i.e., $\bar b = \bar c = 0$). By the same computations as in the proof of Lemma \ref{lemma:explicit}, these terms combine to $\mathcal R_0(\lambda, \rho) + O(\rho^4)$.
The third term, which depends on $D$, requires approximations for its two factors. The first factor is new:
\begin{equation*}
\begin{split}
\frac{p}{p - \kappa \tr\left[\Sigma^{2}(\Sigma+\tau I_p)^{-2}\right]}
&= 1 + \frac{\kappa\operatorname{tr}\left[\Sigma^{2}(\Sigma+\tau I_p)^{-2}\right]}{p-\kappa\operatorname{tr}\left[\Sigma^{2}(\Sigma+\tau I_p)^{-2}\right]} \\
&= 1 + \left( \frac{\kappa}{(1+\tau)^{2}-\kappa} + O(\rho^{2}) \right)
= \frac{(1+\tau)^2}{(1+\tau)^{2}-\kappa} + O(\rho^{2}).
\end{split}
\end{equation*}
For the second factor, we use the trace expansion from Lemma \ref{lemma:explicit}:
\begin{equation*}
\frac{1}{d}\tr\left[\left((\Sigma+\tau I_p)^{-2}\Sigma D(\Sigma+\tau I_p)^{-1}\Sigma\right)_1\right]
=\frac{\bar b}{(1+\tau)^{3}}
+ \rho^{2}\left(\frac{1-3\tau}{(1+\tau)^{5}}\bar b+\frac{\tau(\tau^{2}-1)}{(1+\tau)^{6}}\bar c\right)
+O(\rho^4).
\end{equation*}
We multiply these two factors by $2\tau^2$ (from \eqref{eq:defdetexp-bis}) and keep only the terms of order $O(\bar b)$ and $O(\bar c \rho^2)$:
\begin{align*}
\bar b \overline{A}_1(\lambda) &= \left( \frac{(1+\tau)^2}{(1+\tau)^{2}-\kappa} \right) \left( 2\tau^2 \frac{\bar b}{(1+\tau)^{3}} \right) = \bar b \frac{2\tau^2}{(1+\tau)((1+\tau)^{2}-\kappa)}, \\
\bar c \rho^2 \overline{A}_2(\lambda) &= \left( \frac{(1+\tau)^2}{(1+\tau)^{2}-\kappa} \right) \left( 2\tau^2 \rho^2 \bar c \frac{\tau(\tau^2-1)}{(1+\tau)^6} \right) = \bar c \rho^2 \frac{2\tau^3(\tau^{2}-1)}{(1+\tau)^{4}((1+\tau)^{2}-\kappa)}.
\end{align*}
Adding these terms to $\mathcal R_0(\lambda, \rho)$ yields the claimed expansion.
\end{proof}}

\revised{\begin{lemma}\label{lemma:taustar-bis}
In the setting of Lemma \ref{lemma:explicit-bis}, we have that
\begin{align*}
\tau^*(D, \rho)
&= \tau^{*}_{0}(\rho) + \bar b \left( \overline{B}_3(\sigma, \kappa) + O(\rho^2)\right) + \bar c\left( \rho^2 \overline{C}_3(\sigma, \kappa) + O(\rho^4) \right) + O(\bar b^2+\bar c^2),
\end{align*}
where $\tau_{0}$ and $\tau^{*}_{0}(\rho)$ are given by \eqref{eq:tau0}, and
\begin{equation}\label{eq:tau-coeffs-bis}
\begin{aligned}
\overline{B}_3(\sigma, \kappa) &= - 
\frac{\tau_0 \left( (1+\tau_0)^2 (2-\tau_0) - \kappa (\tau_0+2) \right)}{(1+\tau_0) \left( (1+\tau_0)^2 - \kappa \right)}, \\
\overline{C}_3(\sigma, \kappa) &= - 
\frac{\tau_0^2 \left( (4\tau_0-3)(1+\tau_0)((1+\tau_0)^2-\kappa) - \tau_0(\tau_0-1)(5(1+\tau_0)^2-3\kappa) \right)}{(1+\tau_0)^3 \left( (1+\tau_0)^2 - \kappa \right)}.
\end{aligned}
\end{equation}
\end{lemma}}

\revised{\begin{proof}
A direct differentiation of $\overline{\mathcal R}(D,\lambda,\rho)$ from Lemma \ref{lemma:explicit-bis} gives
\[
\begin{aligned}
\frac{\mathrm{d}}{\mathrm{d}\tau}\overline{\mathcal R}(D,\lambda,\rho)
&= \frac{\mathrm{d}}{\mathrm{d}\tau}\mathcal R_0(\lambda, \rho)
+\bar b \frac{\mathrm{d}}{\mathrm{d}\tau}\overline{A}_1(\lambda)
+\bar c \rho^2 \frac{\mathrm{d}}{\mathrm{d}\tau}\overline{A}_2(\lambda).
\end{aligned}
\]
The first term is identical to that in the proof of Lemma \ref{lemma:taustar}. The new derivatives are:
\[
\begin{aligned}
\frac{\mathrm{d}}{\mathrm{d}\tau}\overline{A}_1(\lambda) &=
\frac{2\tau(1+\tau)^2(2-\tau) - 2\kappa\tau(2+\tau)}{(1+\tau)^2((1+\tau)^2-\kappa)^2}, \\
\frac{\mathrm{d}}{\mathrm{d}\tau}\overline{A}_2(\lambda) &=
\frac{2\tau^2 \left( (4\tau-3)(1+\tau)((1+\tau)^2-\kappa) - \tau(\tau-1)(5(1+\tau)^2-3\kappa) \right)}{(1+\tau)^4((1+\tau)^2-\kappa)^2}.
\end{aligned}
\]
With these explicit derivatives, the stationarity equation $\frac{\mathrm{d}}{\mathrm{d}\tau}\overline{\mathcal R}(D, \lambda, \rho)=0$ is equivalent to
\(
\frac{\overline{F}\left(\tau,\bar b,\bar c,\rho^{2}\right)}{(1 + \tau)^7 ((1+\tau)^2 - \kappa)^3} =0,
\)
where
\[
\overline{F}\left(\tau,\bar b,\bar c,\rho^{2}\right)
= F_{0}(\tau)
+ \rho^2 F_{\rho}(\tau) + \bar b \overline{F}_{b}(\tau)
+ \bar c \rho^2 \overline{F}_{\rho c}(\tau).
\]
The functions $F_{0}(\tau)$ and $F_{\rho}(\tau)$ are identical to those defined in the proof of Lemma \ref{lemma:taustar}. The new functions are
\[
\overline{F}_{b}(\tau)
= 2\tau(1+\tau)^5 ((1+\tau)^2-\kappa) \left( (1+\tau)^2(2-\tau) - \kappa(2+\tau) \right),
\]
\[
\overline{F}_{\rho c}(\tau)
= 2\tau^2(1+\tau)^3 ((1+\tau)^2-\kappa) \left( (4\tau-3)(1+\tau)((1+\tau)^2-\kappa) - \tau(\tau-1)(5(1+\tau)^2-3\kappa) \right).
\]
Setting $\bar b=\bar c=\rho^{2}=0$ yields the same equation for $\tau_{0}$ as in Lemma \ref{lemma:taustar}. The partial derivative $\partial_{\tau}\overline{F}\left(\tau_{0},0,0,0\right)$ is also unchanged:
\[
\partial_{\tau}\overline{F}\left(\tau_{0},0,0,0\right))
= -2(1+\tau_0)^7\left((1+\tau_0)^2-\kappa\right)\sqrt{(1+\kappa+\kappa\sigma^2)^2-4\kappa}\neq 0.
\]
Therefore, the implicit function theorem gives a smooth map
\(
\tau^{*}(\bar b,\bar c,\rho^{2})
\)
with $\tau^{*}(0,0,0)=\tau_{0}$ and $\overline{F}(\tau^{*},\cdot)=0$.
Differentiating $\overline{F}=0$ at $(\tau_{0},0,0,0)$ and dividing by $\partial_{\tau}\overline{F}(\tau_{0},0,0,0)$ yields
\[
\overline{B}_3(\sigma, \kappa) = - \frac{\overline{F_b}(\tau_0)}{\partial_{\tau}F(\tau_{0},0,0,0)}, \qquad
\overline{C}_3(\sigma, \kappa) = - \frac{\overline{F}_{\rho c}(\tau_0)}{\partial_{\tau}F(\tau_{0},0,0,0)}.
\]
Substituting the expressions for $\overline{F_b}(\tau_0)$, $\overline{F}_{\rho c}(\tau_0)$, and $\partial_{\tau}F(\tau_{0},0,0,0)$ and cancelling common factors gives the coefficients as stated in \eqref{eq:tau-coeffs-bis}.
\end{proof}}

\revised{As $\lambda$ and $\tau$ are linked by the fixed point equation \eqref{eq:tau}, an application of Lemma \ref{lemma:taustar-bis} readily gives that
\begin{equation}\label{eq:lambdastar-bis}
\begin{split}
\lambdaeqs(D, \rho)&= \lambdaeqsz(\rho)+\bar b (\overline{B}_1(\sigma, \kappa)+O(\rho^2)) +\bar c \rho^2( \overline{C}_1(\sigma, \kappa)+O(\rho^2))+O(\bar b^2+\bar c^2),
\end{split}
\end{equation}
where $\lambdaeqsz(\rho)$ is unchanged from \eqref{eq:lfor}, and the new coefficients are
\begin{equation}\label{eq:lfor-bis}
\begin{aligned}
\overline{B}_1(\sigma, \kappa) &= -
\frac{\tau_0 \left( (1+\tau_0)^2 (2-\tau_0) - \kappa (\tau_0+2) \right)}{\kappa (1+\tau_0)^3}, \\
\overline{C}_1(\sigma, \kappa) &= -
\frac{\tau_0^2 \left( (4\tau_0-3)(1+\tau_0)((1+\tau_0)^2-\kappa) - \tau_0(\tau_0-1)(5(1+\tau_0)^2-3\kappa) \right)}{\kappa (1+\tau_0)^5}.
\end{aligned}
\end{equation}}

\revised{Next, the corollary below provides the expansion for the optimal equilibrium risk.}

\revised{\begin{corollary}\label{cor:risk-bis}
Consider the setting of Lemma \ref{lemma:explicit-bis} and let $\tau_0$ be given by \eqref{eq:tau0}. Then, we have that
\begin{equation}
\overline{\fixedriskeqs}(D, \rho)= \fixedriskeqs(\rho)+\bar b (\overline{B}_2(\sigma, \kappa)+O(\rho^2))+\bar c \rho^2( \overline{C}_2(\sigma, \kappa)+O(\rho^2))+O(\bar b^2+\bar c^2),
\end{equation}
where $\fixedriskeqs(\rho)$ is given by \eqref{eq:Rfor}, and
\begin{equation}\label{eq:Rfor-bis}
\begin{aligned}
\overline{B}_2(\sigma, \kappa) &= \frac{2\tau_0^2}{(1+\tau_0)((1+\tau_0)^{2}-\kappa)}, \\
\overline{C}_2(\sigma, \kappa) &= \frac{2\tau_0^3(\tau_0^{2}-1)}{(1+\tau_0)^{4}((1+\tau_0)^{2}-\kappa)}.
\end{aligned}
\end{equation}
\end{corollary}}

\revised{\begin{lemma}\label{lemma:B1-bis}
Let $\overline{B}_1(\sigma,\kappa)$ be given by \eqref{eq:lfor-bis}. Then, for any $\kappa \ge 2$ and all $\sigma \ge 0$,
\(
\overline{B}_1(\sigma,\kappa) \ge 0.
\)
\end{lemma}}

\revised{\begin{proof} Let $s(\sigma) = 1+\tau_0$.
By the definition \eqref{eq:lfor-bis}, we can write
\[
\overline{B}_1(\sigma,\kappa)
= - \frac{\tau_0 N_{\overline{B}_1}(s(\sigma),\kappa)}{\kappa s(\sigma)^3},
\qquad
N_{\overline{B}_1}(s,\kappa)
:= s^2(3-s) - \kappa(s+1).
\]
From the construction we have $s(\sigma) \ge \kappa$ and here we assume $\kappa \ge 2$, so in particular $s(\sigma) \ge 2$. Since $\tau_0>0$, $\kappa>0$ and $s(\sigma)>0$, the prefactor
\(
- \frac{\tau_0}{\kappa s(\sigma)^3} < 0.
\)
Therefore, to prove $\overline{B}_1(\sigma,\kappa) > 0$ it suffices to show
\[
N_{\overline{B}_1}(s,\kappa) < 0 \qquad \text{for all } s \ge \kappa \ge 2.
\]
We analyse $N_{\overline{B}_1}$ as a function of $s$ (with $\kappa$ fixed). Its derivatives are
\[
N_{\overline{B}_1}'(s,\kappa) = 6s - 3s^2 - \kappa,
\qquad
N_{\overline{B}_1}''(s,\kappa) = 6 - 6s.
\]
For $s \ge 2$ we have $N_{\overline{B}_1}''(s,\kappa) < 0$, so $N_{\overline{B}_1}$ is concave on $[2,\infty)$, and hence on $[\kappa,\infty)$ since $\kappa \ge 2$.
We first evaluate $N_{\overline{B}_1}$ and its derivative at the boundary point $s=\kappa$:
\[
N_{\overline{B}_1}(\kappa,\kappa)
= \kappa^2(3-\kappa) - \kappa(\kappa+1)
= -\kappa(\kappa-1)^2 < 0,
\]
and
\[
N_{\overline{B}_1}'(\kappa,\kappa)
= 6\kappa - 3\kappa^2 - \kappa
= \kappa(5-3\kappa).
\]
For $\kappa \ge 2$ we have $5-3\kappa < 0$, so
\(
N_{\overline{B}_1}'(\kappa,\kappa) \le 0.
\)
Since $N_{\overline{B}_1}$ is concave on $[\kappa,\infty)$, its derivative $N_{\overline{B}_1}'(s,\kappa)$ is non-increasing in $s$ on this interval. Hence, for all $s \ge \kappa$,
\(
N_{\overline{B}_1}'(s,\kappa) \le N_{\overline{B}_1}'(\kappa,\kappa) \le 0,
\)
so $N_{\overline{B}_1}(\cdot,\kappa)$ is non-increasing on $[\kappa,\infty)$. Together with $N_{\overline{B}_1}(\kappa,\kappa)<0$ this implies
\[
N_{\overline{B}_1}(s,\kappa) \le N_{\overline{B}_1}(\kappa,\kappa) \le 0
\qquad \text{for all } s \ge \kappa \ge 2,
\]
which proves the lemma.
\end{proof}}

\revised{\begin{lemma}\label{lemma:B2-bis}
Let $ \overline{B}_2(\sigma, \kappa)$ be given by \eqref{eq:Rfor-bis}.
Then, for every $\kappa > 1$ and all $\sigma\ge0$,
\(
 \overline{B}_2(\kappa,\sigma) \ge 0.
\)
\end{lemma}}
\revised{\begin{proof}
Recall the definition 
\[ \overline{B}_2(\sigma, \kappa) = \frac{2\tau_0^2}{(1+\tau_0)((1+\tau_0)^{2}-\kappa)}.\]
Both the numerator and the denominator are positive for \(\kappa > 1\) and \(\sigma > 0\) (as shown in the previous lemmas), therefore the claim holds.
\end{proof}}

\revised{\begin{lemma}\label{lemma:C2-bis}
Let $ \overline{C}_2(\sigma, \kappa)$ be given by \eqref{eq:Rfor-bis}.
Then, for every $\kappa \ge 2$ and all $\sigma\ge0$,
\(
 \overline{C}_2(\kappa,\sigma) \ge 0.
\)
\end{lemma}}
\revised{\begin{proof}
Recall the definition \[ \overline{C}_2(\sigma, \kappa) = \frac{2\tau_0^3(\tau_0^{2}-1)}{(1+\tau_0)^{4}((1+\tau_0)^{2}-\kappa)}.\]
Let $s(\sigma) = 1+\tau_0$.  The denominator is strictly positive for $\kappa > 1$. The term $2\tau_0^3$ is also strictly positive.
Thus, the sign of $ \overline{C}_2$ is determined by the sign of $(\tau_0^2 - 1)$.
We rewrite this term as:
\[
\tau_0^2 - 1 = (s(\sigma)-1)^2 - 1 = s(\sigma)^2 - 2s(\sigma) = s(\sigma)(s(\sigma)-2).
\]
Since $s(\sigma) \ge \kappa > 1$, $s(\sigma)$ is positive. The sign is therefore determined by $(s(\sigma) - 2)$.
We are given $\kappa \ge 2$, which gets
\(
s(\sigma) \ge  \kappa \ge 2.
\)
Therefore, $s(\sigma) - 2 \ge 0$ for all $\sigma \ge 0$ and the claim holds.
\end{proof}}

\section{Details for the experimental setup}
\label{app:real}

For both datasets, the curves are obtained by running $100$ equally spaced values of $\lambda$ with the same splits, so that the observations focus on the performative effect. Data is split uniformly at random across the different steps.

\paragraph{Housing.} We keep all features of the dataset and normalize them. We center the target feature since we use a linear regression without intercept. Following \citet{NEURIPS2024_7de66547}, we fix the features affected by the performative effect to be \texttt{MedInc}, \texttt{AveBedrms}, and \texttt{AveOccup}, with all values of $b$ set equal. 

\paragraph{LSAC.} We keep only one feature in cases of redundant encoding, drop features that are too strongly correlated with the target \texttt{GPA} ($\rho > 0.6$), and randomly select roughly half of the features to be affected by the performative effect. The names of these features are reported in \Cref{tab:lsac_features}. We normalize all features and center the target. All coefficients of $b$ are equal. 

\paragraph{Empirical Covariance.} In \Cref{fig:empiricalcovarianceLSAC,fig:empcovHousing}, we observe that the features do not follow the assumptions made on the data matrix $X$ in the theoretical part, despite exhibiting similar behavior in the experiments. This illustrates that our findings on how to scale regularization remain useful for more general datasets.

\begin{table}[h]
\centering
\caption{Features of the LSAC dataset}
\label{tab:lsac_features}
\resizebox{0.99\textwidth}{!}{%
\begin{tabular}{ll}
\toprule
\textbf{Category} & \textbf{Feature name} \\
\midrule
Redundant & \texttt{male} (same as \texttt{sex}), \texttt{parttime} (same as \texttt{fulltime}), \texttt{decile1} (same as \texttt{decile1b}) \\
With $\rho>0.6$ & \texttt{ugpa}, \texttt{index6040}, \texttt{dnn bar pass prediction} \\
With  $b_{\text{feat}} = \bar{b}$ & \texttt{Unnamed0}, \texttt{decile1b}, \texttt{decile3}, \texttt{other}, \texttt{asian}, \texttt{black}, \texttt{hisp}, \texttt{pass bar}, \texttt{tier} \\
\bottomrule
\end{tabular}}
\end{table}

\begin{figure}
\centering
    \begin{subfigure}{0.45\textwidth}
        \includegraphics[width=\linewidth]{mynicecov.pdf}
    \caption{\revised{Empirical covariance of LSAC dataset.}}
    \label{fig:empiricalcovarianceLSAC}
    \end{subfigure}
    \hfill
    \begin{subfigure}{0.45\textwidth}
        \includegraphics[width=\linewidth]{fig/mynicecovhousing.pdf}
        \caption{Empirical covariance of Housing dataset.}
        \label{fig:empcovHousing}
    \end{subfigure}
\end{figure}