source_current/main.tex

\documentclass{article}

\usepackage{microtype}
\usepackage{graphicx}
\usepackage{subcaption}
\usepackage{booktabs} 
\usepackage{arxiv}


\usepackage{mathtools}
\usepackage{amsthm}
\usepackage{natbib}
\usepackage[english]{babel}
\usepackage[utf8]{inputenc} 
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{float}
\usepackage{booktabs} 
\usepackage{multirow} 
\usepackage{graphicx}
\usepackage{subcaption}
\usepackage[colorinlistoftodos]{todonotes}
\usepackage{algorithm}
\usepackage{algorithmic}
\usepackage{pgfplots}
\pgfplotsset{compat=1.12}
\usepackage{mwe}
\usepackage{hyperref} 
\usepackage{tikz}
\usetikzlibrary{shapes.geometric, arrows, positioning, calc, backgrounds, fit}

             
\newcommand{\Qrob}{\mathbb{Q}}
\newcommand{\calP}{\mathcal{P}}
\newcommand{\calM}{\mathcal{M}}
\newcommand{\calZ}{\mathcal{Z}}
\newcommand{\calB}{\mathcal{B}}
\newcommand{\W}{\mathbb{W}}
\newcommand{\X}{\mathcal{X}}
\newcommand{\calK}{\mathcal{K}}
\newcommand{\esssup}{\operatornamewithlimits{ess\,sup}}
\newcommand{\norm}[1]{\left\lVert#1\right\rVert}

\usepackage[capitalize,noabbrev]{cleveref}

\theoremstyle{plain}
\newtheorem{theorem}{Theorem}[section]
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{corollary}[theorem]{Corollary}
\theoremstyle{definition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{assumption}[theorem]{Assumption}
\theoremstyle{remark}
\newtheorem{remark}[theorem]{Remark}



\usepackage{preamble-JZ}

\title{Gradient Flow Sampler-based\\
Distributionally Robust Optimization}

\author{
Zusen Xu\\
Weierstrass Institute \& \\
KTH Royal Institute of Technology\\
Berlin, Germany\\
\texttt{zusen.xu@wias-berlin.de} \\
    \And
Jia-Jie Zhu\\
Weierstrass Institute \& \\
KTH Royal Institute of Technology\\
Stockholm, Sweden\\
\texttt{jiajie@kth.se}\\
}

\begin{document}

\maketitle

\begin{abstract}
We propose a mathematically principled PDE gradient flow framework for distributionally robust optimization (DRO). Exploiting the recent advances in the intersection of Monte Carlo sampling and statistical optimal transport, we show that our theoretical framework can be implemented as practical algorithms for sampling from worst-case distributions and, consequently, DRO.
While numerous previous works have relied on dual reformulation techniques, we contribute a sound and complete gradient flow view based on SDEs or PDEs that can be used to construct new algorithms for general, potentially non-convex, losses.
  As an application, we solve a class of Wasserstein and entropy-regularized DRO problems using the recently-discovered Wasserstein Fisher-Rao and Stein variational gradient flows. Notably, we also show some simple reductions of our framework recover exactly previously proposed popular DRO methods, and provide new insights into their theoretical limits and optimization dynamics of DRO. Numerical studies based on stochastic gradient descent on machine learning tasks provide empirical backing for our theoretical findings.

\end{abstract}



\section{Introduction}\label{sec:intro}

Distributionally robust optimization (DRO)~\citep{delageDistributionallyRobustOptimization2010,kuhnDistributionallyRobustOptimization2024}
is a framework that aims to enhance the robustness of the solution to optimization problems.
  After    the original Wasserstein distributionally robust optimization (DRO) works \citep{mohajerin2018data,zhaoDatadrivenRiskaverseStochastic2018,gaoDistributionallyRobustStochastic2016}, many subsequent works have presented variations of problems with various numerical solutions. 
  In particular, Sinkhorn DRO \citep{wang2021sinkhorn}, also referred to as entropy-regularized Wasserstein DRO \citep{dapogny2023entropy}, demonstrates significant advantages in handling tasks with class imbalance. The entropy-regularized Wasserstein DRO problem can be formulated as a penalized optimization problem:   
\begin{align}
    \min_{\theta\in\Theta} \max_{\rho\in\calP}
    \left\{ \int \ell(\theta, y)\dd\rho{(y)} - \tfrac1{2\tau} W^2_{\epsilon}(\rho,\widehat{\rho_N}) \right\}
    \label{eq:main-ent-dro}
\end{align}
where $\tau > 0$ is the   regularization    parameter, $\widehat{\rho_N}$ is an empirical dataset, and $W^2_{\epsilon}(\mu, \nu) := \inf_{\gamma\in \Pi(\mu,\nu)} \{\mathbb{E}_{\gamma}[c] + \epsilon H(\gamma)\}$ is the
entropy-regularized optimal transport (OT) divergence. Here, $\Pi(\mu,\nu)$ is the set of couplings with marginals $\mu$ and $\nu$,   $c: \mathbb R^d \times \mathbb R^d \rightarrow \mathbb R$ is a cost function,    $H(\gamma) = \int \gamma \log \gamma$ is the negative entropy, and  $\epsilon > 0$ is the entropy regularization parameter.
If we set $\epsilon = 0 $, we recover the penalized version of the Wasserstein DRO as considered by \citet{sinha2020certifyingdistributionalrobustnessprincipled}.
Without loss of generality, we will temporarily focus on the interesting choice of $W^2_\epsilon$, while other choices such as the KL divergences can also be straightforwardly adapted to our framework.


We observe that the inner maximization problem in \eqref{eq:main-ent-dro} is equivalent to applying the entropy-regularized JKO operator (detailed in \eqref{eq:proximal-EOT})   from the PDE literature\citep{jordan1998variational, peyre2015entropic}  , resulting the equivalent formulation to \eqref{eq:main-ent-dro}
\begin{align}
\min_{\theta\in\Theta}
\bbE _ {y\sim \pi_Y} \ell (\theta, y),
\quad 
\pi_Y = \epsjko_{\tau V}(\widehat{\rho_N})
\label{eq:ent-jko-reformulation}
\end{align}
  where $V:=-\ell$.    Exploiting this connection,   our key insight is    solving the inner problem of DRO \eqref{eq:main-ent-dro} can be achieved by sampling from the worst-case distribution $\pi_Y$ using a gradient flow.
This simplification of the problem structure provides intuition for novel algorithmic design, which we use to effortlessly obtain novel algorithms such as Wasserstein-Fisher-Rao and Stein variational gradient-based DRO algorithms.
We term our methodology
\emph{Gradient Flow Sampler-based Distributionally Robust Optimization} (GF-DRO).
We illustrate the main idea in Figure~\ref{fig:gf-dro-intro}. 


\begin{figure}[h!]
    \centering
    \tikzset{
        block/.style={rectangle, draw=black, minimum width=2.0cm, minimum height=0.6cm, text centered, font=\tiny\sffamily, line width=0.4pt},
        img_node/.style={rectangle, draw=black!30, inner sep=0pt, line width=0.4pt},
        arrow/.style={thick, ->, >=stealth, line width=0.6pt},
        label_style/.style={font=\tiny\itshape\sffamily, align=center},
        container/.style={draw=black!15, rectangle, rounded corners=2pt, inner sep=0.15cm, line width=0.4pt}
    }

    \begin{tikzpicture}[node distance=0.8cm]
        \node (data_node) [block, fill=blue!5] {$X \sim \widehat{\rho_N}$};
        
        \node (fig1_node) [img_node, below=0.1cm of data_node] {
            \includegraphics[width=1.6cm]{original.pdf} 
        };
        
        \begin{scope}[on background layer]
            \node (input_box) [container, fill=blue!2, fit=(data_node) (fig1_node)] {};
            \node[above=0pt of input_box, font=\tiny\bfseries\sffamily] {A sample from dataset};
        \end{scope}

        \node (fig_perturbed) [img_node, right=2.4cm of data_node] {
            \includegraphics[width=3.0cm]{perturbed_top3.pdf}
        };

        \node (worst_case_node) [block, below=0.3cm of fig_perturbed, fill=red!5] {$Y_i \sim \pi_{Y|X}$};

        \begin{scope}[on background layer]
            \node (output_box) [container, fill=red!2, fit=(fig_perturbed) (worst_case_node)] {};
            \node[above=0pt of output_box, font=\tiny\bfseries\sffamily, align=center] {Samples from conditional \\ worst-case distribution};
        \end{scope}

        \node (update_node) [block, below=1.2cm of output_box.south, xshift=-1.5cm, minimum width=3.8cm, fill=green!5] {
            \begin{tabular}{c}
                \textbf{Parameter Update} \\
                $\theta \leftarrow \text{Proj}_\Theta(\theta - r \hat{g})$
            \end{tabular}
        };

        \draw [arrow] (input_box.east) -- node[above, label_style] {Gradient Flow \\ Samplers} (output_box.west |- input_box.east);
        
        \draw [arrow] (worst_case_node.south) -- node[right, label_style, xshift=0.05cm] {Estimate \\ gradient $\hat{g}$} (update_node.north -| worst_case_node.south);

        \draw [arrow, dashed, draw=black!30] (update_node.west) -| (input_box.south)
            node[pos=0.3, above, yshift=0.05cm, label_style, color=black!40] {Next Iteration};

    \end{tikzpicture}
    \caption{\textbf{GF-DRO} generates   samples from the conditional worst-case distribution    along the evolution of S/PDE gradient flows, which is fundamentally different from previous DRO methods.   Here $\pi_{Y|X}$ denotes the conditional worst-case distribution defined by the entropic JKO operator (see Proposition~\ref{lem:sb-klform}).    See text for detailed explanation and Section~\ref{sec:experiments} for numerical experiments.}
    \label{fig:gf-dro-intro}
    \vspace{-0.4cm}
\end{figure}

To obtain theoretical guarantees, we connect the stochastic approximation of the gradient w.r.t. $\theta$ for the outer DRO problem
  $\widehat g =
\nabla_\theta \ell (\theta, y), \, y\sim \pi_Y$,
with the analysis of gradient flow-based samplers.   
Subsequently, the gradient estimate is used for the outer DRO problem via SGD-like updates, e.g., $\theta^{s+1} \gets \theta^s - \eta \widehat g$.
The bound of $\widehat g$ deviating from the true gradient can be used in the downstream standard optimization error bound for DRO solution.

\vspace{-0.3cm}
\paragraph*{Contributions.}
Importantly, we distinguish between two different notions:
\begin{tightenum}
\item The DRO problem formulation with various ambiguity sets, e.g., Wasserstein DRO~\citep{zhaoDatadrivenRiskaverseStochastic2018,mohajerin2018data,gaoDistributionallyRobustStochastic2016}, KL-DRO~\citep{hu2013kullback,ben-talRobustSolutionsOptimization2013}, Sinkhorn DRO~\citep{wang2021sinkhorn}, Kernel DRO~\citep{zhu2021kernel}, etc.
\item The optimization algorithm and analysis for solving the DRO problem, e.g., WRM~\citep{sinha2020certifyingdistributionalrobustnessprincipled}, reformulation techniques~\citep{mohajerin2018data}, SGD~\citep{levyLargeScaleMethodsDistributionally2020}, etc.
\end{tightenum}
In this paper, we do not invent new DRO problem formulations or ambiguity sets  (task 1)  . Instead, we propose a principled mathematical framework---GF-DRO---for the analysis and design of novel DRO-solving algorithms  (task 2)   based on the theory of gradient flows and PDE. This framework bridges gradient-flow sampling and DRO, a connection previously missing in the literature. Unlike classical DRO approaches \citep{zhaoDatadrivenRiskaverseStochastic2018,mohajerin2018data,gaoDistributionallyRobustStochastic2016} that usually focus on specific loss functions to derive ad-hoc dual reformulations, GF-DRO provides a general framework to directly solve the variational problem \eqref{eq:proximal-EOT}. This allows the treatment of general, potentially non-convex losses, such as those defined by deep neural networks, for which many classical DRO methods are inapplicable. Our work can be viewed as a general-purpose   \emph{sampler-based bi-level optimization framework}    \citep{nemirovski2009robust}, where we invent novel DRO solvers by exploiting recently proposed gradient flow samplers, such as Wasserstein-Fisher-Rao (WFR) and Stein variational gradient (SVG) flows (i.e. we do not invent new PDE or gradient flow theory), to the energy functionals informed by DRO.
While previous methods like WRM ~\citep{sinha2020certifyingdistributionalrobustnessprincipled}    are fundamentally   discretized    ODEs, our use of S/PDEs and interacting particle systems advances the theoretical depth of the DRO subject. For instance, Wasserstein flows relies on the log-Sobolev inequality (LSI) constant which is problematic in high dimensions, while the WFR gradient flow we introduce provides a new lane of research that may go beyond the limitations of LSI constant.   Among recent sampling-based DRO efforts, FlowDRO \citep{xu2024flow} learns a deterministic transport map via Neural ODEs, and \citet{zhu2024distributionally} analyze a JKO-like iterative scheme; in contrast, GF-DRO leverages the equivalence between the inner DRO problem and a sampling task driven by the \emph{entropy-regularized JKO operator} (Proposition~\ref{lem:sb-klform}), letting us plug in any gradient flow sampler (WGF, WFR, SVG) and naturally handle entropy-regularized Sinkhorn DRO.    Ultimately, we demonstrate that DRO problems can be solved and analyzed in a unified, practical algorithmic framework without   inventing    ad-hoc reformulations.


\section{Preliminaries}
\subsection{Gradient Systems and Their Gradient Flow Equations}
The Wasserstein gradient flow (WGF) framework \citep{otto1996double} was introduced into the sampling literature to provide a theoretical foundation; see
\citet{chewi2024statistical,garcia2018continuum}
for recent surveys.
In that framework, one can write a flow equation formally as
\begin{align}
    \dot \rho =
    -  \mathbb G_W(\rho)^{-1}(\rho) \frac{\delta F}{\delta \rho}[ \rho]
    = \nabla \cdot\left(\rho\nabla \frac{\delta F}{\delta \rho}[ \rho]\right)
    \label{eq:wasserstein-gfe}
\end{align}
using
the inverse of the Wasserstein Riemannian metric tensor:
$\mathbb G_W^{-1}(\rho): T^*_\rho \calM \to T_\rho \calM, \xi \mapsto -\nabla \cdot(\rho\nabla \xi),$
where $\calM$ is a space or a manifold, $T_\rho \calM$ is the tangent space of $\calM$ at $\rho$ and
$T^*_\rho \calM$ is the cotangent space.
With those ingredients, we can formally define the gradient systems that generate gradient flow equations such as the WGF equation~\eqref{eq:wasserstein-gfe}.
We refer to \citet{mielke2023introduction} for more details.
\begin{definition}
    [Gradient system]
    We refer to a tuple $\left( \calM, F, \bbG \right)$
    as a gradient system. 
    It has the gradient structure
    identified by:
\begin{tightenum}
    \item a space or a manifold $\calM$,
    \item an energy functional $F$,
    \item a dissipation geometry given by either:
        a distance metric defined on $\calM$ or
                a Riemannian metric tensor $\mathbb G$.
\end{tightenum}
\label{def:gradient-system}
\end{definition}

\subsection{Gradient Flow Sampler}
For sampling and inference,
a common choice for the energy functional is the KL divergence,
i.e.,
$F(\rho) = \KL(\rho | \pi)$.   Under the Wasserstein metric,   
through elementary calculation, we obtain from \eqref{eq:wasserstein-gfe} the Fokker-Planck equation (FPE)
\begin{align}
    \partial_t \rho =
    \nabla \cdot \left( \rho \nabla \log\frac{\rho}{\pi}  \right)
    =
    { \Delta \rho} - \nabla \cdot \left( \rho \nabla \log \pi  \right)
    .
    \label{eq:fokker-planck}
  \end{align}
When we express the target as $\pi(x) = \frac1Z \exp(-V(x))$
where $Z$ is a normalization constant (partition function), \eqref{eq:fokker-planck} is then 
$\partial_t \rho = { \Delta \rho} + \nabla \cdot \left( \rho \nabla V  \right)$.
Viewed as a dynamical system, the KL divergence energy functional dissipates
along \eqref{eq:fokker-planck} in the steepest descent manner.
Based on Definition~\ref{def:gradient-system}, we say that PDE~\eqref{eq:fokker-planck} has the \emph{gradient structure} that entails the following key ingredients:
\begin{align*}
    \begin{cases}
        \textrm{{Space} :}&
        \text{prob. space }
        \mathcal P ,
        \\
        \textrm{Energy functional} :& {F(\cdot):= \KL(\cdot | \pi)},
        \\
        \textrm{Dissipation Geometry} :& { \text{Wasserstein metric}}.
    \end{cases}
\end{align*}




\subsection{Entropy-regularized Wasserstein DRO}
Works such as \citet{sinha2020certifyingdistributionalrobustnessprincipled,wang2021sinkhorn} considered DRO problems with rather general loss functions as they are based on general-purpose continuous optimization rather than DRO reformulation techniques for special losses.
\citet{wang2021sinkhorn} proposed a specialized algorithm solving the dual formulation of the DRO problem~\eqref{eq:main-ent-dro}. 
Different from their dual approach, this paper develops a unified gradient flow sampler framework that directly samples from the primal worst-case distribution that solves the inner problem.


\section{A Gradient Flow Framework for Sampling from Worst-case Distributions}\label{sec:gf-dro-method}

\subsection{Schr\"odinger Problem Formulation of DRO}
Our starting point is the following entropy-regularized JKO (Jordan-Kinderlehrer-Otto) operator   from PDE\citep{jordan1998variational, peyre2015entropic}   :
\begin{align}
    \epsjko_{\tau V}(\rho_0) :=
    \argmin_{\rho \in \calP}
    \left\{\int V \, \dd \rho + \tfrac{1}{2\tau} W^2_\epsilon(\rho, {\rho_0}) \right\}
    \label{eq:proximal-EOT}
\end{align}
This operation,
hence the inner maximization problem of DRO,
is a special case of the static Schr\"odinger problem with a free marginal, i.e., half bridge or one-sided bridge.
We now provide a variational characterization enabling us to perform the sampling task in \eqref{eq:ent-jko-reformulation}.
\begin{proposition}\label{lem:sb-klform}
    Solving the entropy-regularized JKO operator \eqref{eq:proximal-EOT} is equivalent to solving the Schr\"odinger half bridge problem
\begin{equation}
\label{eq:sb-klform-half-bridge}
\min_{\mathclap{\substack{\Pi\in \calM^2, \\\int \Pi \dd y = \rho_0}}}
\biggl\{ \int V \dd \Pi + \frac{1}{2\tau} \int c(x,y) \dd\Pi + \frac\epsilon{2\tau}\int \log \Pi \dd\Pi \biggr\}
\end{equation}
The optimal marginal distribution of $Y$ in \eqref{eq:sb-klform-half-bridge} is given by a mixture distribution, for some normalization constants $Z_x$:
    \begin{align}\label{eq:worst-dist}
        \pi_Y = \E_{x\sim \rho_0} \left[
            \dfrac{1}{Z_x}\exp
            \left({-\frac{\widetilde V_{x,\tau}(y) }{\epsilon}} \right)    
         \right],
    \end{align}
    where $\widetilde V_{x,\tau}(y) := V(y) + \frac{c(y,x)}{2\tau}.$
    Consequently, \eqref{eq:sb-klform-half-bridge} is equivalent to the minimization of the expected KL divergence with respect to the conditional distribution
    \begin{align}\label{eq:sb-klform}
        \min_{\rho_{Y|X}}
    \mathbb{E}_{x\sim\rho_0}
            \KL
            \left(
            \rho_{Y|X=x}(y) 
            \big|
            \tfrac{1}{Z_x}
            \exp \left(
                       - \tfrac{\widetilde V_{x,\tau}(y)}{\epsilon} 
            \right)
            \right)
            .
    \end{align}
\end{proposition}
The optimal
conditional distribution of \eqref{eq:sb-klform}, denoted by $\pi_{Y|X}$, provides the
$$\text{stochastic entropic transport map}:
\quad
\, x\mapsto \pi_{Y|X=x}(y).$$
This statement gives the overall variational structure of the DRO problem~\eqref{eq:main-ent-dro}.
We are now ready to introduce our proposed method GF-DRO in the next section.
\subsection{Gradient Flow Sampler-based DRO}
  Using the variational problem~\eqref{eq:sb-klform}, we define the energy functional as
the KL divergence energy functional (without expectation):  
\begin{equation}
\label{eq:conditional-sampling-energy}
    \begin{split}
        F(\rho) :&=
        \tfrac{\epsilon}{2\tau}
        \KL
            \left(
            \rho\ 
            \bigg| 
            \tfrac{1}{Z_x}
            \exp \left(
                        -\tfrac{\widetilde V_{x,\tau}(y)}{\epsilon} 
            \right)
            \right) \\
            &=
        \int V\, \dd \rho
        + \frac{1}{2\tau} W_c^2(\rho, \delta_x) 
         + 
         \frac{\epsilon}{2\tau} \int \rho \log \rho 
         + \const
    \end{split}
\end{equation}
  Note that the formulation on the right-hand side was also observed by \citet{chen2022improved} in studying the proximal sampler.

With those ingredients,
in this paper, we propose to achieve
sampling from the conditional distribution $\pi_{Y\mid X}$ by simulating the \emph{gradient system
$\displaystyle\left(
\calP, F, \mathbb G
\right)$}.
This results in the formal gradient flow equation:
\begin{align}
\dot\rho = - \mathbb G^{-1}(\rho)\,  DF(\rho).
\label{eq:general-gradient-flow}
\end{align}
where we can freely choose the dissipation geometry $\mathbb G$ for the gradient flow.
Then,
our central methodology is the following perspective of sampling from conditional distributions
connected to the KL-minimization problem~\eqref{eq:sb-klform}.
Consider solving the inner maximization problem~\eqref{eq:main-ent-dro} via the following two-step sampling procedure:
\begin{algorithm}[h!]
\caption{Worst-case Distribution Sampler via Gradient Flows}
\label{alg:sampler}
\begin{algorithmic}[1]
    \STATE {\bfseries Input:} an initial distribution ${\rho}_0$ to sample from (e.g. empirical distribution $\widehat{\rho}_N$ in data-driven DRO) 
    \STATE Sample $X \sim {\rho}_0$
    \STATE Sample $Y \sim \pi_{Y\mid X}$
    using a gradient flow
    (e.g. with energy functional given by \eqref{eq:conditional-sampling-energy})
    \STATE {\bfseries Output:} sample $Y$ from the worst-case distribution
\end{algorithmic}
\end{algorithm}




Using the gradient flow sampler, we now propose the following general-purpose gradient flow sampler-based DRO framework.
\begin{algorithm}[h!]
\caption{Gradient Flow Sampler-based DRO}
\label{alg:GF-DRO}
\begin{algorithmic}[1]
    \STATE {\bfseries Input:} Initial distribution $\rho_0$, e.g., empirical distribution $\rho_0 = \widehat{\rho_N}$, constraint set $\Theta$, $\tau, \epsilon >0$, stepsize $r_s$.
    \FOR{ iteration count $s = 0, \dots$}
        \STATE Generate a sample from the worst-case distribution $y^s \sim \pi_Y$ by using the gradient flow sampler in Algorithm~\ref{alg:sampler}
        \STATE DRO step: $\theta^{s+1} \leftarrow \text{Proj}_\Theta (\theta^s-r_s \nabla_\theta \ell(\theta^s, y^s))$
    \ENDFOR
\end{algorithmic}
\end{algorithm}
Other straightforward variants, such as using sample average approximation (SAA) instead of stochastic approximation (SA) above, are possible.

\subsubsection{Wasserstein Gradient Flow Sampler}
As the first practical outcome of our theoretical insights, we instantiate a Wasserstein gradient flow (WGF) sampler for the entropy-regularized Wasserstein DRO problem.
We consider the Wasserstein gradient system with the driving functional $F$ and the Wasserstein metric as the dissipation geometry:
    $\left( \calP, F, W_2 \right).$

In sampling algorithms, this gradient flow is typically
implemented by discretizing the Langevin SDE
$$
dX_t = 
-
 \tfrac{2\tau}{\epsilon}\nabla\widetilde V_{x, \tau}(X_t) dt + dW_t,
$$
which results in the following forward Euler discretization known as the unadjusted Langevin algorithm (ULA).
Note that we use a scaled stepsize.
\begin{lemma}
    The forward Euler discretization of the Wasserstein gradient flow equation of the energy functional $F$ in \eqref{eq:conditional-sampling-energy} is given by the difference equation at step $t$:
    \begin{align}
        X_{t+1} = X_t 
        -
         \eta_t \nabla \widetilde V_{x, \tau}(X_t) + \sqrt{\eta_t\frac\epsilon\tau} \xi_t
        \label{eq:ula-proximal-general-D-W2}
    \end{align}
    where $\xi_t$ is a standard normal random variable and
    $\eta_t$ is the stepsize.
\end{lemma}
We note the stepsize scaling in front of the stochastic variable $\xi_t$ is different from the vanilla ULA update rule.

Adapting the above WGF dynamics to our GF-DRO framework in Algorithm~\ref{alg:GF-DRO}   for generating samples from the worst-case distribution   , we obtain the following discrete-time DRO algorithm summarized in Algorithm~\ref{alg:SDRO-NGD}.

\begin{algorithm}[h!]
\caption{Entropy-regularized Wasserstein DRO via WGF}
\label{alg:SDRO-NGD}
\begin{algorithmic}[1]
  \STATE {\bfseries Input:} Empirical distribution $\widehat{\rho_N}$, constraint set $\Theta$, $\tau, \epsilon >0$, stepsize sequence $\{r_s > 0\}_{s=0}^{S-1}$, inner stepsize $\eta$, inner iterations $T$, number of samples $m$.
  
  \FOR{$s = 0, \dots, S-1$}
    \STATE Sample $x^s \sim \widehat{\rho_N}$
    \STATE Initialize $y_{0}^{i,s} \leftarrow x^s, i=1,\dots,m$
    \FOR{$t = 0, \dots, T-1$}
      \STATE Sample $\xi^{i,s}_t \sim\mathcal{N}(0,I)$
      \STATE Update sample using \eqref{eq:ula-proximal-general-D-W2}: \\ $y_{t+1}^{i,s} \leftarrow y_{t}^{i,s} - \eta\nabla\widetilde V_{x^s, \tau}(y_{t}^{i,s}) + \sqrt{\eta\epsilon/\tau} \xi^{i,s}_t$
    \ENDFOR
    \STATE $\theta^{s+1} \leftarrow \text{Proj}_\Theta (\theta^s-r_s \sum_{i=1}^m \frac{1}{m}\nabla_\theta \ell(\theta^{s},y_{T}^{i,s}))$
  \ENDFOR
  \STATE {\bfseries return} $\theta^S$
\end{algorithmic}
\end{algorithm}

\begin{remark}\label{rm:WRM}
    [\citet{sinha2020certifyingdistributionalrobustnessprincipled}'s WRM]
    For the penalized Wasserstein DRO problem,
    i.e.
    $\epsilon=0$ in the entropy-regularized Wasserstein DRO problem   \eqref{eq:main-ent-dro}   , 
    the step in \eqref{eq:ula-proximal-general-D-W2} specializes to the update rule:
    \begin{align}
        X_{t+1} = X_t 
        - \eta_t \nabla \widetilde V_{x, \tau}(X_t)
        \label{eq:ula-proximal-general-D-W2-epsilon-0}
    \end{align}
    which coincides with the inner SGD step used in the WRM algorithm by \citet{sinha2020certifyingdistributionalrobustnessprincipled}.
Hence, their WRM algorithm is a special case (with only ODE) of our GF-DRO framework in Algorithm~\ref{alg:GF-DRO}.
\end{remark}
Through the example above, we see the value of this paper's gradient flow perspective: it can easily generalize the WRM algorithm~\citep{sinha2020certifyingdistributionalrobustnessprincipled} from ODE to a novel SDE-based   or PDE-based    algorithm for the Sinkhorn DRO problem using \eqref{eq:ula-proximal-general-D-W2}.
Therefore, the gradient flow perspective is not simply a theoretical formulation -- it lets us effortlessly design new algorithms to solve DRO without resorting to ad-hoc modifications of other DRO methods.
Moreover, we will later go beyond the standard Wasserstein gradient flow and introduce more advanced gradient flows such as the Wasserstein Fisher-Rao (WFR) and Stein variational gradient (SVG) flows.

\subsubsection{Wasserstein Fisher-Rao Flow Sampler}
Consider the WFR gradient system of the energy functional $F$, i.e., the triple $\left( \calP, F, \WFR \right)$, where $\WFR$ is the Wasserstein-Fisher-Rao metric, a.k.a. the \emph{Hellinger-Kantorovich} metric restricted to the probability space\footnote{Note that there are many cases of misnomer in the machine learning literature: WFR should technically be defined over the space of positive measures, not the space of probability measures; see \citet{mielkeNotesHellingerDistance2025} for a historical account. The latter corresponds to the spherical Hellinger-Kantorovich metric. Although their gradient flow solutions can be easily related to each other via the mass scaling. See \citet{mielke2025hellinger} for technical details.}.
Specifically, we consider the HK/WFR gradient system associated with the reaction-diffusion PDE:
\begin{align}
    \partial_t \rho
    &= \alpha\, \mathrm{div} \left( \rho \nabla \frac{\delta F}{\delta \rho} \right) - \beta\, \rho 
    \left(\frac{\delta F}{\delta \rho} - \int \frac{\delta F}{\delta \rho}\dd \rho \right)
    \label{eq:wfr-gradient-flow}
\end{align}
  where $\alpha,\beta>0$ are the coefficients of the Wasserstein (transport) and Fisher-Rao (reaction) components of the HK/WFR metric, respectively.   
By simulating the interacting particle system associated with \eqref{eq:wfr-gradient-flow},
we propose a WFR-based worst-case distribution sampler for SDRO as detailed in Algorithm \ref{alg:SDRO-WFR}.
The primary distinction of this sampler, compared to the WGF version,   is the addition of a birth-death mechanism. 
Crucially, this mass reallocation mechanism enables the system to concentrate its search within low-energy regions and effectively escape from local optima. See details in Appendix~\ref{sec:wfr-alg}.   


\subsubsection{Stein Variational Gradient Flow Sampler}
In an attempt to simulate \eqref{eq:fokker-planck} using deterministic particle-based methods instead of SDEs,
\citep{liu2016stein}
proposed the Stein variational gradient descent (SVGD) algorithm, which is an 
implementable
deterministic algorithmic version of \eqref{eq:fokker-planck}.
For a target $\pi$, at time step $t$,
it updates the particle locations via the following gradient descent scheme.
\begin{align*}
    X^i_{t+1} = X^i_t + \eta\cdot \Big( \frac{1}{m}\sum_{j=1}^m \nabla \log\pi(X^j_t)k(X^j_t,X^i_t ) +
    \frac{1}{m}\sum_{j=1}^m \nabla_2 k(X^j_t,X^i_t ) \Big).
\end{align*}
Here, $X^i_t$ represents the position of the $i$-th particle at time step $t$, $\eta$ is the stepsize, $m$ is the total number of particles.
$k(\cdot,\cdot)$ is a positive definite kernel function, and $\nabla_2$ denotes the gradient with respect to the second argument of the kernel.
Applying SVGD to the inner problem of DRO, we propose Algorithm~\ref{alg:SDRO-SVG}. See details in Appendix~\ref{sec:svg}.


\subsubsection{Rejection Sampler} 
We also propose Algorithm~\ref{alg:SDRO_rgo},  which is a DRO algorithm    based on rejection sampler (RGO). This approach is based on the backward step of the proximal sampler \citep{lee2021structured,chen2022improved,wibisono2025mixing}, as detailed in Appendix\ref{sec:rgo}. However, this method requires the loss function to be $L$-smooth and cannot be applied when $\tau$ is relatively large, which we will demonstrate in later experiments.

\begin{remark}[\textbf{Dynamic interpolation: merits of a dynamic model}]
  Our gradient flow framework produces a principled family of distributions $\{\rho_t\}_{t\geq 0}$, where each $\rho_t$ solves a well-defined variational problem and corresponds to a specific perturbation intensity controlled by $t$. This dynamic structure has practical value: when the terminal distribution $\rho_{t_T}$ produces unrealistic samples under a ``bad'' hyperparameter (Figure~\ref{fig:evolution-image}), one can directly select an earlier $\rho_{t_i}$ for sample-based SGD without re-running the sampler with a different parameter (e.g.\ a smaller $\tau$). Unlike post-hoc interpolation between independent runs, every $\rho_t$ along the flow inherits structural guarantees from gradient flow theory, in particular monotone energy dissipation along $F$.   
\begin{figure}[h!]
    \centering
    \includegraphics[width=0.8\linewidth]{1_exp_convergence.pdf}
    \caption{Evolution along the gradient flow under a ``bad'' hyperparameter.   The first column displays the original images. The subsequent columns show the evolving states of the particles at intervals of 1000 steps, from Step 1000 to Step 6000.   }
    \label{fig:evolution-image}
    \vspace{-0.3cm}
\end{figure}
\end{remark}

\section{Analysis of Gradient Flows}
For algorithmic design, one paradigm is to first obtain intuition and insights from principled mathematical analysis before constructing practical numerical algorithms.
While the development in machine learning research does not always follow this pattern, we argue in this paper that, by first understanding the theoretical limits of the gradient flow, we can design novel DRO algorithms that performs as expected theoretically. This is done without inventing new theoretical framework, but by leveraging the existing gradient flow and PDE analysis \citep{otto2000generalization, jordan1998variational} specialized to DRO. 
The discrete-time algorithmic analysis will follow in a later section.



In DRO works such as \citep{sinha2020certifyingdistributionalrobustnessprincipled}
and subsequent variants, 
complexity analysis typically involves controlling the gradient estimation error.
That is, to control
the deviation 
from the true gradient used for DRO.
For illustration, let us consider the
DRO problem \eqref{eq:main-ent-dro}.
Note that
setting $\epsilon = 0$ recovers the
problem of \citep{sinha2020certifyingdistributionalrobustnessprincipled}.
Our goal is to analyze the estimation of DRO stochastic gradient $\hat g$
using PDE tools.
With our gradient flow sampler-based DRO algorithm,
we aim to generate samples from the worst-case distribution $\pi_Y$, which is the entropic JKO solution
 
\begin{align}
\label{eq:entropic-jko}
\pi_Y = \epsjko_{\tau V}(\widehat{\rho_N})
\end{align}
  
Following the framework of Algorithm~\ref{alg:sampler},
  for $x\sim \rho_0$,
we generate samples from the worst-case distribution
by sampling from the conditional distribution, $y\sim \rho^*_{Y|X=x}$.
Then, for the outer DRO problem, we can use the sample-based gradient estimate $\widehat g := \nabla_\theta \ell(\theta, y)$    to update the parameter $\theta$.   
The following result uses gradient flow analysis to bound the gradient estimate error.
We note that the results are stated in terms of the ideal continuous-time gradient flow, which is not yet the mixing time of the discrete-time sampler; we will detail the discrete-time algorithmic analysis in a later section.
The gradient flow analysis provides a key insight and guideline for the design of the gradient flow sampler-based DRO algorithm.



For notational simplicity, we abbreviate $\widetilde V_{x,\tau}$ as $\widetilde V$ in this section and, specifically, for the case of quadratic transport cost $c(y,x) = \|x-y\|^2$, we define the following.
Recall the functional $F(\rho) =
\frac{\epsilon}{2\tau}
\KL
    \left(
    \rho\ 
    \bigg| 
    \tfrac{1}{Z_x}
    \exp \left(
                        -\tfrac{\widetilde V_{x,\tau}(y)}{\epsilon} 
            \right)
    \right)$ as in \eqref{eq:conditional-sampling-energy}.
\begin{definition}
\label{as:gf-properties}
\begin{enumerate}
\item   \textbf{$L$-smoothness:} A differentiable function $f$ is $L$-smooth if there exists a constant $L > 0$ such that
    \begin{align}
        \|\nabla f(x) - \nabla f(x')\| \leq L \|x - x'\|, \quad \forall x, x' \in \mathcal{X}.
        \label{eq:L-smoothness}
    \end{align}  
    \item \textbf{Semi-convexity   of the regularized potential $\widetilde V$  :} There exists a constant $\lambda > 0$ such that
    \begin{align}
        \nabla ^2 \widetilde V \geq \lambda I
        \iff 
        \nabla ^2 V + \frac1{2\tau }I \geq \lambda I
        \label{eq:semi-convexity}
    \end{align} 
    \item \textbf{Gradient dominance/Polyak-\L ojaciewicz (PL)
inequality   of $\widetilde V$   :}
    There exists a constant $\lambda > 0$ such that
    \begin{align}
        \big|\nabla \widetilde V \big|^2 \geq \lambda \widetilde V,
        \quad \forall y \in \mathcal X
        \label{eq:gradient-dominance-PL}
    \end{align}
    \item \textbf{Functional PL inequality over measures along GF}: There exists a constant $\lambda > 0$ such that
    \begin{align}
        \left\|\frac{\dd}{\dd t} F(\rho_t)\right\|^2 \geq \lambda F(\rho_t),
        \label{eq:generalized-PL}
    \end{align}
    which is equivalent to the $\lambda$-\textbf{logarithmic Sobolev inequality} (LSI) for $\pi  =     \tfrac{1}{Z_x}
    \exp \left(
                        -\tfrac{\widetilde V_{x,\tau}(y)}{\epsilon} 
            \right)$,
    \begin{align}
        \int \Bigg\| 
        \nabla \log 
        \dfrac{\rho}{\pi} \Bigg\| ^2\dd \rho
        \geq 
        \lambda
        \KL(\rho | \pi)
        \label{eq:LSI-along-GF}
    \end{align}
    along the solution $\rho_t$.
\end{enumerate}
\end{definition}

It is immediate that
$1.\implies 2.$
and
$1.\implies 3$.
We note that
the second condition in Definition~\ref{as:gf-properties}
does not require the function $V$ itself to be convex. Hence, the DRO loss $\ell(\theta, x)$ does not necessarily need to be concave in variable $x$.
In such cases, we can already obtain the geodesic convexity of $F$ in the Wasserstein space, termed displacement convexity.

  Letting    the DRO objective $\Phi(\theta): = \max_{\rho\in\calP}\left\{
    \int \ell(\theta, z)\dd\rho{(z)} - \tfrac1{2\tau} W^2_{\epsilon}(\rho,\widehat{\rho_N})\right\}$, using standard analysis of gradient flows, we obtain the following ideal   gradient    estimate.
\begin{proposition}\label{prop:gradient_oracle_error_control}
    Denote the initial sample $x^i\sim \widehat{\rho_N}$   after running the Wasserstein gradient flow sampler for time $t$ as    $y^i_t\sim \rho^t_{Y|X=x^i}$.
      For a fixed $\theta$,
    suppose that $\nabla_\theta\ell(\theta, y)$ is $L_f$-Lipschitz continuous in $y$ and
    $\widetilde V_{x, \tau}(y) : = -\ell(\theta, y)+\tfrac1{2\tau}c(y, x)$ is $L$-smooth and either
$\lambda$-convex
    with $\lambda > 0$
    or
    satisfies the
    $\lambda$-LSI
    as in Definition~\ref{as:gf-properties}.   
    Then, for sufficiently large $N$, in order to generate an $\epsilon_\text{grad}$-gradient estimate   in expectation  , i.e.,
    \begin{align*}
     \mathbb{E}\!\left[\,  \bigg|\frac1N \sum_{i=1}^N \nabla_\theta \ell(\theta, y^i_t) 
    -
    \nabla_\theta 
    \Phi(\theta)
    \bigg| \,\right]  
    {\leq}
    \epsilon_\text{grad},
    \end{align*}
    the Wasserstein gradient flow sampler needs to run for time at least\;
    $\displaystyle t \gtrsim \calO \left(\frac{1}{\lambda}\log{\frac{L_f}{\sqrt{\lambda}\epsilon_\text{grad}}}\right)$.
\end{proposition}
 
\begin{remark}
    The Lipschitz condition on $y \mapsto \nabla_\theta \ell(\theta, y)$ above is mild and standard, satisfied for example by losses with bounded mixed Hessian $\nabla_y \nabla_\theta \ell$. The same condition will be used in our discrete-time complexity analysis (Section~\ref{sec:complexity}).
\end{remark}
  


Results such as Proposition~\ref{prop:gradient_oracle_error_control} are based on the displacement convexity or PL/LSI in the Wasserstein space.
The value of our general gradient flow perspective is to let us freely choose the gradient flow geometry, not confined to Wasserstein.
This is not just for theoretical considerations --- the estimate such as
\eqref{eq:gf_time_to_epsilon}
depends crucially on the (PL/convexity) constant $\lambda$, which has been a major limitation of the Langevin type samplers in the literature.
  The potential speed-up of WFR over pure Wasserstein gradient flow comes from the Hellinger/Fisher-Rao reaction component when a suitable warm-start initialization is available; see \citet{luBirthdeathDynamicsSampling2023} and Appendix~\ref{sec:dual-review} for the precise warm-start condition.   
It provides a simple insight for algorithmic design:  the unbalanced WFR gradient flow can be used to speed up the sampling process for DRO.
This insight is later validated in the numerical studies.


As noted earlier, Proposition~\ref{prop:gradient_oracle_error_control} 
is aimed at providing the intuition for the behavior of the gradient flow sampler-based DRO algorithm; it is not the practical complexity estimate based on sampler's mixing time.
It serves as a guideline for the design of the gradient flow sampler-based DRO algorithm using existing tools from PDE/SDE.
Next, we provide optimization complexity estimate that characterizes the practical sampler-based DRO algorithm.





\section{Optimization Complexity Analysis}\label{sec:complexity}
The DRO problem \eqref{eq:main-ent-dro} can be written as 
$\min_{\theta\in\Theta} \Phi(\theta)$.
We analyze the computational complexity of our proposed discrete-time algorithms, aiming to find an $\epsilon_{\text{opt}}$-stationary point $\theta^{S}$ at the last iteration $S$,
  i.e., $\mathbb{E}[\|\nabla\Phi(\theta^{S})\|^{2}]\le\epsilon_{\text{opt}}^{2}$.   
Our analysis uses a standard framework for non-convex stochastic optimization \citep{ghadimi2013stochastic}. The outer loop performs SGD on $\theta$, while the inner loop generates samples to approximate the conditional worst-case distribution. The key challenge is controlling the bias in the stochastic gradient. We make the following assumptions regarding the objective function and the sampling process.

\begin{assumption}\label{as:global_assumptions}
    \begin{enumerate}
        \item \label{as:totalsmooth} \textbf{Smoothness of the   DRO    objective   in $\theta$  :}  There exists a constant $L_{\Phi} > 0$ such that
        \begin{equation*}
            \|\nabla\Phi(\theta_1) - \nabla\Phi(\theta_2)\| \le L_{\Phi}\|\theta_1-\theta_2\|, \forall \theta_1,\theta_2\in\Theta
        \end{equation*}
        \item \label{as:fsmooth}\textbf{Lipschitz continuity of the gradient oracle   in $z$  :} There exists a constant $L_{f} > 0$ such that,   for every $\theta \in \Theta$  ,
        \begin{equation*}
             \|\nabla_{\theta}\ell(\theta,   z_1  ) - \nabla_{\theta}\ell(\theta,   z_2  )\| \le L_{f}\|  z_1-z_2  \|, \forall   z_1,z_2\in\mathcal{Z}  
        \end{equation*}
        \item \label{as:gradest}\textbf{Sampler error and bounded variance:} The inner-loop sampler (Algorithm~\ref{alg:sampler}) generates a distribution $\widehat\rho_{Y|X=x}$, and  $\mathbb{E}_{x \sim \widehat{\rho_N}}[W_2(\widehat\rho_{Y|X=x}, \pi_{Y|X=x})] \le \delta_{\text{sample}}$. The stochastic gradient estimator $\widehat g$ has bounded variance $\sigma^2$,   i.e.,  $\mathbb{E}[\|\widehat g-\mathbb{E}[\widehat g ] \|^{2}]\le \sigma^2$   .
    \end{enumerate}
\end{assumption}
Under these standard assumptions, we can bound the gradient bias and establish a general convergence result for the outer loop.

\begin{theorem}[Outer Loop Convergence]
\label{thm:outer_loop}
Let Assumptions \ref{as:global_assumptions} hold. With a stepsize $r = \sqrt \frac{1}{SL_\Phi\sigma^2}$, the outer loop requires $S = O(\frac1{\epsilon_{\text{opt}}^4})$ iterations to find an $\epsilon_{\text{opt}}$-stationary point   (i.e. $\mathbb{E}[\|\nabla\Phi(\theta^{S})\|^{2}]\le\epsilon_{\text{opt}}^{2}$)   , provided the error $\delta_{\text{sample}}$ in Assumption~\ref{as:global_assumptions}.\ref{as:gradest} is controlled such that $\delta_{\text{sample}} = O(\frac {\epsilon_{\text{opt}}} {L_f})$.
\end{theorem}

This complexity aligns with   standard    non-convex stochastic algorithms. Next, we derive the complexity of Algorithm~\ref{alg:SDRO-NGD} as an example. For the analysis of the ULA-based sampler (Algorithm \ref{alg:SDRO-NGD}), we introduce the following assumption on the inner target distribution.

\begin{assumption}\label{as:LSI}
For any fixed $\theta$ and $x$, the conditional distribution $\pi_{Y|X=x}$ is $L_{U}$-smooth and satisfies the $\lambda_U$-log-Sobolev inequality (LSI); see Definition~\ref{as:gf-properties}   for the definitions   .
\end{assumption}






Assumption \ref{as:LSI} provides the necessary framework to determine the number of inner ULA iterations \citep{vempala2019rapid} required to achieve the sampling accuracy $\delta_{\text{sample}}$ (as specified in Assumption~\ref{as:global_assumptions}.\ref{as:gradest}). Crucially, Assumption \ref{as:LSI} relaxes the strict log-convexity assumption commonly adopted in previous works \citep{sinha2020certifyingdistributionalrobustnessprincipled, wang2021sinkhorn}. By combining the derived inner loop complexity analysis with the outer loop convergence rate from Theorem \ref{thm:outer_loop}, we can establish the total computational complexity. 

 
\begin{remark}[Beyond smoothness]
Recent work shows that ULA converges under dissipativity alone \citep{johnston2025performance}, without requiring global smoothness or LSI. Since Theorem~\ref{thm:outer_loop} (outer loop) is sampler-agnostic, any improved inner-loop bound directly translates into improved total complexity. In particular, plugging in such relaxed convergence results would extend our framework to non-smooth losses (e.g., ReLU networks) more rigorously, replacing Assumption~\ref{as:LSI}.
\end{remark}
  

\begin{theorem}[Complexity of   GF-DRO (Algorithm \ref{alg:SDRO-NGD})  ]\label{thm:ula}
Under Assumptions \ref{as:global_assumptions}, and \ref{as:LSI}, the total complexity for Algorithm \ref{alg:SDRO-NGD} to find an $\epsilon_{\text{opt}}$-stationary point is   $\tilde{O}\left(\frac d {\epsilon_{\text{opt}}^6}\right)=O\left(\frac d {\epsilon_{\text{opt}}^6} \cdot\log \frac 1 {\epsilon_{\text{opt}}} \right)$.  
\end{theorem}

The detailed derivations for the gradient bias, and the proofs of Theorem \ref{thm:outer_loop} and Theorem \ref{thm:ula} are provided in Appendix~\ref{ap:complexity_proofs}.



















\section{Experiments}\label{sec:experiments}

We conduct numerical experiments on different tasks to validate our theoretical insights. Our goal is to compare the performance of four different DRO methods: Algorithm~\ref{alg:SDRO-WFR} (WFR-DRO), Algorithm~\ref{alg:SDRO-NGD} (WGF-DRO), the dual method for SDRO (Dual) \citep{wang2021sinkhorn}, and WRM \citep{sinha2020certifyingdistributionalrobustnessprincipled}. We also compare algorithms based on Stein Variational Gradient (SVG-DRO) (Algorithm~\ref{alg:SDRO-SVG}) method and Restricted Gaussian Oracle \citep{lee2021structured} (RGO-DRO) (Algorithm~\ref{alg:SDRO_rgo}) method with those methods in   Appendices~\ref{sec:appx_features} and~\ref{sec:experiment3}  . For methodological consistency across all experiments, the Dual method uses Randomized Truncation MLMC \citep{blanchet2015unbiased} as suggested in \citep{wang2021sinkhorn}, and the WRM solves its inner problem using gradient descent.   Sections~\ref{sec:lenet_mnist} and~\ref{sec:cifar_resnet} evaluate adversarial robustness on increasingly deep convolutional architectures, while Section~\ref{sec:exp_latent_wfr} demonstrates the qualitative behaviour of the WFR sampler on a high-dimensional generative latent space.   
All experiments were conducted on a laptop with an NVIDIA RTX A3000 GPU using Python. The code is available at \url{https://github.com/ZusenXu/GFS-DRO} 




























\subsection{Adversarial Robustness on MNIST with LeNet-5}
\label{sec:lenet_mnist}

We evaluate our methods on MNIST using the LeNet-5 architecture in a high-dimensional, end-to-end training setting where the entire network is trained with DRO algorithms. We assess robustness using PGD attacks under both $L_2$ and $L_\infty$ norm constraints, and compare our proposed WFR-DRO and WGF-DRO ($\tau=1.0, \epsilon=0.05, m=8$) against WRM and the Dual Sinkhorn DRO approach.

\begin{table*}[htbp]
\centering
\begin{minipage}[t]{0.48\textwidth}
\centering
\caption{PGD-$L_2$ Robustness on MNIST LeNet-5 (Test Error \%, mean $\pm$ std)}
\label{tab:lenet_l2}
\resizebox{\linewidth}{!}{%
\begin{tabular}{lcccc}
\toprule
Method & $\Delta=0.00$ & $\Delta=0.025$ & $\Delta=0.05$ & $\Delta=0.075$ \\
\midrule
SAA     & $0.92{\scriptstyle\pm0.09}$ & $3.05{\scriptstyle\pm0.40}$ & $10.80{\scriptstyle\pm1.20}$ & $27.44{\scriptstyle\pm2.86}$ \\
WRM     & $0.98{\scriptstyle\pm0.09}$ & $3.47{\scriptstyle\pm0.39}$ & $11.66{\scriptstyle\pm1.50}$ & $29.22{\scriptstyle\pm3.18}$ \\
Dual    & $1.00{\scriptstyle\pm0.06}$ & $3.21{\scriptstyle\pm0.24}$ & $\phantom{0}9.62{\scriptstyle\pm0.74}$ & $24.20{\scriptstyle\pm1.82}$ \\
WGF-DRO & $0.92{\scriptstyle\pm0.08}$ & $2.97{\scriptstyle\pm0.36}$ & $\phantom{0}8.56{\scriptstyle\pm0.96}$ & $21.57{\scriptstyle\pm1.72}$ \\
WFR-DRO & $0.92{\scriptstyle\pm0.05}$ & $\mathbf{2.86}{\scriptstyle\pm0.18}$ & $\mathbf{\phantom{0}8.27}{\scriptstyle\pm0.29}$ & $\mathbf{20.81}{\scriptstyle\pm0.71}$ \\
\bottomrule
\end{tabular}%
}
\end{minipage}\hfill
\begin{minipage}[t]{0.48\textwidth}
\centering
\caption{PGD-$L_\infty$ Robustness on MNIST LeNet-5 (Test Error \%, mean $\pm$ std)}
\label{tab:lenet_linf}
\resizebox{\linewidth}{!}{%
\begin{tabular}{lcccc}
\toprule
Method & $\Delta=0.00$ & $\Delta=0.05$ & $\Delta=0.10$ & $\Delta=0.15$ \\
\midrule
SAA     & $0.92{\scriptstyle\pm0.09}$ & $3.48{\scriptstyle\pm0.56}$ & $11.05{\scriptstyle\pm1.98}$ & $27.70{\scriptstyle\pm4.52}$ \\
WRM     & $0.98{\scriptstyle\pm0.09}$ & $3.58{\scriptstyle\pm0.54}$ & $11.54{\scriptstyle\pm2.25}$ & $29.18{\scriptstyle\pm5.13}$ \\
Dual    & $1.00{\scriptstyle\pm0.06}$ & $3.13{\scriptstyle\pm0.30}$ & $\phantom{0}8.86{\scriptstyle\pm1.07}$ & $21.29{\scriptstyle\pm2.88}$ \\
WGF-DRO & $0.92{\scriptstyle\pm0.08}$ & $2.81{\scriptstyle\pm0.43}$ & $\phantom{0}7.57{\scriptstyle\pm1.27}$ & $17.79{\scriptstyle\pm2.72}$ \\
WFR-DRO & $0.92{\scriptstyle\pm0.05}$ & $\mathbf{2.68}{\scriptstyle\pm0.23}$ & $\mathbf{\phantom{0}7.41}{\scriptstyle\pm0.56}$ & $\mathbf{17.30}{\scriptstyle\pm1.57}$ \\
\bottomrule
\end{tabular}%
}
\vspace{-0.3cm}
\end{minipage}
\end{table*}

As shown in Tables~\ref{tab:lenet_l2} and~\ref{tab:lenet_linf}, WFR-DRO consistently yields the lowest test error across all non-zero attack intensities $\Delta$, with WGF-DRO and WFR-DRO essentially tied on clean accuracy. WGF-DRO outperforms WRM under non-trivial perturbations, suggesting that entropy regularization help in non-convex landscapes; the birth--death mechanism of WFR provides further improvement over WGF-DRO and outperforms the Dual baseline as well. WRM performs on par with SAA (within one standard deviation), which
is expected in the non-convex setting\citep{sinha2020certifyingdistributionalrobustnessprincipled}. For a complementary comparison on the convex feature-extracted CIFAR-10 setting, including RGO-DRO and SVG-DRO, see Appendix~\ref{sec:appx_features}.

 
\subsection{Adversarial Robustness on CIFAR-10 with ResNet-18}\label{sec:cifar_resnet}
We further validate the scalability of our methods in high-dimensional tasks by fine-tuning a pretrained ResNet-18 end-to-end on CIFAR-10. The DRO hyperparameters are $\tau = 0.1$ and $\epsilon = 0.05$ for Dual method, WGF-DRO and WFR-DRO. To align the comparison between WRM and methods based on SDRO, we set $\tau =0.1$ for WRM. And $m=8$ particles are used in WGF-DRO and WFR-DRO. The inner sampler uses step size $0.01$ for $20$ iterations across all methods. For the Dual baseline, we use the Randomized Truncation MLMC estimator \citep{blanchet2015unbiased} with maximum truncation level $4$. All methods are trained for 20 epochs with batch size 256. 

Tables~\ref{tab:resnet_l2} and~\ref{tab:resnet_linf} report the mean $\pm$ std of PGD-$L_2$ and PGD-$L_\infty$ test error under different attack intensities over 5 seeds. WFR-DRO achieves the lowest error across all non-zero attack intensities under both norms. WGF-DRO comes second, confirming that replacing the deterministic ODE with SDE alone already bring most of the robustness benefit, while WFR-DRO show an additional reduction over WGF-DRO. The Dual baseline performs well compared to WRM, suggesting that entropy regularization helps in this non-convex regime. WRM provides marginal improvement over SAA, consistent with WRM's reliance on convexity \citep{sinha2020certifyingdistributionalrobustnessprincipled} that fails to hold here. Detailed experimental settings, wall-clock timing, outer-loop convergence diagnostics, and a $\tau$-sensitivity analysis can be found in Appendix~\ref{sec:appx_diagnostics}.

\begin{table*}[h!]
\centering
\begin{minipage}[t]{0.48\textwidth}
\centering
\caption{PGD-$L_2$ Robustness on CIFAR-10 with ResNet-18 (Test Error \%, mean $\pm$ std)}
\label{tab:resnet_l2}
\resizebox{\linewidth}{!}{%
\begin{tabular}{lccccc}
\toprule
Method & $\Delta=0.00$ & $\Delta=0.005$ & $\Delta=0.01$ & $\Delta=0.015$ \\
\midrule
SAA      & $6.35{\scriptstyle\pm0.77}$  & $25.79{\scriptstyle\pm1.06}$ & $55.38{\scriptstyle\pm1.02}$ & $78.32{\scriptstyle\pm0.76}$  \\
WRM      & $6.27{\scriptstyle\pm0.55}$  & $22.27{\scriptstyle\pm0.49}$ & $47.76{\scriptstyle\pm1.80}$ & $71.24{\scriptstyle\pm2.28}$  \\
Dual     & $14.12{\scriptstyle\pm0.99}$ & $19.21{\scriptstyle\pm1.10}$ & $25.33{\scriptstyle\pm1.44}$ & $32.00{\scriptstyle\pm1.67}$  \\
WGF-DRO  & $12.09{\scriptstyle\pm0.75}$ & $16.47{\scriptstyle\pm0.69}$ & $21.81{\scriptstyle\pm0.88}$ & $27.70{\scriptstyle\pm1.06}$  \\ 
WFR-DRO  & $11.10{\scriptstyle\pm0.60}$ & $\mathbf{15.39}{\scriptstyle\pm0.73}$ & $\mathbf{20.63}{\scriptstyle\pm0.81}$ & $\mathbf{26.73}{\scriptstyle\pm0.72}$ \\

\bottomrule
\end{tabular}%
}
\end{minipage}\hfill
\begin{minipage}[t]{0.48\textwidth}
\centering
\caption{PGD-$L_\infty$ Robustness on CIFAR-10 ResNet-18 (Test Error \%, mean $\pm$ std)}
\label{tab:resnet_linf}
\resizebox{\linewidth}{!}{%
\begin{tabular}{lccccc}
\toprule
Method & $\Delta=0.00$ & $\Delta=0.005$ & $\Delta=0.01$ & $\Delta=0.015$ \\
\midrule
SAA      & $6.35{\scriptstyle\pm0.77}$  & $36.34{\scriptstyle\pm1.31}$ & $72.46{\scriptstyle\pm1.12}$ & $90.67{\scriptstyle\pm0.79}$ \\
WRM      & $6.27{\scriptstyle\pm0.55}$  & $31.26{\scriptstyle\pm0.88}$ & $65.07{\scriptstyle\pm2.12}$ & $86.35{\scriptstyle\pm1.57}$ \\
Dual     & $14.12{\scriptstyle\pm0.99}$ & $21.42{\scriptstyle\pm1.20}$ & $30.19{\scriptstyle\pm1.66}$ & $39.74{\scriptstyle\pm1.86}$ \\
WGF-DRO  & $12.09{\scriptstyle\pm0.75}$ & $18.48{\scriptstyle\pm0.74}$ & $26.18{\scriptstyle\pm1.11}$ & $35.02{\scriptstyle\pm1.13}$ \\ 
WFR-DRO  & $11.10{\scriptstyle\pm0.60}$ & $\mathbf{17.31}{\scriptstyle\pm0.76}$ & $\mathbf{25.18}{\scriptstyle\pm0.88}$ & $\mathbf{34.06}{\scriptstyle\pm0.83}$ \\

\bottomrule
\end{tabular}%
}
\end{minipage}
\end{table*}
  

\begin{figure}[h!]
  \centering

  \begin{subfigure}[b]{0.48\textwidth}
    \centering
    \includegraphics[width=\textwidth]{wfr_multi_sample_results.pdf}
    \caption{Semantic transitions in the latent space}
    \label{fig:wfr_latent}
  \end{subfigure}
  \hfill
  \begin{subfigure}[b]{0.48\textwidth}
    \centering
    \includegraphics[width=\textwidth]{wfr_topk_comparison.pdf}
    \caption{Comparison of sample diversity}
    \label{fig:wfr_diversity}
  \end{subfigure}
  \caption{\textbf{Performance and diversity of the WFR sampler.} \textbf{(a)} Semantic transitions.   Shown from left to right are the original images, intermediate samples at $T=100$ and $T=200$, and the final result at $T=300$.    \textbf{(b)} Sample diversity comparison.   The first column displays the original images, while the following three columns show the three highest-weighted samples generated at $T=300$.   Unlike deterministic methods that are restricted to a single adversarial solution, the WFR sampler successfully generates multiple distinct samples from the conditional worst-case distribution.}
  \label{fig:wfr_combined}
\end{figure}
\subsection{Human Face Classification}
\label{sec:exp_latent_wfr}

In this section, we apply the WFR gradient flow sampler to the 512-dimensional latent space of a pre-trained ALAE \citep{pidhorskyi2020adversarial} model to investigate how the sampler identifies the worst-case distribution for high-dimensional image data. We employ a linear logistic regression classifier trained on the FFHQ dataset to classify gender attributes in the latent space; the gradient of this classifier then serves as the potential function.   This experiment is qualitative: we use it to visually demonstrate the diversity of worst-case samples generated by the WFR sampler, complementing the quantitative robustness evidence in Sections~\ref{sec:lenet_mnist} and~\ref{sec:cifar_resnet}.   

The WFR sampler is configured with parameters $T=300$, $\eta = 0.05$, $\epsilon=0.2$, and $\tau=5$. As shown in Figure~\ref{fig:wfr_latent}, the sampler produces continuous semantic transitions while maintaining key identity details such as pose and expression. Unlike deterministic baselines such as WRM, which converge to a single adversarial solution, the WFR sampler generates diverse samples. This diversity is illustrated in Figure~\ref{fig:wfr_diversity}, where the top-3 weighted particles represent distinct stylistic variations for the same input, suggesting that the model effectively   generates    multiple plausible perturbations simultaneously.



\section{Other Works and Discussion}

Our work is positioned at the intersection of sampling   via gradient flow    and robust optimization. 
There is a thread of works that considered the (D)RO problems via gradient descent-ascent dynamics \citep{wang2022exponentially,yuFastDistributionallyRobust2022,trillosAdversarialRobustnessUse2023,congerStrategicDistributionShift2023}. 
They typically use proximal algorithms in the Wasserstein space    \emph{iteratively}, i.e.,  the Jordan-Kinderlehrer-Otto (JKO) scheme in PDE    , which can be viewed as one-step discretization of the continuous time PDE
  $\partial_t \rho = \nabla \cdot \left( \rho  \nabla \xi \right)$ for some $\xi$    . In this paper, we shift the perspective and view the inner maximization problem as a regularized variational problem that exactly corresponds to a sampling algorithm, without explicitly resorting to JKO iteratively. This allows our methodology to serve as a general-purpose   sampler-based bi-level optimization framework    \citep{nemirovski2009robust}, enabling a direct approach based on optimizing measures.


This perspective possesses significant unifying power: it reveals that previous methods such as WRM \citep{sinha2020certifyingdistributionalrobustnessprincipled} are fundamentally rooted in ODE analysis, which can be seen as a special case of the S/PDE framework proposed here.   Our framework is also orthogonal to map-based approaches such as FlowDRO \citep{xu2024flow} and JKO-iterative analyses such as~\citet{zhu2024distributionally} (already discussed in Section~\ref{sec:intro}): we address DRO via a sampling methodology built on the equivalence between the inner problem and an entropy-regularized JKO step.    This also allows optimizers to go beyond the standard (Wasserstein) gradient descent type of algorithm, inspiring novel approaches such as SVG and WFR   gradient flows   . Our theory treats general DRO by simply changing the corresponding gradient flow energy function $F$ in \eqref{eq:conditional-sampling-energy}, opening the door for future research into diverse DRO ambiguity notions and gradient flow   dissipation    geometries. 


  A practical SGDA-style variant of our framework is also possible. Instead of running the inner sampler to convergence, one can reuse the perturbed samples from the previous iteration and take only a few inner update steps before each outer parameter update. A full theoretical analysis of this SGDA variant is left to future work.   



\newpage













\appendix
\onecolumn


\section{Further Background}

\subsection{Dual approach to Entropy-regularized Wasserstein DRO}
\label{sec:dual-review}
The Sinkhorn DRO problem, as formulated by \citet{wang2021sinkhorn}, aims to find a decision that minimizes the worst-case expected loss over an ambiguity set defined by Sinkhorn divergence, which is identical to entropy-regularized Wasserstein distance in this problem. Given a loss function $\ell(\theta, z)$, a nominal distribution $\widehat{\rho_N}$, and a radius $r > 0$, the primal inner problem is:
\begin{equation}
\sup_{\rho\in \calB_{r, \epsilon}(\widehat{\rho_N})} \E_{z \sim \rho}[\ell(\theta, z)],
\label{eq:appendix-primal}
\end{equation}
where the ambiguity set is $\calB_{r, \epsilon}(\widehat{\rho_N}) := \{ \rho \in \calP(\calZ) : W_{\epsilon}^2(\widehat{\rho_N}, \rho) \le r \}$, and the entropy-regularized Wasserstein distance,
\[W^2_{\epsilon}(\rho_1, \rho_2) = \inf_{\gamma\in \Gamma(\rho_1,\rho_2)} \{\E_{(x,y)\sim\gamma}[c(x,y)] + H(\gamma)\}.\]

A dual approach, established by \citet{wang2021sinkhorn}, reformulates the inner maximization problem. Using the strong duality of this problem and substituting \eqref{eq:worst-dist} into this problem, the inner problem can be expressed as a minimization over a dual variable $\tau$:
\begin{equation}
\inf_{\tau \ge 0} \left\{ \frac{r}{2\tau} +\frac{\epsilon}{2\tau} \E_{x\sim\widehat{\rho_N}} \left[ \log \E_{y\sim\rho_{x,\epsilon}} \left[ \exp\left(\frac{2\tau\ell(\theta, y)}{\epsilon}\right) \right] \right] \right\},
\label{eq:appendix-dual}
\end{equation}
where $\rho_{x,\epsilon}(y) = \frac{\exp(\frac{-c(x,y)}{\epsilon})}{Z}$ is a kernel distribution centered at $x$, and $Z$ is a normalization constant.

To approximate the value and the gradient of the nested expectation in \eqref{eq:appendix-dual}, \citet{wang2021sinkhorn} employ a two-level sampling procedure for a given $\tau$: first, a sample $x$ is drawn from the nominal distribution $\widehat{\rho_N}$; second, a set of samples are drawn from the kernel distribution $\rho_{x,\epsilon}$ to approximate the inner expectation.

However, a major limitation of this dual method is that the nested expectation structure in the objective leads to a biased subgradient estimator when using standard Monte Carlo sampling, especially for small values of $\epsilon$. This bias can impede the convergence of the optimization procedure. This challenge motivates the exploration of alternative approaches, such as the gradient flow methods discussed in this paper, which can directly sample from the primal worst-case distribution to obtain a less biased estimate of the gradient.

\begin{remark}
    Consider the KL-DRO problem:
    \begin{equation}
    \min_\theta \sup_{\rho \ll \rho_0} \left\{ \mathbb{E}_{y \sim \rho}[\ell(\theta, y)] - \frac{1}{2\tau} \KL(\rho \| \rho_0) \right\} \label{eq:dro_objective_modified}
    \end{equation}
    Let us define a reference distribution $\rho_0(y)$ as a kernel density estimator of the empirical distribution $\widehat{\rho_N} = \frac{1}{N}\Sigma_{i=1}^N \delta(x_i)$: $\rho_0(y) = \frac{1}{N}\Sigma _{i=1}^N K_\epsilon(y,x_i)$, where $K_\epsilon(y,x_i)= \alpha_\epsilon \exp(-c(y,x_i)/\epsilon)$ and $\alpha_\epsilon = \E[\exp(-c(y,x_i)/\epsilon)]^{-1}$. With this choice, and by setting $\tau = \tau'/\epsilon$, the solution to the inner maximization problem is given by \citet{hu2013kullback}:
    \begin{align*}
        \rho^{\KL}(y) &= \rho_0(y) \cdot \frac{\exp(2\tau'\ell(\theta, y)/\epsilon)}{\mathbb{E}_{y \sim \rho_0}[\exp(2\tau'\ell(\theta, y)/\epsilon)]} \\
    &= \frac{1}{N}\Sigma _{i=1}^N \alpha_\epsilon \beta \exp((2\tau'\ell(\theta, y)- c(y,x_i))/\epsilon),
    \end{align*}
    where $\beta = \E_{y\sim \rho_0}[\exp(2\tau'\ell(\theta, y)/\epsilon)]^{-1}$. This resulting distribution bears a strong resemblance to the worst-case distribution for entropy-regularized DRO. However, they are not identical, as the normalization constants differ: $\alpha_\epsilon \beta \neq \alpha_x:= \E[\exp((2\tau'\ell(\theta, y)- c(y,x))/\epsilon]^{-1}$. In fact, the term $\alpha_\epsilon \beta \exp((2\tau'\ell(\theta, y)- c(y,x))/\epsilon)$ does not integrate to one and thus does not define a valid conditional probability measure. Actually, $\rho^{\KL}(y)$ can be interpreted as a weighted expectation of conditional distributions:
    \[
    \rho^{\KL} = \E_{x\sim\widehat{\rho_N}}[\frac{\alpha_\epsilon \beta}{\alpha_x} \rho_{Y|X=x}]
    \]
    Consequently, the two-layer sampling procedure requires computing the corresponding weights, sampling from the resulting weighted empirical distribution, and subsequently sampling from the conditional distribution. However, the term $\alpha_x$ is intractable in practice. Therefore, unlike the entropy-regularized DRO case, the KL-DRO problem cannot be directly framed as a two-level sampling task.
\end{remark}

\subsection{Hellinger-Kantorovich a.k.a. Wasserstein-Fisher-Rao Gradient Flows}

Consider the WFR gradient system of the energy functional $F$, i.e., the triple $\left( \calP, F, \WFR \right)$, where $\WFR$ is the Wasserstein-Fisher-Rao metric, a.k.a. the \emph{Hellinger-Kantorovich} metric restricted to the probability space.
Specifically, we consider the HK/WFR gradient system associated with reaction-diffusion PDE:
\begin{align}
    {\partial_t \rho}
    &= \alpha\, \mathrm{div} \left( \rho \nabla \frac{\delta F}{\delta \rho} \right) - \beta\, \rho 
    \left(\frac{\delta F}{\delta \rho} - \int \frac{\delta F}{\delta \rho}\dd \rho \right)
    .
\end{align}
The hope of this gradient flow is that the added reaction term on the right-hand side will help the gradient flow to converge more rapidly.
This can be formally seen by checking the time derivative of the energy functional $F$ along the gradient flow \eqref{eq:wfr-gradient-flow}, we have
\begin{align*}
    \partial_t F (\rho_t)
    \overset{\text{(EDB)}}{=}
    -\alpha \int \Bigg|\nabla  \log \frac{\rho}{\pi}(x) \Bigg| ^2\dd \rho (x) 
    -\beta \int \Bigg| \log \frac{\rho}{\pi}(x)
    -
    \KL(\rho | \pi)
    \Bigg| ^2\dd \rho (x) 
    \end{align*}
where we used the Energy-Dissipation Balance (EDB, a.k.a. equality) of gradient flows.
We observe the right-hand side is non-positive, hence its dissipation will be faster than the original pure Wasserstein gradient flow.

However, we must note that there is no global PL or strong convexity analogous to the case of the Wasserstein gradient flow, as discussed in the main text.
A ``warm-start'' condition is necessary as it has been shown that global PL/gradient dominance cannot hold for the Fisher-Rao/(spherical)Hellinger gradient flows; see \citet{mielke2025hellinger,carrillo2024fisher} for details.
There exist fine-grained warm-start initialization conditions in the gradient flow literature; see \citet{luBirthdeathDynamicsSampling2023,luAcceleratingLangevinSampling2019,chen2023sampling}.

For example, \citet{luBirthdeathDynamicsSampling2023}
showed that, after a warm-start initialization in the form of a density-ratio lower bound, it is possible to get an improved exponential convergence rate
than the pure Wasserstein gradient flow as characterized in Lemma~\ref{lemma:gradient-dominance-PL-Wasserstein}.
However, it is important to note that
the specific warm-start initialization condition cannot be verified in our DRO problems.
Nonetheless, in practice, we observe significant speedups using the WFR gradient flow as compared to the pure Wasserstein gradient flow.


\section{Further Methods}

\subsection{Wasserstein Fisher-Rao Gradient Flow-based DRO}\label{sec:wfr-alg}
This subsection details the DRO method based on the Wasserstein Fisher-Rao gradient flow. Algorithm \ref{alg:SDRO-WFR} provides a comprehensive summary of this procedure. Compared to the WGF-based method, this approach further refines the DRO framework by introducing birth-death mechanism.

 
The WFR gradient flow associated with the energy $F$ in \eqref{eq:conditional-sampling-energy} is governed by the reaction--diffusion PDE \eqref{eq:wfr-gradient-flow}. Discretizing it via interacting particles, each particle $(y^i, w^i)$ evolves through two coupled updates: a \emph{transport update} identical to ULA, and a \emph{reaction update} on the weights:
\begin{align*}
y^i_{t+1} &= y^i_t - \eta\,\nabla \widetilde V_{x,\tau}(y^i_t) + \sqrt{\eta\epsilon/\tau}\,\xi^i_t, &\textrm{(transport)}\\
w^i_{t+1} &= (w^i_t)^{1-\epsilon\eta_w/(2\tau)}\cdot \exp\!\left(-\eta_w\, \widetilde V_{x,\tau}(y^i_t)\right). &\textrm{(reaction)}
\end{align*}
The reaction term increases or decrease the weights of particles based on their energy, low-energy particles gain mass, while high-energy ones lose mass. Once a particle's weight drops below a threshold $w_{\min}$, it is reborn at the location of a randomly selected surviving particle, with the two particles sharing the original mass equally. This birth--death mechanism prevents the wasted computation of tracking particles stuck in local minima, which we observe empirically in Figure~\ref{fig:perturbation_samples} of Appendix~\ref{sec:experiment3}.

The full algorithm is summarized in Algorithm~\ref{alg:SDRO-WFR}.
  
\begin{algorithm}[h!]
\caption{Entropy-regularized Wasserstein DRO via WFR flow}
\label{alg:SDRO-WFR}
\begin{algorithmic}[1]
   \STATE {\bfseries Input:} Empirical distribution $\widehat{\rho_N}$, constraint set $\Theta$, $\tau, \epsilon >0$, stepsize sequence $\{r_s > 0\}_{s=0}^{S-1}$, inner stepsize $\eta$, weight stepsize $\eta_w$, weight threshold $w_{\text{min}}$, inner iterations $T$, number of samples $m$.
  
  \FOR{$s = 0, \dots, S-1$}
    \STATE Sample $x^s \sim \widehat{\rho_N}$
    \STATE Initialize $y_{0}^{i,s} \leftarrow x^s, i=1,\dots,m$
    \FOR{$t = 0, \dots, T-1$}
      \STATE Sample $\xi^{i,s}_t \sim\mathcal{N}(0,I)$
      \STATE $y_{t+1}^{i,s} \leftarrow y_{t}^{i,s} - \eta\nabla\widetilde V_{x^s, \tau}(y_{t}^{i,s}) + \sqrt{\eta\epsilon/\tau} \xi^{i,s}_t$
      \STATE $w_{t+1}^{i,s} \leftarrow (w_{t}^{i,s})^{1-\epsilon\eta_w/ (2\tau)}e^{-\eta_w \widetilde V_{x^s, \tau}(y_{t}^{i,s})}$
      \STATE Normalize weights: $w_{t+1}^{i,s} \leftarrow w_{t+1}^{i,s}/\sum_{j=1}^m w_{t+1}^{i,s}$
      
      \STATE \textbf{Particle death/rejuvenation}: 
      \IF{$w_{t+1}^{i,s} < w_{\text{min}}$}
        \STATE Uniformly select an index $i'$ from $\{j : j \neq i\}$.
        \STATE $y_{t+1}^{i,s} \leftarrow y_{t+1}^{i',s}$, \, $w_{t+1}^{i',s}, w_{t+1}^{i,s}\leftarrow (w_{t+1}^{i,s} + w_{t+1}^{i',s})/2$.
      \ENDIF
    \ENDFOR
    \STATE $\theta^{s+1} \leftarrow \text{Proj}_\Theta (\theta^s-r_s \sum_{i=1}^m w_{T}^{i,s}\nabla_\theta \ell(\theta^{s},y_{T}^{i,s}))$
  \ENDFOR
  \STATE {\bfseries return} $\theta^S$
\end{algorithmic}
\end{algorithm}
\subsection{Stein Variational Gradient-based DRO}\label{sec:svg}
Stein Variational Gradient Descent (SVGD) is a variational inference algorithm that approximates a target probability density $p(x)$ by iteratively transporting a set of particles $\{ x_i \}_{i=1}^n$ \citep{liu2016stein}. The method is formulated as a functional gradient descent on the Kullback-Leibler (KL) divergence, $\text{KL}(q||p)$, where $q$ represents the empirical distribution of the particles. The particle updates are governed by a velocity field $\phi(x_i)$ that corresponds to the direction of steepest descent. For an empirical measure of $n$ particles, this update is given by:
\begin{equation*}
\boldsymbol{\phi}(x_i) \propto \frac{1}{n} \sum_{j=1}^n \left[ k(x_j, x_i) \nabla_{x_j} \log p(x_j) + \nabla_{x_j} k(x_j, x_i) \right]
\end{equation*}
This update rule can be decomposed into two functional components: (i) a driving force, which is a kernel-smoothed average of the score function $\nabla \log p(x)$ that directs particles towards the modes of the target distribution, and (ii) a repulsive force, which arises from the kernel gradient $\nabla k(\cdot, \cdot)$ and ensures particle diversity to prevent mode collapse \citep{ba2021understanding}.

The iterative application of this update rule forms the basis of the SVGD algorithm. As detailed in \citep{liu2016stein}, its computational complexity is dominated by the pairwise kernel computations, resulting in a cost of $O(m^2d)$ per iteration for $m$ particles in $d$ dimensions. Regarding convergence, theoretical guarantees have been established under certain assumptions. As shown by \citep{salim2022convergence}, for target distributions that satisfy Talagrand's T1 inequality, a finite-iteration complexity bound is derived. A finite-particle convergence analysis is also provided in \citep{shi2023finite}.

By applying SVGD to sample from the worst-case distribution, we arrive at Algorithm \ref{alg:SDRO-SVG}.

\begin{algorithm}[tb]
\caption{Sinkhorn DRO via SVGD}
\label{alg:SDRO-SVG}
\begin{algorithmic}[1]
  \STATE {\bfseries Input:} Empirical distribution $\widehat{\rho_N}$, constraint set $\Theta$, $\tau, \epsilon >0$, stepsize sequence $\{r_s > 0\}_{s=0}^{S-1}$, inner stepsize $\eta$, inner iterations $T$, initial deviation $\sigma$, number of samples $m$.
  
  \FOR{$s = 0, \dots, S-1$}
    \STATE Sample $x^s \sim \widehat{\rho_N}$
    \STATE Sample $\xi^{s,i}\sim\mathcal{N}(0,\sigma I) ,\, i=1,\dots,m$
    \STATE Initialize $y^{s,i}_0 \leftarrow x^s + \xi^{s,i}$
    \FOR{$t = 0, \dots, T-1$}
      \STATE $\phi^{s,i}_t \leftarrow \frac{1}{m}\sum_{j=1}^m\left[-k(y^{s,i}_t, y^{s,j}_t)\cdot \frac{2\tau}{\epsilon}\nabla\widetilde V_{x^s, \tau}(y^{s,j}_t) + \nabla_2 k(y^{s,i}_t, y^{s,j}_t) \right]$
      \STATE $y^{s,i}_{t+1}\leftarrow y^{s,i}_t + \eta \phi^{s,i}_t$
    \ENDFOR
    \STATE $\theta^{s+1} \leftarrow \text{Proj}_\Theta (\theta^s-r_s \sum_{i=1}^m \frac{1}{m}\nabla_\theta \ell(\theta^{s},y^{s,i}_T))$
  \ENDFOR
  \STATE {\bfseries return} $\theta^S$
\end{algorithmic}
\end{algorithm}

However, the empirical performance of Algorithm~\ref{alg:SDRO-SVG} is suboptimal. We observe that the convergence is sensitive to the initialization of the particles. Furthermore, when the number of particles $m$ is small, the magnitude of the driving force to dominate the repulsive force, thereby diminishing the variance. In this regime, the SVGD algorithm degenerates into a simple gradient descent method, losing its sampling capabilities. This behavior is demonstrated empirically in Appendix~\ref{sec:experiment3}.


\subsection{Restricted Gaussian Oracle-based DRO}\label{sec:rgo}

In this section, we explore an alternative approach for sampling from the worst-case distribution based on Proximal Sampler. The \textit{Proximal Sampler} \citep{lee2021structured} was introduced to unbiasedly sample from Gibbs distributions of the form $\nu(x) \propto \exp(-f(x))$. A single iteration of the proximal sampler consists of the following two steps:
\begin{enumerate}
    \item (\textbf{Forward step}): Sample $y_k \mid x_k \sim \nu_\tau^{Y|X}(\cdot \mid x_k) = \mathcal{N}(x_k, \tau I)$. This results in a new iterate $y_k \sim \rho^Y_k$, where $\rho^Y_k = \rho^X_k * \mathcal{N}(0, \tau I)$.
    \item (\textbf{Backward step}): Sample $x_{k+1} \mid y_k \sim \nu_\tau^{X|Y}(\cdot \mid y_k)$. This gives the next iterate $x_{k+1} \sim \rho^X_{k+1}$.
\end{enumerate}


As shown in recent work \citep{chen2022improved}, this forward-backward procedure is equivalent to an entropy-regularized Jordan-Kinderlehrer-Otto (JKO) scheme :
\begin{align}\label{eq:forward_app}
   \rho_k^Y &= \arg\min_{\mu \in P_2(\mathbb{R}^d)} \left\{ \frac{1}{2\tau} W^2_{2\tau}(\rho_{k}^{X}, \mu) \right\}, \\
   \rho_{k+1}^{X} &= \arg\min_{\mu \in P_2(\mathbb{R}^d)} \left\{ \int f \, d\mu + \frac{1}{2\tau} W^2_{2\tau}(\rho_{k}^{Y}, \mu) \right\}, \label{eq:backward_app}
\end{align}
This connection to the entropy-regularized JKO scheme provides a new perspective to our problem. By reformulating our original Lagrangian problem (\eqref{eq:main-ent-dro}) as a minimization and specializing the cost to the squared Euclidean distance, we obtain:
\begin{equation}\label{pb:sinkhornscheme_modified}
    \Phi(\theta)=\min_{\rho\in\calP} \{\E_{y\sim\rho}[-\ell(\theta, y)] + \frac{1}{2\tau} W^2_{\epsilon}(\rho,\widehat{\rho_N})\},
\end{equation}
which precisely matches the form of the problem \eqref{eq:backward_app} solved by backward step. This structural equivalence suggests that methods developed for proximal sampler can be directly applied to solve the entropy-regularized DRO problem. Specifically, we can employ the \textit{Restricted Gaussian Oracle (RGO)} from \citet{lee2021structured} to solve this. The sampling process for a given $x \sim \widehat{\rho_N}$ involves two steps:
\begin{enumerate}\label{alg:rejection-sampling}
    \item Sample $x$ from $\widehat{\rho_N}$, and compute the minimizer \( y^*_{x, \tau} = \arg\min_{y \in \mathbb{R}^d} \{\widetilde V_{x, \tau}(y)\} \).
    \item Repeat until acceptance: draw a sample \( Z \sim \mathcal{N} \left( y^*_{x, \tau}, \frac{\epsilon}{2(1 - L\tau)} I \right) \) and accept it with probability
    \[
    \exp \left(- \widetilde V_{x, \tau}(Z) +\widetilde V_{x, \tau}(y^*_{x, \tau}) + \frac{2(1 - L\tau)}{\epsilon} \| Z - y^*_{x, \tau} \|^2 \right).
    \]
\end{enumerate}


The validity of the RGO sampler requires the loss function $\ell$ to be $L$-smooth, and the penalty parameter $\tau$ must satisfy $\tau < 1/L$. This condition ensures that the auxiliary function $\widetilde V_{x, \tau}(y)$ is $(\frac{2(1-L\tau)}{\epsilon})$-strongly convex, which is crucial for rejection sampling. Applying RGO to the entropy-regularized DRO problem necessitates that the loss function is $L$-smooth and requires solving a convex optimization subproblem in each step. These requirements on the loss function and the computational structure are analogous to the WRM algorithm presented in \citep{sinha2020certifyingdistributionalrobustnessprincipled} for the specific case of a squared Euclidean cost. However, a key difference exists: the WRM algorithm transports each data point to a single worst-case point, whereas in entropy-regularized DRO, each data point induces a full continuous distribution of worst-case scenarios. 
\begin{algorithm}[tb]
\caption{Sinkhorn DRO via RGO}
\label{alg:SDRO_rgo}
\begin{algorithmic}[1]
  \STATE {\bfseries Input:} Empirical distribution $\widehat{\rho_N}$, constraint sets $\Theta$, $\tau, \epsilon > 0$, stepsize sequence $\{r_s > 0\}^{S-1}_{s=0}$
  
  \FOR{$s = 0, \dots, S-1$} 
    \STATE Sample $x^s \sim \widehat{\rho_N}$ and find an $\eta$-approximate minimizer $\hat{y}^s$ of $\widetilde{V}_{\tau,x^s}(y)$
    \STATE Generate samples $\{y^{s,i}\}_{i=1}^m$ via rejection sampling
    \STATE $\theta^{s+1} \leftarrow \text{Proj}_\Theta (\theta^s-\frac{r_s}{m}\sum^m_{i=1} \nabla_\theta \ell(\theta^{s}, y^{s,i}))$
  \ENDFOR
  \STATE {\bfseries return} $\theta^S$
\end{algorithmic}
\end{algorithm}


\begin{remark}
Although WRM\citep{sinha2020certifyingdistributionalrobustnessprincipled} assumes an $L$-smooth loss, its algorithm is “parameter-free” – it never needs to know $L$ and works whenever the inner optimization problem admits a closed-form solution. In contrast, Algorithm \ref{alg:SDRO_rgo} requires explicit knowledge of $L$ and cannot be applied if the loss fails to be $L$-smooth. In practice, we may assume $\tau$ is sufficiently small such that $\tau < 1/L$, and $\ell$ is $L$-smooth for all $y$.
\end{remark}


\section{Proofs of Theoretical Results} \label{ap:complexity_proofs}

\subsection{Proof of Proposition \ref{lem:sb-klform}}
\begin{proof}[Proof of Proposition \ref{lem:sb-klform}]
    We rewrite the problem using the definition of the entropy-regularized OT formulation:
    \begin{align}
    \min_{\rho} \min _{\Pi}
    \biggl\{
    \int V \dd \rho + \frac{1}{2\tau }
     \int c(x,y) \dd\Pi + \frac\epsilon{2\tau}\int \log \Pi \dd\Pi
    \biggr\}
    \end{align}
    subject to the margianl constraints $\int \Pi \dd x = \rho$ and $\int \Pi \dd y = \rho_0$.
    Here,
    $\rho_0$ can be set to empirical distribution $\widehat{\rho_N}$ as in the data-driven DRO problem.
    Using the marginal constraints, 
    we can then write the first linear term as
    $\int \int V(y) \dd \Pi(x, y)$ and eliminate the variable $\rho$, i.e., one end is unconstrained -- hence it's the half bridge problem:
    \begin{align}
         \min _{\Pi}
        \biggl\{
        \int V \dd \Pi + \frac{1}{2\tau }
         \int c(x,y) \dd\Pi + \frac\epsilon{2\tau}\int \log \Pi \dd\Pi
         \, \,
         \bigg |
         \int \Pi \dd y =  \rho_0
         \biggr\}
         .
        \end{align}
    After straightforward linear optimization,
    we obtain the optimal solution to the variational problem~\eqref{eq:proximal-EOT} as
    \begin{align}
        \rho^*(y) = \E_{x\sim \rho_0} \left[
            \frac{1}{Z_x}\exp
            \left[{-\frac{2\tau V(y) +c(y, x) }{\epsilon}} \right]    
         \right],\quad   Z_x = \int \exp\!\left[-\frac{2\tau V(y)+c(y,x)}{\epsilon}\right]\dd y.  
    \end{align}
    Using elementary manipulations, the variational problem \eqref{eq:proximal-EOT}
    can also be written down as the minimization of the expected KL divergence:
    \begin{align}
        \min_{\rho_{Y|X}}
    \mathbb{E}_{x\sim\rho_0}
    		\KL
            \left(
            \rho_{Y|X=x}(y) 
            \bigg| 
    		\dfrac{1}{Z_x}
            \exp \left[
    					-\dfrac{2\tau V(y) +  c(y,x)}{\epsilon} 
    		\right]
            \right)
            .
    \end{align}
\end{proof}

\subsection{Proof of Proposition~\ref{prop:gradient_oracle_error_control}}
First, we recall some standard results.
\begin{lemma}
    \label{lemma:displacement-convexity}
    Suppose the first condition in Definition~\ref{as:gf-properties} holds for some $\lambda > 0$. Then,
    $F$ is $\lambda$-displacement convex.
\end{lemma}

\begin{lemma}[WGF gradient dominance/PL]
    \label{lemma:gradient-dominance-PL-Wasserstein}
    Let $\rho_t$ be a solution to the Wasserstein gradient flow of the KL divergence energy functional $\KL(\rho | \exp(-\widetilde V_{x, \tau}))$. Then,
\begin{align*}
    \text{$\widetilde V$ is $\lambda$-displacement convex or $\lambda$-PL}
    \implies
    \KL(\rho_t| \exp(-\widetilde V_{x, \tau})) 
    \leq
    \ee^{-2\lambda t} \KL(\rho_0| \exp(-\widetilde V_{x, \tau})).
\end{align*}
\end{lemma}

\begin{lemma}[Talagrand's inequality]
    Under the same assumptions as in Lemma \ref{lemma:gradient-dominance-PL-Wasserstein}, we have
    \label{lemma:talagrand-inequality}
\[
W_2^2(\rho_t, \exp(-\widetilde V_{x, \tau})) \leq
\sqrt{\frac2\lambda \KL(\rho_t | \exp(-\widetilde V_{x, \tau}))} 
\]
\end{lemma}
Recall also that $W_1(\rho_t, \pi_{Y|X=x}) \leq W_2(\rho_t, \pi_{Y|X=x})$.


\textbf{From conditional to marginal bound}
We have the following relation between the marginal and conditional KL divergences.
Suppose $\rho_X$ is the marginal distribution of $X$.
Recall the relation $\pi_Y = \int_x \rho^* _{Y|X=x} \dd \rho_X(x)$.
Then,
\[
\KL(\rho_Y | \pi_{Y}) \leq \mathbb{E}_{X \sim \rho_X} \left[ \KL\left(\rho_{Y|X}  \middle| \pi_{Y|X}\right) \right].
\]

Alternatively, the Wasserstein relation gives
\[
W_p(\rho_Y, \pi_Y) \leq \mathbb{E}_{X \sim \rho_X} \left[ W_p(\rho_{Y|X}(\cdot|X), \pi_{Y|X}(\cdot|X)) \right].
\]

\textbf{Marginal distribution error control}
Suppose we have the samples
$y^i_t\sim \rho_{t,Y}$,
Our goal is to use the empirical measure
$\displaystyle\widehat{\rho_{t,Y}} = \frac1N \sum_{i=1}^N \delta_{y^i_t}$
to approximate $\pi_Y$.
 
Taking expectations and applying the triangle inequality, we have
\begin{align}
    \mathbb{E}[W_1(\widehat{\rho_{t,Y}}, \pi_Y)]
    &\leq \mathbb{E}[W_1(\widehat{\rho_{t,Y}}, \rho_{t,Y})] + W_1(\rho_{t,Y}, \pi_Y),
    \label{eq:marginal_w1_decomp}
\end{align}
where the first term is from particle approximation, which vanishes as $N\to\infty$. And the second term is the optimization error along the gradient flow. By Lemma~\ref{lemma:gradient-dominance-PL-Wasserstein} and Lemma~\ref{lemma:talagrand-inequality}, together with the conditional-to-marginal Wasserstein bound,
\begin{align}
    W_1(\rho_{t,Y}, \pi_Y) 
    &\leq \mathbb{E}_{X \sim \rho_X} \left[ W_2(\rho_{t,Y|X}, \pi_{Y|X}) \right]
    \lesssim 
    \frac{{\ee^{-\lambda t}}}{\sqrt{\lambda}}.
    \label{eq:marginal_w1_decay}
\end{align}

Therefore, to reach a $\delta_\text{sample}$-solution in expectation, we need to run the gradient flow (i.e. simulating the PDE/SDE) for time
\begin{align}
    t \gtrsim \frac{1}{\lambda}\log{\frac1{\sqrt{\lambda}\delta_\text{sample}}}
    .
    \label{eq:gf_time_to_epsilon}
\end{align}
  
We will later show the discrete-time gradient descent version of this result.

We emphasize that the continuous-time gradient flow result is important not only for understanding the limit of discrete-time algorithms, but also for serving as a guide for designing new discrete-time algorithms.
Let us get a quick insight from the above estimate:
suppose we discretized algorithm with discretization stepsize $\eta$ for in total $T$ steps. Ignoring the discretization error, we have $t = T\eta$ and the (ideal) number of iterations needed to reach a $\delta_\text{sample}$-solution
of the inner maximization problem~\eqref{eq:entropic-jko}
is $T \gtrsim \frac{1}{\eta  \lambda}\log{\frac1{\sqrt{\lambda}\delta_\text{sample}}}$.

\textbf{DRO gradient oracle error control via gradient flow}
Let us now show how to convert the error estimate in distributions to the error estimate in the gradient oracle, needed for the optimization analysis later.
If   $\mathbb{E}[W_1(\widehat{\rho_{t,Y}}, \pi_Y)] \leq \delta_\text{sample}$    for some $\delta_\text{sample} > 0$ (as controlled by \eqref{eq:marginal_w1_decay}) and $g$ is $L$-Lipschitz (w.r.t. Euclidean norm).
Then,
  by Kantorovich--Rubinstein duality \citep{kantorovich1942translocation},  
we obtain:
\begin{align}
    \label{eq:gradient_oracle_error_control_expectation}
     \mathbb{E}\!\left[\,  \left| \int g\, d(\widehat{\rho_{t,Y}} - \pi_Y) \right| \,\right]  
\leq L  \cdot\mathbb{E}[W_1(\widehat{\rho_{t,Y}}, \pi_Y)]  
\leq L\delta_\text{sample}.
\end{align}

In the optimization analysis, we can later take $g$ to be the gradient oracle of the DRO loss, $g:=\nabla_\theta \ell(\theta, \cdot)$, to control the gradient oracle error.
Recall that $\widehat{\rho_{t,Y}} = \frac1N \sum_{i=1}^N \delta_{y^i_t}$ is the particle approximation obtained by running the gradient flow sampler for $t$ steps, and $\pi_Y$ is the target marginal distribution, which is the worst-case distribution of DRO inner maximization problem~\eqref{eq:entropic-jko}.
Then, 
\eqref{eq:gradient_oracle_error_control_expectation} results in the following
bound on the gradient oracle error:
\begin{multline}
    \label{eq:gradient_oracle_error_control}
 \mathbb{E}\!\left[\,  \bigg|\frac1N \sum_{i=1}^N \nabla_\theta \ell(\theta, y^i_t) 
-
\nabla_\theta 
\underbrace{
\max _{\rho \in \calP} 
\biggl(
    \int \ell(\theta, z)\dd\rho{(z)}
    -\frac1{2\tau} W_\epsilon^2(\rho, \widehat{\rho_N})
\biggr)
}_{\text{DRO objective}}
\bigg| \,\right]  
\\
=
 \mathbb{E}\!\left[\,  \bigg|\frac1N \sum_{i=1}^N \nabla_\theta \ell(\theta, y^i_t) 
-
\mathbb{E}_{y \sim \pi_Y}
\nabla_\theta\ell(\theta, y)
\bigg| \,\right]  
\overset{\eqref{eq:gradient_oracle_error_control_expectation}}{\leq}
L\delta_\text{sample}
\end{multline}

Replacing $L\delta_\text{sample}$ by $\epsilon_\text{grad}$, we obtain the desired bound on the gradient oracle error as in Proposition~\ref{prop:gradient_oracle_error_control}.


\subsection{Proof of Theorem \ref{thm:outer_loop} (Outer Loop Convergence)}
\label{ap:proofth1}
We provide a detailed proof for the convergence of the outer loop, which follows the standard analysis for non-convex Stochastic Gradient Descent with a persistent bias.

\begin{lemma}[Bias of the Stochastic Gradient]
\label{lem:bias_grad}
Under Assumptions \ref{as:global_assumptions}.\ref{as:fsmooth}, \ref{as:global_assumptions}.\ref{as:gradest}, the bias of the stochastic gradient estimator, defined as $B(\theta) := \E_{} [\widehat g] - \nabla\Phi(\theta)$, is bounded as:
$$\|B(\theta)\| \le L_f \cdot \delta_{\text{sample}}.$$
\end{lemma}
\begin{proof}
By definition, the bias is
\begin{align*}
    B(\theta) &= \E_{x \sim \widehat{\rho_N}}\left[\E_{y \sim \widehat\rho_{Y|X=x}}[\nabla_\theta \ell(\theta, y)]\right] - \E_{x \sim \widehat{\rho_N}}\left[\E_{y \sim \pi_{Y|X=x}}[\nabla_\theta \ell(\theta, y)]\right] \\
    &= \E_{x \sim \widehat{\rho_N}}\left[ \E_{y \sim \widehat\rho_{Y|X=x}}[\nabla_\theta \ell(\theta, y)] - \E_{y \sim \pi_{Y|X=x}}[\nabla_\theta \ell(\theta, y)] \right].
\end{align*}
Taking norms and using Jensen's inequality:
$$\|B(\theta)\| \le \E_{x \sim \widehat{\rho_N}}\left[ \| \E_{y \sim \widehat\rho_{Y|X=x}}[\nabla_\theta \ell(\theta, y)] - \E_{y \sim \pi_{Y|X=x}}[\nabla_\theta \ell(\theta, y)] \| \right].$$
  By Assumption~\ref{as:global_assumptions}.\ref{as:fsmooth}, the map $y \mapsto \nabla_\theta \ell(\theta, y)$ is $L_f$-Lipschitz in $y$. Hence by Kantorovich--Rubinstein duality, for any two probability measures $\rho_1, \rho_2$ with finite first moment,
\[\big\|\E_{y\sim\rho_1}[\nabla_\theta \ell(\theta, y)] - \E_{y\sim\rho_2}[\nabla_\theta \ell(\theta, y)]\big\| \le L_f \cdot W_1(\rho_1, \rho_2).\]
Applying this and using $W_1 \le W_2$, we obtain   
\begin{align*}
    \|B(\theta)\| &\le \E_{x \sim \widehat{\rho_N}}\left[ L_f \cdot W_1(\widehat\rho_{Y|X=x}, \pi_{Y|X=x}) \right] \\
    &\le L_f \cdot \E_{x \sim \widehat{\rho_N}}\left[ W_2(\widehat\rho_{Y|X=x}, \pi_{Y|X=x}) \right] \quad (\text{since } W_1 \le W_2) \\
    &\le L_f \cdot \delta_{\text{sample}}.
\end{align*}
\end{proof}

\noindent\textbf{Proof of Theorem \ref{thm:outer_loop}.}
From the $L_\Phi$-smoothness of $\Phi$ (Assumption \ref{as:global_assumptions}.\ref{as:totalsmooth}), we have the standard descent lemma. For simplicity, we assume $\Theta = \mathbb{R}^d$ and a constant stepsize $r_s = r$.
\begin{align*}
    \Phi(\theta^{s+1}) &\le \Phi(\theta^s) + \langle \nabla\Phi(\theta^s), \theta^{s+1} - \theta^s \rangle + \frac{L_\Phi}{2}\|\theta^{s+1} - \theta^s\|^2 \\
    &= \Phi(\theta^s) - r \langle \nabla\Phi(\theta^s), \widehat g^s \rangle + \frac{L_\Phi r^2}{2}\|\widehat{g}^s\|^2.
\end{align*}
Taking expectation conditioned on the filtration $\mathcal{F}_s$:
\begin{align*}
    \E[\Phi(\theta^{s+1})|\mathcal{F}_s] &\le \Phi(\theta^s) - r \langle \nabla\Phi(\theta^s), \E[\widehat{g}^s|\mathcal{F}_s] \rangle + \frac{L_\Phi r^2}{2}\E[\|\widehat{g}^s\|^2|\mathcal{F}_s] \\
    &= \Phi(\theta^s) - r \langle \nabla\Phi(\theta^s), \nabla\Phi(\theta^s) + B(\theta^s) \rangle + \frac{L_\Phi r^2}{2} \left( \|\E[\widehat{g}^s|\mathcal{F}_s]\|^2 + \text{Var}(\widehat{g}^s|\mathcal{F}_s) \right).
\end{align*}
Using the variance bound and the bias definition:
\begin{align*}
    \E[\Phi(\theta^{s+1})|\mathcal{F}_s] &\le \Phi(\theta^s) - r\|\nabla\Phi(\theta^s)\|^2 - r\langle \nabla\Phi(\theta^s), B(\theta^s) \rangle + \frac{L_\Phi r^2}{2} \left( \|\nabla\Phi(\theta^s)+B(\theta^s)\|^2 + \sigma^2 \right).
\end{align*}
Applying Young's inequality to the inner product term:
$$-r\langle \nabla\Phi(\theta^s), B(\theta^s) \rangle \le \frac{r}{2}\|\nabla\Phi(\theta^s)\|^2 + \frac{r}{2}\|B(\theta^s)\|^2.$$
Substituting this in and simplifying with $\|a+b\|^2 \le 2\|a\|^2 + 2\|b\|^2$:
$$\E[\Phi(\theta^{s+1})|\mathcal{F}_s] \le \Phi(\theta^s) - \frac{r}{2}\|\nabla\Phi(\theta^s)\|^2 + \frac{r}{2}\|B(\theta^s)\|^2 + L_\Phi r^2 (\|\nabla\Phi(\theta^s)\|^2 + \|B(\theta^s)\|^2) + \frac{L_\Phi r^2 \sigma^2}{2}.$$
Rearranging terms to isolate $\|\nabla\Phi(\theta^s)\|^2$:
$$r\left(\frac{1}{2} - L_\Phi r\right)\|\nabla\Phi(\theta^s)\|^2 \le \Phi(\theta^s) - \E[\Phi(\theta^{s+1})|\mathcal{F}_s] + \left(\frac{r}{2} + L_\Phi r^2\right)\|B(\theta^s)\|^2 + \frac{L_\Phi r^2 \sigma^2}{2}.$$
Choosing a stepsize $r \le \frac{1}{4L_\Phi}$, we have $(\frac{1}{2} - L_\Phi r) \ge \frac{1}{4}$ and $(\frac{r}{2} + L_\Phi r^2) \le r$. This gives:
$$\frac{r}{4}\|\nabla\Phi(\theta^s)\|^2 \le \Phi(\theta^s) - \E[\Phi(\theta^{s+1})|\mathcal{F}_s] + r\|B(\theta^s)\|^2 + \frac{L_\Phi r^2 \sigma^2}{2}.$$
Taking total expectation and using Lemma~\ref{lem:bias_grad}, $\|B(\theta^s)\|^2 \le (L_f \delta_{\text{sample}})^2$:
$$\frac{r}{4}\E[\|\nabla\Phi(\theta^s)\|^2] \le \E[\Phi(\theta^s)] - \E[\Phi(\theta^{s+1})] + r (L_f \delta_{\text{sample}})^2 + \frac{L_\Phi r^2 \sigma^2}{2}.$$
Summing from $s=0$ to $S-1$ yields a telescoping sum:
$$\frac{r}{4}\sum_{s=0}^{S-1}\E[\|\nabla\Phi(\theta^s)\|^2] \le \Phi(\theta^0) - \E[\Phi(\theta^S)] + Sr(L_f \delta_{\text{sample}})^2 + \frac{S L_\Phi r^2 \sigma^2}{2}.$$
Dividing by $\frac{r S}{4}$ and using $\E[\Phi(\theta^S)] \ge \Phi_{\inf}$:
$$\frac{1}{S}\sum_{s=0}^{S-1}\E[\|\nabla\Phi(\theta^s)|^2] \le \frac{4(\Phi(\theta^0) - \Phi_{\inf})}{r S} + 4(L_f \delta_{\text{sample}})^2 + 2L_\Phi r \sigma^2. $$

Therefore, when $\delta_{\text{sample}}$ is controlled such that $\delta_{\text{sample}}= O(\epsilon_\text{opt}/L_f)$, and step size $r = \sqrt \frac{1}{SL_\Phi\sigma^2}$, and the iteration $S=O(1/\epsilon_\text{opt}^4)$, $\frac{1}{S}\sum_{s=0}^{S-1}\E[\|\nabla\Phi(\theta^s)|^2] \le O(\epsilon_\text{opt}^2)$. 




\subsection{Derivation of ULA Sampler Complexity (for Theorem~\ref{thm:ula})}
\label{ap:ula_derivation}
The complexity of the ULA sampler depends on the geometric properties of the inner target distribution $\pi_{Y|X=x}$.
Under Assumption \ref{as:LSI}, standard results on ULA convergence \citep{vempala2019rapid} state that after $T$ steps, the KL-divergence to the target is bounded. To achieve a final KL-divergence of $\delta_{KL}$, the number of iterations required is $T = O\left(\frac{L_U^2 d}{\lambda_U^2 \delta_{KL}}\log \frac{1}{\delta_{KL}}\right) = \tilde{O}\left(\frac{L_U^2 d}{\lambda_U^2 \delta_{KL}}\right)$.

Our goal is to connect the required outer-loop sampling accuracy $\delta_{\text{sample}}$ (in $W_2$ distance) to the required inner-loop KL-divergence accuracy $\delta_{KL}$. Under the LSI assumption, Talagrand's inequality gives the relationship:
$$ W_2^2(\widehat \rho_{Y|X=x} , \pi_{Y|X=x}) \le \frac{2}{\lambda_U} \KL(\widehat \rho_{Y|X=x} || \pi_{Y|X=x}) $$
From Theorem \ref{thm:outer_loop}, we require $W_2(\widehat \rho_{Y|X=x}, \pi_{Y|X=x}) \le \delta_{\text{sample}} = O(\epsilon_{\text{opt}}/L_f)$. This implies we need to achieve a KL-divergence of:
$$ \delta_{KL} \le \frac{\lambda_U}{2} W_2^2(\widehat \rho_{Y|X=x}, \pi_{Y|X=x}) = O\left(\lambda_U \frac{\epsilon_{\text{opt}}^2}{L_f^2}\right). $$
Substituting this required $\delta_{KL}$ into the ULA iteration complexity gives the number of inner loop steps:
$$ T_{ULA} = \tilde{O}\left(\frac{L_U^2 d}{\lambda_U^2 \cdot \lambda_U \frac{\epsilon_{\text{opt}}^2}{L_f^2}}\right) = \tilde{O}\left(\frac{L_U^2 L_f^2 d}{\lambda_U^3 \epsilon_{\text{opt}}^2}\right). $$
The total complexity is the product of outer iterations $S = O(1/\epsilon_{\text{opt}}^4)$, inner iterations $T_{ULA}$, and the cost per inner gradient step $C_{\nabla_z} = O(d)$.
\begin{align*}
\text{Complexity} &= S \times T \times C_{\nabla_{z}} \\
&= O\left(\frac{1}{\epsilon_{\text{opt}}^{4}}\right) \times \tilde{O}\left(\frac{L_U^2 L_f^2 d}{\lambda_U^3 \epsilon_{\text{opt}}^2}\right) \times O(d) \\
&= \tilde{O}\left(\frac{L_U^2 L_f^2 d^2}{\lambda_U^3 \epsilon_{\text{opt}}^6}\right).
\end{align*}
Note that we absorb constants like $L_{\Phi}$ into the $\tilde{O}(\cdot)$ notation. This completes the derivation for Theorem \ref{thm:ula}. 



\subsection{Complexity of RGO}
We analyze the computational complexity of RGO method under specific regularity conditions on the loss function.

\begin{theorem}[Complexity of RGO Sampling]
\label{thm:rgo_complexity}
Let the loss function $\ell: \mathbb{R}^d \to \mathbb{R}$ be $L$-smooth. For parameter $\tau = \frac{1}{Ld}$ and a target distribution $\pi_{Y|X=x}$, through the rejection sampling procedure described in Appendix~\ref{sec:rgo}, we can draw a sample from a distribution $\tilde{\rho}_{Y|X=x}$ such that the Kullback-Leibler (KL) divergence satisfies $\KL(\tilde{\rho}_{Y|X=x} \,\Vert\, \pi_{Y|X=x}) < \delta$ with iteration complexity   (where $\epsilon$ is the entropy regularization parameter from~\eqref{eq:main-ent-dro})   
\[ O\!\left(\log\!\left(\frac{1}{\epsilon\delta}\right)\right). \]
\end{theorem}

\begin{proof}
Under the assumption that $\ell$ is $L$-smooth and $\tau < \frac{1}{L}$, $\widetilde V_{x, \tau}(y)$, is $\frac{2-2\tau L}{\epsilon}$-strongly convex and $\frac{2+2\tau L}{\epsilon}$-smooth.

Let the minimizer of $\widetilde V_{x, \tau}(y)$ as $y^*_{x, \tau}$. To find the minimum of $\widetilde V_{x, \tau}(y)$, we can apply gradient descent. The number of iterations required to achieve a $\delta'$-approximate optimal solution $\tilde{y}$ is determined by the condition number $\kappa = \frac{1+\tau L}{1-\tau L}$. Specifically, the iteration complexity is:
$$ O\left(\kappa \log\left(\frac{1}{\delta'}\right)\right) = O\left(\frac{1+\tau L}{1-\tau L}\log\left(\frac{1}{\delta'}\right)\right) $$


\textbf{2. Distributional Proximity.}
Let the distribution obtained via rejection sampling with a $\delta'$-approximate optimal solution for the proposal be denoted by $\tilde{\rho}_{Y|X=x}$. The procedure defines the proposal distributions $p(y)$ (using the true minimizer $y^*_{x,\tau}$) and $\tilde{p}(y)$ (using the approximate minimizer $\tilde{y}$) as Gaussian distributions:
\[ p(y) = \mathcal{N} \left( y \mid y^*_{x,\tau}, \frac{\epsilon}{2 - 2\tau L} I \right), \quad \tilde{p}(y) = \mathcal{N} \left( y \mid \tilde{y}, \frac{\epsilon}{2 - 2\tau L} I \right) \]
The corresponding acceptance probabilities $a(y)$ and $\tilde{a}(y)$ are given by:
\begin{align*}
    a(y) &= \exp \left( - \widetilde V_{x, \tau}(y) + \widetilde V_{x, \tau}(y^*_{x,\tau}) + \frac{1 - \tau  L }{\epsilon} \| y - y^*_{x,\tau} \|^2 \right) \\
    \tilde{a}(y) &= \min\left\{\exp \left( - \widetilde V_{x, \tau}(y) + \widetilde V_{x, \tau}(\tilde{y}) + \frac{1 - \tau  L }{\epsilon} \| y - \tilde{y} \|^2 \right), 1\right\}
\end{align*}
The resulting KL divergence from the distribution obtained with the true minimizer, $\rho_{\theta,x}$, is
\begin{align}
    \KL(\tilde{\rho}_{Y|X=x}||\pi_{Y|X=x}) &= \int \frac{\tilde{a}(y)\tilde{p}(y)}{\int \tilde{a}(y)\tilde{p}(y)} \log \left(\frac{\frac{\tilde{a}(y)\tilde{p}(y)}{\int \tilde{a}(y)\tilde{p}(y)}}{\frac{a(y)p(y)}{\int a(y)p(y)}}\right) dy\\
    &= \int \frac{\tilde{a}(y)\tilde{p}(y)}{\int \tilde{a}(y)\tilde{p}(y)} \left(\log \left(\frac{\tilde{a}(y)\tilde{p}(y)}{a(y)p(y)}\right) + \log\left(\frac{\int a(y)p(y)}{\int \tilde{a}(y)\tilde{p}(y) }\right) \right) dy
    \\
    &\leq\int \frac{\tilde{a}(y)\tilde{p}(y)}{\int \tilde{a}(y)\tilde{p}(y)}\cdot 2 \left(\widetilde V_{x, \tau}(\tilde{y})-\widetilde V_{x, \tau}(y^*_{x, \tau}) \right) dy
    \\
    &\leq\int \frac{\tilde{a}(y)\tilde{p}(y)}{\int \tilde{a}(y)\tilde{p}(y)}\cdot \frac{4+4\tau L}{\epsilon} \delta'  dy
    = \frac{4+4\tau L}{\epsilon}\delta' 
\end{align}

To ensure the final KL divergence is less than $\delta$, we must set the optimization accuracy to $\delta' < \frac{\epsilon}{4+4\tau L}\delta$. This implies an optimization complexity of $O\left(\frac{1+\tau L}{1-\tau L}\log\left(\frac{4+4\tau L}{\epsilon\delta}\right)\right)$.

\textbf{3. Rejection Sampling Efficiency.}
The expected number of iterations until acceptance is at most $\left(\frac{1+\tau L}{1-\tau L}\right)^{d/2}$\citep{chewi2022query}. Note that if we choose $\tau = \frac{1}{Ld}$, then the
expected number of iterations until acceptance is at most $\left(\frac{1+\tau L}{1-\tau L}\right)^{d/2}  = \left(\frac{1+1/d}{1-1/d}\right)^{d/2} = O(1)$. 
As $d \to \infty$, this expression converges to $e$. Thus, for this choice of $\tau$, the expected number of rejection sampling trials is $O(1)$.

\textbf{4. Total Complexity.}
The total complexity is the product of the optimization complexity and the expected number of rejection sampling iterations. With $\tau = \frac{1}{Ld}$, the condition number becomes $\kappa = \frac{d+1}{d-1}$.
More precisely, the complexity is:
\[
O\left(\left(\frac{d+1}{d-1}\right)^{\frac{d}{2} + 1} \times\log\left(\frac{4d+4}{d\epsilon\delta}\right)  \right) = O(\log\left(\frac{1}{\epsilon\delta}\right))
\]
This completes the proof.
\end{proof}

\begin{remark}[Practical Limitations]
The theoretical complexity presented in Theorem \ref{thm:rgo_complexity} is highly compelling when compared to alternative sampling methodologies. However, its applicability is constrained by stringent underlying assumptions.
\begin{enumerate}
    \item \textbf{Smoothness Requirement:} The analysis presupposes that the loss function $\ell$ is $L$-smooth. In many practical applications, loss functions are non-smooth or exhibit a very large smoothness constant $L$.
    \item \textbf{Parameter Dependency:} The optimal setting for the parameter, $\tau = \frac{1}{Ld}$, is inversely proportional to both the smoothness constant $L$ and the dimension $d$. If $L$ is large, the resulting $\tau$ will be small. This makes the worst-case distribution very close to the original distribution, thereby limiting the robustness conferred by the method.
\end{enumerate}
Consequently, while the theoretical result is intriguing, the method's performance in empirical settings is often suboptimal due to the difficulty in satisfying these idealized conditions.
\end{remark}


\section{Additional Experiments}
\subsection{Ablation Study for Human Face Classification}
\label{sec:ablation}

In this section, we evaluate the sensitivity of the proposed WFR gradient flow sampler to its key hyperparameters. To ensure a controlled comparison, we vary one parameter at a time while keeping the others fixed at their baseline values. The baseline configuration is defined as follows: regularization $\tau=5$, entropy regularization $\epsilon=0.1$, learning rate $\eta=0.01$, number of particles $m=32$, and total iterations $T=10,000$. All qualitative comparisons are conducted using a fixed representative sample from the FFHQ test set to ensure consistency.

Figure \ref{fig:ablation_grid} presents the qualitative results of varying the regularization strength, learning rate, and the number of particles. As shown in Figure \ref{fig:ab_tau}, a small regularization value ($\tau=0.5$) overly constrains the latent code to the source distribution, causing the model to retain original features such as masculine traits. The baseline value of $\tau=5$ achieves a successful attribute transition while effectively preserving the identity of the source image. However, when $\tau$ is excessively large, the sampler fails to maintain the underlying facial structure, leading to unrealistic results.

The learning rate $\eta$ and the number of particles $m$ also play critical roles in the sampling process. As observed in Figure \ref{fig:ab_lr}, a smaller learning rate can mitigate artifacts and distortions in the generated images, providing more stable updates. Regarding the number of particles, Figure \ref{fig:ab_samples} demonstrates that increasing $m$ consistently improves the quality and stability of the generated samples by providing a better approximation of the gradient flow.

We further analyze the entropy regularization $\epsilon$, which governs the stochasticity and diversity of the generated images. As illustrated in the multi-sample comparison in Figure \ref{fig:ab_epsilon}, low entropy leads to nearly deterministic trajectories with minimal variation between samples. At the intermediate baseline of $\epsilon=0.1$, the sampler produces diverse yet realistic adversarial candidates. Conversely, increasing $\epsilon$ to $1.0$ introduces excessive noise that dominates the drift term, resulting in significant image degradation and loss of semantic content.

\begin{figure}[h!]
    \centering
    \begin{minipage}[c]{0.42\textwidth}
        \centering
        \begin{subfigure}[b]{\textwidth}
            \centering
            \includegraphics[width=\textwidth]{exp_tau.pdf}
            \caption{Effect of Regularization $\tau$}
            \label{fig:ab_tau}
        \end{subfigure}
        \vspace{0.2cm}
        \begin{subfigure}[b]{\textwidth}
            \centering
            \includegraphics[width=\textwidth]{exp_lr.pdf}
            \caption{Effect of learning rate $\eta$}
            \label{fig:ab_lr}
        \end{subfigure}
        \vspace{0.2cm}
        \begin{subfigure}[b]{\textwidth}
            \centering
            \includegraphics[width=\textwidth]{exp_num_samples.pdf}
            \caption{Effect of the number of particles $m$}
            \label{fig:ab_samples}
        \end{subfigure}
    \end{minipage}
    \hfill
    \begin{minipage}[c]{0.54\textwidth}
        \centering
        \begin{subfigure}[b]{\textwidth}
            \centering
            \includegraphics[width=\textwidth]{exp_epsilon.pdf}
            \caption{Effect of Entropy Regularization $\epsilon$ on Diversity}
            \label{fig:ab_epsilon}
        \end{subfigure}
    \end{minipage}
    \caption{Qualitative ablation study on the WFR sampler parameters. (a-c) show the sensitivity of single-sample generation to $\tau$, $\eta$, and $m$. (d) visualizes the Top-3 candidates for each $\epsilon$ to demonstrate sample diversity. The leftmost column in each figure represents the original source image.}
    \label{fig:ablation_grid}
\end{figure}

Finally, we visualize the temporal evolution of the latent code in Figure \ref{fig:convergence}. The progression from iteration 250 to 1,000 demonstrates a stable and smooth transition toward the target attribute, confirming the ability of generating samples from the worst-case distribution of the proposed method.

\begin{figure}[h!]
    \centering
    \includegraphics[width=0.95\textwidth]{exp_convergence.pdf}
    \caption{Visualization of the convergence process. The sampler demonstrates a gradual and stable transition towards the target attribute.}
    \label{fig:convergence}
\end{figure}


\subsection{Multi-class Logistic Regression on CIFAR-10 Features}\label{sec:appx_features}
We assess adversarial robustness using multi-class logistic regression on 512-dimensional features extracted from the CIFAR-10 dataset \citep{krizhevsky2009learning} via a pre-trained ResNet-50. This convex setting is complementary to the deep non-convex experiments in Sections~\ref{sec:lenet_mnist} and~\ref{sec:cifar_resnet}: it allows us to compare the full set of methods (including RGO-DRO and SVG-DRO) head to head across a sweep of $\epsilon$ values. The model minimizes the multi-class logistic loss
\[
\ell(B, x, y) = -y^\top B^\top x + \log\!\left(\mathbf{1}^\top \exp(B^\top x)\right),
\]
where $B \in \mathbb{R}^{512\times 10}$ is the parameter matrix and $y\in\mathbb{R}^{10}$ is the one-hot label. Perturbations affect only the features $x$; labels are fixed. The transport cost is defined as
\[
c((x, y), (x', y')) = \|x - x'\|_2^2 + \infty \cdot \mathbf{1}_{y \neq y'}.
\]
To evaluate robustness, we apply a Projected Gradient Descent (PGD) attack \citep{madry2017towards} with $L_2$-norm constraints to the test data. The perturbation magnitude $\Delta$ is normalized by the average $L_2$-norm of the test features, and we vary $\Delta$ from $0$ to $0.08$. Performance is measured by the misclassification rate.

\begin{figure*}[h!]
\centering
\begin{subfigure}{0.3\textwidth}
    \includegraphics[width=\textwidth]{rgo_lam=10_eps=0.2.pdf}
    \caption{$\tau=0.05, \epsilon=0.2$}
\end{subfigure}
\hfill
\begin{subfigure}{0.3\textwidth}
    \includegraphics[width=\textwidth]{rgo_lam=10_eps=0.02.pdf}
    \caption{$\tau=0.05, \epsilon=0.02$}
\end{subfigure}
\hfill
\begin{subfigure}{0.3\textwidth}
    \includegraphics[width=\textwidth]{rgo_lam=10_eps=0.002.pdf}
    \caption{$\tau=0.05, \epsilon=0.002$}
\end{subfigure}
\caption{CIFAR-10 feature classification error under $L_2$-norm PGD attack. The plots show test error (\%) versus normalized perturbation $\Delta$. Models are trained for 10 epochs; inner loops use a stepsize of 0.01 for 100 iterations.}
\label{fig:robustness}
\end{figure*}

As shown in Figure~\ref{fig:robustness}, RGO-DRO fails to provide robustness, performing similarly to the SAA baseline. This suggests the inefficiency of rejection sampling on non-smooth loss functions, which is further supported by the results in Appendix~\ref{sec:experiment3}. The Dual method is sensitive to $\epsilon$ and becomes less robust as $\epsilon$ decreases. Notably, SVG-DRO performs nearly identically to WRM, showing a mode collapse issue (detailed in Appendix~\ref{sec:experiment3}). As observed in Figure~\ref{fig:svg_force}, with a small number of particles $m$, the repulsive force fails to prevent convergence to a single adversarial point. In contrast, the SDE-based methods (WFR-DRO and WGF-DRO) avoid this issue and achieve high robustness across all settings.

 
\subsection{Additional Diagnostics for CNN Experiments}\label{sec:appx_diagnostics}
This section provides training details of the experiments in Section~\ref{sec:cifar_resnet}, including convergence curves and wall-clock
cost, and a sensitivity analysis with respect to $\tau$ based on the experiments in Section~\ref{sec:cifar_resnet}.
\paragraph{Experimental setup.}
To accommodate the $32\!\times\!32$ input resolution, we apply the standard CIFAR-10 adaptation of ResNet-18: the first convolution is replaced with a $3\!\times\!3$, stride-$1$ kernel, the initial max-pooling layer is removed, and the final fully connected layer is replaced with a $10$-way classifier. 
All methods share this architecture and pretraining. Training uses SGD with learning rate $0.01$, momentum $0.9$, and weight decay $5\!\times\!10^{-4}$, batch size $256$, for $20$ epochs. 

\paragraph{Convergence diagnostics} Figure~\ref{fig:loss_curves} shows the outer-loop training loss. WRM
behaves almost identically to SAA. WGF-DRO, WFR-DRO and Dual exhibit
a similar overall trend, but WGF-DRO and WFR-DRO decay faster than
Dual, as the Dual gradient estimator suffers exponential bias in
$B\tau/\epsilon$ (where $B := \sup\ell(\theta,x)$, see
\citet{wang2021sinkhorn}) when the loss is large. WFR-DRO is
consistently a bit faster than WGF-DRO, which is consistent with
our theory.

\begin{figure}[h]
\centering
\includegraphics[width=0.5\linewidth]{loss_curves.pdf}
\caption{Training loss convergence on CIFAR-10 ResNet-18 (mean $\pm$ std over 5 runs).}
\label{fig:loss_curves}
\end{figure}

\paragraph{Computational cost and \texorpdfstring{$\tau$}{tau}-sensitivity.}
Table~\ref{tab:wallclock} reports wall-clock time per epoch; Table~\ref{tab:tau_sweep} reports a $\tau$-sweep on MNIST LeNet-5 under PGD-$L_\infty$ at $\Delta=0.10$. The overhead relative to WRM scales with $m$; the $m$ particle updates are independent and fully parallelizable on GPU, so the wall-clock overhead is sub-linear in $m=8$ in practice. The Dual method is cheapest per epoch among all DRO methods. The $\tau$-sweep confirms WFR-DRO's advantage holds across the full hyperparameter range: WFR-DRO at its worst-case $\tau=0.05$ ($10.92\%$) still beats WRM at its best $\tau=1.0$ ($12.94\%$).

\begin{table*}[h!]
\centering
\begin{minipage}[t]{0.48\textwidth}
\centering
\caption{Wall-clock time per epoch (seconds).}
\label{tab:wallclock}
\resizebox{\linewidth}{!}{%
\begin{tabular}{lcc}
\toprule
Method & MNIST LeNet-5 & CIFAR-10 ResNet-18 \\
\midrule
SAA      & $20.43 \pm 0.56$     & $19.01 \pm 0.22$ \\
Dual     & $21.99 \pm 0.89$  & $84.66 \pm 5.63$ \\
WRM      & $24.64 \pm 0.69$  & $195.11 \pm 0.65$ \\
WGF-DRO  & $32.66 \pm 0.51$  & $711.29 \pm 7.53$\\
WFR-DRO  & $52.57 \pm 1.42$  & $1022.15 \pm 11.69$ \\
\bottomrule
\end{tabular}%
}
\end{minipage}\hfill
\begin{minipage}[t]{0.48\textwidth}
\centering
\caption{$\tau$-sweep error (\%) on MNIST LeNet-5 under PGD-$L_\infty$ at $\Delta=0.10$.}
\label{tab:tau_sweep}
\resizebox{\linewidth}{!}{%
\begin{tabular}{ccccc}
\toprule
$\tau$ & WFR-DRO & WGF-DRO & WRM & Dual \\
\midrule
$0.05$ & $10.92$ & $12.05$ & $13.51$ & $11.09$ \\
$0.5$  & $\phantom{0}8.58$  & $\phantom{0}9.40$  & $13.34$ & $11.14$ \\
$1.0$  & $\phantom{0}8.78$  & $\phantom{0}8.83$  & $12.94$ & $11.36$ \\
\bottomrule
\end{tabular}%
}
\end{minipage}
\end{table*}
  

\subsection{Two Moon Classification}\label{sec:experiment3}
In this section, we compare methods on an imbalanced binary classification problem using the 'two moons' dataset. The model is a three-layer neural network with ReLU activations and a hidden dimension of 16. The objective is to minimize the cross-entropy loss. The cost function for samples $(x, y)$ and $(x', y')$ is defined as $c((x,y),(x',y'))=\|x-x'\|_2^2 + \infty \cdot \mathbf{1}_{y\neq y'}$.

The training data is generated by \texttt{sklearn.make\_moons} with a noise level of 0.1. To introduce class imbalance, the training set of $N_{\text{train}}=200$ samples consists of 90\% positive ($y=1$) and 10\% negative ($y=0$) samples.

\begin{figure}[h!]
    \centering
    \includegraphics[scale=0.7]{boundary_5_0.01.pdf}
    \caption{Decision boundary comparison for all methods on the two-moon classification task. For all DRO methods, we set $\tau=0.1$, and for SDRO methods, we set $\epsilon=0.01$. In each inner loop, WFR and WGF generate $m=5$ particles.}
    \label{fig:final_boundaries}
\end{figure}


Figure~\ref{fig:final_boundaries} illustrates the decision boundaries learned by various algorithms against the ideal boundary. Notably, the boundary from the WRM method fails to correctly separate the training data. The Dual method, while separating the classes in high-density regions, learns a boundary that deviates significantly from the ground truth. In contrast, WFR, and WGF learn boundaries that more closely approximate the ideal curve. Among them, WFR and WGF achieve the best results, tracking the ideal separator with high fidelity, which indicates superior performance in this setting.


To evaluate the capacity of generating the worst-case distribution of WRM, WGF, WFR, SVG, and RGO, we conducted an experiment on a pre-trained SAA model. This setup isolates the inner-loop optimization process used to find the worst-case distribution. The evolution of the inner objective function, $\mathbb{E}[-\widetilde V_{x, \tau}(z)]$, is depicted in Figure~\ref{fig:loss_twomoon}. 


The efficacy of WGF, WFR, and SVG in approximating the worst-case distribution is demonstrated by a significant increase in the objective value. The reweighting mechanism in WFR contributes to its faster convergence compared to WGF. In contrast, SVG's performance is highly dependent on its initialization parameters. An initial standard deviation of 0.1 leads to slower convergence than a standard deviation of 0.2, which achieves a rate similar to WFR. This sensitivity is attributed to the ability of a larger initial standard deviation to propel particles across the decision boundary, thereby facilitating a more thorough exploration of the perturbation space, as illustrated later in Figure~\ref{fig:perturbation_samples}. In contrast, WGF and WFR initiate their optimization from the empirical data distribution, which obviates the need to select initialization parameters.
\begin{figure}[h!]
\centering
    \includegraphics[scale=0.6]{saa_perturb_objective_comparison.pdf}
    \caption{Evolution of $\mathbb{E}[\widetilde V_{x, \tau}(z)]$. We run all methods for 300 steps with a stepsize of 0.01. For RGO (blue), we run a rejection sampling procedure after solving the inner optimization problem. SVG\_0.1 and SVG\_0.2 denote initial distributions with a standard deviation of 0.1 and 0.2, respectively.}
    \label{fig:loss_twomoon}
\end{figure}

\begin{figure*}[h!]
\centering
    \begin{subfigure}{0.32\textwidth}
        \includegraphics[width=\textwidth]{wrm_frame.pdf}
        \caption{Perturbation by WRM}
        \label{fig:wrm_perturb}
    \end{subfigure}
    \hfill
    \begin{subfigure}{0.32\textwidth}
        \includegraphics[width=\textwidth]{rgo_frame.pdf}
        \caption{Perturbation by RGO}
    \end{subfigure}
    \hfill
    \begin{subfigure}{0.32\textwidth}
        \includegraphics[width=\textwidth]{wgf_frame.pdf}
        \caption{Perturbation by WGF}
        \label{fig:wgf_perturb}
    \end{subfigure}
    \hfill
    \begin{subfigure}{0.32\textwidth}
        \includegraphics[width=\textwidth]{wfr_frame.pdf}
        \caption{Perturbation by WFR}
        \label{fig:wfr_perturb}
    \end{subfigure}
    \hfill
    \begin{subfigure}{0.32\textwidth}
        \includegraphics[width=\textwidth]{svg_frame_0.1.pdf}
        \caption{Perturbation by SVG ($\sigma=0.1$)}
        \label{fig:svg0.1}
    \end{subfigure}
    \hfill
    \begin{subfigure}{0.32\textwidth}
        \includegraphics[width=\textwidth]{svg_frame_0.2.pdf}
        \caption{Perturbation by SVG ($\sigma=0.2$)}
        \label{fig:svg0.2}
    \end{subfigure}
    \caption{Visualization of perturbed samples (gray-edged smaller circles) generated from original data (black-edged bigger circles) against the SAA boundary at the final step. For all methods, we use a stepsize of 0.01 and run for 300 iterations. For WFR, the intensity of points visualizes the sample weights. We show perturbations by SVG-DRO with different initializations.}
    \label{fig:perturbation_samples}
\end{figure*}

Conversely, WRM and RGO fail to significantly increase the objective. The non-convex nature of the objective, evidenced by the initial dip and subsequent rise for WGF and WFR, likely presents a challenge for the optimization procedures in WRM and RGO. Additionally, the performance of RGO is hindered by its rejection sampling stage, which is inefficient for non-smooth objectives.



Figure~\ref{fig:perturbation_samples} provides a visual confirmation of these results, displaying the perturbed samples at the final iteration of the inner loop. WRM generates minimal perturbations, with particles remaining in close proximity to the original data points. The perturbations from RGO are qualitatively similar, appearing as slightly noisier versions of the WRM results. In contrast, WGF, WFR, and SVG generate a diverse set of adversarial examples, effectively pushing samples across the decision boundary, including those initially distant from it. Notably, the extent of perturbation for SVG is dependent on the initialization; an initial standard deviation of 0.2 results in more significant perturbations than a standard deviation of 0.1. And the particles generated by SVG for a single data point tend to be highly concentrated.

This phenomenon arises from the interplay between the two forces governing SVGD: a driving force that pushes particles toward regions of higher loss and a repulsive force from the kernel that prevents particle collapse. The optimization dynamics, visualized in Figure~\ref{fig:svg_force}, show that the driving force initially dominates, causing the particles to converge. As the particles draw closer, the repulsive force increases to counteract this convergence. However, since the number of particles is relatively small, the repulsive force only dominates when particles are very close, leading to the observed particle concentration. This result also explains the reason why SVG shows nearly identical performance as WRM in Appendix~\ref{sec:appx_features}.

\begin{figure*}[h!]
\centering
    \begin{subfigure}{0.32\textwidth}
        \includegraphics[width=\textwidth]{frame_020.pdf}
        \caption{Iteration 20}
    \end{subfigure}
    \hfill
    \begin{subfigure}{0.32\textwidth}
        \includegraphics[width=\textwidth]{frame_100.pdf}
        \caption{Iteration 100}
    \end{subfigure}
    \hfill
    \begin{subfigure}{0.32\textwidth}
        \includegraphics[width=\textwidth]{frame_288.pdf}
        \caption{Iteration 300}
    \end{subfigure}
    \caption{Evolution of particle positions in the SVG method. The driving force guides initial convergence, while the repulsive force prevents complete collapse. But in this experiment, the repulsive force fails to push particles apart efficiently, leading to the mode collapse. }
    \label{fig:svg_force}
\end{figure*}








\subsection{Classification under Data Imbalance}\label{sec:circle-exp}

\begin{figure*}[h!]
\begin{subfigure}[b]{0.32\textwidth}
        \includegraphics[width=\textwidth]{circle_boundary_comparison.pdf}
        \caption{Decision boundaries}
        \label{fig:circle-boundary}
    \end{subfigure}
    \hfill
    \begin{subfigure}[b]{0.32\textwidth}
        \includegraphics[width=\textwidth]{WFR_frame_000_samples.pdf}
        \caption{Perturbation by WFR}
        \label{fig:WFR_circle}
    \end{subfigure}
    \hfill
    \begin{subfigure}[b]{0.32\textwidth}
        \includegraphics[width=\textwidth]{WRM_frame_000_samples.pdf}
        \caption{Perturbation by WRM}
        \label{fig:WRM_circle}
    \end{subfigure}
    \caption{\textbf{Robust Decision Boundaries with Biased Data.} \textbf{(a)} Decision boundaries learned by different methods on the biased circle dataset. The training data are shown as orange (positive class) and blue (negative class) points. The final classification boundaries are shown for the WRM (green), WGF (purple), WFR (red), and Dual (brown) models. All models were trained for 40 epochs. We set the regularization parameter $\tau=2.5$ for all methods and the entropy regularization $\epsilon=0.15$ for methods based on entropy-regularized Wasserstein DRO problem. \textbf{(b)} Samples from the worst-case distribution generated by WFR sampler at the first epoch. 
    \textbf{(c)} Worst-case samples generated by WRM method at the first epoch. WRM can only generate discrete distributions as worst-case distribution while entropy-regularized DRO uses potentially continuous distributions as worst-case distribution. The original data points are shown as circles with black edge and the worst-case samples are shown by circles with a shallower color.}
    \label{fig:circle-experiment}
\end{figure*}
This experiment demonstrates the robustness of our proposed methods to data imbalance. The experimental setup is adapted from \citep{sinha2020certifyingdistributionalrobustnessprincipled}. We generate a dataset where features $X \in \mathbb{R}^2$ are drawn from a Gaussian distribution. The labels are assigned based on the rule $Y = \text{sign}(\|X\|_2 - \sqrt{2})$, which creates two classes separated by a circle of radius $\sqrt{2}$. To establish a clear margin, all data points satisfying $\|X\|_2 \in (\sqrt{2}/1.3, 1.3\sqrt{2})$ are excluded. To simulate a biased training distribution, we remove all samples from the first quadrant. This removal introduces a significant imbalance, testing the ability of each algorithm to learn a generalizable decision boundary rather than overfitting to the biased training data. The model is a neural network with a single hidden layer of 4 units.



Figure~\ref{fig:circle-boundary} visualizes the learned decision boundaries. WFR learns a decision boundary that closely matches the true circular boundary, correctly classifying the held-out data in the first quadrant and thus demonstrating robustness to the distributional shift. In contrast, the WRM boundary is overly expansive, misclassifying large regions of the feature space. This indicates a failure to generalize from the biased training set, resulting in a less reliable classifier. The boundary from WGF lies between those of WFR and WRM, highlighting the benefit of the weight flow mechanism in WFR for achieving a more robust solution. The dual method \citep{wang2021sinkhorn} does not perform as well as WFR in this setting. We attribute this to the high sensitivity of the dual method to the choice of hyperparameters ($\epsilon$ and $\lambda$), which can significantly impact the bias of the gradient estimator. 


To further investigate the differing behaviors, Figure~\ref{fig:WFR_circle} and Figure~\ref{fig:WRM_circle} visualize the samples generated by WFR and WRM at the first epoch. We observe that the samples generated by WFR algorithm begin to recover parts of the missing data distribution in the first quadrant. Conversely, the samples generated by WRM fail to do so, providing insights into why it learns a less robust boundary.


We also conducted an experiment with an increased training set of 2000 samples while keeping all other settings unchanged. With a larger dataset, the decision boundaries of all methods are expected to converge toward the true circular boundary. In this high-data regime, WGF, Dual, and WRM learn similar boundaries that are nearly circular but still exhibit noticeable deviations. WFR, however, learns a tight and circular boundary, demonstrating its consistent superior performance (see Figure~\ref{fig:circle_2000}).
\begin{figure}[h]
    \centering
    \includegraphics[width=0.45\linewidth]{2000.pdf}
    \caption{Decision boundaries on the Biased Circle dataset with 2000 training samples. WFR still learns a more accurate circular boundary compared to other methods.}
    \label{fig:circle_2000}
\end{figure}













\subsection{Uncertain Least Square}
We evaluate our methods on a distributionally robust least squares problem, following the setup from \citep{zhu2021kernel}, which was adapted from \citep{el1997robust}. The objective is to find a parameter vector $\theta \in \mathbb{R}^{10}$ that minimizes the loss $f_{\theta}(\xi) = \|A(\xi)\theta - b\|_2^2$, where the system matrix $A(\xi)$ is subject to uncertainty. The matrix $A(\xi) = A_0 + \xi A_1 \in \mathbb{R}^{10 \times 10}$ is an affine function of an uncertain scalar parameter $\xi \in [-1, 1]$. The matrices $A_0, A_1$ and the vector $b$ are fixed, with entries drawn independently from a standard normal distribution $\mathcal{N}(0, 1)$.

\begin{figure}[h!]
    \centering
    \includegraphics[scale=0.6]{uncertain_least_squares_results.pdf}
    \caption{Test loss as a function of the perturbation level $\Delta$ for the uncertain least squares problem. All methods were trained for 10 epochs with $\lambda=0.1$. For methods with an inner loop, the stepsize was $0.0001$ for $3000$ iterations. SDRO-based methods used $\epsilon=0.1$. WGF and WFR used $m=8$ particles.}
    \label{fig:uls_results}
\end{figure}
The training set comprises $N=10$ samples $\{\xi_i\}_{i=1}^{N}$ drawn uniformly from $[-0.5, 0.5]$. To evaluate robustness, test samples are drawn from a shifted distribution, specifically uniform on $[-0.5(1+\Delta), 0.5(1+\Delta)]$. We vary the shift magnitude $\Delta$ from $0$ to $10$, where a larger $\Delta$ signifies a greater departure from the training distribution. 



Figure~\ref{fig:uls_results} shows the test loss as a function of the perturbation level $\Delta$. All distributionally robust methods maintain a significantly lower test loss than the empirical risk minimization baseline, demonstrating their robustness to distributional shifts. At low perturbation levels, all robust methods perform comparably. As $\Delta$ increases, the performance of the dual method degrades more rapidly than the others. WFR shows a marginal improvement over WGF and WRM.   This limited advantage is attributable to the one-dimensional, bounded nature of the inner problem, which constrains worst-case samples to lie near the boundary of $[-1, 1]$, thus minimizing the differences between the generated adversarial distributions.   



\paragraph*{Acknowledgement.}
We thank Jie Wang~\citep{wang2025iterative} for pointing out a flaw in our original proof of Theorem~\ref{thm:ula}, which we have now fixed.


\bibliographystyle{apalike}
\bibliography{Bibliography}





\end{document}