Initial release: UGTC - Uncertainty-Gated Temporal Credit

.github/workflows/ci.yml

@@ -0,0 +1,69 @@
+name: CI
+
+on:
+  push:
+    branches: [main, develop]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    name: Unit Tests — Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Cache pip
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('pyproject.toml') }}-${{ matrix.python-version }}
+
+      - name: Install core dependencies (CPU-only torch)
+        run: |
+          pip install --upgrade pip
+          pip install torch --index-url https://download.pytorch.org/whl/cpu
+          pip install numpy
+          pip install -e ".[dev]"
+
+      - name: Lint with ruff
+        run: ruff check ugtc/ tests/
+
+      - name: Type check with mypy
+        run: mypy ugtc/ --ignore-missing-imports
+
+      - name: Run tests
+        run: pytest tests/ -v --tb=short --cov=ugtc --cov-report=term-missing
+
+      - name: Upload coverage
+        if: matrix.python-version == '3.11'
+        uses: codecov/codecov-action@v4
+        with:
+          fail_ci_if_error: false
+
+  lint:
+    name: Code Quality
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Install lint tools
+        run: pip install ruff black isort
+      - name: Check formatting (black)
+        run: black --check ugtc/ tests/ examples/
+      - name: Check imports (isort)
+        run: isort --check ugtc/ tests/ examples/
+      - name: Lint (ruff)
+        run: ruff check ugtc/ tests/ examples/

.github/workflows/pages.yml

@@ -0,0 +1,42 @@
+name: GitHub Pages
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  build:
+    name: Build Documentation
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Pages
+        uses: actions/configure-pages@v5
+
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: "./docs"
+
+  deploy:
+    name: Deploy to GitHub Pages
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

CITATION.cff

@@ -0,0 +1,42 @@
+cff-version: 1.2.0
+message: "If you use UGTC in your research, please cite it using this metadata."
+type: software
+title: "UGTC: Uncertainty-Gated Temporal Credit"
+abstract: >
+  UGTC is a backbone-agnostic advantage estimator for actor-critic reinforcement
+  learning. It maintains dual critics with different GAE lambda values and blends
+  their estimates using a sigmoid uncertainty gate driven by ensemble disagreement,
+  resolving the bias-variance trade-off in temporal credit assignment.
+authors:
+  - family-names: Dalar
+    given-names: Yağız Ekrem
+    affiliation: "Ethosoft AI"
+repository-code: "https://github.com/ethosoftai/ugtc"
+url: "https://ethosoftai.github.io/ugtc"
+license: MIT
+version: "1.0.0"
+date-released: "2026-06-15"
+doi: "10.5281/zenodo.19715116"
+keywords:
+  - reinforcement learning
+  - advantage estimation
+  - temporal credit assignment
+  - uncertainty quantification
+  - actor-critic
+  - PPO
+  - TD3
+  - SAC
+  - DDPG
+  - ensemble methods
+preferred-citation:
+  type: article
+  authors:
+    - family-names: Dalar
+      given-names: Yağız Ekrem
+      affiliation: "Ethosoft AI"
+  title: "UGTC: Uncertainty-Gated Temporal Credit"
+  year: 2026
+  journal: "Ulysseus Young Explorers in Science (UYES)"
+  doi: "10.5281/zenodo.19715116"
+  url: "https://doi.org/10.5281/zenodo.19715116"
+  note: "Journal DOI forthcoming upon publication."

CONTRIBUTING.md

@@ -0,0 +1,84 @@
+# Contributing to UGTC
+
+Thank you for your interest in contributing to UGTC!
+
+## Getting Started
+
+```bash
+git clone https://github.com/ethosoftai/ugtc.git
+cd ugtc
+pip install -e ".[dev]"
+```
+
+## Development Setup
+
+```bash
+pip install ruff black isort mypy pytest pytest-cov
+```
+
+## Running Tests
+
+```bash
+pytest tests/ -v
+pytest tests/ -v --cov=ugtc --cov-report=html
+```
+
+## Code Style
+
+We use:
+- **black** for formatting (line length 100)
+- **isort** for import ordering
+- **ruff** for linting
+- **mypy** for type checking
+
+```bash
+black ugtc/ tests/ examples/
+isort ugtc/ tests/ examples/
+ruff check ugtc/ tests/ examples/
+mypy ugtc/ --ignore-missing-imports
+```
+
+## Types of Contributions
+
+### Bug Reports
+Open an issue with:
+- Minimal reproducible example
+- Expected vs actual behavior
+- Python/PyTorch/OS version
+
+### New RL Algorithm Integrations
+When adding a new RL backbone integration (e.g., UGTC-IMPALA):
+1. Create `ugtc/impala.py` following the pattern of `ugtc/ppo.py`
+2. Add integration tests to `tests/test_integrations.py`
+3. Add an example to `examples/`
+4. Clearly label if the integration is from the paper or a proposed extension
+
+### New Benchmark Scripts
+Add to `benchmarks/` with:
+- Clear docstring explaining the experiment
+- Config at the top of the file
+- Full standalone runnable script
+
+### Documentation
+- Improve `docs/index.html`
+- Fix typos in README or pseudocode
+
+## Transparency Requirements
+
+UGTC is a published research codebase. Please:
+
+1. **Do not fabricate benchmark numbers.** If you run experiments and get results, share them as your own empirical findings, not as paper claims.
+2. **Label assumptions.** If you implement an extension not described in the paper, mark it with `# NOTE: implementation extension — not in paper`.
+3. **Cite correctly.** Reference [the paper](https://doi.org/10.5281/zenodo.19715116) when using this work.
+
+## Pull Request Process
+
+1. Fork the repository
+2. Create a feature branch: `git checkout -b feature/my-feature`
+3. Commit changes with clear messages
+4. Ensure all tests pass and linting is clean
+5. Open a PR with a description of what changed and why
+
+## Code of Conduct
+
+This project follows the [Contributor Covenant](CODE_OF_CONDUCT.md).

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Yağız Ekrem Dalar / Ethosoft AI
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,139 @@
+---
+license: mit
+language:
+- en
+tags:
+- reinforcement-learning
+- advantage-estimation
+- temporal-credit
+- uncertainty
+- actor-critic
+- PPO
+- TD3
+- SAC
+- DDPG
+- pytorch
+pipeline_tag: reinforcement-learning
+---
+
+# UGTC: Uncertainty-Gated Temporal Credit
+
+[![Paper](https://img.shields.io/badge/Paper-Zenodo%2019715116-blue)](https://doi.org/10.5281/zenodo.19715116)
+[![UYES](https://img.shields.io/badge/Published-UYES%20Journal-green)](https://doi.org/10.5281/zenodo.19715116)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow)](LICENSE)
+[![GitHub](https://img.shields.io/badge/GitHub-ethosoftai%2Fugtc-black)](https://github.com/ethosoftai/ugtc)
+
+> **Accepted — Ulysseus Young Explorers in Science (UYES) Journal**  
+> Preprint DOI: [10.5281/zenodo.19715116](https://doi.org/10.5281/zenodo.19715116) · Journal DOI forthcoming  
+> Author: Yağız Ekrem Dalar | Ethosoft AI
+
+---
+
+## What is UGTC?
+
+**UGTC** is a backbone-agnostic plug-in advantage estimator for actor-critic reinforcement learning. It resolves the bias–variance trade-off in temporal credit assignment by maintaining two critics with different GAE λ values and blending their estimates using a sigmoid uncertainty gate:
+
+```
+A^UGTC_t = u(sₜ) · A^slow_t + (1 - u(sₜ)) · A^fast_t
+
+u(s) = sigmoid(-β · (σ̂(s) - 1))
+σ̂(s) = std(V¹_slow, ..., Vᴹ_slow)(s) / σ_EMA
+```
+
+- **Low ensemble disagreement** → `u → 1` → use slow critic (accurate, λ=0.99)  
+- **High ensemble disagreement** → `u → 0` → use fast critic (stable, λ=0.80)
+
+## Fixed Hyperparameters (same across all benchmarks)
+
+| Parameter | Value |
+|-----------|-------|
+| λ_fast | 0.80 |
+| λ_slow | 0.99 |
+| Ensemble size M | 3 |
+| Gate temperature β | 5.0 |
+| EMA momentum | 0.99 |
+
+## Installation
+
+```bash
+git clone https://github.com/ethosoftai/ugtc.git
+cd ugtc
+pip install -e .
+```
+
+Or from this repo:
+
+```bash
+pip install huggingface_hub
+python -c "from huggingface_hub import snapshot_download; snapshot_download('Ethosoft/ugtc', local_dir='ugtc')"
+cd ugtc && pip install -e .
+```
+
+## Quick Usage
+
+```python
+from ugtc import UGTCModule
+
+ugtc = UGTCModule(obs_dim=17)  # e.g. Hopper-v4
+
+# In your actor-critic update — replace standard GAE with:
+advantages = ugtc.compute_advantages(
+    obs=obs,           # (T, obs_dim)
+    next_obs=next_obs, # (T, obs_dim)
+    rewards=rewards,   # (T,)
+    dones=dones,       # (T,)
+    gamma=0.99,
+)
+```
+
+## Supported Algorithms
+
+| Algorithm | Key Change |
+|-----------|-----------|
+| **UGTC-PPO** | A^UGTC replaces standard GAE in clipped surrogate |
+| **UGTC-TD3** | UGTC baseline correction on actor gradient |
+| **UGTC-SAC** | V^UGTC replaces value baseline in actor loss |
+| **UGTC-DDPG** | UGTC advantage scales actor update *(extension)* |
+
+## Repository Structure
+
+```
+ugtc/               Core Python package
+  module.py         UGTCModule — backbone-agnostic core
+  ppo.py            UGTC-PPO integration
+  td3.py            UGTC-TD3 integration
+  sac.py            UGTC-SAC integration
+  ddpg.py           UGTC-DDPG integration (extension)
+  utils.py          Evaluation utilities (IQM, bootstrap CI, AUC)
+examples/           Runnable examples (CartPole, Pendulum, MuJoCo)
+benchmarks/         Procgen + MuJoCo benchmark scripts
+tests/              Unit and integration tests
+implementations/
+  cpp/ugtc.hpp      C++ header-only reference
+  java/UGTCModule.java  Java reference
+pseudocode/         Algorithm pseudocode (PPO, TD3, SAC)
+configs/            YAML configs for all benchmarks
+docs/               GitHub Pages documentation source
+```
+
+## Citation
+
+```bibtex
+@misc{dalar2026ugtc,
+  author    = {Dalar, Yağız Ekrem},
+  title     = {{UGTC}: Uncertainty-Gated Temporal Credit},
+  year      = {2026},
+  publisher = {Zenodo},
+  doi       = {10.5281/zenodo.19715116},
+  url       = {https://doi.org/10.5281/zenodo.19715116},
+  note      = {Accepted — Ulysseus Young Explorers in Science (UYES) Journal.
+               Journal DOI forthcoming.}
+}
+```
+
+## Links
+
+- **Paper:** https://doi.org/10.5281/zenodo.19715116
+- **GitHub:** https://github.com/ethosoftai/ugtc
+- **Docs:** https://ethosoftai.github.io/ugtc
+- **Demo Space:** https://huggingface.co/spaces/Ethosoft/ugtc

benchmarks/mujoco/train_ugtc_td3_mujoco.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+"""
+UGTC-TD3 on MuJoCo continuous control — Ant-v5 default.
+
+Fixed UGTC hyperparameters (same across all benchmarks):
+    λ_fast = 0.80, λ_slow = 0.99, M = 3, β = 5.0, α_EMA = 0.99
+
+NOTE: eta=0.5 (UGTC correction weight) is an implementation default.
+It is not a fixed UGTC hyperparameter and may benefit from tuning.
+
+Requirements:
+    pip install torch gymnasium mujoco
+
+Usage:
+    python benchmarks/mujoco/train_ugtc_td3_mujoco.py --env Ant-v5
+    python benchmarks/mujoco/train_ugtc_td3_mujoco.py --env Hopper-v4
+"""
+
+import argparse
+import numpy as np
+import torch
+
+import gymnasium as gym
+
+from ugtc import UGTCTD3
+from ugtc.td3 import ReplayBuffer
+from ugtc.utils import bootstrap_ci, interquartile_mean
+
+
+def evaluate(agent, env_name, n_episodes=20):
+    env = gym.make(env_name)
+    returns = []
+    for _ in range(n_episodes):
+        obs, _ = env.reset()
+        done, ret = False, 0.0
+        while not done:
+            action = agent.select_action(obs, noise=0.0)
+            obs, r, te, tr, _ = env.step(action)
+            ret += r
+            done = te or tr
+        returns.append(ret)
+    env.close()
+    return returns
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--env", type=str, default="Ant-v5")
+    parser.add_argument("--total_steps", type=int, default=3_000_000)
+    parser.add_argument("--start_steps", type=int, default=25_000)
+    parser.add_argument("--batch_size", type=int, default=256)
+    parser.add_argument("--eval_freq", type=int, default=50_000)
+    parser.add_argument("--eval_episodes", type=int, default=20)
+    parser.add_argument("--seed", type=int, default=0)
+    parser.add_argument("--hidden", type=int, default=256)
+    parser.add_argument("--eta", type=float, default=0.5)
+    args = parser.parse_args()
+
+    torch.manual_seed(args.seed)
+    np.random.seed(args.seed)
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+
+    env = gym.make(args.env)
+    obs_dim = env.observation_space.shape[0]
+    act_dim = env.action_space.shape[0]
+    max_action = float(env.action_space.high[0])
+
+    agent = UGTCTD3(
+        obs_dim=obs_dim,
+        act_dim=act_dim,
+        max_action=max_action,
+        hidden=args.hidden,
+        eta=args.eta,
+        M=3,
+        beta=5.0,
+        device=device,
+    )
+    replay = ReplayBuffer(obs_dim, act_dim, capacity=1_000_000)
+
+    obs, _ = env.reset(seed=args.seed)
+    all_returns = []
+    best = -1e9
+
+    print(f"UGTC-TD3 | {args.env} | device={device} | η={args.eta}")
+    print(f"{'Steps':>10}  {'Mean':>10}  {'Std':>8}  {'Gate':>8}  {'Best':>10}")
+    print("-" * 56)
+
+    gate_metric = {}
+    for step in range(args.total_steps):
+        if step < args.start_steps:
+            action = env.action_space.sample()
+        else:
+            action = agent.select_action(obs, noise=0.1)
+
+        next_obs, reward, te, tr, _ = env.step(action)
+        done = te or tr
+        replay.add(obs, action, reward, next_obs, done)
+        obs = next_obs if not done else env.reset()[0]
+
+        if step >= args.start_steps:
+            gate_metric = agent.update(replay, args.batch_size)
+
+        if (step + 1) % args.eval_freq == 0:
+            ep_returns = evaluate(agent, args.env, args.eval_episodes)
+            mean_ret = float(np.mean(ep_returns))
+            std_ret = float(np.std(ep_returns))
+            best = max(best, mean_ret)
+            all_returns.append(ep_returns)
+            gate = gate_metric.get("gate_mean", float("nan"))
+            print(f"{step+1:>10}  {mean_ret:>10.1f}  {std_ret:>8.1f}  {gate:>8.3f}  {best:>10.1f}")
+
+    env.close()
+
+    if all_returns:
+        final_returns = [r[-1] for r in [ep for ep in all_returns[-1:]]]
+        flat = [r for ep_list in all_returns[-3:] for r in ep_list]
+        iqm = interquartile_mean(flat)
+        lo, hi = bootstrap_ci(flat)
+        print(f"\nFinal IQM:  {iqm:.1f}")
+        print(f"95% CI:     [{lo:.1f}, {hi:.1f}]")
+
+
+if __name__ == "__main__":
+    main()

benchmarks/procgen/train_ugtc_ppo_procgen.py

@@ -0,0 +1,445 @@
+#!/usr/bin/env python3
+"""
+UGTC-PPO on Procgen Benchmark (hard mode) — 25M environment steps.
+
+Default configuration matches the paper:
+    - env:               coinrun (hard mode, 500 training levels)
+    - evaluation:        500 unseen levels from level 10,000+
+    - total timesteps:   25,000,000
+    - vectorized envs:   64
+
+UGTC hyperparameters (fixed across ALL benchmarks):
+    λ_fast = 0.80   (fast critic GAE lambda)
+    λ_slow = 0.99   (slow ensemble GAE lambda)
+    M = 3           (ensemble size)
+    β = 5.0         (gate temperature)
+    α_EMA = 0.99    (EMA momentum)
+
+This script is derived from the benchmark code used in the paper experiments.
+Results may vary by hardware, random seed, and library version.
+
+Requirements:
+    pip install torch procgen gymnasium pandas psutil
+
+Usage:
+    python benchmarks/procgen/train_ugtc_ppo_procgen.py
+    python benchmarks/procgen/train_ugtc_ppo_procgen.py --env_name bigfish
+"""
+
+import json
+import math
+import os
+import random
+import time
+import argparse
+from pathlib import Path
+from typing import Any, Dict
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.optim as optim
+from torch.distributions.categorical import Categorical
+
+import gymnasium as gym
+
+try:
+    import procgen2  # noqa
+except ImportError:
+    try:
+        import procgen  # noqa
+    except ImportError:
+        raise ImportError("Install procgen or procgen2: pip install procgen")
+
+try:
+    import pandas as pd
+    HAS_PANDAS = True
+except ImportError:
+    HAS_PANDAS = False
+
+try:
+    import psutil
+    HAS_PSUTIL = True
+except ImportError:
+    HAS_PSUTIL = False
+
+
+# ── Default config ────────────────────────────────────────────────────────────
+
+DEFAULT_CONFIG: Dict[str, Any] = {
+    "env_name": "coinrun",
+    "distribution_mode": "hard",
+    "num_levels": 500,
+    "start_level": 0,
+    "eval_start_level": 10000,
+    "eval_num_levels": 500,
+    "seed": 1,
+    "total_timesteps": 25_000_000,
+    "num_envs": 64,
+    "num_steps": 256,
+    "update_epochs": 3,
+    "num_minibatches": 8,
+    "gamma": 0.999,
+    # UGTC hyperparameters (fixed across all benchmarks)
+    "gae_lambda_fast": 0.80,
+    "gae_lambda_slow": 0.99,
+    "slow_critics": 3,
+    "uncertainty_beta": 5.0,
+    "running_unc_momentum": 0.99,
+    # PPO hyperparameters
+    "clip_coef": 0.2,
+    "ent_coef": 0.01,
+    "vf_coef_fast": 0.5,
+    "vf_coef_slow": 0.5,
+    "max_grad_norm": 0.5,
+    "learning_rate": 5e-4,
+    "anneal_lr": True,
+    "feature_dim": 256,
+    "hidden_size": 64,
+    # Eval / logging
+    "eval_freq": 1_000_000,
+    "n_eval_episodes": 20,
+    "log_freq": 10,
+    "save_model": True,
+    "root_dir": "runs_ugtc_ppo_procgen",
+    "device": "cuda" if torch.cuda.is_available() else "cpu",
+}
+
+
+# ── Utils ─────────────────────────────────────────────────────────────────────
+
+def set_seed(seed: int):
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    if torch.cuda.is_available():
+        torch.cuda.manual_seed_all(seed)
+
+
+def make_procgen_env(env_name, distribution_mode, start_level, num_levels, seed):
+    for env_id in [f"procgen:procgen-{env_name}-v0", f"procgen-{env_name}-v0"]:
+        try:
+            env = gym.make(env_id, start_level=start_level, num_levels=num_levels,
+                           distribution_mode=distribution_mode)
+            try:
+                env.reset(seed=seed)
+            except TypeError:
+                pass
+            return env
+        except Exception:
+            continue
+    raise RuntimeError(f"Cannot create Procgen env '{env_name}'")
+
+
+def make_vector_envs(cfg, train=True):
+    start = cfg["start_level"] if train else cfg["eval_start_level"]
+    n_levels = cfg["num_levels"] if train else cfg["eval_num_levels"]
+    n_envs = cfg["num_envs"] if train else 1
+
+    def thunk(rank):
+        def _init():
+            return make_procgen_env(cfg["env_name"], cfg["distribution_mode"],
+                                    start_level=start, num_levels=n_levels,
+                                    seed=cfg["seed"] + rank)
+        return _init
+
+    return gym.vector.SyncVectorEnv([thunk(i) for i in range(n_envs)])
+
+
+def obs_to_tensor(obs, device):
+    x = np.asarray(obs)
+    if x.ndim == 3:
+        x = x[None, ...]
+    return torch.as_tensor(np.transpose(x, (0, 3, 1, 2)).astype(np.float32) / 255.0,
+                           dtype=torch.float32, device=device)
+
+
+# ── Model ─────────────────────────────────────────────────────────────────────
+
+class ResidualBlock(nn.Module):
+    def __init__(self, ch):
+        super().__init__()
+        self.block = nn.Sequential(
+            nn.ReLU(), nn.Conv2d(ch, ch, 3, 1, 1),
+            nn.ReLU(), nn.Conv2d(ch, ch, 3, 1, 1),
+        )
+
+    def forward(self, x):
+        return x + self.block(x)
+
+
+class ImpalaBlock(nn.Module):
+    def __init__(self, in_ch, out_ch):
+        super().__init__()
+        self.conv = nn.Conv2d(in_ch, out_ch, 3, 1, 1)
+        self.pool = nn.MaxPool2d(3, 2, 1)
+        self.res = nn.Sequential(ResidualBlock(out_ch), ResidualBlock(out_ch))
+
+    def forward(self, x):
+        return self.res(self.pool(self.conv(x)))
+
+
+class VisualEncoder(nn.Module):
+    def __init__(self, feature_dim):
+        super().__init__()
+        self.net = nn.Sequential(
+            ImpalaBlock(3, 16), ImpalaBlock(16, 32), ImpalaBlock(32, 32),
+            nn.ReLU(), nn.Flatten(),
+        )
+        with torch.no_grad():
+            flat = self.net(torch.zeros(1, 3, 64, 64)).shape[1]
+        self.proj = nn.Sequential(nn.Linear(flat, feature_dim), nn.ReLU())
+
+    def forward(self, x):
+        return self.proj(self.net(x))
+
+
+class ValueHead(nn.Module):
+    def __init__(self, feature_dim, hidden):
+        super().__init__()
+        self.net = nn.Sequential(
+            nn.Linear(feature_dim, hidden), nn.Tanh(), nn.Linear(hidden, 1),
+        )
+        for m in self.modules():
+            if isinstance(m, nn.Linear):
+                nn.init.orthogonal_(m.weight, math.sqrt(2))
+                nn.init.constant_(m.bias, 0)
+        nn.init.orthogonal_(self.net[-1].weight, 1.0)
+
+    def forward(self, feat):
+        return self.net(feat).squeeze(-1)
+
+
+class UGTCPPOAgent(nn.Module):
+    def __init__(self, n_actions, feature_dim, hidden, slow_critics):
+        super().__init__()
+        self.encoder = VisualEncoder(feature_dim)
+        self.policy_head = nn.Sequential(
+            nn.Linear(feature_dim, hidden), nn.Tanh(), nn.Linear(hidden, n_actions),
+        )
+        self.fast_value = ValueHead(feature_dim, hidden)
+        self.slow_values = nn.ModuleList(
+            [ValueHead(feature_dim, hidden) for _ in range(slow_critics)]
+        )
+        nn.init.orthogonal_(self.policy_head[-1].weight, 0.01)
+
+    def get_features(self, obs):
+        return self.encoder(obs)
+
+    def get_policy(self, feat):
+        return Categorical(logits=self.policy_head(feat))
+
+    def get_fast_value(self, feat):
+        return self.fast_value(feat)
+
+    def get_slow_values(self, feat):
+        return torch.stack([h(feat) for h in self.slow_values], dim=0)  # [M, B]
+
+    def get_action_and_values(self, obs, action=None):
+        feat = self.get_features(obs)
+        dist = self.get_policy(feat)
+        if action is None:
+            action = dist.sample()
+        return action, dist.log_prob(action), dist.entropy(), self.get_fast_value(feat), self.get_slow_values(feat)
+
+
+# ── GAE ───────────────────────────────────────────────────────────────────────
+
+def compute_gae(rewards, dones, values, next_value, gamma, gae_lambda):
+    T, N = rewards.shape
+    adv = torch.zeros_like(rewards)
+    gae = torch.zeros(N, device=rewards.device)
+    for t in reversed(range(T)):
+        nxt = next_value if t == T - 1 else values[t + 1]
+        nt = 1.0 - (dones[t] if t == T - 1 else dones[t + 1])
+        delta = rewards[t] + gamma * nxt * nt - values[t]
+        gae = delta + gamma * gae_lambda * nt * gae
+        adv[t] = gae
+    return adv, adv + values
+
+
+# ── Eval ──────────────────────────────────────────────────────────────────────
+
+@torch.no_grad()
+def evaluate(agent, cfg, device):
+    env = make_vector_envs(cfg, train=False)
+    returns = []
+    for ep in range(cfg["n_eval_episodes"]):
+        obs, _ = env.reset(seed=cfg["seed"] + 100000 + ep)
+        done = np.array([False])
+        ret = 0.0
+        while not done[0]:
+            obs_t = obs_to_tensor(obs, device)
+            feat = agent.get_features(obs_t)
+            action = torch.argmax(agent.get_policy(feat).logits, dim=-1)
+            out = env.step(action.cpu().numpy())
+            obs, reward = out[0], out[1]
+            done = np.logical_or(out[2], out[3]) if len(out) == 5 else out[2]
+            ret += float(reward[0])
+        returns.append(ret)
+    env.close()
+    return {"eval_mean": float(np.mean(returns)), "eval_std": float(np.std(returns))}
+
+
+# ── Train ─────────────────────────────────────────────────────────────────────
+
+def train(cfg: Dict[str, Any]) -> None:
+    set_seed(cfg["seed"])
+    device = torch.device(cfg["device"])
+    root = Path(cfg["root_dir"])
+    root.mkdir(parents=True, exist_ok=True)
+
+    envs = make_vector_envs(cfg, train=True)
+    n_actions = int(envs.single_action_space.n)
+    agent = UGTCPPOAgent(n_actions, cfg["feature_dim"], cfg["hidden_size"], cfg["slow_critics"]).to(device)
+    optimizer = optim.Adam(agent.parameters(), lr=cfg["learning_rate"], eps=1e-5)
+
+    num_updates = cfg["total_timesteps"] // (cfg["num_envs"] * cfg["num_steps"])
+    mb_size = (cfg["num_envs"] * cfg["num_steps"]) // cfg["num_minibatches"]
+
+    # Buffers
+    obs_buf    = torch.zeros((cfg["num_steps"], cfg["num_envs"], 3, 64, 64), device=device)
+    actions_buf = torch.zeros((cfg["num_steps"], cfg["num_envs"]), dtype=torch.long, device=device)
+    logp_buf   = torch.zeros((cfg["num_steps"], cfg["num_envs"]), device=device)
+    rew_buf    = torch.zeros((cfg["num_steps"], cfg["num_envs"]), device=device)
+    done_buf   = torch.zeros((cfg["num_steps"], cfg["num_envs"]), device=device)
+    fv_buf     = torch.zeros((cfg["num_steps"], cfg["num_envs"]), device=device)
+    sv_buf     = torch.zeros((cfg["num_steps"], cfg["slow_critics"], cfg["num_envs"]), device=device)
+
+    obs_raw, _ = envs.reset(seed=cfg["seed"])
+    next_obs = obs_to_tensor(obs_raw, device)
+    next_done = torch.zeros(cfg["num_envs"], device=device)
+
+    global_step = 0
+    next_eval = cfg["eval_freq"]
+    running_unc = 1.0
+    start_time = time.time()
+    train_logs, eval_logs = [], []
+
+    print("=" * 80)
+    print(f"UGTC-PPO | Procgen {cfg['env_name']} | {cfg['distribution_mode']} | {cfg['total_timesteps']:,} steps")
+    print(f"λ_fast={cfg['gae_lambda_fast']}  λ_slow={cfg['gae_lambda_slow']}  M={cfg['slow_critics']}  β={cfg['uncertainty_beta']}")
+    print("=" * 80)
+
+    for update in range(1, num_updates + 1):
+        if cfg["anneal_lr"]:
+            frac = 1.0 - (update - 1) / num_updates
+            optimizer.param_groups[0]["lr"] = frac * cfg["learning_rate"]
+
+        for step in range(cfg["num_steps"]):
+            global_step += cfg["num_envs"]
+            obs_buf[step] = next_obs
+            done_buf[step] = next_done
+            with torch.no_grad():
+                action, logp, _, fv, sv = agent.get_action_and_values(next_obs)
+            actions_buf[step], logp_buf[step] = action, logp
+            fv_buf[step], sv_buf[step] = fv, sv
+            out = envs.step(action.cpu().numpy())
+            obs_raw, reward = out[0], out[1]
+            terminated, truncated = (out[2], out[3]) if len(out) == 5 else (out[2], out[2])
+            rew_buf[step] = torch.as_tensor(reward, dtype=torch.float32, device=device)
+            next_obs = obs_to_tensor(obs_raw, device)
+            next_done = torch.as_tensor(np.logical_or(terminated, truncated), dtype=torch.float32, device=device)
+
+        with torch.no_grad():
+            feat_next = agent.get_features(next_obs)
+            nfv = agent.get_fast_value(feat_next)
+            nsv = agent.get_slow_values(feat_next).mean(dim=0)
+            sv_mean = sv_buf.mean(dim=1)
+
+            adv_fast, ret_fast = compute_gae(rew_buf, done_buf, fv_buf, nfv, cfg["gamma"], cfg["gae_lambda_fast"])
+            adv_slow, ret_slow = compute_gae(rew_buf, done_buf, sv_mean, nsv, cfg["gamma"], cfg["gae_lambda_slow"])
+
+            sigma = sv_buf.var(dim=1).sqrt()
+            cur_unc = float(sigma.mean().item())
+            running_unc = cfg["running_unc_momentum"] * running_unc + (1 - cfg["running_unc_momentum"]) * cur_unc
+            gate = torch.sigmoid(-cfg["uncertainty_beta"] * (sigma / (running_unc + 1e-8) - 1.0))
+            blended_adv = gate * adv_slow + (1.0 - gate) * adv_fast
+
+        b_obs   = obs_buf.reshape(-1, 3, 64, 64)
+        b_act   = actions_buf.reshape(-1)
+        b_lp    = logp_buf.reshape(-1)
+        b_fret  = ret_fast.reshape(-1)
+        b_sret  = ret_slow.reshape(-1)
+        b_adv   = blended_adv.reshape(-1)
+        b_sv    = sv_buf.permute(0, 2, 1).reshape(-1, cfg["slow_critics"])
+        b_gate  = gate.reshape(-1)
+        b_adv   = (b_adv - b_adv.mean()) / (b_adv.std() + 1e-8)
+        inds    = np.arange(cfg["num_envs"] * cfg["num_steps"])
+
+        for _ in range(cfg["update_epochs"]):
+            np.random.shuffle(inds)
+            for start in range(0, len(inds), mb_size):
+                mb = inds[start:start + mb_size]
+                feat = agent.get_features(b_obs[mb])
+                dist = agent.get_policy(feat)
+                new_lp = dist.log_prob(b_act[mb])
+                entropy = dist.entropy()
+                fv_new = agent.get_fast_value(feat)
+                sv_new = agent.get_slow_values(feat).transpose(0, 1)
+
+                ratio = (new_lp - b_lp[mb]).exp()
+                mb_adv = b_adv[mb]
+                pg_loss = torch.max(
+                    -mb_adv * ratio,
+                    -mb_adv * torch.clamp(ratio, 1 - cfg["clip_coef"], 1 + cfg["clip_coef"])
+                ).mean()
+                fv_loss = 0.5 * ((fv_new - b_fret[mb]) ** 2).mean()
+                sv_loss = 0.5 * ((sv_new - b_sret[mb].unsqueeze(-1)) ** 2).mean()
+                loss = pg_loss - cfg["ent_coef"] * entropy.mean() + cfg["vf_coef_fast"] * fv_loss + cfg["vf_coef_slow"] * sv_loss
+                optimizer.zero_grad(set_to_none=True)
+                loss.backward()
+                nn.utils.clip_grad_norm_(agent.parameters(), cfg["max_grad_norm"])
+                optimizer.step()
+
+        if update % cfg["log_freq"] == 0:
+            elapsed = time.time() - start_time
+            log = {
+                "update": update, "timesteps": global_step,
+                "mean_gate": float(b_gate.mean().item()),
+                "running_unc": running_unc,
+                "sps": int(global_step / max(elapsed, 1e-9)),
+            }
+            train_logs.append(log)
+            print(f"[{update:5d}/{num_updates}] steps={global_step:>9,} gate={log['mean_gate']:.3f} unc={running_unc:.4f} sps={log['sps']}")
+
+        if global_step >= next_eval or update == num_updates:
+            ev = evaluate(agent, cfg, device)
+            eval_logs.append({"timesteps": global_step, **ev})
+            print(f"  ► EVAL  mean={ev['eval_mean']:.3f}  std={ev['eval_std']:.3f}")
+            next_eval += cfg["eval_freq"]
+
+    envs.close()
+
+    if HAS_PANDAS:
+        pd.DataFrame(eval_logs).to_csv(root / "evals.csv", index=False)
+        pd.DataFrame(train_logs).to_csv(root / "train.csv", index=False)
+
+    summary = {
+        "algo": "UGTC-PPO", "env_name": cfg["env_name"],
+        "lambda_fast": cfg["gae_lambda_fast"], "lambda_slow": cfg["gae_lambda_slow"],
+        "M": cfg["slow_critics"], "beta": cfg["uncertainty_beta"],
+        "total_timesteps": global_step,
+        "best_eval": max((e["eval_mean"] for e in eval_logs), default=float("nan")),
+        "elapsed_sec": time.time() - start_time,
+    }
+    with open(root / "summary.json", "w") as f:
+        json.dump(summary, f, indent=2)
+
+    if cfg["save_model"]:
+        torch.save({"state_dict": agent.state_dict(), "config": cfg}, root / "model.pt")
+
+    print("\nTraining complete.")
+    print(f"  Best eval return: {summary['best_eval']:.3f}")
+    print(f"  Results saved to: {root}/")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--env_name", type=str, default="coinrun")
+    parser.add_argument("--seed", type=int, default=1)
+    parser.add_argument("--total_timesteps", type=int, default=25_000_000)
+    parser.add_argument("--device", type=str, default="cuda" if torch.cuda.is_available() else "cpu")
+    args = parser.parse_args()
+    cfg = {**DEFAULT_CONFIG, **vars(args)}
+    train(cfg)

configs/ant.yaml

@@ -0,0 +1,8 @@
+env_name: "Ant-v5"
+hidden_dim: 256
+lr: 3e-4
+gamma: 0.99
+batch_size: 2048
+total_steps: 10000000
+eval_interval: 50000
+eval_episodes: 20

configs/carracing.yaml

@@ -0,0 +1,8 @@
+env_name: "CarRacing-v3"
+hidden_dim: 128
+lr: 2.5e-4
+gamma: 0.99
+batch_size: 2048
+total_steps: 2200000
+eval_interval: 50000
+eval_episodes: 20

configs/crafter.yaml

@@ -0,0 +1,8 @@
+env_name: "Crafter"
+hidden_dim: 128
+lr: 3e-4
+gamma: 0.99
+batch_size: 128
+total_steps: 3000000
+eval_interval: 50000
+eval_episodes: 20

configs/hopper.yaml

@@ -0,0 +1,8 @@
+env_name: "Hopper-v4"
+hidden_dim: 64
+lr: 3e-4
+gamma: 0.99
+batch_size: 2048
+total_steps: 15000000
+eval_interval: 50000
+eval_episodes: 20

configs/metaworld.yaml

@@ -0,0 +1,9 @@
+env_name: "MetaWorld-ML45"
+hidden_dim: 256
+lr: 3e-4
+gamma: 0.99
+batch_size: 512
+total_steps: 25000000
+eval_interval: 500000
+eval_episodes: 10
+backbone: "dreamerv3"

configs/procgen.yaml

@@ -0,0 +1,9 @@
+env_name: "procgen:CoinRun"
+hidden_dim: 256
+lr: 2.5e-4
+gamma: 0.999
+batch_size: 4096
+total_steps: 200000000
+eval_interval: 50000
+eval_episodes: 20
+num_envs: 16

docs/index.html

@@ -0,0 +1,504 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="UTF-8">
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+<title>UGTC — Uncertainty-Gated Temporal Credit</title>
+<meta name="description" content="UGTC: A backbone-agnostic advantage estimator for actor-critic reinforcement learning, published at UYES Journal.">
+<link rel="preconnect" href="https://fonts.googleapis.com">
+<link href="https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet">
+<script src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/3.2.2/es5/tex-mml-chtml.min.js"></script>
+<style>
+  :root {
+    --bg: #0a0e17;
+    --bg2: #111827;
+    --bg3: #1f2937;
+    --border: #374151;
+    --accent: #6366f1;
+    --accent2: #8b5cf6;
+    --accent3: #06b6d4;
+    --text: #f1f5f9;
+    --muted: #94a3b8;
+    --green: #10b981;
+    --orange: #f59e0b;
+    --red: #ef4444;
+  }
+  * { margin: 0; padding: 0; box-sizing: border-box; }
+  body { font-family: 'Inter', sans-serif; background: var(--bg); color: var(--text); line-height: 1.7; }
+  code, pre, .mono { font-family: 'JetBrains Mono', monospace; }
+
+  /* Nav */
+  nav {
+    position: sticky; top: 0; z-index: 100;
+    background: rgba(10,14,23,0.95);
+    backdrop-filter: blur(12px);
+    border-bottom: 1px solid var(--border);
+    padding: 0 2rem;
+    display: flex; align-items: center; justify-content: space-between;
+    height: 60px;
+  }
+  .nav-logo { font-weight: 700; font-size: 1.1rem; color: var(--accent); letter-spacing: -0.02em; }
+  .nav-links { display: flex; gap: 1.5rem; list-style: none; }
+  .nav-links a { color: var(--muted); text-decoration: none; font-size: 0.9rem; transition: color 0.2s; }
+  .nav-links a:hover { color: var(--text); }
+
+  /* Hero */
+  .hero {
+    text-align: center;
+    padding: 6rem 2rem 4rem;
+    max-width: 900px;
+    margin: 0 auto;
+  }
+  .hero-badge {
+    display: inline-flex; align-items: center; gap: 0.5rem;
+    background: rgba(99,102,241,0.12);
+    border: 1px solid rgba(99,102,241,0.3);
+    color: var(--accent);
+    padding: 0.4rem 1rem; border-radius: 9999px;
+    font-size: 0.8rem; font-weight: 600; letter-spacing: 0.05em;
+    text-transform: uppercase; margin-bottom: 1.5rem;
+  }
+  h1 {
+    font-size: clamp(2.2rem, 5vw, 3.5rem);
+    font-weight: 700;
+    letter-spacing: -0.03em;
+    background: linear-gradient(135deg, #fff 0%, #a5b4fc 100%);
+    -webkit-background-clip: text; -webkit-text-fill-color: transparent;
+    margin-bottom: 0.5rem;
+  }
+  .hero-subtitle {
+    font-size: 1.2rem; color: var(--muted); margin-bottom: 1.5rem; max-width: 600px; margin-left: auto; margin-right: auto;
+  }
+  .hero-tagline {
+    font-size: 0.95rem; color: var(--muted); margin-bottom: 2rem;
+    max-width: 700px; margin-left: auto; margin-right: auto;
+  }
+  .badges { display: flex; flex-wrap: wrap; gap: 0.5rem; justify-content: center; margin-bottom: 2.5rem; }
+  .badge { display: inline-block; }
+  .badge img { height: 24px; }
+
+  .btn-group { display: flex; gap: 1rem; justify-content: center; flex-wrap: wrap; }
+  .btn {
+    padding: 0.75rem 1.75rem; border-radius: 8px; text-decoration: none;
+    font-weight: 600; font-size: 0.95rem; transition: all 0.2s;
+    display: inline-flex; align-items: center; gap: 0.5rem;
+  }
+  .btn-primary {
+    background: linear-gradient(135deg, var(--accent), var(--accent2));
+    color: white;
+  }
+  .btn-primary:hover { transform: translateY(-2px); box-shadow: 0 8px 25px rgba(99,102,241,0.4); }
+  .btn-secondary {
+    background: var(--bg3); border: 1px solid var(--border); color: var(--text);
+  }
+  .btn-secondary:hover { border-color: var(--accent); color: var(--accent); }
+
+  /* Main layout */
+  main { max-width: 1100px; margin: 0 auto; padding: 0 2rem 6rem; }
+
+  /* Section */
+  section { margin-bottom: 4rem; }
+  h2 {
+    font-size: 1.7rem; font-weight: 700; letter-spacing: -0.02em;
+    margin-bottom: 1.5rem; color: var(--text);
+    padding-bottom: 0.75rem;
+    border-bottom: 1px solid var(--border);
+  }
+  h3 { font-size: 1.2rem; font-weight: 600; margin: 1.5rem 0 0.75rem; color: var(--text); }
+
+  /* Cards */
+  .cards { display: grid; grid-template-columns: repeat(auto-fit, minmax(260px, 1fr)); gap: 1.25rem; }
+  .card {
+    background: var(--bg2); border: 1px solid var(--border);
+    border-radius: 12px; padding: 1.5rem;
+    transition: border-color 0.2s, transform 0.2s;
+  }
+  .card:hover { border-color: var(--accent); transform: translateY(-2px); }
+  .card-icon { font-size: 1.75rem; margin-bottom: 0.75rem; }
+  .card h3 { margin: 0 0 0.5rem; font-size: 1rem; }
+  .card p { font-size: 0.9rem; color: var(--muted); margin: 0; }
+
+  /* Architecture diagram */
+  .arch-box {
+    background: var(--bg2); border: 1px solid var(--border);
+    border-radius: 12px; padding: 2rem; overflow-x: auto;
+  }
+  .arch-box pre { font-size: 0.82rem; line-height: 1.6; color: var(--text); }
+
+  /* Math */
+  .math-block {
+    background: var(--bg2); border: 1px solid var(--border);
+    border-radius: 8px; padding: 1.5rem; margin: 1rem 0;
+    overflow-x: auto;
+  }
+
+  /* Table */
+  table { width: 100%; border-collapse: collapse; }
+  th { background: var(--bg3); padding: 0.75rem 1rem; text-align: left; font-size: 0.85rem; color: var(--muted); font-weight: 600; }
+  td { padding: 0.75rem 1rem; border-top: 1px solid var(--border); font-size: 0.9rem; }
+  tr:hover td { background: var(--bg2); }
+  .tag {
+    display: inline-block; padding: 0.2rem 0.6rem; border-radius: 4px;
+    font-size: 0.75rem; font-weight: 600; font-family: 'JetBrains Mono', monospace;
+  }
+  .tag-green { background: rgba(16,185,129,0.12); color: var(--green); }
+  .tag-blue { background: rgba(6,182,212,0.12); color: var(--accent3); }
+  .tag-purple { background: rgba(99,102,241,0.12); color: var(--accent); }
+
+  /* Code block */
+  .code-block {
+    background: #0d1117; border: 1px solid var(--border);
+    border-radius: 8px; padding: 1.25rem 1.5rem;
+    overflow-x: auto; position: relative;
+  }
+  .code-block pre { font-size: 0.85rem; line-height: 1.7; color: #e6edf3; }
+  .code-label {
+    position: absolute; top: 0.75rem; right: 1rem;
+    font-size: 0.7rem; color: var(--muted); font-family: 'JetBrains Mono', monospace;
+  }
+
+  /* Gate visualizer */
+  .gate-viz {
+    display: flex; flex-direction: column; gap: 0.5rem; padding: 1.5rem;
+    background: var(--bg2); border: 1px solid var(--border); border-radius: 12px;
+  }
+  .gate-row { display: flex; align-items: center; gap: 1rem; }
+  .gate-label { font-size: 0.85rem; color: var(--muted); width: 120px; flex-shrink: 0; }
+  .gate-bar-bg { flex: 1; height: 8px; background: var(--bg3); border-radius: 4px; overflow: hidden; }
+  .gate-bar { height: 100%; border-radius: 4px; transition: width 0.5s ease; }
+  .gate-value { font-size: 0.8rem; font-family: monospace; color: var(--muted); width: 40px; text-align: right; }
+
+  /* Footer */
+  footer {
+    border-top: 1px solid var(--border);
+    padding: 2rem;
+    text-align: center;
+    color: var(--muted);
+    font-size: 0.85rem;
+  }
+  footer a { color: var(--accent); text-decoration: none; }
+  footer a:hover { text-decoration: underline; }
+
+  @media (max-width: 640px) {
+    nav { padding: 0 1rem; }
+    .nav-links { display: none; }
+    .hero { padding: 4rem 1rem 3rem; }
+    main { padding: 0 1rem 4rem; }
+  }
+</style>
+</head>
+<body>
+
+<nav>
+  <div class="nav-logo">UGTC</div>
+  <ul class="nav-links">
+    <li><a href="#architecture">Architecture</a></li>
+    <li><a href="#math">Mathematics</a></li>
+    <li><a href="#algorithms">Algorithms</a></li>
+    <li><a href="#quickstart">Quick Start</a></li>
+    <li><a href="https://github.com/ethosoftai/ugtc">GitHub</a></li>
+    <li><a href="https://doi.org/10.5281/zenodo.19715116">Paper</a></li>
+  </ul>
+</nav>
+
+<div class="hero">
+  <div class="hero-badge">📄 Published · UYES Journal · 2026</div>
+  <h1>Uncertainty-Gated Temporal Credit</h1>
+  <p class="hero-subtitle">A plug-in advantage estimator for actor-critic reinforcement learning</p>
+  <p class="hero-tagline">
+    UGTC dynamically blends short-horizon (low-variance) and long-horizon (low-bias) advantage
+    estimates using a sigmoid gate driven by critic ensemble disagreement — resolving the
+    bias–variance trade-off in temporal credit assignment.
+  </p>
+  <div class="badges">
+    <span class="badge"><img src="https://img.shields.io/badge/Paper-Zenodo%2019715116-blue?style=flat-square&logo=zenodo" alt="Paper"></span>
+    <span class="badge"><img src="https://img.shields.io/badge/Published-UYES%20Journal-green?style=flat-square" alt="UYES"></span>
+    <span class="badge"><img src="https://img.shields.io/badge/License-MIT-yellow?style=flat-square" alt="License"></span>
+    <span class="badge"><img src="https://img.shields.io/badge/Python-3.10%2B-blue?style=flat-square&logo=python" alt="Python"></span>
+    <span class="badge"><img src="https://img.shields.io/badge/PyTorch-2.2%2B-ee4c2c?style=flat-square&logo=pytorch" alt="PyTorch"></span>
+  </div>
+  <div class="btn-group">
+    <a href="https://github.com/ethosoftai/ugtc" class="btn btn-primary">⭐ View on GitHub</a>
+    <a href="https://doi.org/10.5281/zenodo.19715116" class="btn btn-secondary">📄 Read Paper</a>
+    <a href="https://huggingface.co/spaces/Ethosoft/ugtc" class="btn btn-secondary">🤗 Live Demo</a>
+  </div>
+</div>
+
+<main>
+
+  <!-- Key Features -->
+  <section>
+    <h2>Key Features</h2>
+    <div class="cards">
+      <div class="card">
+        <div class="card-icon">🔌</div>
+        <h3>Backbone-Agnostic</h3>
+        <p>Drop UGTC into any actor-critic algorithm by replacing the advantage computation. Tested with PPO, TD3, SAC.</p>
+      </div>
+      <div class="card">
+        <div class="card-icon">🎯</div>
+        <h3>Adaptive Credit Assignment</h3>
+        <p>Automatically selects between short-horizon and long-horizon GAE estimates based on per-state uncertainty.</p>
+      </div>
+      <div class="card">
+        <div class="card-icon">📐</div>
+        <h3>Fixed Hyperparameters</h3>
+        <p>λ_fast=0.80, λ_slow=0.99, M=3, β=5.0. Same across all benchmarks — no per-task tuning required.</p>
+      </div>
+      <div class="card">
+        <div class="card-icon">🔬</div>
+        <h3>Ensemble Uncertainty</h3>
+        <p>Slow critic ensemble disagreement provides calibrated uncertainty estimates without Bayesian inference.</p>
+      </div>
+      <div class="card">
+        <div class="card-icon">⚡</div>
+        <h3>Lightweight Overhead</h3>
+        <p>Three small MLP value heads. Minimal parameter and compute overhead relative to actor network.</p>
+      </div>
+      <div class="card">
+        <div class="card-icon">🌐</div>
+        <h3>Multi-Language</h3>
+        <p>Reference implementations in Python, C++ (header-only), and Java for portability.</p>
+      </div>
+    </div>
+  </section>
+
+  <!-- Architecture -->
+  <section id="architecture">
+    <h2>Architecture</h2>
+    <div class="arch-box">
+      <pre>
+┌─────────────────────────────────────────────────────────────────────────────┐
+│                              UGTC MODULE                                    │
+│                                                                             │
+│   Input: s (observation)                                                    │
+│                                                                             │
+│   ┌──────────────────┐      ┌────────────────────────────────────────────┐  │
+│   │   Fast Critic    │      │          Slow Ensemble (M=3)               │  │
+│   │   V_fast(s)      │      │   V¹(s)    V²(s)    V³(s)                 │  │
+│   │   λ_fast = 0.80  │      │   (independent parameters, λ = 0.99)      │  │
+│   └────────┬─────────┘      └──────────────────┬──────────────────────── ┘  │
+│            │                                   │                            │
+│            │                     ┌─────────────┴───────────────┐            │
+│            │                     │  σ(s) = std(V¹,V²,V³)(s)   │            │
+│            │                     │  Ensemble Disagreement       │            │
+│            │                     └─────────────┬───────────────┘            │
+│            │                                   │                            │
+│            │                     ┌─────────────▼───────────────┐            │
+│            │                     │  EMA Normalization           │            │
+│            │                     │  σ_EMA ← α·σ_EMA + (1-α)·σ  │            │
+│            │                     │  σ̂(s) = σ(s) / (σ_EMA + ε)  │            │
+│            │                     └─────────────┬───────────────┘            │
+│            │                                   │                            │
+│            │                     ┌─────────────▼───────────────┐            │
+│            │                     │   Sigmoid Gate               │            │
+│            │                     │   u(s) = σ(-β·(σ̂(s) - 1))   │            │
+│            │                     └─────────────┬───────────────┘            │
+│            │                                   │                            │
+│   ┌────────▼───────────────────────────────────▼─────────────────────────┐  │
+│   │   A^UGTC = u(s) · A^slow  +  (1 - u(s)) · A^fast                    │  │
+│   │   Blended Advantage Estimate                                          │  │
+│   └───────────────────────────────────────────────────────────────────────┘  │
+└─────────────────────────────────────────────────────────────────────────────┘
+      </pre>
+    </div>
+
+    <h3>Gate Behavior</h3>
+    <div class="gate-viz">
+      <div class="gate-row">
+        <span class="gate-label">Low uncertainty</span>
+        <div class="gate-bar-bg"><div class="gate-bar" style="width:92%;background:linear-gradient(90deg,#6366f1,#8b5cf6)"></div></div>
+        <span class="gate-value">u → 1</span>
+        <span style="font-size:0.8rem;color:#10b981;">→ use A^slow (accurate)</span>
+      </div>
+      <div class="gate-row">
+        <span class="gate-label">Medium uncertainty</span>
+        <div class="gate-bar-bg"><div class="gate-bar" style="width:50%;background:linear-gradient(90deg,#6366f1,#06b6d4)"></div></div>
+        <span class="gate-value">u = 0.5</span>
+        <span style="font-size:0.8rem;color:#94a3b8;">→ equal blend</span>
+      </div>
+      <div class="gate-row">
+        <span class="gate-label">High uncertainty</span>
+        <div class="gate-bar-bg"><div class="gate-bar" style="width:8%;background:linear-gradient(90deg,#f59e0b,#ef4444)"></div></div>
+        <span class="gate-value">u → 0</span>
+        <span style="font-size:0.8rem;color:#f59e0b;">→ use A^fast (stable)</span>
+      </div>
+    </div>
+  </section>
+
+  <!-- Mathematics -->
+  <section id="math">
+    <h2>Mathematical Foundation</h2>
+
+    <h3>Generalized Advantage Estimation</h3>
+    <div class="math-block">
+      \[
+        \delta_t = r_t + \gamma V(s_{t+1})(1 - d_t) - V(s_t)
+      \]
+      \[
+        A_t^{\text{GAE}} = \sum_{k=0}^{\infty} (\gamma\lambda)^k \delta_{t+k}
+      \]
+    </div>
+
+    <h3>UGTC Dual-Stream Computation</h3>
+    <div class="math-block">
+      \[
+        A_t^{\text{fast}} = \text{GAE}\!\left(\tau,\, V_{\text{fast}},\, \lambda_{\text{fast}} = 0.80\right)
+      \]
+      \[
+        A_t^{\text{slow}} = \text{GAE}\!\left(\tau,\, \bar{V}_{\text{slow}},\, \lambda_{\text{slow}} = 0.99\right)
+      \]
+      <p style="color:var(--muted);font-size:0.85rem;margin-top:0.75rem;">
+        where \(\bar{V}_{\text{slow}} = \frac{1}{M}\sum_{m=1}^{M} V^m_{\text{slow}}\) (ensemble mean, M = 3)
+      </p>
+    </div>
+
+    <h3>Uncertainty Gate</h3>
+    <div class="math-block">
+      \[
+        \sigma(s) = \text{std}\!\left(V^1_{\text{slow}}(s),\, \ldots,\, V^M_{\text{slow}}(s)\right)
+      \]
+      \[
+        \hat{\sigma}(s) = \frac{\sigma(s)}{\sigma_{\text{EMA}} + \varepsilon}, \qquad
+        \sigma_{\text{EMA}} \leftarrow \alpha \cdot \sigma_{\text{EMA}} + (1-\alpha)\cdot\mathbb{E}[\sigma(s)]
+      \]
+      \[
+        u(s) = \sigma\!\left(-\beta \cdot (\hat{\sigma}(s) - 1)\right)
+      \]
+    </div>
+
+    <h3>Blended Advantage</h3>
+    <div class="math-block">
+      \[
+        \boxed{A_t^{\text{UGTC}} = u(s_t) \cdot A_t^{\text{slow}} + (1 - u(s_t)) \cdot A_t^{\text{fast}}}
+      \]
+    </div>
+
+    <h3>Fixed Hyperparameters</h3>
+    <table>
+      <thead>
+        <tr><th>Parameter</th><th>Symbol</th><th>Value</th><th>Description</th></tr>
+      </thead>
+      <tbody>
+        <tr><td>Fast λ</td><td>\(\lambda_{\text{fast}}\)</td><td><span class="tag tag-green">0.80</span></td><td>GAE lambda for fast critic (low variance)</td></tr>
+        <tr><td>Slow λ</td><td>\(\lambda_{\text{slow}}\)</td><td><span class="tag tag-green">0.99</span></td><td>GAE lambda for slow ensemble (low bias)</td></tr>
+        <tr><td>Ensemble size</td><td>M</td><td><span class="tag tag-blue">3</span></td><td>Number of slow critic heads</td></tr>
+        <tr><td>Gate temperature</td><td>β</td><td><span class="tag tag-purple">5.0</span></td><td>Sigmoid sharpness</td></tr>
+        <tr><td>EMA momentum</td><td>α</td><td><span class="tag tag-green">0.99</span></td><td>Running uncertainty normalization</td></tr>
+      </tbody>
+    </table>
+  </section>
+
+  <!-- Algorithms -->
+  <section id="algorithms">
+    <h2>RL Algorithm Integrations</h2>
+    <div class="cards">
+      <div class="card">
+        <h3>UGTC-PPO</h3>
+        <p style="color:var(--muted);font-size:0.85rem;margin-bottom:0.75rem;">
+          <span class="tag tag-green">On-policy</span>
+        </p>
+        <p>A^UGTC replaces standard GAE in the clipped surrogate objective. All UGTC critics trained via same regression pipeline.</p>
+      </div>
+      <div class="card">
+        <h3>UGTC-TD3</h3>
+        <p style="color:var(--muted);font-size:0.85rem;margin-bottom:0.75rem;">
+          <span class="tag tag-blue">Off-policy</span>
+        </p>
+        <p>UGTC provides baseline correction for the actor: L = -(Q_min + η·A^UGTC). Twin-Q and delayed update preserved.</p>
+      </div>
+      <div class="card">
+        <h3>UGTC-SAC</h3>
+        <p style="color:var(--muted);font-size:0.85rem;margin-bottom:0.75rem;">
+          <span class="tag tag-blue">Off-policy</span>
+        </p>
+        <p>V^UGTC replaces implicit value baseline in the entropy-regularized actor loss. Auto-α entropy tuning unchanged.</p>
+      </div>
+      <div class="card">
+        <h3>UGTC-DDPG</h3>
+        <p style="color:var(--muted);font-size:0.85rem;margin-bottom:0.75rem;">
+          <span class="tag tag-purple">Extension</span>
+        </p>
+        <p>Proposed extension following TD3 integration logic. Not benchmarked in the paper — labeled as implementation assumption.</p>
+      </div>
+    </div>
+  </section>
+
+  <!-- Quick Start -->
+  <section id="quickstart">
+    <h2>Quick Start</h2>
+
+    <h3>Installation</h3>
+    <div class="code-block">
+      <span class="code-label">bash</span>
+      <pre>git clone https://github.com/ethosoftai/ugtc.git
+cd ugtc
+pip install -e .</pre>
+    </div>
+
+    <h3>Minimal Usage</h3>
+    <div class="code-block">
+      <span class="code-label">python</span>
+      <pre>from ugtc import UGTCModule
+
+# Create UGTC module (obs_dim=17 for Hopper-v4)
+ugtc = UGTCModule(obs_dim=17)
+
+# Replace standard GAE in your PPO update:
+advantages = ugtc.compute_advantages(
+    obs=obs,            # (T, obs_dim)
+    next_obs=next_obs,  # (T, obs_dim)
+    rewards=rewards,    # (T,)
+    dones=dones,        # (T,)
+    gamma=0.99,
+)
+
+# Same as before: normalize and use in clipped surrogate
+advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-8)</pre>
+    </div>
+
+    <h3>Run an Example</h3>
+    <div class="code-block">
+      <span class="code-label">bash</span>
+      <pre># UGTC-PPO on CartPole-v1 (no MuJoCo needed)
+python examples/ugtc_ppo_cartpole.py
+
+# UGTC-PPO on Hopper-v4 (requires MuJoCo)
+python examples/ugtc_ppo_mujoco.py --env Hopper-v4
+
+# UGTC-TD3 on Pendulum-v1
+python examples/ugtc_td3_pendulum.py</pre>
+    </div>
+  </section>
+
+  <!-- Citation -->
+  <section>
+    <h2>Citation</h2>
+    <div class="code-block">
+      <pre>@misc{dalar2026ugtc,
+  author    = {Dalar, Yağız Ekrem},
+  title     = {{UGTC}: Uncertainty-Gated Temporal Credit},
+  year      = {2026},
+  publisher = {Zenodo},
+  doi       = {10.5281/zenodo.19715116},
+  url       = {https://doi.org/10.5281/zenodo.19715116},
+  note      = {Accepted — Ulysseus Young Explorers in Science (UYES) Journal.
+               Journal DOI forthcoming.}
+}</pre>
+    </div>
+  </section>
+
+</main>
+
+<footer>
+  <p>
+    UGTC · <a href="https://github.com/ethosoftai">Ethosoft AI</a> ·
+    <a href="https://doi.org/10.5281/zenodo.19715116">Paper</a> ·
+    <a href="https://github.com/ethosoftai/ugtc">GitHub</a> ·
+    <a href="https://huggingface.co/spaces/Ethosoft/ugtc">HuggingFace</a>
+  </p>
+  <p style="margin-top:0.5rem;">MIT License · Accepted at Ulysseus Young Explorers in Science (UYES) Journal</p>
+</footer>
+
+</body>
+</html>

examples/ugtc_ppo_cartpole.py

@@ -0,0 +1,197 @@
+#!/usr/bin/env python3
+"""
+UGTC-PPO on CartPole-v1 — minimal runnable example.
+
+Demonstrates the drop-in nature of UGTC: the only change from vanilla PPO
+is passing advantages through UGTCModule.compute_advantages() instead of
+standard single-critic GAE.
+
+Requirements:
+    pip install torch gymnasium
+
+Usage:
+    python examples/ugtc_ppo_cartpole.py
+    python examples/ugtc_ppo_cartpole.py --episodes 500 --seed 42
+"""
+
+import argparse
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.optim as optim
+from torch.distributions import Categorical
+
+import gymnasium as gym
+
+from ugtc import UGTCModule
+
+
+# ── Config ──────────────────────────────────────────────────────────────────
+
+def get_args():
+    p = argparse.ArgumentParser()
+    p.add_argument("--episodes", type=int, default=300)
+    p.add_argument("--steps_per_update", type=int, default=128)
+    p.add_argument("--hidden", type=int, default=64)
+    p.add_argument("--lr", type=float, default=3e-4)
+    p.add_argument("--gamma", type=float, default=0.99)
+    p.add_argument("--clip_eps", type=float, default=0.2)
+    p.add_argument("--ppo_epochs", type=int, default=4)
+    p.add_argument("--seed", type=int, default=0)
+    # UGTC hyperparameters (fixed across all benchmarks in the paper)
+    p.add_argument("--lambda_fast", type=float, default=0.80)
+    p.add_argument("--lambda_slow", type=float, default=0.99)
+    p.add_argument("--M", type=int, default=3)
+    p.add_argument("--beta", type=float, default=5.0)
+    return p.parse_args()
+
+
+# ── Policy ───────────────────────────────────────────────────────────────────
+
+class DiscretePolicy(nn.Module):
+    def __init__(self, obs_dim, n_actions, hidden):
+        super().__init__()
+        self.net = nn.Sequential(
+            nn.Linear(obs_dim, hidden), nn.Tanh(),
+            nn.Linear(hidden, hidden), nn.Tanh(),
+            nn.Linear(hidden, n_actions),
+        )
+
+    def forward(self, obs):
+        return Categorical(logits=self.net(obs))
+
+
+# ── Training ─────────────────────────────────────────────────────────────────
+
+def collect_rollout(env, policy, ugtc, steps, gamma, device):
+    obs_list, action_list, reward_list, done_list, logp_list, next_obs_list = [], [], [], [], [], []
+    obs, _ = env.reset()
+
+    for _ in range(steps):
+        obs_t = torch.FloatTensor(obs).unsqueeze(0).to(device)
+        with torch.no_grad():
+            dist = policy(obs_t)
+            action = dist.sample()
+            log_prob = dist.log_prob(action)
+
+        next_obs, reward, terminated, truncated, _ = env.step(action.item())
+        done = terminated or truncated
+
+        obs_list.append(obs)
+        action_list.append(action.item())
+        reward_list.append(reward)
+        done_list.append(float(done))
+        logp_list.append(log_prob.item())
+        next_obs_list.append(next_obs)
+
+        obs = next_obs if not done else env.reset()[0]
+
+    return {
+        "obs":       torch.FloatTensor(np.array(obs_list)).to(device),
+        "actions":   torch.LongTensor(action_list).to(device),
+        "rewards":   torch.FloatTensor(reward_list).to(device),
+        "dones":     torch.FloatTensor(done_list).to(device),
+        "log_probs": torch.FloatTensor(logp_list).to(device),
+        "next_obs":  torch.FloatTensor(np.array(next_obs_list)).to(device),
+    }
+
+
+def main():
+    args = get_args()
+    torch.manual_seed(args.seed)
+    np.random.seed(args.seed)
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    env = gym.make("CartPole-v1")
+    obs_dim = env.observation_space.shape[0]
+    n_actions = env.action_space.n
+
+    policy = DiscretePolicy(obs_dim, n_actions, args.hidden).to(device)
+    ugtc = UGTCModule(
+        obs_dim=obs_dim,
+        hidden_dim=args.hidden,
+        M=args.M,
+        lambda_fast=args.lambda_fast,
+        lambda_slow=args.lambda_slow,
+        beta=args.beta,
+    ).to(device)
+
+    optimizer = optim.Adam(
+        list(policy.parameters()) + list(ugtc.parameters()), lr=args.lr
+    )
+
+    print(f"UGTC-PPO on CartPole-v1 | device={device}")
+    print(f"λ_fast={args.lambda_fast}  λ_slow={args.lambda_slow}  M={args.M}  β={args.beta}")
+    print(f"{'Episode':>8}  {'Return':>8}  {'Gate':>8}  {'σ_EMA':>8}")
+    print("-" * 44)
+
+    episode_returns = []
+    ep_return = 0.0
+    ep_count = 0
+
+    for update in range(1, args.episodes + 1):
+        rollout = collect_rollout(env, policy, ugtc, args.steps_per_update, args.gamma, device)
+
+        # === KEY CHANGE: UGTC advantages ===
+        ugtc.train()
+        advantages = ugtc.compute_advantages(
+            obs=rollout["obs"],
+            next_obs=rollout["next_obs"],
+            rewards=rollout["rewards"],
+            dones=rollout["dones"],
+            gamma=args.gamma,
+        )
+        advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-8)
+
+        returns = (advantages + ugtc.get_value_ugtc(rollout["obs"])).detach()
+
+        for _ in range(args.ppo_epochs):
+            dist = policy(rollout["obs"])
+            new_log_probs = dist.log_prob(rollout["actions"])
+            ratio = (new_log_probs - rollout["log_probs"]).exp()
+
+            surr1 = ratio * advantages
+            surr2 = ratio.clamp(1 - args.clip_eps, 1 + args.clip_eps) * advantages
+            policy_loss = -torch.min(surr1, surr2).mean()
+
+            fast_loss = (ugtc.fast_critic(rollout["obs"]) - returns).pow(2).mean()
+            slow_loss = torch.stack([
+                (m(rollout["obs"]) - returns).pow(2).mean()
+                for m in ugtc.slow_ensemble.members
+            ]).mean()
+
+            loss = policy_loss + 0.5 * (fast_loss + slow_loss)
+            optimizer.zero_grad()
+            loss.backward()
+            nn.utils.clip_grad_norm_(
+                list(policy.parameters()) + list(ugtc.parameters()), 0.5
+            )
+            optimizer.step()
+
+        # Eval
+        if update % 20 == 0:
+            eval_returns = []
+            for _ in range(10):
+                obs_e, _ = env.reset()
+                done_e, ret_e = False, 0.0
+                while not done_e:
+                    with torch.no_grad():
+                        obs_t = torch.FloatTensor(obs_e).unsqueeze(0).to(device)
+                        action_e = policy(obs_t).probs.argmax().item()
+                    obs_e, r, te, tr, _ = env.step(action_e)
+                    ret_e += r
+                    done_e = te or tr
+                eval_returns.append(ret_e)
+
+            gate_mean = ugtc.get_gate_stats(rollout["obs"])["gate_mean"]
+            sigma_ema = ugtc.sigma_ema.item()
+            mean_ret = np.mean(eval_returns)
+            episode_returns.append(mean_ret)
+            print(f"{update:>8}  {mean_ret:>8.1f}  {gate_mean:>8.3f}  {sigma_ema:>8.4f}")
+
+    env.close()
+    print(f"\nFinal mean return (last 5 evals): {np.mean(episode_returns[-5:]):.1f}")
+
+
+if __name__ == "__main__":
+    main()

examples/ugtc_ppo_mujoco.py

@@ -0,0 +1,170 @@
+#!/usr/bin/env python3
+"""
+UGTC-PPO on MuJoCo continuous control.
+
+Targets Hopper-v4 and Ant-v5 by default (as evaluated in the paper).
+Uses the same fixed UGTC hyperparameters as all benchmarks.
+
+Requirements:
+    pip install torch gymnasium mujoco
+
+Usage:
+    python examples/ugtc_ppo_mujoco.py --env Hopper-v4
+    python examples/ugtc_ppo_mujoco.py --env Ant-v5 --total_steps 5000000
+"""
+
+import argparse
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.optim as optim
+
+import gymnasium as gym
+
+from ugtc import UGTCModule
+
+
+def get_args():
+    p = argparse.ArgumentParser()
+    p.add_argument("--env", type=str, default="Hopper-v4")
+    p.add_argument("--total_steps", type=int, default=1_000_000)
+    p.add_argument("--num_steps", type=int, default=2048)
+    p.add_argument("--hidden", type=int, default=64)
+    p.add_argument("--lr", type=float, default=3e-4)
+    p.add_argument("--gamma", type=float, default=0.99)
+    p.add_argument("--clip_eps", type=float, default=0.2)
+    p.add_argument("--ppo_epochs", type=int, default=10)
+    p.add_argument("--seed", type=int, default=0)
+    p.add_argument("--eval_interval", type=int, default=50_000)
+    return p.parse_args()
+
+
+class ContinuousPolicy(nn.Module):
+    def __init__(self, obs_dim, act_dim, hidden):
+        super().__init__()
+        self.trunk = nn.Sequential(
+            nn.Linear(obs_dim, hidden), nn.Tanh(),
+            nn.Linear(hidden, hidden), nn.Tanh(),
+        )
+        self.mean = nn.Linear(hidden, act_dim)
+        self.log_std = nn.Parameter(torch.zeros(act_dim))
+        nn.init.orthogonal_(self.mean.weight, 0.01)
+
+    def forward(self, obs):
+        h = self.trunk(obs)
+        mean = self.mean(h)
+        std = self.log_std.clamp(-4, 2).exp().expand_as(mean)
+        return torch.distributions.Normal(mean, std)
+
+
+def evaluate(policy, env_name, n_eps=20, device="cpu"):
+    env = gym.make(env_name)
+    rets = []
+    for _ in range(n_eps):
+        obs, _ = env.reset()
+        done, ret = False, 0.0
+        while not done:
+            with torch.no_grad():
+                dist = policy(torch.FloatTensor(obs).to(device))
+                action = dist.mean
+            obs, r, te, tr, _ = env.step(action.cpu().numpy())
+            ret += r
+            done = te or tr
+        rets.append(ret)
+    env.close()
+    return np.mean(rets), np.std(rets)
+
+
+def main():
+    args = get_args()
+    torch.manual_seed(args.seed)
+    np.random.seed(args.seed)
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    env = gym.make(args.env)
+    obs_dim = env.observation_space.shape[0]
+    act_dim = env.action_space.shape[0]
+
+    policy = ContinuousPolicy(obs_dim, act_dim, args.hidden).to(device)
+    ugtc = UGTCModule(obs_dim, args.hidden, M=3, lambda_fast=0.80, lambda_slow=0.99, beta=5.0).to(device)
+
+    optimizer = optim.Adam(
+        list(policy.parameters()) + list(ugtc.parameters()), lr=args.lr
+    )
+
+    print(f"UGTC-PPO | {args.env} | device={device} | seed={args.seed}")
+    print(f"{'Steps':>10}  {'Mean Return':>12}  {'Std':>8}  {'Gate':>8}")
+    print("-" * 48)
+
+    obs, _ = env.reset(seed=args.seed)
+    step = 0
+    next_eval = args.eval_interval
+
+    while step < args.total_steps:
+        obs_buf, act_buf, rew_buf, done_buf, logp_buf, nobs_buf = [], [], [], [], [], []
+        current_obs = obs
+
+        for _ in range(args.num_steps):
+            obs_t = torch.FloatTensor(current_obs).unsqueeze(0).to(device)
+            with torch.no_grad():
+                dist = policy(obs_t)
+                action = dist.sample()
+                log_prob = dist.log_prob(action).sum(-1)
+
+            next_obs, reward, terminated, truncated, _ = env.step(action.squeeze(0).cpu().numpy())
+            done = terminated or truncated
+
+            obs_buf.append(current_obs)
+            act_buf.append(action.squeeze(0).cpu().numpy())
+            rew_buf.append(reward)
+            done_buf.append(float(done))
+            logp_buf.append(log_prob.item())
+            nobs_buf.append(next_obs)
+
+            current_obs = next_obs if not done else env.reset()[0]
+            step += 1
+
+        obs = current_obs
+
+        obs_t = torch.FloatTensor(np.array(obs_buf)).to(device)
+        act_t = torch.FloatTensor(np.array(act_buf)).to(device)
+        rew_t = torch.FloatTensor(rew_buf).to(device)
+        done_t = torch.FloatTensor(done_buf).to(device)
+        logp_t = torch.FloatTensor(logp_buf).to(device)
+        nobs_t = torch.FloatTensor(np.array(nobs_buf)).to(device)
+
+        ugtc.train()
+        advantages = ugtc.compute_advantages(obs_t, nobs_t, rew_t, done_t, args.gamma)
+        advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-8)
+        returns = (advantages + ugtc.get_value_ugtc(obs_t)).detach()
+
+        for _ in range(args.ppo_epochs):
+            dist = policy(obs_t)
+            new_lp = dist.log_prob(act_t).sum(-1)
+            ratio = (new_lp - logp_t).exp()
+            surr1 = ratio * advantages
+            surr2 = ratio.clamp(1 - args.clip_eps, 1 + args.clip_eps) * advantages
+            policy_loss = -torch.min(surr1, surr2).mean()
+            fast_loss = (ugtc.fast_critic(obs_t) - returns).pow(2).mean()
+            slow_loss = torch.stack([
+                (m(obs_t) - returns).pow(2).mean() for m in ugtc.slow_ensemble.members
+            ]).mean()
+            loss = policy_loss + 0.5 * (fast_loss + slow_loss)
+            optimizer.zero_grad()
+            loss.backward()
+            nn.utils.clip_grad_norm_(
+                list(policy.parameters()) + list(ugtc.parameters()), 0.5
+            )
+            optimizer.step()
+
+        if step >= next_eval:
+            mean_ret, std_ret = evaluate(policy, args.env, n_eps=20, device=str(device))
+            gate = ugtc.get_gate_stats(obs_t)["gate_mean"]
+            print(f"{step:>10}  {mean_ret:>12.1f}  {std_ret:>8.1f}  {gate:>8.3f}")
+            next_eval += args.eval_interval
+
+    env.close()
+
+
+if __name__ == "__main__":
+    main()

examples/ugtc_td3_pendulum.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+"""
+UGTC-TD3 on Pendulum-v1 — minimal runnable example.
+
+Shows UGTC integrated with TD3: backbone's twin-Q and delayed policy update
+are preserved; UGTC adds a value baseline correction to the actor gradient.
+
+Requirements:
+    pip install torch gymnasium
+
+Usage:
+    python examples/ugtc_td3_pendulum.py
+"""
+
+import argparse
+import numpy as np
+import torch
+
+import gymnasium as gym
+
+from ugtc import UGTCTD3
+from ugtc.td3 import ReplayBuffer
+
+
+def get_args():
+    p = argparse.ArgumentParser()
+    p.add_argument("--total_steps", type=int, default=100_000)
+    p.add_argument("--start_steps", type=int, default=1_000)
+    p.add_argument("--batch_size", type=int, default=256)
+    p.add_argument("--seed", type=int, default=0)
+    p.add_argument("--hidden", type=int, default=256)
+    p.add_argument("--eta", type=float, default=0.5,
+                   help="UGTC correction weight in actor loss (implementation default, not fixed in paper)")
+    return p.parse_args()
+
+
+def evaluate(agent, env_name="Pendulum-v1", n_episodes=10):
+    env = gym.make(env_name)
+    returns = []
+    for _ in range(n_episodes):
+        obs, _ = env.reset()
+        done, total = False, 0.0
+        while not done:
+            action = agent.select_action(obs, noise=0.0)
+            obs, r, terminated, truncated, _ = env.step(action)
+            total += r
+            done = terminated or truncated
+        returns.append(total)
+    env.close()
+    return float(np.mean(returns)), float(np.std(returns))
+
+
+def main():
+    args = get_args()
+    torch.manual_seed(args.seed)
+    np.random.seed(args.seed)
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+
+    env = gym.make("Pendulum-v1")
+    obs_dim = env.observation_space.shape[0]
+    act_dim = env.action_space.shape[0]
+    max_action = float(env.action_space.high[0])
+
+    agent = UGTCTD3(
+        obs_dim=obs_dim,
+        act_dim=act_dim,
+        max_action=max_action,
+        hidden=args.hidden,
+        eta=args.eta,
+        device=device,
+    )
+    replay = ReplayBuffer(obs_dim, act_dim)
+
+    obs, _ = env.reset(seed=args.seed)
+    print(f"UGTC-TD3 on Pendulum-v1 | device={device} | η={args.eta}")
+    print(f"{'Step':>8}  {'Return':>10}  {'Gate':>8}")
+    print("-" * 34)
+
+    for step in range(args.total_steps):
+        if step < args.start_steps:
+            action = env.action_space.sample()
+        else:
+            action = agent.select_action(obs, noise=0.1)
+
+        next_obs, reward, terminated, truncated, _ = env.step(action)
+        done = terminated or truncated
+        replay.add(obs, action, reward, next_obs, done)
+        obs = next_obs if not done else env.reset()[0]
+
+        if step >= args.start_steps:
+            metrics = agent.update(replay, args.batch_size)
+
+        if (step + 1) % 10_000 == 0:
+            mean_ret, std_ret = evaluate(agent)
+            gate = metrics.get("gate_mean", float("nan"))
+            print(f"{step+1:>8}  {mean_ret:>10.1f}  {gate:>8.3f}")
+
+    env.close()
+
+
+if __name__ == "__main__":
+    main()

implementations/cpp/ugtc.hpp

@@ -0,0 +1,310 @@
+/**
+ * UGTC: Uncertainty-Gated Temporal Credit — C++ Header-Only Reference Implementation
+ * ====================================================================================
+ *
+ * A minimal, dependency-free reference implementation of the UGTC module.
+ * Uses Eigen3 for matrix operations. No RL framework dependency.
+ *
+ * Requirements:
+ *   - C++17 or later
+ *   - Eigen3 (https://eigen.tuxfamily.org/)
+ *
+ * Usage:
+ *   #include "ugtc.hpp"
+ *
+ *   UGTC::Config cfg;
+ *   UGTC::Module ugtc(obs_dim, cfg);
+ *   auto advantages = ugtc.computeAdvantages(obs, next_obs, rewards, dones, gamma);
+ *
+ * Paper: https://doi.org/10.5281/zenodo.19715116
+ */
+
+#pragma once
+
+#include <vector>
+#include <cmath>
+#include <numeric>
+#include <random>
+#include <cassert>
+#include <algorithm>
+#include <Eigen/Dense>
+
+namespace UGTC {
+
+using Matrix = Eigen::MatrixXf;
+using Vector = Eigen::VectorXf;
+
+// ──────────────────────────────────────────────────────────────────────────────
+// Configuration
+// ──────────────────────────────────────────────────────────────────────────────
+
+struct Config {
+    int   hidden_dim    = 64;      ///< Hidden layer width
+    int   M             = 3;       ///< Ensemble size (slow critic)
+    float lambda_fast   = 0.80f;   ///< GAE lambda for fast critic
+    float lambda_slow   = 0.99f;   ///< GAE lambda for slow ensemble
+    float beta          = 5.0f;    ///< Gate temperature
+    float ema_momentum  = 0.99f;   ///< EMA momentum for uncertainty normalization
+    float eps           = 1e-8f;   ///< Numerical stability epsilon
+};
+
+// ──────────────────────────────────────────────────────────────────────────────
+// Activation functions
+// ──────────────────────────────────────────────────────────────────────────────
+
+inline float sigmoid(float x) {
+    return 1.0f / (1.0f + std::exp(-x));
+}
+
+inline float tanh_activation(float x) {
+    return std::tanh(x);
+}
+
+inline Vector tanh_vec(const Vector& x) {
+    return x.unaryExpr([](float v) { return std::tanh(v); });
+}
+
+// ──────────────────────────────────────────────────────────────────────────────
+// Linear layer (weight matrix + bias vector)
+// ──────────────────────────────────────────────────────────────────────────────
+
+struct Linear {
+    Matrix W;  ///< (out_dim, in_dim)
+    Vector b;  ///< (out_dim,)
+
+    Linear() = default;
+
+    Linear(int in_dim, int out_dim, std::mt19937& rng) {
+        W = Matrix::Random(out_dim, in_dim);
+        b = Vector::Zero(out_dim);
+        // Orthogonal-ish initialization via scaled random
+        float scale = std::sqrt(2.0f / in_dim);
+        W *= scale;
+    }
+
+    Vector forward(const Vector& x) const {
+        return W * x + b;
+    }
+};
+
+// ──────────────────────────────────────────────────────────────────────────────
+// Value network: obs → hidden → hidden → scalar
+// Architecture: Linear → Tanh → Linear → Tanh → Linear
+// ──────────────────────────────────────────────────────────────────────────────
+
+struct ValueNetwork {
+    Linear fc1, fc2, fc3;
+
+    ValueNetwork() = default;
+
+    ValueNetwork(int obs_dim, int hidden_dim, std::mt19937& rng)
+        : fc1(obs_dim, hidden_dim, rng)
+        , fc2(hidden_dim, hidden_dim, rng)
+        , fc3(hidden_dim, 1, rng)
+    {}
+
+    float forward(const Vector& obs) const {
+        Vector h1 = tanh_vec(fc1.forward(obs));
+        Vector h2 = tanh_vec(fc2.forward(h1));
+        return fc3.forward(h2)(0);
+    }
+};
+
+// ──────────────────────────────────────────────────────────────────────────────
+// Ensemble value network: M independent ValueNetworks
+// ─────────��────────────────────────────────────────────────────────────────────
+
+struct EnsembleValueNetwork {
+    std::vector<ValueNetwork> members;
+    int M;
+
+    EnsembleValueNetwork() = default;
+
+    EnsembleValueNetwork(int obs_dim, int hidden_dim, int M, std::mt19937& rng)
+        : M(M)
+    {
+        members.reserve(M);
+        for (int i = 0; i < M; ++i) {
+            members.emplace_back(obs_dim, hidden_dim, rng);
+        }
+    }
+
+    /// Returns (mean, std) of ensemble predictions for a single observation.
+    std::pair<float, float> forward(const Vector& obs) const {
+        std::vector<float> vals;
+        vals.reserve(M);
+        for (auto& m : members) vals.push_back(m.forward(obs));
+
+        float mean = std::accumulate(vals.begin(), vals.end(), 0.0f) / M;
+        float var = 0.0f;
+        for (float v : vals) var += (v - mean) * (v - mean);
+        var /= (M > 1 ? M - 1 : 1);
+
+        return { mean, std::sqrt(var) };
+    }
+};
+
+// ──────────────────────────────────────────────────────────────────────────────
+// Gate statistics output
+// ──────────────────────────────────────────────────────────────────────────────
+
+struct GateResult {
+    float gate;       ///< u(s) ∈ [0, 1]
+    float v_fast;     ///< Fast critic value
+    float v_slow;     ///< Slow ensemble mean value
+    float sigma;      ///< Ensemble disagreement (std)
+};
+
+// ──────────────────────────────────────────────────────────────────────────────
+// UGTC Module
+// ──────────────────────────────────────────────────────────────────────────────
+
+class Module {
+public:
+    Module(int obs_dim, const Config& cfg = Config{})
+        : cfg_(cfg)
+        , sigma_ema_(1.0f)
+    {
+        std::mt19937 rng(42);
+        fast_critic_  = ValueNetwork(obs_dim, cfg.hidden_dim, rng);
+        slow_ensemble_ = EnsembleValueNetwork(obs_dim, cfg.hidden_dim, cfg.M, rng);
+    }
+
+    // ── Gate computation ──────────────────────────────────────────────────────
+
+    /**
+     * Compute the uncertainty gate u(s) for a single observation.
+     *
+     * Steps:
+     *   1. Evaluate fast critic: v_fast = V_fast(s)
+     *   2. Evaluate slow ensemble: (v̄_slow, σ) = ensemble(s)
+     *   3. EMA-normalize: σ̂ = σ / σ_EMA
+     *   4. Sigmoid gate: u(s) = sigmoid(-β · (σ̂ - 1))
+     *
+     * @param obs  Observation vector (obs_dim,)
+     * @param train Whether to update EMA (true during training)
+     * @return GateResult with gate, v_fast, v_slow, sigma
+     */
+    GateResult computeGate(const Vector& obs, bool train = false) {
+        float v_fast = fast_critic_.forward(obs);
+        auto [v_slow, sigma] = slow_ensemble_.forward(obs);
+
+        if (train) {
+            sigma_ema_ = cfg_.ema_momentum * sigma_ema_
+                       + (1.0f - cfg_.ema_momentum) * sigma;
+        }
+
+        float normalized_sigma = sigma / (sigma_ema_ + cfg_.eps);
+        float gate = sigmoid(-cfg_.beta * (normalized_sigma - 1.0f));
+
+        return { gate, v_fast, v_slow, sigma };
+    }
+
+    // ── Value estimation ──────────────────────────────────────────────────────
+
+    /**
+     * Blended value estimate V^UGTC(s) = u(s)·V̄_slow(s) + (1-u(s))·V_fast(s)
+     */
+    float getValueUGTC(const Vector& obs, bool train = false) {
+        auto r = computeGate(obs, train);
+        return r.gate * r.v_slow + (1.0f - r.gate) * r.v_fast;
+    }
+
+    // ── GAE computation ───────────────────────────────────────────────────────
+
+    /**
+     * Standard Generalized Advantage Estimation.
+     *
+     * δₜ = rₜ + γ·V(sₜ₊₁)·(1-dₜ) - V(sₜ)
+     * Aₜ = δₜ + γλ·(1-dₜ)·Aₜ₊₁
+     *
+     * @param rewards   (T,) reward sequence
+     * @param values    (T,) current-state values
+     * @param next_vals (T,) next-state values
+     * @param dones     (T,) episode termination flags
+     * @param gamma     discount factor
+     * @param lam       GAE lambda
+     * @return          (T,) advantage estimates
+     */
+    static std::vector<float> computeGAE(
+        const std::vector<float>& rewards,
+        const std::vector<float>& values,
+        const std::vector<float>& next_vals,
+        const std::vector<float>& dones,
+        float gamma,
+        float lam
+    ) {
+        int T = static_cast<int>(rewards.size());
+        std::vector<float> advantages(T, 0.0f);
+
+        float gae = 0.0f;
+        for (int t = T - 1; t >= 0; --t) {
+            float delta = rewards[t] + gamma * next_vals[t] * (1.0f - dones[t]) - values[t];
+            gae = delta + gamma * lam * (1.0f - dones[t]) * gae;
+            advantages[t] = gae;
+        }
+        return advantages;
+    }
+
+    // ── UGTC advantage ────────────────────────────────────────────────────────
+
+    /**
+     * Compute UGTC blended advantages for a trajectory.
+     *
+     * A^UGTC_t = u(sₜ)·A^slow_t + (1-u(sₜ))·A^fast_t
+     *
+     * @param obs_seq      Sequence of observations (T × obs_dim)
+     * @param next_obs_seq Sequence of next observations (T × obs_dim)
+     * @param rewards      (T,) rewards
+     * @param dones        (T,) done flags
+     * @param gamma        Discount factor
+     * @param train        Whether to update EMA
+     * @return             (T,) UGTC blended advantages
+     */
+    std::vector<float> computeAdvantages(
+        const std::vector<Vector>& obs_seq,
+        const std::vector<Vector>& next_obs_seq,
+        const std::vector<float>& rewards,
+        const std::vector<float>& dones,
+        float gamma = 0.99f,
+        bool  train = false
+    ) {
+        int T = static_cast<int>(obs_seq.size());
+        assert(T == static_cast<int>(rewards.size()));
+
+        std::vector<float> gates(T), v_fast_arr(T), v_slow_arr(T);
+        std::vector<float> v_fast_next(T), v_slow_next(T);
+
+        for (int t = 0; t < T; ++t) {
+            auto r     = computeGate(obs_seq[t], train);
+            auto r_next = computeGate(next_obs_seq[t], false);
+            gates[t]       = r.gate;
+            v_fast_arr[t]  = r.v_fast;
+            v_slow_arr[t]  = r.v_slow;
+            v_fast_next[t] = r_next.v_fast;
+            v_slow_next[t] = r_next.v_slow;
+        }
+
+        auto adv_fast = computeGAE(rewards, v_fast_arr, v_fast_next, dones, gamma, cfg_.lambda_fast);
+        auto adv_slow = computeGAE(rewards, v_slow_arr, v_slow_next, dones, gamma, cfg_.lambda_slow);
+
+        std::vector<float> advantages(T);
+        for (int t = 0; t < T; ++t) {
+            advantages[t] = gates[t] * adv_slow[t] + (1.0f - gates[t]) * adv_fast[t];
+        }
+        return advantages;
+    }
+
+    // ── Accessors ─────────────────────────────────────────────────────────────
+
+    float getSigmaEMA() const { return sigma_ema_; }
+    const Config& getConfig() const { return cfg_; }
+
+private:
+    Config              cfg_;
+    ValueNetwork        fast_critic_;
+    EnsembleValueNetwork slow_ensemble_;
+    float               sigma_ema_;
+};
+
+} // namespace UGTC

implementations/java/UGTCModule.java

@@ -0,0 +1,372 @@
+/**
+ * UGTC: Uncertainty-Gated Temporal Credit — Java Reference Implementation
+ * =========================================================================
+ *
+ * A pure Java reference implementation of the UGTC module.
+ * No external dependencies. Uses simple float[][] arrays for matrix ops.
+ *
+ * This is a reference implementation for readability and portability.
+ * For production use, consider using a Java deep learning framework
+ * (DL4J, PyTorch Java API) with proper GPU support.
+ *
+ * Paper: https://doi.org/10.5281/zenodo.19715116
+ */
+
+package ai.ethosoft.ugtc;
+
+import java.util.Random;
+
+public class UGTCModule {
+
+    // ──────────────────────────────────────────────────────────────────────────
+    // Configuration
+    // ──────────────────────────────────────────────────────────────────────────
+
+    public static class Config {
+        public int   hiddenDim    = 64;
+        public int   M            = 3;
+        public float lambdaFast   = 0.80f;
+        public float lambdaSlow   = 0.99f;
+        public float beta         = 5.0f;
+        public float emaMomentum  = 0.99f;
+        public float eps          = 1e-8f;
+
+        public Config() {}
+
+        public Config(int hiddenDim, int M, float lambdaFast, float lambdaSlow,
+                      float beta, float emaMomentum) {
+            this.hiddenDim   = hiddenDim;
+            this.M           = M;
+            this.lambdaFast  = lambdaFast;
+            this.lambdaSlow  = lambdaSlow;
+            this.beta        = beta;
+            this.emaMomentum = emaMomentum;
+        }
+    }
+
+    // ──────────────────────────────────────────────────────────────────────────
+    // Linear layer
+    // ──────────────────────────────────────────────────────────────────────────
+
+    private static class Linear {
+        final float[][] W;   // [outDim][inDim]
+        final float[]   b;   // [outDim]
+
+        Linear(int inDim, int outDim, Random rng) {
+            W = new float[outDim][inDim];
+            b = new float[outDim];
+            float scale = (float) Math.sqrt(2.0 / inDim);
+            for (int i = 0; i < outDim; i++)
+                for (int j = 0; j < inDim; j++)
+                    W[i][j] = rng.nextGaussian() > 0 ? scale : -scale;
+        }
+
+        float[] forward(float[] x) {
+            int outDim = W.length;
+            int inDim  = x.length;
+            float[] out = new float[outDim];
+            for (int i = 0; i < outDim; i++) {
+                out[i] = b[i];
+                for (int j = 0; j < inDim; j++)
+                    out[i] += W[i][j] * x[j];
+            }
+            return out;
+        }
+    }
+
+    // ──────────────────────────────────────────────────────────────────────────
+    // Value network: obs → h → h → scalar
+    // ──────────────────────────────────────────────────────────────────────────
+
+    private static class ValueNetwork {
+        final Linear fc1, fc2, fc3;
+
+        ValueNetwork(int obsDim, int hiddenDim, Random rng) {
+            fc1 = new Linear(obsDim,    hiddenDim, rng);
+            fc2 = new Linear(hiddenDim, hiddenDim, rng);
+            fc3 = new Linear(hiddenDim, 1,         rng);
+        }
+
+        float forward(float[] obs) {
+            float[] h1 = applyTanh(fc1.forward(obs));
+            float[] h2 = applyTanh(fc2.forward(h1));
+            return fc3.forward(h2)[0];
+        }
+    }
+
+    // ──────────────────────────────────────────────────────────────────────────
+    // Ensemble value network
+    // ──────────────────────────────────────────────────────────────────────────
+
+    private static class EnsembleValueNetwork {
+        final ValueNetwork[] members;
+
+        EnsembleValueNetwork(int obsDim, int hiddenDim, int M, Random rng) {
+            members = new ValueNetwork[M];
+            for (int i = 0; i < M; i++)
+                members[i] = new ValueNetwork(obsDim, hiddenDim, rng);
+        }
+
+        /** @return float[] {mean, std} of ensemble predictions */
+        float[] forward(float[] obs) {
+            int M = members.length;
+            float[] vals = new float[M];
+            float mean   = 0.0f;
+
+            for (int i = 0; i < M; i++) {
+                vals[i] = members[i].forward(obs);
+                mean    += vals[i];
+            }
+            mean /= M;
+
+            float var = 0.0f;
+            for (float v : vals) var += (v - mean) * (v - mean);
+            var /= (M > 1 ? M - 1 : 1);
+
+            return new float[]{ mean, (float) Math.sqrt(var) };
+        }
+    }
+
+    // ──────────────────────────────────────────────────────────────────────────
+    // Gate result
+    // ──────────────────────────────────────────────────────────────────────────
+
+    public static class GateResult {
+        public final float gate;
+        public final float vFast;
+        public final float vSlow;
+        public final float sigma;
+
+        GateResult(float gate, float vFast, float vSlow, float sigma) {
+            this.gate  = gate;
+            this.vFast = vFast;
+            this.vSlow = vSlow;
+            this.sigma = sigma;
+        }
+    }
+
+    // ──────────────────────────────────────────────────────────────────────────
+    // Module fields
+    // ──────────────────────────────────────────────────────────────────────────
+
+    private final Config config;
+    private final ValueNetwork fastCritic;
+    private final EnsembleValueNetwork slowEnsemble;
+    private float sigmaEMA;
+
+    // ──────────────────────────────────────────────────────────────────────────
+    // Constructor
+    // ──────────────────────────────────────────────────────────────────────────
+
+    public UGTCModule(int obsDim) {
+        this(obsDim, new Config());
+    }
+
+    public UGTCModule(int obsDim, Config config) {
+        this.config = config;
+        this.sigmaEMA = 1.0f;
+        Random rng = new Random(42);
+        this.fastCritic   = new ValueNetwork(obsDim, config.hiddenDim, rng);
+        this.slowEnsemble = new EnsembleValueNetwork(obsDim, config.hiddenDim, config.M, rng);
+    }
+
+    // ──────────────────────────────────────────────────────────────────────────
+    // Gate computation
+    // ──────────────────────────────────────────────────────────────────────────
+
+    /**
+     * Compute the uncertainty gate u(s) for a single observation.
+     *
+     * u(s) = sigmoid(-β · (σ̂(s) - 1))
+     * where σ̂(s) = σ(s) / σ_EMA
+     *
+     * @param obs   Observation vector
+     * @param train Whether to update EMA statistics
+     * @return GateResult containing gate, v_fast, v_slow, sigma
+     */
+    public GateResult computeGate(float[] obs, boolean train) {
+        float vFast     = fastCritic.forward(obs);
+        float[] ensOut  = slowEnsemble.forward(obs);
+        float vSlow     = ensOut[0];
+        float sigma     = ensOut[1];
+
+        if (train) {
+            sigmaEMA = config.emaMomentum * sigmaEMA
+                     + (1.0f - config.emaMomentum) * sigma;
+        }
+
+        float normalizedSigma = sigma / (sigmaEMA + config.eps);
+        float gate = sigmoid(-config.beta * (normalizedSigma - 1.0f));
+
+        return new GateResult(gate, vFast, vSlow, sigma);
+    }
+
+    // ──────────────────────────────────────────────────────────────────────────
+    // Value estimation
+    // ──────────────────────────────────────────────────────────────────────────
+
+    /**
+     * Blended value estimate: V^UGTC(s) = u(s)·V̄_slow(s) + (1-u(s))·V_fast(s)
+     *
+     * @param obs   Observation vector
+     * @param train Whether to update EMA
+     * @return Scalar blended value
+     */
+    public float getValueUGTC(float[] obs, boolean train) {
+        GateResult r = computeGate(obs, train);
+        return r.gate * r.vSlow + (1.0f - r.gate) * r.vFast;
+    }
+
+    // ──────────────────────────────────────────────────────────────────────────
+    // GAE computation
+    // ──────────────────────────────────────────────────────────────────────────
+
+    /**
+     * Standard Generalized Advantage Estimation.
+     *
+     * δₜ = rₜ + γ·V(sₜ₊₁)·(1-dₜ) - V(sₜ)
+     * Aₜ = δₜ + γλ·(1-dₜ)·Aₜ₊₁
+     *
+     * @param rewards   Reward sequence
+     * @param values    Current-state values
+     * @param nextVals  Next-state values
+     * @param dones     Episode termination flags (1.0 = done)
+     * @param gamma     Discount factor
+     * @param lam       GAE lambda
+     * @return Array of advantage estimates
+     */
+    public static float[] computeGAE(
+        float[] rewards, float[] values, float[] nextVals, float[] dones,
+        float gamma, float lam
+    ) {
+        int T = rewards.length;
+        float[] advantages = new float[T];
+        float gae = 0.0f;
+
+        for (int t = T - 1; t >= 0; t--) {
+            float delta = rewards[t] + gamma * nextVals[t] * (1.0f - dones[t]) - values[t];
+            gae = delta + gamma * lam * (1.0f - dones[t]) * gae;
+            advantages[t] = gae;
+        }
+        return advantages;
+    }
+
+    // ──────────────────────────────────────────────────────────────────────────
+    // UGTC advantage computation
+    // ──────────────────────────────────────────────────────────────────────────
+
+    /**
+     * Compute UGTC blended advantages for a trajectory.
+     *
+     * A^UGTC_t = u(sₜ)·A^slow_t + (1-u(sₜ))·A^fast_t
+     *
+     * @param obsSeq     Sequence of observations (T × obsDim)
+     * @param nextObsSeq Sequence of next observations (T × obsDim)
+     * @param rewards    Reward sequence (T,)
+     * @param dones      Done flags (T,)
+     * @param gamma      Discount factor
+     * @param train      Whether to update EMA
+     * @return           UGTC blended advantages (T,)
+     */
+    public float[] computeAdvantages(
+        float[][] obsSeq, float[][] nextObsSeq,
+        float[] rewards, float[] dones,
+        float gamma, boolean train
+    ) {
+        int T = rewards.length;
+        float[] gates      = new float[T];
+        float[] vFastArr   = new float[T];
+        float[] vSlowArr   = new float[T];
+        float[] vFastNext  = new float[T];
+        float[] vSlowNext  = new float[T];
+
+        for (int t = 0; t < T; t++) {
+            GateResult r     = computeGate(obsSeq[t], train);
+            GateResult rNext = computeGate(nextObsSeq[t], false);
+            gates[t]      = r.gate;
+            vFastArr[t]   = r.vFast;
+            vSlowArr[t]   = r.vSlow;
+            vFastNext[t]  = rNext.vFast;
+            vSlowNext[t]  = rNext.vSlow;
+        }
+
+        float[] advFast = computeGAE(rewards, vFastArr, vFastNext, dones, gamma, config.lambdaFast);
+        float[] advSlow = computeGAE(rewards, vSlowArr, vSlowNext, dones, gamma, config.lambdaSlow);
+
+        float[] advantages = new float[T];
+        for (int t = 0; t < T; t++) {
+            advantages[t] = gates[t] * advSlow[t] + (1.0f - gates[t]) * advFast[t];
+        }
+        return advantages;
+    }
+
+    // Convenience overload with training=false and gamma=0.99
+    public float[] computeAdvantages(float[][] obsSeq, float[][] nextObsSeq,
+                                     float[] rewards, float[] dones) {
+        return computeAdvantages(obsSeq, nextObsSeq, rewards, dones, 0.99f, false);
+    }
+
+    // ──────────────────────────────────────────────────────────────────────────
+    // Accessors
+    // ──────────────────────────────────────────────────────────────────────────
+
+    public float getSigmaEMA() { return sigmaEMA; }
+    public Config getConfig()  { return config; }
+
+    // ────────────────────────���─────────────────────────────────────────────────
+    // Utility functions
+    // ──────────────────────────────────────────────────────────────────────────
+
+    private static float sigmoid(float x) {
+        return 1.0f / (1.0f + (float) Math.exp(-x));
+    }
+
+    private static float[] applyTanh(float[] x) {
+        float[] out = new float[x.length];
+        for (int i = 0; i < x.length; i++) out[i] = (float) Math.tanh(x[i]);
+        return out;
+    }
+
+    // ──────────────────────────────────────────────────────────────────────────
+    // Main — minimal smoke test
+    // ──────────────────────────────────────────────────────────────────────────
+
+    public static void main(String[] args) {
+        System.out.println("UGTC Java Reference Implementation");
+        System.out.println("Paper: https://doi.org/10.5281/zenodo.19715116");
+        System.out.println();
+
+        int obsDim = 17;
+        int T      = 32;
+        UGTCModule ugtc = new UGTCModule(obsDim);
+
+        // Random trajectory
+        Random rng = new Random(0);
+        float[][] obs      = new float[T][obsDim];
+        float[][] nextObs  = new float[T][obsDim];
+        float[]   rewards  = new float[T];
+        float[]   dones    = new float[T];
+
+        for (int t = 0; t < T; t++) {
+            for (int d = 0; d < obsDim; d++) {
+                obs[t][d]     = (float) rng.nextGaussian();
+                nextObs[t][d] = (float) rng.nextGaussian();
+            }
+            rewards[t] = (float) rng.nextGaussian();
+            dones[t]   = (t == T - 1) ? 1.0f : 0.0f;
+        }
+
+        float[] advantages = ugtc.computeAdvantages(obs, nextObs, rewards, dones, 0.99f, true);
+
+        System.out.printf("obs_dim: %d  T: %d%n", obsDim, T);
+        System.out.printf("Advantages: [%.4f, %.4f, %.4f, ...]%n",
+            advantages[0], advantages[1], advantages[2]);
+
+        // Gate check
+        GateResult gate = ugtc.computeGate(obs[0], false);
+        System.out.printf("Gate u(s₀): %.4f  σ_EMA: %.4f%n",
+            gate.gate, ugtc.getSigmaEMA());
+
+        System.out.println("\nSmoke test passed.");
+    }
+}

pseudocode/ugtc_pseudocode.md

@@ -0,0 +1,197 @@
+# UGTC: Algorithm Pseudocode
+
+## Algorithm 1: UGTC Module (Core)
+
+```
+INPUT:
+    obs_dim      — observation space dimension
+    M            — ensemble size (default: 3)
+    λ_fast       — fast critic GAE lambda (default: 0.80)
+    λ_slow       — slow ensemble GAE lambda (default: 0.99)
+    β            — gate temperature (default: 5.0)
+    α_EMA        — EMA momentum (default: 0.99)
+
+PARAMETERS:
+    θ_fast       — fast critic V_fast: obs_dim → 64 → 64 → 1
+    θ¹_slow, ..., θᴹ_slow — slow ensemble members (same arch, independent init)
+    σ_EMA        — EMA buffer (initialized to 1.0)
+
+──────────────────────────────────────────────────────────────
+
+PROCEDURE compute_gate(s):
+    v_fast       ← V_fast(s; θ_fast)
+    v¹, ..., vᴹ ← V¹_slow(s), ..., Vᴹ_slow(s)     # slow ensemble
+    v̄_slow       ← mean(v¹, ..., vᴹ)
+    σ(s)         ← std(v¹, ..., vᴹ)
+
+    // EMA normalization (updated only during training)
+    σ_EMA        ← α_EMA · σ_EMA + (1 - α_EMA) · mean(σ(s))
+    σ̂(s)         ← σ(s) / (σ_EMA + ε)
+
+    // Sigmoid gate
+    u(s)         ← sigmoid(-β · (σ̂(s) - 1.0))
+
+    RETURN u(s), v_fast, v̄_slow
+
+──────────────────────────────────────────────────────────────
+
+PROCEDURE compute_advantages(τ = {s₀, a₀, r₀, s₁, ..., sₜ}, γ):
+    FOR t = 0 TO T-1:
+        u(sₜ), v_fast(sₜ), v̄_slow(sₜ)   ← compute_gate(sₜ)
+        u(sₜ₊₁), v_fast(sₜ₊₁), v̄_slow(sₜ₊₁) ← compute_gate(sₜ₊₁)
+
+    // Fast GAE stream (λ = λ_fast)
+    A^fast ← GAE(τ, V=v_fast, λ=λ_fast, γ=γ)
+
+    // Slow GAE stream (λ = λ_slow)
+    A^slow ← GAE(τ, V=v̄_slow, λ=λ_slow, γ=γ)
+
+    // Blended advantage
+    FOR t = 0 TO T-1:
+        A^UGTC_t ← u(sₜ) · A^slow_t + (1 - u(sₜ)) · A^fast_t
+
+    RETURN A^UGTC
+
+──────────────────────────────────────────────────────────────
+
+SUBROUTINE GAE(τ, V, λ, γ):
+    // Standard Generalized Advantage Estimation
+    gae ← 0
+    FOR t = T-1 DOWNTO 0:
+        δₜ   ← rₜ + γ · V(sₜ₊₁) · (1 - dₜ) - V(sₜ)
+        gaeₜ ← δₜ + γ · λ · (1 - dₜ) · gae
+        gae  ← gaeₜ
+    RETURN {gae₀, gae₁, ..., gaeₜ}
+```
+
+---
+
+## Algorithm 2: UGTC-PPO
+
+```
+HYPERPARAMETERS (fixed):
+    λ_fast=0.80, λ_slow=0.99, M=3, β=5.0, α_EMA=0.99
+PPO HYPERPARAMETERS:
+    ε=0.2 (clip), K epochs, γ, lr
+
+INITIALIZE:
+    π_θ          — policy (Gaussian or Categorical)
+    UGTC         — UGTCModule(obs_dim, M, λ_fast, λ_slow, β)
+
+REPEAT until convergence:
+    // Collect rollout
+    τ ← {s₀, a₀, r₀, ..., sₜ}  using π_θ
+
+    // === KEY CHANGE: UGTC replaces standard GAE ===
+    A^UGTC ← UGTC.compute_advantages(τ, γ)
+    Â      ← normalize(A^UGTC)             // mean=0, std=1
+    R̂      ← Â + V^UGTC(s)                 // value targets
+
+    // K gradient epochs
+    FOR k = 1 TO K:
+        // Policy loss (standard PPO clipped surrogate)
+        ratio       ← π_θ(a|s) / π_θ_old(a|s)
+        L_policy    ← -mean(min(ratio·Â, clip(ratio, 1±ε)·Â))
+
+        // Critic losses (train all UGTC critics)
+        L_fast      ← MSE(V_fast(s), R̂)
+        L_slow      ← mean over m of MSE(Vᵐ_slow(s), R̂)
+        L_total     ← L_policy + 0.5·(L_fast + L_slow)
+
+        θ, UGTC ← Adam step on L_total
+```
+
+---
+
+## Algorithm 3: UGTC-TD3
+
+```
+INITIALIZE:
+    π_φ (actor), Q_ψ (twin-Q critic), target networks
+    UGTC — UGTCModule(obs_dim)
+    Replay buffer D
+
+REPEAT until convergence:
+    // Collect experience
+    aₜ ← π_φ(sₜ) + ε (exploration noise)
+    Store (sₜ, aₜ, rₜ, sₜ₊₁) in D
+
+    // Sample batch
+    B = {(s, a, r, s')} ← sample(D)
+
+    // Critic update (standard TD3 twin-Q, unchanged)
+    ã ← π_φ_target(s') + clip(ε, -c, c)    // target policy smoothing
+    y ← r + γ · min(Q¹_target(s',ã), Q²_target(s',ã))
+    L_critic ← MSE(Q¹(s,a), y) + MSE(Q²(s,a), y)
+    ψ ← Adam step on L_critic
+
+    // Delayed actor update (every d steps)
+    IF update_count % d == 0:
+        u(s), v_fast, v̄_slow ← UGTC.compute_gate(s)
+        V^UGTC(s)  ← u(s)·v̄_slow + (1-u(s))·v_fast
+        Q_min      ← min(Q¹(s, π(s)), Q²(s, π(s)))
+        A^UGTC(s)  ← Q_min - V^UGTC(s)     // advantage correction
+
+        // === KEY CHANGE: DPG + UGTC baseline correction ===
+        L_actor ← -mean(Q_min + η · A^UGTC(s))
+        φ ← Adam step on L_actor
+
+        // UGTC critic training
+        L_UGTC ← MSE(V^UGTC(s), r + γ·V^UGTC(s'))
+        UGTC ← Adam step on L_UGTC
+
+        // Soft target updates
+        ψ_target ← τ·ψ + (1-τ)·ψ_target
+        φ_target ← τ·φ + (1-τ)·φ_target
+```
+
+---
+
+## Algorithm 4: UGTC-SAC
+
+```
+INITIALIZE:
+    π_φ (stochastic actor), Q_ψ (twin-Q), α (entropy coefficient)
+    UGTC — UGTCModule(obs_dim)
+
+REPEAT until convergence:
+    Sample B from replay buffer
+
+    // Critic update (standard SAC twin-Q)
+    ã', log_π(ã'|s') ← sample π(·|s')
+    y  ← r + γ(min(Q¹_target, Q²_target)(s',ã') - α·log_π(ã'|s'))
+    L_critic ← MSE(Q¹(s,a), y) + MSE(Q²(s,a), y)
+
+    // Actor update with UGTC baseline
+    ã, log_π(ã|s) ← sample π(·|s)
+    Q_min          ← min(Q¹(s,ã), Q²(s,ã))
+    V^UGTC(s)      ← u(s)·V̄_slow(s) + (1-u(s))·V_fast(s)
+
+    // === KEY CHANGE: V^UGTC replaces implicit value baseline ===
+    // Standard SAC:  L_π = mean(α·log π - Q_min)
+    // UGTC-SAC:      L_π = mean(α·log π - Q_min + V^UGTC)
+    L_actor ← mean(α·log π(ã|s) - Q_min + V^UGTC(s))
+
+    // Entropy coefficient update (auto-tuning)
+    L_α ← mean(-log_α · (log π(ã|s) + H̄))   // H̄ = target entropy
+
+    // UGTC critic update
+    L_UGTC ← MSE(V^UGTC(s), r + γ·V^UGTC(s'))
+```
+
+---
+
+## Gate Behavior Summary
+
+```
+                      σ̂(s)                     u(s)        Action
+───────────────────────────────────────────────────────────────────
+σ̂ << 1   (low uncertainty, ensemble agrees)  → u → 1.0   use A^slow (accurate, high λ)
+σ̂ = 1    (uncertainty at running average)   → u = 0.5   equal blend
+σ̂ >> 1   (high uncertainty, ensemble disagrees) → u → 0.0  use A^fast (stable, low λ)
+```
+
+The gate temperature β controls the sharpness of this transition:
+- β = 1: gradual blend
+- β = 5: moderate sharpness (paper default)
+- β = 20: near-binary switching

pyproject.toml

@@ -0,0 +1,100 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.backends.legacy:build"
+
+[project]
+name = "ugtc"
+version = "1.0.0"
+description = "UGTC: Uncertainty-Gated Temporal Credit — plug-in advantage estimator for actor-critic RL"
+readme = "README.md"
+license = { file = "LICENSE" }
+authors = [
+    { name = "Yağız Ekrem Dalar", email = "pertevniyalai@gmail.com" }
+]
+keywords = [
+    "reinforcement-learning",
+    "advantage-estimation",
+    "temporal-credit",
+    "uncertainty",
+    "actor-critic",
+    "PPO",
+    "TD3",
+    "SAC",
+    "DDPG",
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+requires-python = ">=3.10"
+dependencies = [
+    "torch>=2.2.0",
+    "numpy>=1.24.0",
+]
+
+[project.optional-dependencies]
+mujoco = [
+    "gymnasium>=1.0.0",
+    "mujoco>=3.1.0",
+]
+procgen = [
+    "gymnasium>=1.0.0",
+    "procgen>=0.10.7",
+]
+all = [
+    "gymnasium>=1.0.0",
+    "mujoco>=3.1.0",
+    "procgen>=0.10.7",
+    "metaworld>=2.0.0",
+    "crafter>=1.8.0",
+    "pyyaml>=6.0",
+    "matplotlib>=3.7.0",
+    "pandas>=2.0.0",
+    "psutil>=5.9.0",
+]
+dev = [
+    "pytest>=7.4",
+    "pytest-cov>=4.1",
+    "black>=24.0",
+    "isort>=5.12",
+    "mypy>=1.8",
+    "ruff>=0.3",
+]
+
+[project.urls]
+Homepage = "https://ethosoftai.github.io/ugtc"
+Repository = "https://github.com/ethosoftai/ugtc"
+Paper = "https://doi.org/10.5281/zenodo.19715116"
+Demo = "https://huggingface.co/spaces/Ethosoft/ugtc"
+"Bug Tracker" = "https://github.com/ethosoftai/ugtc/issues"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["ugtc*"]
+
+[tool.black]
+line-length = 100
+target-version = ["py310", "py311"]
+
+[tool.isort]
+profile = "black"
+line_length = 100
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-v --tb=short"
+
+[tool.mypy]
+python_version = "3.10"
+warn_return_any = true
+warn_unused_configs = true

requirements.txt

@@ -0,0 +1,11 @@
+torch>=2.2.0
+numpy>=1.24.0
+gymnasium>=1.0.0
+mujoco>=3.1.0
+pyyaml>=6.0
+procgen>=0.10.7
+metaworld>=2.0.0
+crafter>=1.8.0
+matplotlib>=3.7.0
+pandas>=2.0.0
+psutil>=5.9.0

tests/test_integrations.py

@@ -0,0 +1,216 @@
+"""
+Integration tests for UGTC algorithm wrappers (PPO, TD3, SAC, DDPG).
+
+These tests verify that each integration:
+    - Initializes without error
+    - Produces valid losses (finite, not NaN)
+    - Returns expected metric keys
+    - Updates parameters (gradient flows)
+
+Note: Full training runs are not tested here — see examples/ for that.
+"""
+
+import pytest
+import torch
+import numpy as np
+
+from ugtc import UGTCPPO, UGTCTD3, UGTCSAC, UGTCDDPG
+from ugtc.td3 import ReplayBuffer
+
+
+OBS_DIM = 8
+ACT_DIM = 2
+BATCH = 32
+HIDDEN = 32
+
+
+def make_ppo_rollout(T=64, device="cpu"):
+    obs = torch.randn(T, OBS_DIM)
+    actions = torch.randn(T, ACT_DIM)
+    return {
+        "obs": obs,
+        "actions": actions,
+        "rewards": torch.randn(T),
+        "next_obs": torch.randn(T, OBS_DIM),
+        "dones": torch.zeros(T),
+        "log_probs": torch.randn(T),
+    }
+
+
+def make_replay_buffer():
+    buf = ReplayBuffer(OBS_DIM, ACT_DIM, capacity=1000)
+    for _ in range(BATCH * 2):
+        obs = np.random.randn(OBS_DIM).astype(np.float32)
+        action = np.random.randn(ACT_DIM).astype(np.float32)
+        reward = float(np.random.randn())
+        next_obs = np.random.randn(OBS_DIM).astype(np.float32)
+        done = False
+        buf.add(obs, action, reward, next_obs, done)
+    return buf
+
+
+# ── UGTC-PPO ──────────────────────────────────────────────────────────────────
+
+class TestUGTCPPO:
+    @pytest.fixture
+    def agent(self):
+        return UGTCPPO(OBS_DIM, ACT_DIM, hidden_dim=HIDDEN, epochs=2)
+
+    def test_init(self, agent):
+        assert agent.policy is not None
+        assert agent.ugtc is not None
+
+    def test_select_action_shape(self, agent):
+        obs = torch.randn(1, OBS_DIM)
+        action, log_prob = agent.select_action(obs)
+        assert action.shape == (1, ACT_DIM)
+        assert log_prob.shape == (1,)
+
+    def test_update_returns_dict(self, agent):
+        rollout = make_ppo_rollout()
+        metrics = agent.update(rollout)
+        assert isinstance(metrics, dict)
+
+    def test_update_metrics_finite(self, agent):
+        rollout = make_ppo_rollout()
+        metrics = agent.update(rollout)
+        for key, val in metrics.items():
+            assert np.isfinite(val), f"Metric {key} = {val} is not finite"
+
+    def test_update_metrics_keys(self, agent):
+        rollout = make_ppo_rollout()
+        metrics = agent.update(rollout)
+        assert "policy_loss" in metrics
+        assert "fast_value_loss" in metrics
+        assert "gate_mean" in metrics
+
+    def test_parameters_update(self, agent):
+        """Verify that an update step actually modifies parameters."""
+        initial_params = [p.clone() for p in agent.policy.parameters()]
+        rollout = make_ppo_rollout()
+        agent.update(rollout)
+        updated_params = list(agent.policy.parameters())
+        changed = any(
+            not torch.allclose(i, u) for i, u in zip(initial_params, updated_params)
+        )
+        assert changed, "Policy parameters should change after update"
+
+    def test_save_load(self, agent, tmp_path):
+        path = str(tmp_path / "ppo.pt")
+        agent.save(path)
+        agent.load(path)
+
+
+# ── UGTC-TD3 ──────────────────────────────────────────────────────────────────
+
+class TestUGTCTD3:
+    @pytest.fixture
+    def agent(self):
+        return UGTCTD3(OBS_DIM, ACT_DIM, hidden=HIDDEN, device="cpu")
+
+    def test_init(self, agent):
+        assert agent.actor is not None
+        assert agent.ugtc is not None
+
+    def test_select_action_shape(self, agent):
+        obs = np.random.randn(OBS_DIM).astype(np.float32)
+        action = agent.select_action(obs, noise=0.0)
+        assert action.shape == (ACT_DIM,)
+
+    def test_update_returns_dict(self, agent):
+        buf = make_replay_buffer()
+        metrics = agent.update(buf, batch_size=BATCH)
+        assert isinstance(metrics, dict)
+
+    def test_critic_loss_finite(self, agent):
+        buf = make_replay_buffer()
+        metrics = agent.update(buf, batch_size=BATCH)
+        assert np.isfinite(metrics["critic_loss"])
+
+    def test_actor_loss_after_delay(self, agent):
+        """Actor loss should appear after policy_delay updates."""
+        buf = make_replay_buffer()
+        # Run enough updates to trigger delayed actor update
+        for _ in range(agent.policy_delay + 1):
+            metrics = agent.update(buf, batch_size=BATCH)
+        assert "actor_loss" in metrics
+
+    def test_action_clipped(self, agent):
+        obs = np.random.randn(OBS_DIM).astype(np.float32) * 10
+        action = agent.select_action(obs, noise=0.0)
+        assert np.all(np.abs(action) <= agent.max_action + 1e-6)
+
+
+# ── UGTC-SAC ───────────────────���──────────────────────────────────────────────
+
+class TestUGTCSAC:
+    @pytest.fixture
+    def agent(self):
+        return UGTCSAC(OBS_DIM, ACT_DIM, hidden=HIDDEN, auto_alpha=True, device="cpu")
+
+    def test_init(self, agent):
+        assert agent.policy is not None
+        assert agent.ugtc is not None
+
+    def test_select_action_stochastic(self, agent):
+        obs = np.random.randn(OBS_DIM).astype(np.float32)
+        a1 = agent.select_action(obs, deterministic=False)
+        a2 = agent.select_action(obs, deterministic=False)
+        assert a1.shape == (ACT_DIM,)
+        # Stochastic: two samples should (almost certainly) differ
+        assert not np.allclose(a1, a2)
+
+    def test_select_action_deterministic(self, agent):
+        obs = np.random.randn(OBS_DIM).astype(np.float32)
+        a1 = agent.select_action(obs, deterministic=True)
+        a2 = agent.select_action(obs, deterministic=True)
+        assert np.allclose(a1, a2)
+
+    def test_update_returns_dict(self, agent):
+        buf = make_replay_buffer()
+        metrics = agent.update(buf, batch_size=BATCH)
+        assert isinstance(metrics, dict)
+
+    def test_all_metrics_finite(self, agent):
+        buf = make_replay_buffer()
+        metrics = agent.update(buf, batch_size=BATCH)
+        for k, v in metrics.items():
+            assert np.isfinite(v), f"Metric {k} = {v}"
+
+    def test_auto_alpha_changes(self, agent):
+        buf = make_replay_buffer()
+        initial_alpha = agent.alpha
+        for _ in range(5):
+            agent.update(buf, batch_size=BATCH)
+        # Alpha should move (unless already converged)
+        assert isinstance(agent.alpha, float)
+
+
+# ── UGTC-DDPG ─────────────────────────────────────────────────────────────────
+
+class TestUGTCDDPG:
+    @pytest.fixture
+    def agent(self):
+        return UGTCDDPG(OBS_DIM, ACT_DIM, hidden=HIDDEN, device="cpu")
+
+    def test_init(self, agent):
+        assert agent.actor is not None
+        assert agent.ugtc is not None
+
+    def test_select_action(self, agent):
+        obs = np.random.randn(OBS_DIM).astype(np.float32)
+        action = agent.select_action(obs, add_noise=False)
+        assert action.shape == (ACT_DIM,)
+
+    def test_update_finite(self, agent):
+        buf = make_replay_buffer()
+        metrics = agent.update(buf, batch_size=BATCH)
+        for k, v in metrics.items():
+            assert np.isfinite(v), f"Metric {k} = {v}"
+
+    def test_update_keys(self, agent):
+        buf = make_replay_buffer()
+        metrics = agent.update(buf, batch_size=BATCH)
+        assert "critic_loss" in metrics
+        assert "actor_loss" in metrics
+        assert "gate_mean" in metrics

tests/test_module.py

@@ -0,0 +1,243 @@
+"""
+Unit tests for UGTCModule.
+
+Tests cover:
+    - Gate computation correctness
+    - GAE computation
+    - Advantage shapes and dtype
+    - EMA normalization
+    - Parameter counting
+    - Value blending
+"""
+
+import pytest
+import torch
+import numpy as np
+
+from ugtc.module import ValueNetwork, EnsembleValueNetwork, UGTCModule
+
+
+OBS_DIM = 17
+BATCH = 32
+HIDDEN = 32
+
+
+@pytest.fixture
+def ugtc():
+    return UGTCModule(obs_dim=OBS_DIM, hidden_dim=HIDDEN, M=3)
+
+
+@pytest.fixture
+def obs():
+    return torch.randn(BATCH, OBS_DIM)
+
+
+# ── ValueNetwork ─────────────────────────────────────────────────────────────
+
+class TestValueNetwork:
+    def test_output_shape(self):
+        net = ValueNetwork(OBS_DIM, HIDDEN)
+        obs = torch.randn(BATCH, OBS_DIM)
+        out = net(obs)
+        assert out.shape == (BATCH,), f"Expected ({BATCH},), got {out.shape}"
+
+    def test_single_sample(self):
+        net = ValueNetwork(OBS_DIM, HIDDEN)
+        obs = torch.randn(1, OBS_DIM)
+        out = net(obs)
+        assert out.shape == (1,)
+
+    def test_grad_flows(self):
+        net = ValueNetwork(OBS_DIM, HIDDEN)
+        obs = torch.randn(BATCH, OBS_DIM, requires_grad=False)
+        loss = net(obs).mean()
+        loss.backward()
+        for p in net.parameters():
+            if p.requires_grad:
+                assert p.grad is not None
+
+
+# ── EnsembleValueNetwork ──────────────────────────────────────────────────────
+
+class TestEnsembleValueNetwork:
+    def test_output_shapes(self):
+        ens = EnsembleValueNetwork(OBS_DIM, HIDDEN, M=3)
+        obs = torch.randn(BATCH, OBS_DIM)
+        mean, sigma = ens(obs)
+        assert mean.shape == (BATCH,)
+        assert sigma.shape == (BATCH,)
+
+    def test_uncertainty_nonneg(self):
+        ens = EnsembleValueNetwork(OBS_DIM, HIDDEN, M=3)
+        obs = torch.randn(BATCH, OBS_DIM)
+        _, sigma = ens(obs)
+        assert (sigma >= 0).all(), "Uncertainty (std) must be non-negative"
+
+    def test_diversity(self):
+        """Members should produce different outputs (different random init)."""
+        ens = EnsembleValueNetwork(OBS_DIM, HIDDEN, M=5)
+        obs = torch.randn(BATCH, OBS_DIM)
+        all_vals = ens.forward_all(obs)  # (M, batch)
+        # At least one pair should differ
+        diffs = all_vals[0] - all_vals[1]
+        assert diffs.abs().mean() > 0, "Ensemble members should differ"
+
+    def test_forward_all_shape(self):
+        M = 4
+        ens = EnsembleValueNetwork(OBS_DIM, HIDDEN, M=M)
+        out = ens.forward_all(torch.randn(BATCH, OBS_DIM))
+        assert out.shape == (M, BATCH)
+
+
+# ── UGTCModule ────────────────────────────────────────────────────────────────
+
+class TestUGTCModule:
+    def test_gate_shape(self, ugtc, obs):
+        gate, v_fast, v_slow = ugtc.compute_gate(obs)
+        assert gate.shape == (BATCH,)
+        assert v_fast.shape == (BATCH,)
+        assert v_slow.shape == (BATCH,)
+
+    def test_gate_in_unit_interval(self, ugtc, obs):
+        gate, _, _ = ugtc.compute_gate(obs)
+        assert (gate >= 0.0).all(), "Gate must be >= 0"
+        assert (gate <= 1.0).all(), "Gate must be <= 1"
+
+    def test_advantage_shape(self, ugtc, obs):
+        next_obs = torch.randn(BATCH, OBS_DIM)
+        rewards = torch.randn(BATCH)
+        dones = torch.zeros(BATCH)
+        adv = ugtc.compute_advantages(obs, next_obs, rewards, dones, gamma=0.99)
+        assert adv.shape == (BATCH,), f"Expected ({BATCH},), got {adv.shape}"
+
+    def test_advantage_finite(self, ugtc, obs):
+        next_obs = torch.randn(BATCH, OBS_DIM)
+        rewards = torch.randn(BATCH)
+        dones = torch.zeros(BATCH)
+        adv = ugtc.compute_advantages(obs, next_obs, rewards, dones)
+        assert torch.isfinite(adv).all(), "All advantages should be finite"
+
+    def test_value_shape(self, ugtc, obs):
+        v = ugtc.get_value_ugtc(obs)
+        assert v.shape == (BATCH,)
+
+    def test_value_finite(self, ugtc, obs):
+        v = ugtc.get_value_ugtc(obs)
+        assert torch.isfinite(v).all()
+
+    def test_gae_zero_reward(self, ugtc):
+        """Zero rewards with no termination should give near-zero advantages."""
+        T = 16
+        obs = torch.randn(T, OBS_DIM)
+        next_obs = torch.randn(T, OBS_DIM)
+        rewards = torch.zeros(T)
+        dones = torch.zeros(T)
+        adv = ugtc.compute_advantages(obs, next_obs, rewards, dones, gamma=0.99)
+        # GAE with zero rewards is not necessarily zero (depends on value differences)
+        assert adv.shape == (T,)
+
+    def test_done_masks(self, ugtc):
+        """Done flags should prevent bootstrapping across episodes."""
+        T = 4
+        obs = torch.randn(T, OBS_DIM)
+        next_obs = torch.randn(T, OBS_DIM)
+        rewards = torch.ones(T)
+        dones_all = torch.ones(T)  # every step terminates
+        dones_none = torch.zeros(T)
+
+        adv_all = ugtc.compute_advantages(obs, next_obs, rewards, dones_all)
+        adv_none = ugtc.compute_advantages(obs, next_obs, rewards, dones_none)
+        # These should differ because done=1 zeroes out future bootstrapping
+        assert not torch.allclose(adv_all, adv_none)
+
+    def test_parameter_count(self, ugtc):
+        counts = ugtc.parameter_count()
+        assert "fast_critic" in counts
+        assert "slow_ensemble" in counts
+        assert "total" in counts
+        assert counts["total"] == counts["fast_critic"] + counts["slow_ensemble"]
+        assert counts["total"] > 0
+
+    def test_gate_stats_keys(self, ugtc, obs):
+        stats = ugtc.get_gate_stats(obs)
+        for key in ("gate_mean", "gate_std", "gate_min", "gate_max", "sigma_ema"):
+            assert key in stats, f"Missing key: {key}"
+
+    def test_ema_updates_during_training(self, ugtc, obs):
+        ugtc.train()
+        initial_ema = ugtc.sigma_ema.item()
+        for _ in range(10):
+            ugtc.compute_gate(obs)
+        updated_ema = ugtc.sigma_ema.item()
+        # EMA should change (unless uncertainty is exactly 1.0 from the start)
+        assert isinstance(updated_ema, float)
+
+    def test_ema_frozen_in_eval(self, ugtc, obs):
+        ugtc.eval()
+        initial_ema = ugtc.sigma_ema.item()
+        for _ in range(10):
+            ugtc.compute_gate(obs)
+        assert ugtc.sigma_ema.item() == initial_ema, "EMA should not update in eval mode"
+
+    def test_different_lambda_values(self):
+        """Verify different lambda values produce different advantages."""
+        ugtc_low = UGTCModule(OBS_DIM, HIDDEN, lambda_fast=0.1, lambda_slow=0.2)
+        ugtc_high = UGTCModule(OBS_DIM, HIDDEN, lambda_fast=0.8, lambda_slow=0.99)
+        obs = torch.randn(16, OBS_DIM)
+        next_obs = torch.randn(16, OBS_DIM)
+        rewards = torch.randn(16)
+        dones = torch.zeros(16)
+        adv_low = ugtc_low.compute_advantages(obs, next_obs, rewards, dones)
+        adv_high = ugtc_high.compute_advantages(obs, next_obs, rewards, dones)
+        assert not torch.allclose(adv_low, adv_high)
+
+    def test_beta_affects_gate_sharpness(self):
+        """Higher beta should produce sharper gate transitions."""
+        ugtc_low_beta = UGTCModule(OBS_DIM, HIDDEN, beta=0.1)
+        ugtc_high_beta = UGTCModule(OBS_DIM, HIDDEN, beta=20.0)
+        obs = torch.randn(64, OBS_DIM)
+        gate_low, _, _ = ugtc_low_beta.compute_gate(obs)
+        gate_high, _, _ = ugtc_high_beta.compute_gate(obs)
+        # High beta should have more extreme values (closer to 0 or 1)
+        extremity_low = ((gate_low - 0.5).abs()).mean()
+        extremity_high = ((gate_high - 0.5).abs()).mean()
+        assert extremity_high >= extremity_low, "Higher beta should produce sharper gate"
+
+    @pytest.mark.parametrize("M", [1, 2, 5, 10])
+    def test_ensemble_sizes(self, M):
+        ugtc = UGTCModule(OBS_DIM, HIDDEN, M=M)
+        obs = torch.randn(BATCH, OBS_DIM)
+        gate, v_fast, v_slow = ugtc.compute_gate(obs)
+        assert gate.shape == (BATCH,)
+
+    def test_no_grad_in_advantage_computation(self, ugtc, obs):
+        """compute_advantages should not retain gradients on the output."""
+        next_obs = torch.randn(BATCH, OBS_DIM)
+        rewards = torch.randn(BATCH)
+        dones = torch.zeros(BATCH)
+        adv = ugtc.compute_advantages(obs, next_obs, rewards, dones)
+        assert not adv.requires_grad
+
+
+# ── GAE computation ────────────────────────────────────────────────────────────
+
+class TestGAEComputation:
+    def test_single_step_gae(self):
+        """Single step: advantage = δ = r + γV(s') - V(s)."""
+        rewards = torch.tensor([1.0])
+        values = torch.tensor([0.5])
+        next_values = torch.tensor([0.5])
+        dones = torch.tensor([0.0])
+        adv = UGTCModule._compute_gae(rewards, values, next_values, dones, gamma=0.99, lam=0.95)
+        expected = 1.0 + 0.99 * 0.5 - 0.5
+        assert abs(adv[0].item() - expected) < 1e-5
+
+    def test_terminal_step(self):
+        """Done=1 should zero out future value bootstrap."""
+        rewards = torch.tensor([1.0])
+        values = torch.tensor([0.0])
+        next_values = torch.tensor([100.0])  # large, should be masked
+        dones = torch.tensor([1.0])
+        adv = UGTCModule._compute_gae(rewards, values, next_values, dones, gamma=0.99, lam=0.95)
+        expected = 1.0 + 0.99 * 100.0 * 0.0 - 0.0  # next_values masked out
+        assert abs(adv[0].item() - expected) < 1e-5

ugtc/__init__.py

@@ -0,0 +1,41 @@
+"""
+UGTC: Uncertainty-Gated Temporal Credit
+========================================
+
+A backbone-agnostic plug-in advantage estimator for actor-critic RL.
+
+Core exports:
+    UGTCModule       — main module (use this in your algorithm)
+    ValueNetwork     — single value network (fast critic)
+    EnsembleValueNetwork — ensemble of value networks (slow critic)
+
+Algorithm integrations:
+    UGTCPPO          — UGTC + Proximal Policy Optimization
+    UGTCTD3          — UGTC + Twin Delayed DDPG
+    UGTCSAC          — UGTC + Soft Actor-Critic
+    UGTCDDPG         — UGTC + Deep Deterministic Policy Gradient
+
+Paper: https://doi.org/10.5281/zenodo.19715116
+Docs:  https://ethosoftai.github.io/ugtc
+"""
+
+from ugtc.module import UGTCModule, ValueNetwork, EnsembleValueNetwork
+from ugtc.ppo import UGTCPPO
+from ugtc.td3 import UGTCTD3
+from ugtc.sac import UGTCSAC
+from ugtc.ddpg import UGTCDDPG
+
+__version__ = "1.0.0"
+__author__ = "Yağız Ekrem Dalar"
+__license__ = "MIT"
+__paper__ = "https://doi.org/10.5281/zenodo.19715116"
+
+__all__ = [
+    "UGTCModule",
+    "ValueNetwork",
+    "EnsembleValueNetwork",
+    "UGTCPPO",
+    "UGTCTD3",
+    "UGTCSAC",
+    "UGTCDDPG",
+]

ugtc/ddpg.py

@@ -0,0 +1,199 @@
+"""
+UGTC-DDPG: Deep Deterministic Policy Gradient with Uncertainty-Gated Temporal Credit
+======================================================================================
+
+Integration strategy:
+    UGTC provides an uncertainty-aware advantage correction to the DDPG actor.
+    This is an implementation extension — DDPG is not explicitly benchmarked in
+    the paper, but the integration follows the same principle as UGTC-TD3
+    (removing twin-Q and delayed policy update).
+
+    Actor loss:
+        L_actor = -(Q(s, π(s)) + η · A^UGTC(s)).mean()
+
+    where A^UGTC(s) = Q(s, π(s)) - V^UGTC(s)
+
+    ⚠️  Assumption: This integration is a proposed extension based on the
+    UGTC principle. It is not directly evaluated in the paper. Results may
+    differ from TD3/PPO integrations.
+
+Reference: https://doi.org/10.5281/zenodo.19715116
+"""
+
+import copy
+import torch
+import torch.nn as nn
+import torch.optim as optim
+import numpy as np
+from typing import Dict
+
+from ugtc.module import UGTCModule
+from ugtc.td3 import ReplayBuffer
+
+
+class DDPGActor(nn.Module):
+    def __init__(self, obs_dim: int, act_dim: int, hidden: int = 256, max_action: float = 1.0):
+        super().__init__()
+        self.max_action = max_action
+        self.net = nn.Sequential(
+            nn.Linear(obs_dim, hidden), nn.ReLU(),
+            nn.Linear(hidden, hidden), nn.ReLU(),
+            nn.Linear(hidden, act_dim), nn.Tanh(),
+        )
+
+    def forward(self, obs: torch.Tensor) -> torch.Tensor:
+        return self.max_action * self.net(obs)
+
+
+class DDPGCritic(nn.Module):
+    """Single Q-network (DDPG, unlike TD3 which uses twin-Q)."""
+
+    def __init__(self, obs_dim: int, act_dim: int, hidden: int = 256):
+        super().__init__()
+        self.net = nn.Sequential(
+            nn.Linear(obs_dim + act_dim, hidden), nn.ReLU(),
+            nn.Linear(hidden, hidden), nn.ReLU(),
+            nn.Linear(hidden, 1),
+        )
+
+    def forward(self, obs: torch.Tensor, action: torch.Tensor) -> torch.Tensor:
+        return self.net(torch.cat([obs, action], dim=-1)).squeeze(-1)
+
+
+class OUNoise:
+    """Ornstein-Uhlenbeck noise for DDPG exploration."""
+
+    def __init__(self, act_dim: int, mu: float = 0.0, theta: float = 0.15, sigma: float = 0.2):
+        self.mu = mu * np.ones(act_dim)
+        self.theta = theta
+        self.sigma = sigma
+        self.state = self.mu.copy()
+
+    def reset(self):
+        self.state = self.mu.copy()
+
+    def sample(self) -> np.ndarray:
+        dx = self.theta * (self.mu - self.state) + self.sigma * np.random.randn(*self.state.shape)
+        self.state += dx
+        return self.state
+
+
+class UGTCDDPG:
+    """
+    UGTC-DDPG trainer.
+
+    ⚠️  Implementation assumption: DDPG is not benchmarked in the paper.
+    This follows the same UGTC integration logic as TD3 but without
+    twin-Q critics or delayed policy update.
+
+    Args:
+        obs_dim:     Observation space dimension.
+        act_dim:     Action space dimension.
+        max_action:  Action space bound (default 1.0).
+        hidden:      Hidden layer width (default 256).
+        lr:          Learning rate (default 1e-3).
+        gamma:       Discount factor (default 0.99).
+        tau:         Soft target update coefficient (default 0.005).
+        eta:         UGTC correction weight in actor loss (default 0.5).
+        M:           UGTC ensemble size (default 3).
+        beta:        UGTC gate temperature (default 5.0).
+        device:      PyTorch device string (default "cpu").
+    """
+
+    def __init__(
+        self,
+        obs_dim: int,
+        act_dim: int,
+        max_action: float = 1.0,
+        hidden: int = 256,
+        lr: float = 1e-3,
+        gamma: float = 0.99,
+        tau: float = 0.005,
+        eta: float = 0.5,
+        M: int = 3,
+        beta: float = 5.0,
+        device: str = "cpu",
+    ):
+        self.gamma = gamma
+        self.tau = tau
+        self.eta = eta
+        self.max_action = max_action
+        self.device = torch.device(device)
+
+        self.actor = DDPGActor(obs_dim, act_dim, hidden, max_action).to(self.device)
+        self.actor_target = copy.deepcopy(self.actor)
+
+        self.critic = DDPGCritic(obs_dim, act_dim, hidden).to(self.device)
+        self.critic_target = copy.deepcopy(self.critic)
+
+        self.ugtc = UGTCModule(obs_dim, hidden, M=M, beta=beta).to(self.device)
+        self.noise = OUNoise(act_dim)
+
+        self.actor_opt = optim.Adam(self.actor.parameters(), lr=lr)
+        self.critic_opt = optim.Adam(self.critic.parameters(), lr=lr)
+        self.ugtc_opt = optim.Adam(self.ugtc.parameters(), lr=lr)
+
+    def select_action(self, obs: np.ndarray, add_noise: bool = True) -> np.ndarray:
+        obs_t = torch.FloatTensor(obs).unsqueeze(0).to(self.device)
+        with torch.no_grad():
+            action = self.actor(obs_t).squeeze(0).cpu().numpy()
+        if add_noise:
+            action += self.noise.sample()
+        return action.clip(-self.max_action, self.max_action)
+
+    def update(self, replay: ReplayBuffer, batch_size: int = 256) -> Dict[str, float]:
+        """One DDPG gradient step with UGTC baseline correction."""
+        batch = replay.sample(batch_size, self.device)
+        obs = batch["obs"]
+        actions = batch["actions"]
+        rewards = batch["rewards"]
+        next_obs = batch["next_obs"]
+        dones = batch["dones"]
+
+        # --- Critic update ---
+        with torch.no_grad():
+            next_actions = self.actor_target(next_obs)
+            q_target = rewards + (1.0 - dones) * self.gamma * self.critic_target(next_obs, next_actions)
+
+        q = self.critic(obs, actions)
+        critic_loss = (q - q_target).pow(2).mean()
+
+        self.critic_opt.zero_grad()
+        critic_loss.backward()
+        self.critic_opt.step()
+
+        # --- Actor update with UGTC baseline ---
+        pi = self.actor(obs)
+        q_pi = self.critic(obs, pi)
+
+        gate, v_fast, v_slow = self.ugtc.compute_gate(obs)
+        v_ugtc = (gate * v_slow + (1.0 - gate) * v_fast).detach()
+        advantage = q_pi - v_ugtc
+
+        actor_loss = -(q_pi + self.eta * advantage).mean()
+
+        self.actor_opt.zero_grad()
+        actor_loss.backward()
+        self.actor_opt.step()
+
+        # --- UGTC critic update ---
+        with torch.no_grad():
+            v_target = rewards + (1.0 - dones) * self.gamma * self.ugtc.get_value_ugtc(next_obs)
+        ugtc_loss = (self.ugtc.get_value_ugtc(obs) - v_target).pow(2).mean()
+
+        self.ugtc_opt.zero_grad()
+        ugtc_loss.backward()
+        self.ugtc_opt.step()
+
+        # Soft target updates
+        for sp, tp in zip(self.actor.parameters(), self.actor_target.parameters()):
+            tp.data.copy_(self.tau * sp.data + (1.0 - self.tau) * tp.data)
+        for sp, tp in zip(self.critic.parameters(), self.critic_target.parameters()):
+            tp.data.copy_(self.tau * sp.data + (1.0 - self.tau) * tp.data)
+
+        return {
+            "critic_loss": critic_loss.item(),
+            "actor_loss": actor_loss.item(),
+            "ugtc_loss": ugtc_loss.item(),
+            **self.ugtc.get_gate_stats(obs),
+        }

ugtc/module.py

@@ -0,0 +1,289 @@
+"""
+UGTC: Uncertainty-Gated Temporal Credit — Core Module
+======================================================
+
+Backbone-agnostic advantage estimator. Drop this into any actor-critic
+algorithm by replacing the advantage computation step.
+
+Mathematical summary:
+
+    Fast GAE:   A^fast_t  computed with λ_fast = 0.80
+    Slow GAE:   A^slow_t  computed with λ_slow = 0.99, using ensemble mean
+    Gate:       u(s)  = σ(-β · (σ̂(s) - 1.0))
+                σ̂(s) = std(V¹,...,Vᴹ)(s) / σ_EMA
+    Blended:    A^UGTC_t = u(sₜ)·A^slow_t + (1-u(sₜ))·A^fast_t
+
+Reference: https://doi.org/10.5281/zenodo.19715116
+"""
+
+import torch
+import torch.nn as nn
+import numpy as np
+from typing import Tuple, Dict
+
+
+class ValueNetwork(nn.Module):
+    """
+    Single feed-forward value network V(s).
+
+    Used as the fast critic. Architecture: two hidden layers with Tanh,
+    orthogonal initialization (following PPO conventions).
+
+    Args:
+        obs_dim:    Observation space dimension.
+        hidden_dim: Hidden layer width (default 64).
+    """
+
+    def __init__(self, obs_dim: int, hidden_dim: int = 64):
+        super().__init__()
+        self.net = nn.Sequential(
+            nn.Linear(obs_dim, hidden_dim),
+            nn.Tanh(),
+            nn.Linear(hidden_dim, hidden_dim),
+            nn.Tanh(),
+            nn.Linear(hidden_dim, 1),
+        )
+        self._init_weights()
+
+    def _init_weights(self) -> None:
+        for m in self.modules():
+            if isinstance(m, nn.Linear):
+                nn.init.orthogonal_(m.weight, gain=np.sqrt(2))
+                nn.init.constant_(m.bias, 0.0)
+        nn.init.orthogonal_(self.net[-1].weight, gain=1.0)
+
+    def forward(self, obs: torch.Tensor) -> torch.Tensor:
+        return self.net(obs).squeeze(-1)
+
+
+class EnsembleValueNetwork(nn.Module):
+    """
+    Ensemble of M independently initialized value networks (slow critic).
+
+    Each member has its own parameters — no shared trunk — to maximise
+    functional diversity and produce meaningful disagreement estimates.
+
+    Args:
+        obs_dim:    Observation space dimension.
+        hidden_dim: Hidden layer width (default 64).
+        M:          Ensemble size (default 3).
+    """
+
+    def __init__(self, obs_dim: int, hidden_dim: int = 64, M: int = 3):
+        super().__init__()
+        self.M = M
+        self.members = nn.ModuleList(
+            [ValueNetwork(obs_dim, hidden_dim) for _ in range(M)]
+        )
+
+    def forward(self, obs: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
+        """
+        Args:
+            obs: (batch, obs_dim)
+
+        Returns:
+            mean_value:  (batch,)  — ensemble mean V̄_slow(s)
+            uncertainty: (batch,)  — per-state std dev σ(s)
+        """
+        values = torch.stack([m(obs) for m in self.members], dim=0)  # (M, batch)
+        return values.mean(dim=0), values.std(dim=0)
+
+    def forward_all(self, obs: torch.Tensor) -> torch.Tensor:
+        """Returns all member predictions as (M, batch) tensor."""
+        return torch.stack([m(obs) for m in self.members], dim=0)
+
+
+class UGTCModule(nn.Module):
+    """
+    Uncertainty-Gated Temporal Credit module.
+
+    Drop-in replacement for standard GAE. Integrates with any actor-critic
+    algorithm by substituting the advantage computation step.
+
+    Architecture:
+        - Fast critic: single ValueNetwork, λ = lambda_fast
+        - Slow ensemble: M independent ValueNetworks, λ = lambda_slow
+        - Gate: sigmoid of EMA-normalized ensemble disagreement
+
+    All hyperparameters are fixed across benchmarks (no per-task tuning).
+
+    Args:
+        obs_dim:       Observation space dimension.
+        hidden_dim:    Hidden layer width for all critics (default 64).
+        M:             Ensemble size for slow critic (default 3).
+        lambda_fast:   GAE lambda for fast critic (default 0.80).
+        lambda_slow:   GAE lambda for slow ensemble (default 0.99).
+        beta:          Gate temperature — higher = sharper gate (default 5.0).
+        ema_momentum:  EMA momentum for uncertainty normalization (default 0.99).
+
+    Example::
+
+        ugtc = UGTCModule(obs_dim=17)
+        advantages = ugtc.compute_advantages(obs, next_obs, rewards, dones, gamma=0.99)
+    """
+
+    def __init__(
+        self,
+        obs_dim: int,
+        hidden_dim: int = 64,
+        M: int = 3,
+        lambda_fast: float = 0.80,
+        lambda_slow: float = 0.99,
+        beta: float = 5.0,
+        ema_momentum: float = 0.99,
+    ):
+        super().__init__()
+
+        self.lambda_fast = lambda_fast
+        self.lambda_slow = lambda_slow
+        self.beta = beta
+        self.ema_momentum = ema_momentum
+
+        self.fast_critic = ValueNetwork(obs_dim, hidden_dim)
+        self.slow_ensemble = EnsembleValueNetwork(obs_dim, hidden_dim, M)
+
+        self.register_buffer("sigma_ema", torch.ones(1))
+        self.eps = 1e-8
+
+    # ------------------------------------------------------------------
+    # Gate computation
+    # ------------------------------------------------------------------
+
+    def compute_gate(
+        self, obs: torch.Tensor
+    ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+        """
+        Compute per-state uncertainty gate u(s).
+
+        Steps:
+            1. Evaluate fast critic:  v_fast = V_fast(s)
+            2. Evaluate slow ensemble: v_slow_mean, σ = V̄_slow(s), std(V¹,...,Vᴹ)(s)
+            3. EMA-normalize:         σ̂ = σ / σ_EMA
+            4. Sigmoid gate:          u(s) = σ(-β · (σ̂ - 1))
+
+        Returns:
+            gate:        (batch,) — blend weight u(s) ∈ [0, 1]
+            v_fast:      (batch,) — fast critic values
+            v_slow_mean: (batch,) — slow ensemble mean values
+        """
+        v_fast = self.fast_critic(obs)
+        v_slow_mean, sigma = self.slow_ensemble(obs)
+
+        if self.training:
+            self.sigma_ema = (
+                self.ema_momentum * self.sigma_ema
+                + (1.0 - self.ema_momentum) * sigma.mean().detach()
+            )
+
+        normalized_sigma = sigma / (self.sigma_ema + self.eps)
+        gate = torch.sigmoid(-self.beta * (normalized_sigma - 1.0))
+
+        return gate, v_fast, v_slow_mean
+
+    # ------------------------------------------------------------------
+    # Advantage computation
+    # ------------------------------------------------------------------
+
+    def compute_advantages(
+        self,
+        obs: torch.Tensor,
+        next_obs: torch.Tensor,
+        rewards: torch.Tensor,
+        dones: torch.Tensor,
+        gamma: float = 0.99,
+    ) -> torch.Tensor:
+        """
+        Compute UGTC blended advantages for a batch of transitions.
+
+        Formula:
+            A^UGTC_t = u(sₜ)·A^slow_t + (1-u(sₜ))·A^fast_t
+
+        Args:
+            obs:      (T, obs_dim) — current observations
+            next_obs: (T, obs_dim) — next observations
+            rewards:  (T,) — reward signal
+            dones:    (T,) — episode termination flags (1.0 = done)
+            gamma:    Discount factor (default 0.99)
+
+        Returns:
+            advantages: (T,) — UGTC blended advantages
+        """
+        with torch.no_grad():
+            gate, v_fast, v_slow = self.compute_gate(obs)
+            _, v_fast_next, v_slow_next = self.compute_gate(next_obs)
+
+        adv_fast = self._compute_gae(
+            rewards, v_fast, v_fast_next, dones, gamma, self.lambda_fast
+        )
+        adv_slow = self._compute_gae(
+            rewards, v_slow, v_slow_next, dones, gamma, self.lambda_slow
+        )
+
+        return gate * adv_slow + (1.0 - gate) * adv_fast
+
+    @staticmethod
+    def _compute_gae(
+        rewards: torch.Tensor,
+        values: torch.Tensor,
+        next_values: torch.Tensor,
+        dones: torch.Tensor,
+        gamma: float,
+        lam: float,
+    ) -> torch.Tensor:
+        """
+        Standard GAE computation (vectorized reverse accumulation).
+
+        δₜ = rₜ + γ·V(sₜ₊₁)·(1-dₜ) - V(sₜ)
+        Aₜ = δₜ + γλ·(1-dₜ)·Aₜ₊₁
+        """
+        T = rewards.shape[0]
+        advantages = torch.zeros_like(rewards)
+        deltas = rewards + gamma * next_values * (1.0 - dones) - values
+
+        gae = 0.0
+        for t in reversed(range(T)):
+            gae = deltas[t] + gamma * lam * (1.0 - dones[t]) * gae
+            advantages[t] = gae
+
+        return advantages
+
+    # ------------------------------------------------------------------
+    # Value estimation
+    # ------------------------------------------------------------------
+
+    def get_value_ugtc(self, obs: torch.Tensor) -> torch.Tensor:
+        """
+        Blended value estimate.
+
+        V^UGTC(s) = u(s)·V̄_slow(s) + (1-u(s))·V_fast(s)
+
+        Args:
+            obs: (batch, obs_dim)
+
+        Returns:
+            (batch,) blended value
+        """
+        gate, v_fast, v_slow = self.compute_gate(obs)
+        return gate * v_slow + (1.0 - gate) * v_fast
+
+    # ------------------------------------------------------------------
+    # Diagnostics
+    # ------------------------------------------------------------------
+
+    def get_gate_stats(self, obs: torch.Tensor) -> Dict[str, float]:
+        """Return gate statistics for logging/debugging."""
+        with torch.no_grad():
+            gate, v_fast, v_slow = self.compute_gate(obs)
+        return {
+            "gate_mean": gate.mean().item(),
+            "gate_std": gate.std().item(),
+            "gate_min": gate.min().item(),
+            "gate_max": gate.max().item(),
+            "sigma_ema": self.sigma_ema.item(),
+        }
+
+    def parameter_count(self) -> Dict[str, int]:
+        """Report parameter counts for overhead analysis."""
+        fast = sum(p.numel() for p in self.fast_critic.parameters())
+        slow = sum(p.numel() for p in self.slow_ensemble.parameters())
+        return {"fast_critic": fast, "slow_ensemble": slow, "total": fast + slow}

ugtc/ppo.py

@@ -0,0 +1,228 @@
+"""
+UGTC-PPO: Proximal Policy Optimization with Uncertainty-Gated Temporal Credit
+==============================================================================
+
+Integration strategy:
+    The only structural change from vanilla PPO is the advantage computation:
+    A^UGTC replaces the standard single-critic GAE advantage.
+
+    All UGTC critics are trained via the same regression loss pipeline
+    used by the vanilla PPO critic (MSE against λ-returns).
+
+    Algorithm (one update step):
+        1. Collect rollout with current policy
+        2. Compute A^UGTC using UGTCModule.compute_advantages()
+        3. Normalize advantages
+        4. Run K epochs of clipped surrogate + critic regression
+        5. Gradient clip and step
+
+Reference: https://doi.org/10.5281/zenodo.19715116
+"""
+
+import torch
+import torch.nn as nn
+import torch.optim as optim
+import numpy as np
+from typing import Dict, Optional
+
+from ugtc.module import UGTCModule
+
+
+class PPOPolicy(nn.Module):
+    """
+    Gaussian policy for continuous control (shared trunk, separate heads).
+
+    For discrete action spaces, replace Normal distribution with Categorical
+    and remove log_std parameter.
+    """
+
+    def __init__(self, obs_dim: int, act_dim: int, hidden_dim: int = 64):
+        super().__init__()
+        self.trunk = nn.Sequential(
+            nn.Linear(obs_dim, hidden_dim),
+            nn.Tanh(),
+            nn.Linear(hidden_dim, hidden_dim),
+            nn.Tanh(),
+        )
+        self.mean_head = nn.Linear(hidden_dim, act_dim)
+        self.log_std = nn.Parameter(torch.zeros(act_dim))
+        self._init_weights()
+
+    def _init_weights(self) -> None:
+        for m in self.trunk.modules():
+            if isinstance(m, nn.Linear):
+                nn.init.orthogonal_(m.weight, gain=np.sqrt(2))
+                nn.init.constant_(m.bias, 0.0)
+        nn.init.orthogonal_(self.mean_head.weight, gain=0.01)
+        nn.init.constant_(self.mean_head.bias, 0.0)
+
+    def forward(self, obs: torch.Tensor) -> torch.distributions.Normal:
+        h = self.trunk(obs)
+        mean = self.mean_head(h)
+        std = self.log_std.clamp(-4.0, 2.0).exp().expand_as(mean)
+        return torch.distributions.Normal(mean, std)
+
+    def get_log_prob(self, obs: torch.Tensor, actions: torch.Tensor) -> torch.Tensor:
+        return self.forward(obs).log_prob(actions).sum(-1)
+
+
+class UGTCPPO:
+    """
+    UGTC-PPO trainer.
+
+    The sole algorithmic difference from vanilla PPO is that advantages are
+    computed by UGTCModule.compute_advantages() rather than single-critic GAE.
+
+    Args:
+        obs_dim:      Observation space dimension.
+        act_dim:      Action space dimension.
+        hidden_dim:   Hidden layer width (default 64).
+        lr:           Learning rate (default 3e-4).
+        gamma:        Discount factor (default 0.99).
+        clip_eps:     PPO clipping coefficient ε (default 0.2).
+        epochs:       Number of update epochs per rollout (default 10).
+        vf_coef:      Value function loss coefficient (default 0.5).
+        ent_coef:     Entropy bonus coefficient (default 0.0).
+        max_grad_norm: Gradient norm clip (default 0.5).
+        lambda_fast:  UGTC fast critic λ (default 0.80).
+        lambda_slow:  UGTC slow ensemble λ (default 0.99).
+        M:            UGTC ensemble size (default 3).
+        beta:         UGTC gate temperature (default 5.0).
+        ema_momentum: UGTC EMA momentum (default 0.99).
+    """
+
+    def __init__(
+        self,
+        obs_dim: int,
+        act_dim: int,
+        hidden_dim: int = 64,
+        lr: float = 3e-4,
+        gamma: float = 0.99,
+        clip_eps: float = 0.2,
+        epochs: int = 10,
+        vf_coef: float = 0.5,
+        ent_coef: float = 0.0,
+        max_grad_norm: float = 0.5,
+        # UGTC hyperparameters — fixed across ALL benchmarks
+        lambda_fast: float = 0.80,
+        lambda_slow: float = 0.99,
+        M: int = 3,
+        beta: float = 5.0,
+        ema_momentum: float = 0.99,
+    ):
+        self.gamma = gamma
+        self.clip_eps = clip_eps
+        self.epochs = epochs
+        self.vf_coef = vf_coef
+        self.ent_coef = ent_coef
+        self.max_grad_norm = max_grad_norm
+
+        self.policy = PPOPolicy(obs_dim, act_dim, hidden_dim)
+        self.ugtc = UGTCModule(
+            obs_dim, hidden_dim, M, lambda_fast, lambda_slow, beta, ema_momentum
+        )
+
+        self.optimizer = optim.Adam(
+            list(self.policy.parameters()) + list(self.ugtc.parameters()),
+            lr=lr,
+            eps=1e-5,
+        )
+
+    def select_action(
+        self, obs: torch.Tensor
+    ) -> tuple[torch.Tensor, torch.Tensor]:
+        """Sample action and log-prob for rollout collection."""
+        with torch.no_grad():
+            dist = self.policy(obs)
+            action = dist.sample()
+            log_prob = dist.log_prob(action).sum(-1)
+        return action, log_prob
+
+    def update(self, rollout: Dict[str, torch.Tensor]) -> Dict[str, float]:
+        """
+        One PPO update using UGTC advantages.
+
+        Args:
+            rollout: dict with keys:
+                'obs'       (T, obs_dim)
+                'actions'   (T, act_dim)
+                'rewards'   (T,)
+                'next_obs'  (T, obs_dim)
+                'dones'     (T,)
+                'log_probs' (T,)
+
+        Returns:
+            dict with logged scalar metrics.
+        """
+        obs = rollout["obs"]
+        actions = rollout["actions"]
+        old_log_probs = rollout["log_probs"]
+        rewards = rollout["rewards"]
+        next_obs = rollout["next_obs"]
+        dones = rollout["dones"]
+
+        self.ugtc.train()
+
+        # === KEY CHANGE: UGTC advantages replaces standard single-critic GAE ===
+        advantages = self.ugtc.compute_advantages(
+            obs, next_obs, rewards, dones, self.gamma
+        )
+        advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-8)
+
+        # Value targets for critic training (using blended value + advantages)
+        with torch.no_grad():
+            returns = advantages + self.ugtc.get_value_ugtc(obs)
+
+        metrics: Dict[str, float] = {}
+
+        for epoch in range(self.epochs):
+            # --- Policy loss (PPO clipped surrogate) ---
+            dist = self.policy(obs)
+            new_log_probs = dist.log_prob(actions).sum(-1)
+            entropy = dist.entropy().sum(-1)
+            ratio = (new_log_probs - old_log_probs).exp()
+
+            surr1 = ratio * advantages
+            surr2 = ratio.clamp(1.0 - self.clip_eps, 1.0 + self.clip_eps) * advantages
+            policy_loss = -torch.min(surr1, surr2).mean()
+
+            # --- Critic losses (train all UGTC critics against same targets) ---
+            fast_loss = (self.ugtc.fast_critic(obs) - returns).pow(2).mean()
+
+            ensemble_loss = torch.stack([
+                (m(obs) - returns).pow(2).mean()
+                for m in self.ugtc.slow_ensemble.members
+            ]).mean()
+
+            entropy_loss = -self.ent_coef * entropy.mean()
+
+            loss = policy_loss + self.vf_coef * (fast_loss + ensemble_loss) + entropy_loss
+
+            self.optimizer.zero_grad()
+            loss.backward()
+            nn.utils.clip_grad_norm_(
+                list(self.policy.parameters()) + list(self.ugtc.parameters()),
+                self.max_grad_norm,
+            )
+            self.optimizer.step()
+
+        gate_stats = self.ugtc.get_gate_stats(obs)
+        metrics.update({
+            "policy_loss": policy_loss.item(),
+            "fast_value_loss": fast_loss.item(),
+            "ensemble_value_loss": ensemble_loss.item(),
+            **gate_stats,
+        })
+
+        return metrics
+
+    def save(self, path: str) -> None:
+        torch.save(
+            {"policy": self.policy.state_dict(), "ugtc": self.ugtc.state_dict()},
+            path,
+        )
+
+    def load(self, path: str) -> None:
+        ckpt = torch.load(path, map_location="cpu")
+        self.policy.load_state_dict(ckpt["policy"])
+        self.ugtc.load_state_dict(ckpt["ugtc"])

ugtc/sac.py

@@ -0,0 +1,232 @@
+"""
+UGTC-SAC: Soft Actor-Critic with Uncertainty-Gated Temporal Credit
+===================================================================
+
+Integration strategy:
+    UGTC substitutes the value baseline in the SAC actor loss.
+    Entropy coefficient α, twin-Q training, and target networks are unchanged.
+
+    Standard SAC actor loss:
+        J_π = E[α·log π(a|s) - Q_min(s, a)]
+
+    UGTC-SAC actor loss:
+        J_π^UGTC = E[α·log π(a|s) - Q_min(s, a) + V^UGTC(s)]
+
+    where V^UGTC(s) = u(s)·V̄_slow(s) + (1-u(s))·V_fast(s)
+
+    This substitution replaces the implicit value baseline with an
+    uncertainty-aware estimate, providing variance reduction in the gradient.
+
+Reference: https://doi.org/10.5281/zenodo.19715116
+"""
+
+import copy
+import torch
+import torch.nn as nn
+import torch.optim as optim
+import torch.nn.functional as F
+import numpy as np
+from typing import Dict, Tuple
+
+from ugtc.module import UGTCModule
+from ugtc.td3 import ReplayBuffer
+
+
+LOG_SIG_MAX = 2
+LOG_SIG_MIN = -20
+EPSILON = 1e-6
+
+
+class SACGaussianPolicy(nn.Module):
+    """Squashed Gaussian policy for SAC."""
+
+    def __init__(self, obs_dim: int, act_dim: int, hidden: int = 256):
+        super().__init__()
+        self.net = nn.Sequential(
+            nn.Linear(obs_dim, hidden), nn.ReLU(),
+            nn.Linear(hidden, hidden), nn.ReLU(),
+        )
+        self.mean_head = nn.Linear(hidden, act_dim)
+        self.log_std_head = nn.Linear(hidden, act_dim)
+
+    def forward(self, obs: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
+        h = self.net(obs)
+        mean = self.mean_head(h)
+        log_std = self.log_std_head(h).clamp(LOG_SIG_MIN, LOG_SIG_MAX)
+        return mean, log_std
+
+    def sample(self, obs: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
+        mean, log_std = self.forward(obs)
+        std = log_std.exp()
+        normal = torch.distributions.Normal(mean, std)
+        x_t = normal.rsample()
+        y_t = torch.tanh(x_t)
+        log_prob = normal.log_prob(x_t) - torch.log(1 - y_t.pow(2) + EPSILON)
+        return y_t, log_prob.sum(-1)
+
+
+class SACCritic(nn.Module):
+    """Twin Q-networks — same as TD3 critic."""
+
+    def __init__(self, obs_dim: int, act_dim: int, hidden: int = 256):
+        super().__init__()
+        in_dim = obs_dim + act_dim
+        self.q1 = nn.Sequential(
+            nn.Linear(in_dim, hidden), nn.ReLU(),
+            nn.Linear(hidden, hidden), nn.ReLU(),
+            nn.Linear(hidden, 1),
+        )
+        self.q2 = nn.Sequential(
+            nn.Linear(in_dim, hidden), nn.ReLU(),
+            nn.Linear(hidden, hidden), nn.ReLU(),
+            nn.Linear(hidden, 1),
+        )
+
+    def forward(self, obs, action):
+        x = torch.cat([obs, action], dim=-1)
+        return self.q1(x).squeeze(-1), self.q2(x).squeeze(-1)
+
+
+class UGTCSAC:
+    """
+    UGTC-SAC trainer.
+
+    Entropy tuning, twin-Q critic training, and target network updates
+    are all standard SAC. UGTC modifies only the actor's value baseline.
+
+    Args:
+        obs_dim:       Observation space dimension.
+        act_dim:       Action space dimension.
+        hidden:        Hidden layer width (default 256).
+        lr:            Learning rate (default 3e-4).
+        gamma:         Discount factor (default 0.99).
+        tau:           Soft target update coefficient (default 0.005).
+        alpha:         Initial entropy coefficient (default 0.2).
+        auto_alpha:    Enable automatic entropy tuning (default True).
+        target_entropy: Target entropy for auto-tuning (default -act_dim).
+        M:             UGTC ensemble size (default 3).
+        beta:          UGTC gate temperature (default 5.0).
+        device:        PyTorch device string (default "cpu").
+    """
+
+    def __init__(
+        self,
+        obs_dim: int,
+        act_dim: int,
+        hidden: int = 256,
+        lr: float = 3e-4,
+        gamma: float = 0.99,
+        tau: float = 0.005,
+        alpha: float = 0.2,
+        auto_alpha: bool = True,
+        target_entropy: float = None,
+        M: int = 3,
+        beta: float = 5.0,
+        device: str = "cpu",
+    ):
+        self.gamma = gamma
+        self.tau = tau
+        self.act_dim = act_dim
+        self.device = torch.device(device)
+
+        # Policy and critics
+        self.policy = SACGaussianPolicy(obs_dim, act_dim, hidden).to(self.device)
+        self.critic = SACCritic(obs_dim, act_dim, hidden).to(self.device)
+        self.critic_target = copy.deepcopy(self.critic)
+
+        # UGTC module
+        self.ugtc = UGTCModule(obs_dim, hidden, M=M, beta=beta).to(self.device)
+
+        # Entropy coefficient
+        self.auto_alpha = auto_alpha
+        if auto_alpha:
+            self.target_entropy = target_entropy or -float(act_dim)
+            self.log_alpha = torch.zeros(1, requires_grad=True, device=self.device)
+            self.alpha = self.log_alpha.exp().item()
+            self.alpha_opt = optim.Adam([self.log_alpha], lr=lr)
+        else:
+            self.alpha = alpha
+
+        self.policy_opt = optim.Adam(self.policy.parameters(), lr=lr)
+        self.critic_opt = optim.Adam(self.critic.parameters(), lr=lr)
+        self.ugtc_opt = optim.Adam(self.ugtc.parameters(), lr=lr)
+
+    def select_action(self, obs: np.ndarray, deterministic: bool = False) -> np.ndarray:
+        obs_t = torch.FloatTensor(obs).unsqueeze(0).to(self.device)
+        with torch.no_grad():
+            if deterministic:
+                mean, _ = self.policy(obs_t)
+                return torch.tanh(mean).squeeze(0).cpu().numpy()
+            action, _ = self.policy.sample(obs_t)
+        return action.squeeze(0).cpu().numpy()
+
+    def update(self, replay: ReplayBuffer, batch_size: int = 256) -> Dict[str, float]:
+        """One SAC gradient step with UGTC baseline substitution."""
+        batch = replay.sample(batch_size, self.device)
+        obs = batch["obs"]
+        actions = batch["actions"]
+        rewards = batch["rewards"]
+        next_obs = batch["next_obs"]
+        dones = batch["dones"]
+
+        # --- Critic update (standard SAC twin-Q, unchanged) ---
+        with torch.no_grad():
+            next_actions, next_log_pi = self.policy.sample(next_obs)
+            q1_next, q2_next = self.critic_target(next_obs, next_actions)
+            q_next = torch.min(q1_next, q2_next) - self.alpha * next_log_pi
+            q_target = rewards + (1.0 - dones) * self.gamma * q_next
+
+        q1, q2 = self.critic(obs, actions)
+        critic_loss = F.mse_loss(q1, q_target) + F.mse_loss(q2, q_target)
+
+        self.critic_opt.zero_grad()
+        critic_loss.backward()
+        self.critic_opt.step()
+
+        # --- Actor update (UGTC baseline substitution) ---
+        pi, log_pi = self.policy.sample(obs)
+        q1_pi, q2_pi = self.critic(obs, pi)
+        q_min_pi = torch.min(q1_pi, q2_pi)
+
+        # UGTC value baseline (state-only)
+        self.ugtc.train()
+        gate, v_fast, v_slow = self.ugtc.compute_gate(obs)
+        v_ugtc = (gate * v_slow + (1.0 - gate) * v_fast).detach()
+
+        # KEY CHANGE: V^UGTC replaces implicit value baseline in SAC actor loss
+        # Standard:   J_π = (α·log π - Q_min).mean()
+        # UGTC-SAC:   J_π = (α·log π - Q_min + V^UGTC).mean()
+        policy_loss = (self.alpha * log_pi - q_min_pi + v_ugtc).mean()
+
+        self.policy_opt.zero_grad()
+        policy_loss.backward()
+        self.policy_opt.step()
+
+        # --- UGTC critic training ---
+        with torch.no_grad():
+            v_target = (rewards + (1.0 - dones) * self.gamma * self.ugtc.get_value_ugtc(next_obs))
+        ugtc_loss = (self.ugtc.get_value_ugtc(obs) - v_target).pow(2).mean()
+
+        self.ugtc_opt.zero_grad()
+        ugtc_loss.backward()
+        self.ugtc_opt.step()
+
+        # --- Entropy coefficient update ---
+        if self.auto_alpha:
+            alpha_loss = -(self.log_alpha * (log_pi + self.target_entropy).detach()).mean()
+            self.alpha_opt.zero_grad()
+            alpha_loss.backward()
+            self.alpha_opt.step()
+            self.alpha = self.log_alpha.exp().item()
+
+        # Soft target update
+        for sp, tp in zip(self.critic.parameters(), self.critic_target.parameters()):
+            tp.data.copy_(self.tau * sp.data + (1.0 - self.tau) * tp.data)
+
+        return {
+            "critic_loss": critic_loss.item(),
+            "policy_loss": policy_loss.item(),
+            "ugtc_loss": ugtc_loss.item(),
+            "alpha": self.alpha,
+            **self.ugtc.get_gate_stats(obs),
+        }

ugtc/td3.py

@@ -0,0 +1,252 @@
+"""
+UGTC-TD3: Twin Delayed DDPG with Uncertainty-Gated Temporal Credit
+===================================================================
+
+Integration strategy:
+    UGTC provides a baseline correction for the actor gradient while the
+    backbone's clipped double-Q and delayed policy update are preserved.
+
+    Actor loss:
+        L_actor = -(Q_min(s, π(s)) + η · A^UGTC(s, π(s))).mean()
+
+    where A^UGTC(s, a) = Q_min(s, a) - V^UGTC(s)
+          V^UGTC(s)    = u(s)·V̄_slow(s) + (1-u(s))·V_fast(s)
+
+    The critic (twin-Q) update is standard TD3: unmodified.
+
+Reference: https://doi.org/10.5281/zenodo.19715116
+"""
+
+import copy
+import torch
+import torch.nn as nn
+import torch.optim as optim
+import numpy as np
+from typing import Dict, Tuple
+
+from ugtc.module import UGTCModule
+
+
+class TD3Actor(nn.Module):
+    def __init__(self, obs_dim: int, act_dim: int, hidden: int = 256, max_action: float = 1.0):
+        super().__init__()
+        self.max_action = max_action
+        self.net = nn.Sequential(
+            nn.Linear(obs_dim, hidden), nn.ReLU(),
+            nn.Linear(hidden, hidden), nn.ReLU(),
+            nn.Linear(hidden, act_dim), nn.Tanh(),
+        )
+
+    def forward(self, obs: torch.Tensor) -> torch.Tensor:
+        return self.max_action * self.net(obs)
+
+
+class TD3Critic(nn.Module):
+    """Twin Q-networks — standard TD3 critic."""
+
+    def __init__(self, obs_dim: int, act_dim: int, hidden: int = 256):
+        super().__init__()
+        in_dim = obs_dim + act_dim
+        self.q1 = nn.Sequential(
+            nn.Linear(in_dim, hidden), nn.ReLU(),
+            nn.Linear(hidden, hidden), nn.ReLU(),
+            nn.Linear(hidden, 1),
+        )
+        self.q2 = nn.Sequential(
+            nn.Linear(in_dim, hidden), nn.ReLU(),
+            nn.Linear(hidden, hidden), nn.ReLU(),
+            nn.Linear(hidden, 1),
+        )
+
+    def forward(self, obs: torch.Tensor, action: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
+        x = torch.cat([obs, action], dim=-1)
+        return self.q1(x).squeeze(-1), self.q2(x).squeeze(-1)
+
+    def q_min(self, obs: torch.Tensor, action: torch.Tensor) -> torch.Tensor:
+        q1, q2 = self.forward(obs, action)
+        return torch.min(q1, q2)
+
+
+class ReplayBuffer:
+    """Simple experience replay buffer."""
+
+    def __init__(self, obs_dim: int, act_dim: int, capacity: int = 1_000_000):
+        self.capacity = capacity
+        self.ptr = 0
+        self.size = 0
+        self.obs = np.zeros((capacity, obs_dim), dtype=np.float32)
+        self.next_obs = np.zeros((capacity, obs_dim), dtype=np.float32)
+        self.actions = np.zeros((capacity, act_dim), dtype=np.float32)
+        self.rewards = np.zeros((capacity, 1), dtype=np.float32)
+        self.dones = np.zeros((capacity, 1), dtype=np.float32)
+
+    def add(self, obs, action, reward, next_obs, done):
+        self.obs[self.ptr] = obs
+        self.actions[self.ptr] = action
+        self.rewards[self.ptr] = reward
+        self.next_obs[self.ptr] = next_obs
+        self.dones[self.ptr] = float(done)
+        self.ptr = (self.ptr + 1) % self.capacity
+        self.size = min(self.size + 1, self.capacity)
+
+    def sample(self, batch_size: int, device: torch.device) -> Dict[str, torch.Tensor]:
+        idx = np.random.randint(0, self.size, size=batch_size)
+        return {
+            "obs": torch.FloatTensor(self.obs[idx]).to(device),
+            "actions": torch.FloatTensor(self.actions[idx]).to(device),
+            "rewards": torch.FloatTensor(self.rewards[idx]).to(device),
+            "next_obs": torch.FloatTensor(self.next_obs[idx]).to(device),
+            "dones": torch.FloatTensor(self.dones[idx]).to(device),
+        }
+
+
+class UGTCTD3:
+    """
+    UGTC-TD3 trainer.
+
+    The backbone's twin-Q, target networks, and delayed policy update are
+    fully preserved. UGTC modifies only the actor gradient via a value
+    baseline correction.
+
+    Args:
+        obs_dim:      Observation space dimension.
+        act_dim:      Action space dimension.
+        max_action:   Action space bound (default 1.0).
+        hidden:       Hidden layer width (default 256).
+        lr:           Learning rate for all networks (default 3e-4).
+        gamma:        Discount factor (default 0.99).
+        tau:          Soft target update coefficient (default 0.005).
+        policy_noise: Target policy smoothing std (default 0.2).
+        noise_clip:   Target noise clip (default 0.5).
+        policy_delay: Actor update frequency in critic steps (default 2).
+        eta:          UGTC correction weight in actor loss (default 0.5).
+        M:            UGTC ensemble size (default 3).
+        beta:         UGTC gate temperature (default 5.0).
+
+    Note: eta=0.5 is the suggested default. This hyperparameter controls the
+    strength of the UGTC baseline correction in the actor loss and may benefit
+    from tuning per environment. It is not a fixed UGTC hyperparameter.
+    """
+
+    def __init__(
+        self,
+        obs_dim: int,
+        act_dim: int,
+        max_action: float = 1.0,
+        hidden: int = 256,
+        lr: float = 3e-4,
+        gamma: float = 0.99,
+        tau: float = 0.005,
+        policy_noise: float = 0.2,
+        noise_clip: float = 0.5,
+        policy_delay: int = 2,
+        eta: float = 0.5,
+        M: int = 3,
+        beta: float = 5.0,
+        device: str = "cpu",
+    ):
+        self.gamma = gamma
+        self.tau = tau
+        self.policy_noise = policy_noise
+        self.noise_clip = noise_clip
+        self.policy_delay = policy_delay
+        self.eta = eta
+        self.max_action = max_action
+        self.device = torch.device(device)
+        self._update_count = 0
+
+        self.actor = TD3Actor(obs_dim, act_dim, hidden, max_action).to(self.device)
+        self.actor_target = copy.deepcopy(self.actor)
+
+        self.critic = TD3Critic(obs_dim, act_dim, hidden).to(self.device)
+        self.critic_target = copy.deepcopy(self.critic)
+
+        # UGTC module — obs_dim only (state-based value baseline)
+        self.ugtc = UGTCModule(obs_dim, hidden, M=M, beta=beta).to(self.device)
+
+        self.actor_opt = optim.Adam(self.actor.parameters(), lr=lr)
+        self.critic_opt = optim.Adam(self.critic.parameters(), lr=lr)
+        self.ugtc_opt = optim.Adam(self.ugtc.parameters(), lr=lr)
+
+    def select_action(self, obs: np.ndarray, noise: float = 0.1) -> np.ndarray:
+        obs_t = torch.FloatTensor(obs).unsqueeze(0).to(self.device)
+        with torch.no_grad():
+            action = self.actor(obs_t).squeeze(0).cpu().numpy()
+        if noise > 0:
+            action += np.random.normal(0, noise, size=action.shape)
+        return action.clip(-self.max_action, self.max_action)
+
+    def update(self, replay: ReplayBuffer, batch_size: int = 256) -> Dict[str, float]:
+        """One TD3 gradient step with UGTC baseline correction."""
+        self._update_count += 1
+        batch = replay.sample(batch_size, self.device)
+        obs = batch["obs"]
+        actions = batch["actions"]
+        rewards = batch["rewards"]
+        next_obs = batch["next_obs"]
+        dones = batch["dones"]
+
+        # --- Critic update (standard TD3, unchanged) ---
+        with torch.no_grad():
+            noise = (
+                torch.randn_like(actions) * self.policy_noise
+            ).clamp(-self.noise_clip, self.noise_clip)
+            next_actions = (self.actor_target(next_obs) + noise).clamp(
+                -self.max_action, self.max_action
+            )
+            q1_next, q2_next = self.critic_target(next_obs, next_actions)
+            q_target = rewards + (1.0 - dones) * self.gamma * torch.min(q1_next, q2_next)
+
+        q1, q2 = self.critic(obs, actions)
+        critic_loss = (q1 - q_target).pow(2).mean() + (q2 - q_target).pow(2).mean()
+
+        self.critic_opt.zero_grad()
+        critic_loss.backward()
+        self.critic_opt.step()
+
+        metrics = {"critic_loss": critic_loss.item()}
+
+        # --- Actor update (delayed, with UGTC baseline correction) ---
+        if self._update_count % self.policy_delay == 0:
+            pi = self.actor(obs)
+            q_min = self.critic.q_min(obs, pi)
+
+            # UGTC value baseline
+            self.ugtc.train()
+            gate, v_fast, v_slow = self.ugtc.compute_gate(obs)
+            v_ugtc = gate * v_slow + (1.0 - gate) * v_fast
+
+            # A^UGTC(s, π(s)) = Q_min(s, π(s)) - V^UGTC(s)
+            advantage_ugtc = q_min - v_ugtc.detach()
+
+            # Actor loss: DPG + UGTC baseline correction
+            actor_loss = -(q_min + self.eta * advantage_ugtc).mean()
+
+            self.actor_opt.zero_grad()
+            actor_loss.backward()
+            self.actor_opt.step()
+
+            # Train UGTC critics against TD targets
+            v_ugtc_pred = self.ugtc.get_value_ugtc(obs)
+            ugtc_value_target = (rewards + (1.0 - dones) * self.gamma * self.ugtc.get_value_ugtc(next_obs)).detach()
+            ugtc_loss = (v_ugtc_pred - ugtc_value_target).pow(2).mean()
+
+            self.ugtc_opt.zero_grad()
+            ugtc_loss.backward()
+            self.ugtc_opt.step()
+
+            # Soft target updates
+            self._soft_update(self.actor, self.actor_target)
+            self._soft_update(self.critic, self.critic_target)
+
+            metrics.update({
+                "actor_loss": actor_loss.item(),
+                "ugtc_value_loss": ugtc_loss.item(),
+                **self.ugtc.get_gate_stats(obs),
+            })
+
+        return metrics
+
+    def _soft_update(self, source: nn.Module, target: nn.Module) -> None:
+        for sp, tp in zip(source.parameters(), target.parameters()):
+            tp.data.copy_(self.tau * sp.data + (1.0 - self.tau) * tp.data)

ugtc/utils.py

@@ -0,0 +1,134 @@
+"""
+UGTC utility functions — evaluation metrics and logging helpers.
+"""
+
+import numpy as np
+from typing import List, Dict, Sequence
+
+
+def bootstrap_ci(
+    data: Sequence[float],
+    n_bootstrap: int = 10_000,
+    ci: float = 0.95,
+) -> tuple[float, float]:
+    """
+    Compute bootstrap confidence interval (percentile method).
+
+    Args:
+        data:        1-D sequence of scalar values (e.g., final returns per seed).
+        n_bootstrap: Number of bootstrap samples (default 10,000).
+        ci:          Confidence level (default 0.95).
+
+    Returns:
+        (lower, upper) confidence interval bounds.
+    """
+    data = np.asarray(data, dtype=float)
+    rng = np.random.default_rng()
+    boot_means = np.array([
+        rng.choice(data, size=len(data), replace=True).mean()
+        for _ in range(n_bootstrap)
+    ])
+    alpha = (1.0 - ci) / 2.0
+    return float(np.percentile(boot_means, 100 * alpha)), float(np.percentile(boot_means, 100 * (1 - alpha)))
+
+
+def interquartile_mean(data: Sequence[float]) -> float:
+    """
+    Compute IQM (Interquartile Mean) — robust to outlier seeds.
+
+    IQM discards the bottom and top 25% of seeds before averaging,
+    following the rliable evaluation protocol.
+
+    Args:
+        data: 1-D sequence of scalar values.
+
+    Returns:
+        IQM value.
+    """
+    data = np.sort(np.asarray(data, dtype=float))
+    n = len(data)
+    q1 = n // 4
+    q3 = 3 * n // 4
+    return float(data[q1:q3 + 1].mean())
+
+
+def area_under_curve(
+    curves: List[List[Dict[str, float]]],
+    step_key: str = "step",
+    return_key: str = "mean_return",
+    normalize_by: float = None,
+) -> float:
+    """
+    Compute mean area under the learning curve across seeds.
+
+    Args:
+        curves:       List of per-seed curves, each a list of {step_key: ..., return_key: ...} dicts.
+        step_key:     Key for timestep values.
+        return_key:   Key for return values.
+        normalize_by: Normalise AUC by this step count (e.g., total_steps).
+
+    Returns:
+        Mean AUC across seeds.
+    """
+    aucs = []
+    for curve in curves:
+        steps = np.array([r[step_key] for r in curve])
+        returns = np.array([r[return_key] for r in curve])
+        auc = np.trapz(returns, steps)
+        if normalize_by is not None:
+            auc /= normalize_by
+        aucs.append(auc)
+    return float(np.mean(aucs))
+
+
+def print_eval_table(
+    results: Dict[str, List[float]],
+    n_bootstrap: int = 10_000,
+) -> None:
+    """
+    Pretty-print evaluation table with IQM, mean±std, and 95% CI.
+
+    Args:
+        results:     Dict mapping method name → list of final returns per seed.
+        n_bootstrap: Bootstrap samples for CI.
+    """
+    header = f"{'Method':<20} {'Mean':>8} {'Std':>8} {'IQM':>8} {'95% CI':>18}"
+    print(header)
+    print("-" * len(header))
+    for method, returns in sorted(results.items()):
+        mean = float(np.mean(returns))
+        std = float(np.std(returns))
+        iqm = interquartile_mean(returns)
+        lo, hi = bootstrap_ci(returns, n_bootstrap)
+        print(f"{method:<20} {mean:>8.1f} {std:>8.1f} {iqm:>8.1f} [{lo:>7.1f}, {hi:>7.1f}]")
+
+
+def compute_sample_efficiency(
+    ugtc_curve: List[Dict],
+    baseline_curve: List[Dict],
+    target_return: float,
+    step_key: str = "step",
+    return_key: str = "mean_return",
+) -> Dict[str, float]:
+    """
+    Compute steps-to-threshold: how many steps to first exceed target_return.
+
+    Returns -1 for a curve that never reaches the target.
+    """
+
+    def steps_to(curve):
+        for point in curve:
+            if point[return_key] >= target_return:
+                return float(point[step_key])
+        return -1.0
+
+    ugtc_steps = steps_to(ugtc_curve)
+    base_steps = steps_to(baseline_curve)
+
+    result = {
+        "ugtc_steps_to_target": ugtc_steps,
+        "baseline_steps_to_target": base_steps,
+    }
+    if ugtc_steps > 0 and base_steps > 0:
+        result["efficiency_ratio"] = base_steps / ugtc_steps
+    return result