scripts/compute_match.py

from __future__ import annotations
import argparse
import math

def round_to(x: float, multiple: int) -> int:
    return int(multiple * round(x / multiple))

def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--stem_layers", type=int, required=True)
    ap.add_argument("--hemi_layers", type=int, required=True)
    ap.add_argument("--dff_dense", type=int, required=True)
    ap.add_argument("--dff_expert", type=int, required=True)
    ap.add_argument("--topk", type=int, default=1)
    ap.add_argument("--multiple", type=int, default=256, help="round dense_dff to this multiple")
    args = ap.parse_args()

    Ls = args.stem_layers
    Lh = args.hemi_layers

    # Matches attention layer count by setting dense_layers = Ls + 2*Lh
    dense_layers = Ls + 2 * Lh

    # In our current StructuredBiHMoE implementation, MoE occurs on odd i => floor(Lh/2) MoE layers per hemisphere
    moe_per_hemi = Lh // 2
    dense_per_hemi = Lh - moe_per_hemi  # ceil(Lh/2)

    # Equivalent FFN "d_ff units" across the whole structured forward (both hemispheres + stem)
    ffn_units = (Ls * args.dff_dense) + (2 * dense_per_hemi * args.dff_dense) + (2 * moe_per_hemi * args.topk * args.dff_expert)

    dense_dff = ffn_units / dense_layers
    dense_dff_rounded = max(args.multiple, round_to(dense_dff, args.multiple))

    print("=== D_a (compute-ish match) suggestion ===")
    print(f"dense_layers = {dense_layers}   (matches attention depth: stem + 2*hemi)")
    print(f"dense_dff    ≈ {dense_dff:.2f}  -> rounded to {dense_dff_rounded} (multiple={args.multiple})")
    print()
    print("Details:")
    print(f"  stem_layers={Ls}")
    print(f"  hemi_layers={Lh} -> dense_per_hemi={dense_per_hemi} moe_per_hemi={moe_per_hemi}")
    print(f"  dff_dense={args.dff_dense} dff_expert={args.dff_expert} topk={args.topk}")
    print(f"  ffn_units(total)={ffn_units} over dense_layers={dense_layers}")

if __name__ == "__main__":
    main()