about_native.html

<style>

#about-native *, #about-native *::before, #about-native *::after{ box-sizing: border-box; margin: 0; padding: 0; }
#about-native{
      background: #f8f9fb;
      color: #2d3142;
      font-family: Helvetica, Arial, sans-serif;
      font-size: 14px;
      line-height: 1.75;
    }
#about-native .paper-header{
      max-width: 760px;
      margin: 64px auto 0;
      padding: 0 24px;
      text-align: center;
    }
#about-native .paper-title{
      font-size: 26px;
      font-weight: 700;
      letter-spacing: 0;
      line-height: 1.35;
      color: #111;
      margin-bottom: 24px;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .authors{
      font-size: 15px;
      color: #333;
      margin-bottom: 5px;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .authors .name{
      font-weight: 600;
    }
#about-native .authors sup{
      font-size: 9px;
      color: #555;
      font-weight: 600;
      vertical-align: super;
      margin-left: 1px;
    }
#about-native .affiliation{
      font-size: 13px;
      color: #666;
      margin-bottom: 36px;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .affiliation sup{
      font-size: 9px;
      font-weight: 600;
      color: #555;
      vertical-align: super;
      margin-right: 2px;
    }
#about-native .divider{
      border: none;
      border-top: 1px solid #bbb;
      max-width: 760px;
      margin: 0 auto 44px;
    }
#about-native .content{
      max-width: none;
      margin: 0 auto;
      padding: 0 0 72px;
    }
#about-native .section{
      margin-bottom: 56px;
    }
#about-native .section-heading{
      display: flex;
      align-items: baseline;
      gap: 0;
      width: fit-content;
      margin-bottom: 20px;
      padding-bottom: 8px;
      border-bottom: 1px solid #ccc;
    }
#about-native .section-accent{
      display: none;
    }
#about-native .section-heading h2{
      font-size: 20px;
      font-weight: 700;
      color: #2d3142;
      font-family: Helvetica, Arial, sans-serif;
      letter-spacing: 0;
    }
#about-native .lead{
      font-size: 14px;
      color: #222;
      line-height: 1.8;
      background: none;
      border-left: none;
      border-radius: 0;
      padding: 0;
      margin-bottom: 20px;
      font-style: normal;
    }
#about-native .body-text{
      font-size: 14px;
      color: #222;
      line-height: 1.8;
      margin-bottom: 18px;
    }
#about-native .hl{
      background: none;
      color: inherit;
      border-radius: 0;
      padding: 0;
      font-weight: 600;
      font-size: inherit;
      font-style: normal;
    }
#about-native .card-grid{
      display: grid;
      grid-template-columns: 1fr 1fr 1fr;
      gap: 16px;
      margin-top: 24px;
    }
#about-native .card{
      background: #fff;
      border: 1px solid #bcc3cd;
      border-radius: 0;
      padding: 16px 14px;
    }
#about-native .card-title{
      font-size: 11px;
      font-weight: 700;
      letter-spacing: 0.6px;
      text-transform: uppercase;
      color: #444;
      margin-bottom: 10px;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .card-body{
      font-size: 14px;
      color: #444;
      line-height: 1.65;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .console-label{
      font-size: 12px;
      font-weight: 700;
      letter-spacing: 0.5px;
      text-transform: uppercase;
      color: #666;
      margin-bottom: 8px;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .console-model-tag{
      display: inline-block;
      background: #f0f0f0;
      color: #333;
      border-radius: 0;
      padding: 2px 8px;
      font-size: 11px;
      font-weight: 600;
      margin-bottom: 12px;
      font-family: Helvetica, Arial, sans-serif;
      border: 1px solid #ccc;
    }
#about-native .chat-window{
      border: 1px solid #bcc3cd;
      border-radius: 0;
      background: #f8f9fb;
      padding: 20px;
      display: flex;
      flex-direction: column;
      gap: 20px;
      max-height: 500px;
      overflow-y: auto;
    }
#about-native .chat-user-row{
      display: flex;
      justify-content: flex-end;
    }
#about-native .chat-user-bubble{
      background: #eef1f6;
      color: #2d3142;
      padding: 10px 14px;
      border-radius: 0;
      max-width: 76%;
      font-size: 14px;
      line-height: 1.65;
      border: 1px solid #bcc3cd;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .chat-assistant-row{
      display: flex;
      gap: 10px;
      align-items: flex-start;
    }
#about-native .chat-avatar{
      min-width: 28px;
      height: 28px;
      border-radius: 0;
      background: #333;
      font-size: 11px;
      font-weight: 700;
      color: #fff;
      display: flex;
      align-items: center;
      justify-content: center;
      flex-shrink: 0;
      margin-top: 2px;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .chat-assistant-bubble{
      background: #fff;
      border: 1px solid #bcc3cd;
      padding: 12px 16px;
      border-radius: 0;
      font-size: 14px;
      line-height: 1.78;
      color: #2d3142;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .degen-span{
      background: rgba(180,0,0,0.07);
      border-bottom: 1.5px solid #b91c1c;
      border-radius: 0;
    }
#about-native .degen-note{
      display: block;
      color: #666;
      font-style: normal;
      font-size: 12px;
      margin-top: 10px;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .boxed-val{
      display: inline-block;
      border: 1.5px solid currentColor;
      border-radius: 0;
      padding: 0 4px;
      font-weight: 600;
      font-family: 'JetBrains Mono', monospace;
      margin: 0 2px;
    }
#about-native .dataset-grid{
      display: grid;
      grid-template-columns: 1fr 1fr;
      gap: 12px;
      margin: 20px 0;
    }
#about-native .dataset-card{
      background: #fff;
      border: 1px solid #bcc3cd;
      border-radius: 0;
      padding: 12px 14px;
      display: flex;
      gap: 12px;
      align-items: flex-start;
    }
#about-native .dataset-dot{
      min-width: 8px;
      height: 8px;
      border-radius: 50%;
      margin-top: 5px;
      flex-shrink: 0;
    }
#about-native .dataset-name{
      font-weight: 700;
      font-size: 14px;
      color: #111;
      margin-bottom: 3px;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .dataset-desc{
      font-size: 12px;
      color: #555;
      line-height: 1.55;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .stats-row{
      display: grid;
      grid-template-columns: repeat(4, 1fr);
      gap: 12px;
      margin: 24px 0;
    }
#about-native .stat-box{
      background: #f7f7f7;
      border: 1px solid #ccc;
      border-radius: 0;
      padding: 14px 16px;
      text-align: center;
    }
#about-native .stat-number{
      font-size: 20px;
      font-weight: 700;
      color: #111;
      font-family: 'JetBrains Mono', monospace;
      line-height: 1.2;
      margin-bottom: 4px;
    }
#about-native .stat-label{
      font-size: 11px;
      color: #666;
      font-weight: 500;
      letter-spacing: 0.3px;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .split-table{
      width: 100%;
      border-collapse: collapse;
      font-size: 14px;
      margin-top: 16px;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .split-table th{
      text-align: left;
      padding: 7px 12px;
      font-size: 12px;
      font-weight: 700;
      letter-spacing: 0.5px;
      text-transform: uppercase;
      color: #555;
      border-top: 1px solid #2d3142;
      border-bottom: 1px solid #2d3142;
    }
#about-native .split-table td{
      padding: 9px 12px;
      border-bottom: 1px solid #d4d9e0;
      color: #222;
      font-family: 'JetBrains Mono', monospace;
      font-size: 14px;
    }
#about-native .split-table tr:last-child td{
      border-bottom: 1px solid #2d3142;
    }
#about-native .badge-pos{
      display: inline-block;
      background: none;
      color: #b91c1c;
      border-radius: 0;
      padding: 0;
      font-size: 12px;
      font-weight: 600;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .badge-neg{
      display: inline-block;
      background: none;
      color: #1e3a6b;
      border-radius: 0;
      padding: 0;
      font-size: 12px;
      font-weight: 600;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .imbalance-bar{
      margin: 20px 0 8px;
    }
#about-native .imbalance-bar-label{
      display: flex;
      justify-content: space-between;
      font-size: 12px;
      color: #666;
      margin-bottom: 6px;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .imbalance-track{
      height: 16px;
      border-radius: 0;
      background: #ddd;
      overflow: hidden;
      display: flex;
    }
#about-native .imbalance-fill-pos{
      height: 100%;
      background: #b91c1c;
      display: flex;
      align-items: center;
      justify-content: center;
      font-size: 11px;
      font-weight: 700;
      color: #fff;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .imbalance-fill-neg{
      height: 100%;
      background: #1e3a6b;
      display: flex;
      align-items: center;
      justify-content: center;
      font-size: 11px;
      font-weight: 700;
      color: #fff;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .imbalance-pos-wrapper{
      position: relative;
      height: 100%;
    }
#about-native .bar-callout{
      position: absolute;
      bottom: calc(100% + 6px);
      left: 0;
      color: #fff;
      font-size: 11px;
      font-weight: 700;
      padding: 2px 5px;
      border-radius: 0;
      white-space: nowrap;
      font-family: Helvetica, Arial, sans-serif;
      line-height: 1.4;
    }
#about-native .bar-callout.pos{ background: #b91c1c; }
#about-native .bar-callout.neg{ background: #1e3a6b; right: 0; left: auto; }
#about-native .bar-callout.pos::after{
      content: '';
      position: absolute;
      top: 100%;
      left: 6px;
      border: 4px solid transparent;
      border-top-color: #b91c1c;
    }
#about-native .bar-callout.neg::after{
      content: '';
      position: absolute;
      top: 100%;
      right: 6px;
      border: 4px solid transparent;
      border-top-color: #1e3a6b;
    }
#about-native .imbalance-neg-wrapper{
      position: relative;
      height: 100%;
    }
#about-native .arrow-row{
      display: flex;
      align-items: center;
      gap: 16px;
      margin: 16px 0;
    }
#about-native .arrow-row .bar-wrap{ flex: 1; }
#about-native .arrow-icon{
      font-size: 20px;
      color: #555;
      flex-shrink: 0;
    }
#about-native .formula-block{
      background: #f8f9fb;
      border-radius: 0;
      border: 1px solid #bcc3cd;
      border-left: 3px solid #2d3142;
      padding: 12px 18px;
      font-family: 'JetBrains Mono', monospace;
      font-size: 14px;
      color: #2d3142;
      margin: 16px 0;
      display: flex;
      align-items: center;
      gap: 12px;
    }
#about-native .formula-block .formula-label{
      font-size: 11px;
      font-weight: 700;
      letter-spacing: 0.5px;
      text-transform: uppercase;
      color: #666;
      flex-shrink: 0;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .pca-embed-wrap{
      border: 1px solid #bcc3cd;
      border-radius: 0;
      background: #fff;
      overflow: hidden;
      padding: 8px 4px;
    }
#about-native .probe-diagram-embed{
      width: 100%;
      overflow: hidden;
      display: flex;
      flex-direction: column;
      align-items: center;
      padding: 20px 0 0;
    }
#about-native .probe-diagram-scaler{
      transform-origin: top center;
      width: 100%;
      display: flex;
      justify-content: center;
    }
#about-native .probe-diagram-embed .diagram{
      position: relative;
      width: 1200px;
    }
#about-native .probe-diagram-embed svg.connectors{
      position: absolute;
      inset: 0;
      width: 100%;
      height: 100%;
      overflow: visible;
      pointer-events: none;
      z-index: 0;
    }
#about-native .probe-diagram-embed .row{
      display: flex;
      align-items: center;
      justify-content: space-between;
      gap: 0;
      position: relative;
      z-index: 1;
    }
#about-native .probe-diagram-embed .box{
      border-radius: 0;
      padding: 16px 18px;
      overflow: hidden;
      border: 1px solid #ccc;
      position: relative;
      background: #fff;
      transition: none;
    }
#about-native .probe-diagram-embed .box-label{
      font-size: 0.72rem;
      font-weight: 700;
      letter-spacing: .07em;
      text-transform: uppercase;
      margin-bottom: 10px;
      opacity: .5;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .probe-diagram-embed .box-input{ width: 185px; border-color: #aab4d4; background: #f5f6fa; }
#about-native .probe-diagram-embed .box-llm{ width: 310px; border-color: #ccc;    background: #fafafa; }
#about-native .probe-diagram-embed .box-probe{ width: 210px; border-color: #bba8cc; background: #f8f5fc; }
#about-native .probe-diagram-embed .box-loss{ width: 210px; border-color: #c8b870; background: #fdfaf0; }
#about-native .probe-diagram-embed .tokens{ display: flex; flex-wrap: wrap; gap: 4px; }
#about-native .probe-diagram-embed .token{
      font-size: 0.72rem; font-weight: 600;
      padding: 3px 7px; border-radius: 0; letter-spacing: .02em;
      opacity: 0; transform: translateY(6px);
      transition: opacity .4s, transform .4s;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .probe-diagram-embed .token.visible{ opacity: 1; transform: translateY(0); }
#about-native .probe-diagram-embed .token-prompt{ background: #e0e7ff; color: #3730a3; }
#about-native .probe-diagram-embed .token-ok{ background: #dcfce7; color: #166534; }
#about-native .probe-diagram-embed .token-bad{ background: #fee2e2; color: #991b1b; }
#about-native .probe-diagram-embed .token-section-label{
      font-size: 0.66rem; color: #888; font-weight: 600;
      letter-spacing: .06em; text-transform: uppercase;
      width: 100%; margin-top: 6px; margin-bottom: 2px;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .probe-diagram-embed .layers{ display: flex; flex-direction: column; gap: 5px; }
#about-native .probe-diagram-embed .layer{
      border-radius: 0; padding: 7px 12px;
      font-size: 0.8rem; font-weight: 600;
      background: #f3f4f6; color: #555;
      border: 1px solid #ddd;
      transition: background .4s, color .4s, border-color .4s, transform .3s;
      position: relative;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .probe-diagram-embed .layer.active{ background: #eef2fb; color: #1e3a6b; border-color: #9eb0d4; }
#about-native .probe-diagram-embed .layer.probe-layer{
      background: #fef3c7; color: #78350f;
      border: 1.5px solid #d97706; font-weight: 700;
      transform: scale(1);
    }
#about-native .probe-diagram-embed .layer.probe-layer.pulse{ animation: layerPulse 1.2s ease-in-out; }
@keyframes layerPulse{
      0%   { box-shadow: 0 0 0 0 rgba(217,119,6,.4); transform: scale(1); }
      40%  { box-shadow: 0 0 0 6px rgba(217,119,6,.15); transform: scale(1.03); }
      100% { box-shadow: 0 0 0 0 rgba(217,119,6,0); transform: scale(1); }
    }
#about-native .probe-diagram-embed .hook-badge{
      position: absolute; right: -10px; top: 50%; transform: translateY(-50%);
      background: #d97706; color: #fff; font-size: 0.5rem; font-weight: 800;
      padding: 2px 5px; border-radius: 0; letter-spacing: .04em;
      opacity: 0; transition: opacity .4s; z-index: 10;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .probe-diagram-embed .hook-badge.show{ opacity: 1; }
#about-native .probe-diagram-embed .act-dots{ display: inline-flex; gap: 3px; margin-left: 8px; vertical-align: middle; }
#about-native .probe-diagram-embed .act-dot{
      width: 6px; height: 6px; border-radius: 50%;
      background: #ccc; transition: background .3s, transform .3s; flex-shrink: 0;
    }
#about-native .probe-diagram-embed .probe-explainer{
      display: flex; align-items: center; gap: 5px; margin-bottom: 10px;
      font-size: 0.62rem; color: #555; font-weight: 600;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .probe-diagram-embed .probe-explainer .arrow-mini{ color: #888; font-size: .8rem; }
#about-native .probe-diagram-embed .score-grid{ display: flex; flex-direction: column; gap: 5px; }
#about-native .probe-diagram-embed .score-header{
      display: flex; align-items: center; gap: 6px;
      font-size: 0.66rem; font-weight: 700; letter-spacing: .05em; text-transform: uppercase;
      color: #888; padding-bottom: 3px;
      border-bottom: 1px solid #ddd; margin-bottom: 2px;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .probe-diagram-embed .score-header .sh-tok{ width: 48px; flex-shrink: 0; }
#about-native .probe-diagram-embed .score-header .sh-bar{ flex: 1; }
#about-native .probe-diagram-embed .score-header .sh-val{ width: 32px; text-align: right; flex-shrink: 0; }
#about-native .probe-diagram-embed .score-row{
      display: flex; align-items: center; gap: 6px;
      opacity: 0; transform: translateX(8px);
      transition: opacity .35s, transform .35s;
    }
#about-native .probe-diagram-embed .score-row.visible{ opacity: 1; transform: translateX(0); }
#about-native .probe-diagram-embed .score-tok{
      font-size: 0.70rem; font-weight: 600; width: 48px;
      border-radius: 0; padding: 2px 4px; text-align: center; flex-shrink: 0;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .probe-diagram-embed .score-bar-wrap{ flex: 1; background: #eee; border-radius: 0; height: 6px; overflow: hidden; }
#about-native .probe-diagram-embed .score-bar{ height: 100%; border-radius: 0; width: 0%; transition: width 0.8s cubic-bezier(.4,0,.2,1); }
#about-native .probe-diagram-embed .score-val{
      font-size: 0.72rem; font-weight: 700; width: 32px; text-align: right;
      flex-shrink: 0; font-variant-numeric: tabular-nums;
      font-family: 'JetBrains Mono', monospace;
    }
#about-native .probe-diagram-embed .cmp-table{
      width: 100%; border-collapse: collapse; font-size: 0.72rem;
      opacity: 0; transition: opacity .5s;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .probe-diagram-embed .cmp-table.visible{ opacity: 1; }
#about-native .probe-diagram-embed .cmp-table th{
      font-size: 0.66rem; font-weight: 700; letter-spacing: .05em; text-transform: uppercase;
      padding: 3px 6px 5px; text-align: center;
      border-bottom: 1px solid #ccc; color: #666;
    }
#about-native .probe-diagram-embed .cmp-table th.th-tok{ text-align: left; }
#about-native .probe-diagram-embed .cmp-table td{
      padding: 2px 6px; text-align: center;
      border-bottom: 1px solid #eee; font-variant-numeric: tabular-nums;
    }
#about-native .probe-diagram-embed .cmp-table td.td-tok{ text-align: left; font-weight: 600; border-radius: 0; }
#about-native .probe-diagram-embed .cmp-table .diff-cell{ font-weight: 700; border-radius: 0; transition: background .5s; }
#about-native .probe-diagram-embed .cmp-row{ opacity: 0; transition: opacity .3s; }
#about-native .probe-diagram-embed .cmp-row.visible{ opacity: 1; }
#about-native .probe-diagram-embed .loss-formula{
      font-size: 0.85rem; font-weight: 700; color: #78350f;
      text-align: center; padding: 8px 0 4px; opacity: 0; transition: opacity .5s;
      font-family: 'JetBrains Mono', monospace;
    }
#about-native .probe-diagram-embed .loss-formula.visible{ opacity: 1; }
#about-native .probe-diagram-embed .grad-note{
      margin-top: 8px; font-size: 0.58rem; color: #555;
      padding: 5px 8px; background: #f7f7f7; border-radius: 0;
      border: 1px solid #ddd; opacity: 0; transition: opacity .5s; line-height: 1.55;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .probe-diagram-embed .grad-note.visible{ opacity: 1; }
#about-native .probe-diagram-embed .phase-badge{
      font-size: 0.7rem; font-weight: 600; color: #555;
      background: #f7f7f7; border: 1px solid #ccc;
      padding: 4px 14px; border-radius: 0; white-space: nowrap;
      letter-spacing: .02em; margin-top: 16px;
      transition: opacity .4s;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .probe-diagram-embed .col-gap{ width: 60px; flex-shrink: 0; }
#about-native .probe-diagram-embed .col-gap-sm{ width: 30px; flex-shrink: 0; }
#about-native .cite{
      font-size: 12px;
      font-weight: 700;
      color: #1e3a6b;
      vertical-align: super;
      line-height: 0;
      text-decoration: none;
      margin-left: 1px;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .cite:hover{ text-decoration: underline; }
#about-native .references-section{
      max-width: none;
      margin: 0 auto;
      padding: 0 0 72px;
      border-top: 1px solid #bbb;
      margin-top: 8px;
    }
#about-native .references-section h2{
      font-size: 20px;
      font-weight: 700;
      color: #2d3142;
      margin: 36px 0 20px;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .ref-list{
      list-style: none;
      display: flex;
      flex-direction: column;
      gap: 14px;
    }
#about-native .ref-list li{
      display: flex;
      gap: 12px;
      font-size: 14px;
      color: #333;
      line-height: 1.6;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .ref-num{
      min-width: 26px;
      font-weight: 700;
      color: #333;
      padding-top: 1px;
    }
#about-native .ref-list a{
      color: #1e3a6b;
      text-decoration: none;
      word-break: break-all;
    }
#about-native .ref-list a:hover{ text-decoration: underline; }
#about-native .step-title{
      font-size: 14px;
      font-weight: 700;
      color: #333;
      margin-bottom: 8px;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .caption-text{
      font-size: 12px;
      color: #888;
      font-family: Helvetica, Arial, sans-serif;
    }
#about-native .plot-iframe{
      width: 100%;
      border: none;
      display: block;
    }
#about-native em, #about-native i, #about-native cite{ font-style: normal; }
#about-native .cb-toggle{
      position: fixed;
      top: 14px;
      right: 14px;
      z-index: 1000;
      display: flex;
      align-items: center;
      gap: 8px;
      padding: 7px 12px;
      background: #fff;
      border: 1.5px solid #2d3142;
      border-radius: 0;
      font-size: 11px;
      font-weight: 600;
      font-family: Helvetica, Arial, sans-serif;
      color: #2d3142;
      cursor: pointer;
      box-shadow: 0 1px 4px rgba(0,0,0,0.08);
    }
#about-native .cb-toggle .cb-switch{
      position: relative;
      width: 30px;
      height: 16px;
      flex-shrink: 0;
      background: #ddd;
      border: 1px solid #2d3142;
      border-radius: 0;
      transition: background .15s;
    }
#about-native .cb-toggle .cb-switch::after{
      content: "";
      position: absolute;
      top: 1px;
      left: 1px;
      width: 12px;
      height: 12px;
      background: #fff;
      border: 1px solid #2d3142;
      transition: transform .15s;
    }
#about-native .cb-toggle[aria-checked="true"] .cb-switch{
      background: #1e3a6b;
    }
#about-native .cb-toggle[aria-checked="true"] .cb-switch::after{
      transform: translateX(14px);
    }
#about-native body.colorblind-mode .degen-span, #about-native body.colorblind-mode .degen-note, #about-native body.colorblind-mode .pca-embed-wrap, #about-native body.colorblind-mode .plot-iframe, #about-native body.colorblind-mode .cb-recolor{
      filter: url(#cb-recolor-filter);
    }
@media all{
      #about-native.colorblind-mode .degen-span,
      #about-native.colorblind-mode .degen-note,
      #about-native.colorblind-mode .cb-recolor {
        filter: url(#cb-recolor-filter);
      }
    }

</style>
<div id="about-native">
  <header class="paper-header">
    <h1 class="paper-title">Detecting Model Degeneration with Probe</h1>

    <p class="authors">
      <span class="name">Lorenzo Baggi</span><sup>1</sup>&ensp;
      <span class="name">Marco De Negri</span><sup>1</sup>&ensp;
      <span class="name">Luca Sartori</span><sup>1</sup>&ensp;
      <span class="name">Moritz Reihs</span><sup>1</sup>
    </p>

    <p class="affiliation">
      <sup>1</sup>ETH Zurich
    </p>
  </header>

  <hr class="divider" />

  <main class="content">

    <!-- ── Section 1: What is model degeneration? ───────── -->
    <section class="section">
      <div class="section-heading">
        <div class="section-accent"></div>
        <h2>What is model degeneration?</h2>
      </div>

      <p class="lead">
        Model degeneration is a failure mode of autoregressive language models in which
        generation collapses into <strong>repetitive, low-information output</strong>.
        The model gets trapped in a loop, producing the same phrase, sentence, or reasoning
        pattern over and over, while the text loses all semantic value.
        It does not stop generating; it simply stops <em>communicating</em>.
      </p>

      <p class="body-text">
        Autoregressive models generate text one token at a time: at each step the model
        looks at all previous tokens and picks the next one according to a probability
        distribution. When that distribution becomes too concentrated on a narrow set of
        tokens, often ones that are already present in context, the model enters a
        self-reinforcing cycle it cannot escape on its own.<a href="#ref-2" class="cite">[2,&thinsp;3]</a>
      </p>

      <p class="body-text">
        Across the completions we collected, degeneration takes several recurring forms.
        The repeated unit may be a single token, a digit or a word emitted hundreds of
        times, a short phrase, a full sentence reproduced verbatim, a mathematical
        expression, or a structural marker such as a row of asterisks or hashtags. In each
        case the model stops advancing the task and instead recycles material already
        present in its own output.
      </p>

      <p class="body-text">
        The effect is strongly correlated with the difficulty of the prompt: simple
        questions are answered cleanly, whereas hard, multi-step reasoning, competition
        mathematics above all, degenerates far more often. Once a loop sets in, the model
        frequently, though not always, fails to recover a natural stopping signal and keeps
        emitting tokens at full confidence until the generation is cut off by the token
        limit, as illustrated by
        <span class="hl">Apertus-8B-Instruct</span><a href="#ref-1" class="cite">[1]</a>
        in the example below.
      </p>
    </section>

    <!-- ── Section 2: Concrete example ──────────────────── -->
    <section class="section">
      <div class="section-heading">
        <div class="section-accent"></div>
        <h2>A concrete example</h2>
      </div>

      <p class="body-text">
        The exchange below was generated by
        <span class="hl">Apertus-8B-Instruct</span><a href="#ref-1" class="cite">[1]</a>
        on a one-line arithmetic question drawn from our balanced dataset. The model
        factorises the number correctly and even spots that 13 is prime, the answer is
        right there, but as it starts to list the factors it falls into a loop, repeating
        the digit "2" thousands of times and never delivering the answer.
        Text highlighted in
        <span class="degen-span" style="padding:2px 7px;">red</span>
        marks the degenerate region.
      </p>

      <p class="console-label">Model output: Apertus-8B-Instruct</p>

      <div class="chat-window">

        <!-- User bubble -->
        <div class="chat-user-row">
          <div class="chat-user-bubble">
            What is the largest prime factor of 3328?
          </div>
        </div>

        <!-- Assistant bubble -->
        <div class="chat-assistant-row">
          <div class="chat-avatar">A</div>
          <div class="chat-assistant-bubble">

            To find the largest prime factor of 3328, we first need to find the prime
            factors of 3328. We can do this by dividing 3328 by the smallest prime number,
            which is 2.<br><br>

            3328 &divide; 2 = 1664<br>
            1664 &divide; 2 = 832<br>
            832 &divide; 2 = 416<br>
            416 &divide; 2 = 208<br>
            208 &divide; 2 = 104<br>
            104 &divide; 2 = 52<br>
            52 &divide; 2 = 26<br>
            26 &divide; 2 = 13<br><br>

            Since 13 is a prime number, we have found all the prime factors of 3328.<br><br>

            The prime factors of 3328 are
            <span class="degen-span">
              2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
              2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
              2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
              2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,

              <span class="degen-note">
                [&hellip; the digit "2" repeats until the generation
                hits its length limit, the correct answer, 13, is never given]
              </span>
            </span>

          </div>
        </div>

      </div>
    </section>

    <!-- ── Section 3: Dataset ────────────────────────────── -->
    <section class="section">
      <div class="section-heading">
        <div class="section-accent"></div>
        <h2>Dataset</h2>
      </div>

      <p class="body-text">
        The starting point is the dataset published on Hugging Face by Luca Sartori
        (<a href="https://huggingface.co/datasets/luca-sartori/degeneration-probe-instruct-token-level"
            target="_blank" style="color:#2c4a7c;text-decoration:none;font-weight:500;">
          luca-sartori/degeneration-probe-instruct-token-level</a>).
        It contains <strong>83,350 completions</strong> generated by
        <span class="hl">Apertus-8B-Instruct</span><a href="#ref-1" class="cite">[1]</a> across five public instruction datasets,
        one response per prompt, each annotated at the token level with a
        continuous repetition score.
      </p>

      <!-- 5 source datasets -->
      <div class="dataset-grid">
        <div class="dataset-card">
          <div class="dataset-dot" style="background:#2c4a7c;"></div>
          <div>
            <div class="dataset-name">NuminaMath-1.5</div>
            <div class="dataset-desc">AI-MO: Competition mathematics. High degeneration rate due to complex multi-step reasoning.</div>
          </div>
        </div>
        <div class="dataset-card">
          <div class="dataset-dot" style="background:#7c3aed;"></div>
          <div>
            <div class="dataset-name">Medical-O1</div>
            <div class="dataset-desc">FreedomIntelligence: Verifiable medical reasoning requiring clinical knowledge.</div>
          </div>
        </div>
        <div class="dataset-card">
          <div class="dataset-dot" style="background:#0891b2;"></div>
          <div>
            <div class="dataset-name">DeepMath-103K</div>
            <div class="dataset-desc">zwhe99: Deep mathematical reasoning. Among the hardest prompts; very high degeneration likelihood.</div>
          </div>
        </div>
        <div class="dataset-card">
          <div class="dataset-dot" style="background:#059669;"></div>
          <div>
            <div class="dataset-name">Llama-Nemotron</div>
            <div class="dataset-desc">NVIDIA: Post-training instruction data. Mixed difficulty; moderate degeneration rate.</div>
          </div>
        </div>
        <div class="dataset-card">
          <div class="dataset-dot" style="background:#d97706;"></div>
          <div>
            <div class="dataset-name">IF-SFT</div>
            <div class="dataset-desc">AllenAI: Instruction-following data. Lower difficulty; low-degeneration baseline.</div>
          </div>
        </div>
      </div>

      <!-- Class imbalance -->
      <p class="body-text" style="margin-top:28px;">
        A generation is labeled <strong>positive</strong> (degenerating) if it contains at
        least one token whose averaged repetition score is &ge;&nbsp;0.8; otherwise it is
        labeled <strong>negative</strong> (clean). In the full 83k corpus, the vast majority
        of completions are clean: the natural positive-to-negative ratio is roughly
        <strong>1&thinsp;:&thinsp;72</strong>.
        Training a probe on such an imbalanced set would bias the model towards always
        predicting the negative class, so we downsampled the negative examples.
      </p>

      <!-- Before / After bars -->
      <div class="imbalance-bar">
        <div class="imbalance-bar-label">
          <span>Before downsampling: 83,350 generations: ratio 1&thinsp;:&thinsp;72</span>
          <span><span class="badge-pos">positive</span> &nbsp;<span class="badge-neg">negative</span></span>
        </div>
        <div class="imbalance-track" style="overflow:visible; margin-top:26px;">
          <div class="imbalance-pos-wrapper" style="width:1.4%;">
            <div class="imbalance-fill-pos" style="width:100%; height:16px;"></div>
            <div class="bar-callout pos">1.4%</div>
          </div>
          <div class="imbalance-neg-wrapper" style="width:98.6%;">
            <div class="imbalance-fill-neg" style="width:100%; height:16px;"></div>
            <div class="bar-callout neg">98.6%</div>
          </div>
        </div>
      </div>

      <div style="text-align:center;font-size:20px;color:#2c4a7c;margin:8px 0;">&#8595;</div>

      <div class="imbalance-bar">
        <div class="imbalance-bar-label">
          <span>After downsampling: 2,973 generations: ratio ~1&thinsp;:&thinsp;1.6</span>
          <span><span class="badge-pos">positive</span> &nbsp;<span class="badge-neg">negative</span></span>
        </div>
        <div class="imbalance-track" style="overflow:visible; margin-top:26px;">
          <div class="imbalance-pos-wrapper" style="width:38%;">
            <div class="imbalance-fill-pos" style="width:100%; height:16px;"></div>
            <div class="bar-callout pos">38%</div>
          </div>
          <div class="imbalance-neg-wrapper" style="width:62%;">
            <div class="imbalance-fill-neg" style="width:100%; height:16px;"></div>
            <div class="bar-callout neg">62%</div>
          </div>
        </div>
      </div>

      <p class="body-text" style="margin-top:16px;">
        Negative examples were randomly removed using a fixed seed (42) to keep the
        experiment reproducible. After a cleanup step that discarded rows with no scored
        tokens, the final ratio settled at approximately <strong>1&thinsp;:&thinsp;1.6</strong>.
        The balanced dataset is published at
        <a href="https://huggingface.co/datasets/lorenzo0312/degeneration-probe-instruct-token-level-balanced"
           target="_blank" style="color:#2c4a7c;text-decoration:none;font-weight:500;">
          lorenzo0312/degeneration-probe-instruct-token-level-balanced</a>.
      </p>

      <!-- Overall split table -->
      <table class="split-table">
        <thead>
          <tr>
            <th>Split</th>
            <th>Total</th>
            <th>Positive</th>
            <th>Negative</th>
            <th>Ratio</th>
          </tr>
        </thead>
        <tbody>
          <tr>
            <td>Train</td>
            <td>2,379</td>
            <td><span class="badge-pos">915</span></td>
            <td><span class="badge-neg">1,464</span></td>
            <td>1&thinsp;:&thinsp;1.6</td>
          </tr>
          <tr>
            <td>Validation</td>
            <td>299</td>
            <td><span class="badge-pos">113</span></td>
            <td><span class="badge-neg">186</span></td>
            <td>1&thinsp;:&thinsp;1.6</td>
          </tr>
          <tr>
            <td>Test</td>
            <td>295</td>
            <td><span class="badge-pos">113</span></td>
            <td><span class="badge-neg">182</span></td>
            <td>1&thinsp;:&thinsp;1.6</td>
          </tr>
        </tbody>
      </table>

      <!-- Per-dataset breakdown -->
      <div style="margin-top:28px;">
        <div class="step-title">
          Prompts per source dataset
        </div>
        <table class="split-table">
          <thead>
            <tr>
              <th>Dataset</th>
              <th>Train</th>
              <th>Val</th>
              <th>Test</th>
              <th>Total</th>
              <th>%</th>
            </tr>
          </thead>
          <tbody>
            <tr>
              <td><span style="display:inline-block;width:8px;height:8px;border-radius:50%;background:#0891b2;margin-right:6px;vertical-align:middle;"></span>DeepMath-103K</td>
              <td>654</td><td>83</td><td>78</td><td>815</td><td>27.4%</td>
            </tr>
            <tr>
              <td><span style="display:inline-block;width:8px;height:8px;border-radius:50%;background:#2c4a7c;margin-right:6px;vertical-align:middle;"></span>NuminaMath-1.5</td>
              <td>610</td><td>88</td><td>76</td><td>774</td><td>26.0%</td>
            </tr>
            <tr>
              <td><span style="display:inline-block;width:8px;height:8px;border-radius:50%;background:#d97706;margin-right:6px;vertical-align:middle;"></span>IF-SFT</td>
              <td>563</td><td>63</td><td>70</td><td>696</td><td>23.4%</td>
            </tr>
            <tr>
              <td><span style="display:inline-block;width:8px;height:8px;border-radius:50%;background:#059669;margin-right:6px;vertical-align:middle;"></span>Llama-Nemotron</td>
              <td>459</td><td>52</td><td>60</td><td>571</td><td>19.2%</td>
            </tr>
            <tr>
              <td><span style="display:inline-block;width:8px;height:8px;border-radius:50%;background:#7c3aed;margin-right:6px;vertical-align:middle;"></span>Medical-O1</td>
              <td>93</td><td>13</td><td>11</td><td>117</td><td>3.9%</td>
            </tr>
            <tr style="font-weight:700;border-top:2px solid #e2e6ea;">
              <td>Total</td>
              <td>2,379</td><td>299</td><td>295</td><td>2,973</td><td>100%</td>
            </tr>
          </tbody>
        </table>
        <p class="caption-text" style="margin-top:10px;">
          Math datasets (DeepMath + NuminaMath) together account for ~53% of each split.
          Medical-O1 is under-represented (3.9%) because it rarely triggers degeneration,
          none of its tokens cross the 0.8 threshold, making it largely a negative-class source.
        </p>
      </div>

    </section>

    <!-- ── Section 3: Labeling ───────────────────────────── -->
    <section class="section">
      <div class="section-heading">
        <div class="section-accent"></div>
        <h2>How we labeled the data</h2>
      </div>

      <p class="lead">
        Every completion in the corpus was generated by
        <span class="hl">Apertus-8B-Instruct</span><a href="#ref-1" class="cite">[1]</a>.
        Each token then receives a continuous <strong>repetition score</strong>
        between 0 and 1. A score of <strong>0</strong> means every bigram in the surrounding
        context is unique: healthy, diverse output. A score of <strong>1</strong>
        means all bigrams repeat: a fully degenerate loop.
        A token is flagged as <em>degenerating</em> when its score is &ge;&nbsp;<strong>0.8</strong>.
      </p>

      <p class="body-text">
        The core idea is simple: rather than looking at a single token in isolation, we look
        at the <em>neighbourhood</em> around it. Concretely, we slide a window of
        <strong>256 tokens</strong> along the generation and compute a bigram repetition
        score for each window position. A token then inherits the average score of all
        windows that contain it.
      </p>

      <!-- Step 1 -->
      <div style="margin-bottom:20px;">
        <div class="step-title">
          Step 1: Bigram repetition score per window
        </div>
        <p class="body-text">
          For every 256-token window starting at position <em>t</em> we compute:
        </p>
        <div class="formula-block">
          <span>\( \mathrm{rep}(t) = 1 - \mathrm{TTR}(\text{bigrams in tokens}[\,t \dots t+255\,]) \)</span>
        </div>
        <p class="body-text">
          TTR (Type-Token Ratio) = unique bigrams &divide; total bigrams.
          Intuitively: if you keep seeing the same two-word pair over and over,
          the TTR is low and therefore rep(t) is high. A window where every consecutive
          pair of tokens is different gives TTR&nbsp;=&nbsp;1 and rep(t)&nbsp;=&nbsp;0.
        </p>
      </div>

      <!-- Step 2 -->
      <div style="margin-bottom:24px;">
        <div class="step-title">
          Step 2: Average across all windows containing the token
        </div>
        <p class="body-text">
          Token at position <em>i</em> belongs to every window starting between
          <em>i</em>&minus;255 and <em>i</em> (up to 256 windows for tokens deep in a long
          generation). Its final score is the <strong>mean of all those window scores</strong>.
          This smoothing prevents a single high-repetition burst from unfairly tainting
          isolated tokens far from the loop.
          If score &ge; 0.8, the token is flagged as degenerating.
        </p>
      </div>

      <!-- SVG diagram -->
      <div style="background:#f8f9fb;border:1px solid #e2e6ea;border-radius:0;padding:20px 16px 16px;">
        <div style="font-size:11px;color:#8a8fa0;text-align:center;font-weight:600;letter-spacing:0.6px;margin-bottom:12px;">SLIDING WINDOW TOKEN SCORING</div>
        <svg viewBox="0 0 524 184" xmlns="http://www.w3.org/2000/svg" style="width:100%;max-width:580px;display:block;margin:0 auto;">
          <defs>
            <marker id="arrowBlue" markerWidth="6" markerHeight="6" refX="3" refY="3" orient="auto">
              <path d="M0,0 L6,3 L0,6 Z" fill="#2c4a7c"/>
            </marker>
          </defs>
          <!-- tok 0 -->
          <rect x="10"  y="12" width="46" height="26" rx="0" fill="#eef0f6" stroke="#ced3e8" stroke-width="1"/>
          <text x="33"  y="29" text-anchor="middle" font-size="9.5" fill="#a0a8c0" font-family="monospace">t&#8208;3</text>
          <!-- tok 1 -->
          <rect x="64"  y="12" width="46" height="26" rx="0" fill="#eef0f6" stroke="#ced3e8" stroke-width="1"/>
          <text x="87"  y="29" text-anchor="middle" font-size="9.5" fill="#a0a8c0" font-family="monospace">t&#8208;2</text>
          <!-- tok 2 -->
          <rect x="118" y="12" width="46" height="26" rx="0" fill="#eef0f6" stroke="#ced3e8" stroke-width="1"/>
          <text x="141" y="29" text-anchor="middle" font-size="9.5" fill="#a0a8c0" font-family="monospace">t&#8208;1</text>
          <!-- tok 3 HIGHLIGHTED -->
          <rect x="172" y="7"  width="46" height="36" rx="0" fill="#2c4a7c" stroke="#3a5ce0" stroke-width="1.5"/>
          <text x="195" y="29" text-anchor="middle" font-size="11" font-weight="700" fill="#fff" font-family="monospace">t&#7522;</text>
          <!-- tok 4 -->
          <rect x="226" y="12" width="46" height="26" rx="0" fill="#eef0f6" stroke="#ced3e8" stroke-width="1"/>
          <text x="249" y="29" text-anchor="middle" font-size="9.5" fill="#a0a8c0" font-family="monospace">t+1</text>
          <!-- tok 5 -->
          <rect x="280" y="12" width="46" height="26" rx="0" fill="#eef0f6" stroke="#ced3e8" stroke-width="1"/>
          <text x="303" y="29" text-anchor="middle" font-size="9.5" fill="#a0a8c0" font-family="monospace">t+2</text>
          <!-- tok 6 -->
          <rect x="334" y="12" width="46" height="26" rx="0" fill="#eef0f6" stroke="#ced3e8" stroke-width="1"/>
          <text x="357" y="29" text-anchor="middle" font-size="9.5" fill="#a0a8c0" font-family="monospace">t+3</text>
          <!-- tok 7 -->
          <rect x="388" y="12" width="46" height="26" rx="0" fill="#eef0f6" stroke="#ced3e8" stroke-width="1"/>
          <text x="411" y="29" text-anchor="middle" font-size="9.5" fill="#a0a8c0" font-family="monospace">t+4</text>
          <!-- tok 8 -->
          <rect x="442" y="12" width="46" height="26" rx="0" fill="#eef0f6" stroke="#ced3e8" stroke-width="1"/>
          <text x="465" y="29" text-anchor="middle" font-size="9.5" fill="#a0a8c0" font-family="monospace">t+5</text>
          <!-- Window bars -->
          <rect x="10"  y="50" width="262" height="16" rx="0" fill="rgba(74,108,247,0.14)" stroke="#2c4a7c" stroke-width="1"/>
          <text x="16"  y="62" font-size="9.5" fill="#2c4a7c" font-weight="600" font-family="Helvetica, Arial, sans-serif">w&#8321;  &#x2192;  rep&#8321; = 0.71</text>
          <rect x="64"  y="68" width="262" height="16" rx="0" fill="rgba(124,58,237,0.14)" stroke="#7c3aed" stroke-width="1"/>
          <text x="70"  y="80" font-size="9.5" fill="#7c3aed" font-weight="600" font-family="Helvetica, Arial, sans-serif">w&#8322;  &#x2192;  rep&#8322; = 0.45</text>
          <rect x="118" y="86" width="262" height="16" rx="0" fill="rgba(16,185,129,0.14)" stroke="#10b981" stroke-width="1"/>
          <text x="124" y="98" font-size="9.5" fill="#10b981" font-weight="600" font-family="Helvetica, Arial, sans-serif">w&#8323;  &#x2192;  rep&#8323; = 0.60</text>
          <rect x="172" y="104" width="262" height="16" rx="0" fill="rgba(8,145,178,0.14)" stroke="#0891b2" stroke-width="1"/>
          <text x="178" y="116" font-size="9.5" fill="#0891b2" font-weight="600" font-family="Helvetica, Arial, sans-serif">w&#8324;  &#x2192;  rep&#8324; = 0.68</text>
          <!-- Vertical dashed line -->
          <line x1="195" y1="43" x2="195" y2="128" stroke="#2c4a7c" stroke-width="1.5" stroke-dasharray="3 2" opacity="0.65"/>
          <line x1="195" y1="128" x2="195" y2="136" stroke="#2c4a7c" stroke-width="1.5" marker-end="url(#arrowBlue)"/>
          <!-- Formula box -->
          <rect x="10" y="138" width="504" height="40" rx="0" fill="#ffffff" stroke="#e2e6ea" stroke-width="1"/>
          <text x="262" y="153" text-anchor="middle" font-size="10" fill="#4a5568" font-family="Helvetica, Arial, sans-serif">score(t&#7522;) = mean(rep&#8321;, rep&#8322;, rep&#8323;, rep&#8324;, &#8230;) = (0.71 + 0.45 + 0.60 + 0.68) / 4</text>
          <text x="262" y="170" text-anchor="middle" font-size="11" font-weight="700" fill="#2c4a7c" font-family="Helvetica, Arial, sans-serif">= 0.61  &rarr;  below 0.8  &rarr;  not degenerating</text>
        </svg>
      </div>

      <p class="body-text" style="margin-top:16px;font-size:12px;color:#8a8fa0;">
        Tokens that fall outside any complete 256-token window (only possible in
        generations shorter than 256 tokens) receive <code style="background:#eef1f6;padding:1px 5px;border-radius:0;">repetition: null</code>
        and are excluded from training.
      </p>

    </section>

    <!-- ── Section 5: Training ───────────────────────────── -->
    <section class="section">
      <div class="section-heading">
        <div class="section-accent"></div>
        <h2>Training the probe</h2>
      </div>

      <p class="lead">
        A <strong>probe</strong> is a lightweight linear head attached to one internal layer
        of the language model. During a forward pass the prompt and response are fed through
        <span class="hl">Apertus-8B-Instruct</span><a href="#ref-1" class="cite">[1]</a>; at a chosen layer&nbsp;X the probe
        intercepts the hidden state <strong>\(\mathbf{h}_t\)</strong> for every token&nbsp;<em>t</em>
        and maps it to a scalar score in [0,&nbsp;1] via a linear projection followed by sigmoid.
        The backbone is kept frozen except for lightweight
        <span class="hl">LoRA</span><a href="#ref-4" class="cite">[4]</a> adapters.
        LoRA (Low-Rank Adaptation) works by injecting two small trainable matrices alongside
        each frozen attention weight matrix&nbsp;W: a down-projection
        \(A \in \mathbb{R}^{d \times r}\) and an up-projection
        \(B \in \mathbb{R}^{r \times d}\), so the effective weight update is
        \(\Delta W = BA\) with rank \(r \ll d\) (typically \(r = 8\text{–}64\)).
        Only A and B are trained; W stays unchanged. This reduces trainable parameters by
        several orders of magnitude while still letting the representations adapt to the probe task.
      </p>

      <p class="body-text">
        The training signal is <strong>Mean Squared Error</strong> between the probe&rsquo;s
        predicted score and the pre-computed repetition label \(y_t\) for each token:
      </p>

      <div class="formula-block">
        <span class="formula-label">Loss</span>
        <span>\( \mathcal{L} = \frac{1}{N} \sum_t (\hat{y}_t - y_t)^2 \)</span>
      </div>

      <p class="body-text">
        Tokens with <code style="background:#eef1f6;padding:1px 5px;border-radius:0;">repetition: null</code>
        are excluded. The regression formulation gives the probe a continuous gradient signal
        across the full score range rather than a hard binary boundary at 0.8.
        The pipeline is illustrated in the diagram below.
      </p>

      <!-- Inline probe diagram -->
      <div style="margin:28px 0 8px;border:1px solid #bcc3cd;border-radius:0;overflow:hidden;">
        <iframe src="/gradio_api/file=probe_setup_interactive.html" width="100%" height="570"
                style="border:none;display:block;" scrolling="no"></iframe>
      </div>
      <p style="font-size:12px;color:#888;text-align:center;margin-bottom:8px;font-family:Helvetica,Arial,sans-serif;">
        The probe attaches to a chosen layer, reads its 4,096-dimensional activations, and predicts a
        per-token repetition score ŷ[t]; the MSE loss against the deterministic target y[t] trains the probe
        weights W. Hover a layer to see how the activations and scores change with depth.
      </p>

      <!-- Multi-layer results -->
      <div style="margin-top:32px;">
        <p class="body-text">
          Each probe is trained independently on a single transformer layer. We swept across
          ten layers (10, 14, 18, 22, 24, 26, 28, 29, 30, 31) and found that layers from
          layer&nbsp;20 onwards all achieve similar detection performance, with only small
          differences in AUC across sources. Earlier layers are slightly weaker, and
          <strong>layer&nbsp;31</strong> (the very last) is the worst among the later ones.
        </p>
      </div>


    </section>

    <section class="section">
      <iframe id="about-sec6" src="/gradio_api/file=about_section6.html?v=SECTION6_V" scrolling="no"
              style="width:100%;border:none;display:block;height:1400px;"
              onload="(function(f){function fit(){try{var d=f.contentWindow.document;f.style.height=Math.max(d.body.scrollHeight,d.documentElement.scrollHeight)+'px';}catch(e){}}fit();try{var w=f.contentWindow,d=w.document;if(w.ResizeObserver)new w.ResizeObserver(fit).observe(d.body);if(d.fonts&&d.fonts.ready)d.fonts.ready.then(fit);}catch(e){}[300,800,1600,3000,5000].forEach(function(t){setTimeout(fit,t);});})(this)"></iframe>
    </section>
  </main>

  <!-- ── References ──────────────────────────────────────── -->
  <div class="references-section">
    <h2>References</h2>
    <ol class="ref-list">
      <li>
        <span class="ref-num">[1]</span>
        <span>Swiss AI Initiative. <em>Apertus: an open multilingual large language model from ETH Zurich and EPFL</em>. 2024.
          <a href="https://huggingface.co/swiss-ai/Apertus-8B-Instruct-2509" target="_blank" id="ref-1">
            huggingface.co/swiss-ai/Apertus-8B-Instruct-2509</a></span>
      </li>
      <li>
        <span class="ref-num">[2]</span>
        <span id="ref-2">Holtzman, A., Buys, J., Du, L., Forbes, M., &amp; Choi, Y.
          <em>The Curious Case of Neural Text Degeneration</em>. ICLR 2020.
          <a href="https://arxiv.org/abs/1904.09751" target="_blank">arxiv.org/abs/1904.09751</a></span>
      </li>
      <li>
        <span class="ref-num">[3]</span>
        <span id="ref-3">Pipis, C., Garg, S., Kontonis, V., Shrivastava, V., Krishnamurthy, A., &amp; Papailiopoulos, D.
          <em>Wait, Wait, Wait&hellip; Why Do Reasoning Models Loop?</em> arXiv, 2025.
          <a href="https://arxiv.org/abs/2512.12895v1" target="_blank">arxiv.org/abs/2512.12895v1</a></span>
      </li>
      <li>
        <span class="ref-num">[4]</span>
        <span id="ref-4">Hu, E. J., Shen, Y., Wallis, P., Allen-Zhu, Z., Li, Y., Wang, S., Wang, L., &amp; Chen, W.
          <em>LoRA: Low-Rank Adaptation of Large Language Models.</em> ICLR 2022.
          <a href="https://arxiv.org/abs/2106.09685" target="_blank">arxiv.org/abs/2106.09685</a></span>
      </li>
    </ol>
  </div>


</div>