File size: 1,784 Bytes
c6f3061 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 | """Bidirectional bigram baseline using only visible immediate neighbours."""
import numpy as np
from .model import PAD, BOS, MASK
def bigram_accuracy(train, noisy, targets, selected, vocab_size):
counts = np.zeros((vocab_size, vocab_size), dtype=np.float32)
unigram = np.ones(vocab_size, dtype=np.float64)
for start in range(0, len(train), 2048):
rows = np.asarray(train[start:start+2048], dtype=np.int64)
left, right = rows[:, :-1].reshape(-1), rows[:, 1:].reshape(-1)
valid = (left != PAD) & (right != PAD)
np.add.at(counts, (left[valid], right[valid]), 1)
unigram += np.bincount(rows[rows != PAD], minlength=vocab_size)
unigram[[PAD, BOS, MASK]] = 1e-9
prior = unigram / unigram.sum()
row_total = counts.sum(-1) + 1
correct = total = 0
for source, truth, mask in zip(noisy, targets, selected):
for i in np.flatnonzero(mask):
# P(candidate | left) * P(right | candidate), with a unigram
# backoff if a neighbour is masked. No clean neighbour is consulted.
probabilities = prior.copy()
if i > 0 and source[i-1] not in (PAD, MASK):
probabilities = (counts[source[i-1]].astype(np.float64) + prior) / row_total[source[i-1]]
if i+1 < len(source) and source[i+1] not in (PAD, MASK):
probabilities *= (counts[:, source[i+1]].astype(np.float64) + prior[source[i+1]]) / row_total
probabilities[[PAD, BOS, MASK]] = 0
correct += int(probabilities.argmax() == truth[i])
total += 1
return {"policy": "bidirectional_bigram", "masked_accuracy": correct / total, "masked_tokens": total,
"description": "Add-one unigram-prior smoothing; immediate visible neighbours only"}
|