| """Bidirectional bigram baseline using only visible immediate neighbours.""" |
| import numpy as np |
| from .model import PAD, BOS, MASK |
|
|
|
|
| def bigram_accuracy(train, noisy, targets, selected, vocab_size): |
| counts = np.zeros((vocab_size, vocab_size), dtype=np.float32) |
| unigram = np.ones(vocab_size, dtype=np.float64) |
| for start in range(0, len(train), 2048): |
| rows = np.asarray(train[start:start+2048], dtype=np.int64) |
| left, right = rows[:, :-1].reshape(-1), rows[:, 1:].reshape(-1) |
| valid = (left != PAD) & (right != PAD) |
| np.add.at(counts, (left[valid], right[valid]), 1) |
| unigram += np.bincount(rows[rows != PAD], minlength=vocab_size) |
| unigram[[PAD, BOS, MASK]] = 1e-9 |
| prior = unigram / unigram.sum() |
| row_total = counts.sum(-1) + 1 |
| correct = total = 0 |
| for source, truth, mask in zip(noisy, targets, selected): |
| for i in np.flatnonzero(mask): |
| |
| |
| probabilities = prior.copy() |
| if i > 0 and source[i-1] not in (PAD, MASK): |
| probabilities = (counts[source[i-1]].astype(np.float64) + prior) / row_total[source[i-1]] |
| if i+1 < len(source) and source[i+1] not in (PAD, MASK): |
| probabilities *= (counts[:, source[i+1]].astype(np.float64) + prior[source[i+1]]) / row_total |
| probabilities[[PAD, BOS, MASK]] = 0 |
| correct += int(probabilities.argmax() == truth[i]) |
| total += 1 |
| return {"policy": "bidirectional_bigram", "masked_accuracy": correct / total, "masked_tokens": total, |
| "description": "Add-one unigram-prior smoothing; immediate visible neighbours only"} |
|
|
