File size: 2,206 Bytes
9d2d895 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 | import type { NewsItem } from '../types';
import { decodeHtmlEntities } from '../utils/html-entities';
export function normalizeHeadlineKey(title: string): string {
return decodeHtmlEntities(title)
.toLowerCase()
.normalize('NFKD')
.replace(/[\u0300-\u036f]/g, '')
.replace(/[^a-z0-9]+/g, ' ')
.trim()
.split(' ')
.filter((w) => w.length > 2)
.slice(0, 8)
.join(' ');
}
function fallbackHeadlineKey(title: string): string {
return decodeHtmlEntities(title).toLowerCase().trim().replace(/\s+/g, ' ');
}
export interface DedupedHeadline {
item: NewsItem;
extraSources: string[];
}
export type TierLookup = (item: NewsItem) => number;
export function dedupeHeadlines(
items: NewsItem[],
tierOf?: TierLookup,
): DedupedHeadline[] {
const getTier: TierLookup = tierOf ?? ((it) => (typeof it.tier === 'number' ? it.tier : 4));
const groups = new Map<string, NewsItem[]>();
const order: string[] = [];
for (const it of items) {
let key = normalizeHeadlineKey(it.title);
if (!key) key = fallbackHeadlineKey(it.title);
if (!key) key = it.link || `__idx_${order.length}`;
const existing = groups.get(key);
if (existing) {
existing.push(it);
} else {
groups.set(key, [it]);
order.push(key);
}
}
const out: DedupedHeadline[] = [];
for (const key of order) {
const group = groups.get(key);
if (!group || group.length === 0) continue;
const primary = [...group].sort((a, b) => {
const ta = getTier(a);
const tb = getTier(b);
if (ta !== tb) return ta - tb;
const da = a.pubDate instanceof Date ? a.pubDate.getTime() : new Date(a.pubDate).getTime();
const db = b.pubDate instanceof Date ? b.pubDate.getTime() : new Date(b.pubDate).getTime();
return (Number.isFinite(db) ? db : 0) - (Number.isFinite(da) ? da : 0);
})[0]!;
const extraSources: string[] = [];
for (const other of group) {
if (other === primary) continue;
if (other.source && other.source !== primary.source && !extraSources.includes(other.source)) {
extraSources.push(other.source);
}
}
out.push({ item: primary, extraSources });
}
return out;
}
|