multilingual-profanity.js

"use strict";

const fs = require("fs");
const path = require("path");

const DATA_DIRECTORY = path.join(__dirname, "profanity-data");
const WORD_PATTERN = /[\p{L}\p{N}\p{M}]+/gu;
const CONTINUOUS_SCRIPT_PATTERN = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Thai}\p{Script=Khmer}\p{Script=Myanmar}]/u;
const REGEX_CHUNK_SIZE = 400;
const SOURCE_LANGUAGE_COUNT = 75;
const SOURCE_ENTRY_COUNT = 57191;

const LANGUAGE_ALIASES = Object.freeze({
  "zh-cn": "zh",
  "zh-hans": "zh",
  "zh-hant": "zh",
  "zh-hk": "zh",
  "zh-tw": "zh",
  km: "kh",
  nb: "no",
  nn: "no",
  tl: "fil",
  yi: "yid"
});

const dictionaries = new Map();
let totalEntries = 0;

function normalizeToken(value) {
  return String(value || "")
    .normalize("NFKD")
    .toLocaleLowerCase()
    .replace(/\p{M}+/gu, "");
}

function tokenize(value) {
  const text = String(value || "");
  const tokens = [];
  WORD_PATTERN.lastIndex = 0;

  let match;
  while ((match = WORD_PATTERN.exec(text)) !== null) {
    const normalized = normalizeToken(match[0]);
    if (!normalized) continue;
    tokens.push({
      value: normalized,
      start: match.index,
      end: match.index + match[0].length
    });
  }

  return tokens;
}

function escapeRegex(value) {
  return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}

function buildRegexChunks(terms) {
  const unique = [...new Set(terms)]
    .filter((term) => Array.from(term).length >= 2)
    .sort((left, right) => right.length - left.length);
  const chunks = [];

  for (let index = 0; index < unique.length; index += REGEX_CHUNK_SIZE) {
    const alternatives = unique
      .slice(index, index + REGEX_CHUNK_SIZE)
      .map(escapeRegex)
      .join("|");
    if (alternatives) chunks.push(new RegExp(alternatives, "giu"));
  }

  return chunks;
}

function buildWildcardRegex(entry) {
  const normalized = normalizeToken(entry);
  if (!normalized.includes("*")) return null;

  const source = normalized
    .split("*")
    .map(escapeRegex)
    .join("[\\p{L}\\p{N}]*");
  return source ? new RegExp(`^${source}$`, "u") : null;
}

function loadDictionary(filePath, language) {
  const lines = fs.readFileSync(filePath, "utf8")
    .split(/\r?\n/)
    .map((entry) => entry.trim())
    .filter(Boolean);
  const termsByTokenCount = new Map();
  const continuousTerms = [];
  const wildcardPatterns = [];

  for (const entry of lines) {
    if (entry.includes("*")) {
      const wildcard = buildWildcardRegex(entry);
      if (wildcard) wildcardPatterns.push(wildcard);
      continue;
    }

    const entryTokens = tokenize(entry).map((token) => token.value);
    if (entryTokens.length === 0) continue;

    const tokenCount = entryTokens.length;
    if (!termsByTokenCount.has(tokenCount)) {
      termsByTokenCount.set(tokenCount, new Set());
    }
    termsByTokenCount.get(tokenCount).add(entryTokens.join(" "));

    if (CONTINUOUS_SCRIPT_PATTERN.test(entry)) {
      continuousTerms.push(entry.normalize("NFC"));
    }
  }

  return {
    language,
    entries: lines.length,
    termsByTokenCount,
    tokenCounts: [...termsByTokenCount.keys()].sort((a, b) => a - b),
    wildcardPatterns,
    continuousPatterns: buildRegexChunks(continuousTerms)
  };
}

function loadDictionaries() {
  try {
    const files = fs.readdirSync(DATA_DIRECTORY, { withFileTypes: true })
      .filter((entry) => entry.isFile() && /^[a-z]{2,3}\.txt$/i.test(entry.name));

    for (const file of files) {
      const language = path.basename(file.name, ".txt").toLowerCase();
      // The source English list includes common words such as "class". The
      // server's curated English roots are safer for live game chat.
      if (language === "en") continue;
      const dictionary = loadDictionary(path.join(DATA_DIRECTORY, file.name), language);
      dictionaries.set(language, dictionary);
      totalEntries += dictionary.entries;
    }
  } catch (error) {
    console.error(`[moderation] Multilingual dictionaries could not be loaded: ${error.message}`);
  }
}

function cleanLanguage(value) {
  const raw = String(value || "")
    .trim()
    .toLowerCase()
    .replace(/_/g, "-");
  if (!raw) return "";

  if (LANGUAGE_ALIASES[raw]) return LANGUAGE_ALIASES[raw];
  const base = raw.split("-")[0];
  return LANGUAGE_ALIASES[base] || base;
}

function addIfSupported(target, language) {
  if (dictionaries.has(language) && !target.includes(language)) target.push(language);
}

function selectedLanguages(text, requestedLanguage) {
  const selected = [];
  addIfSupported(selected, cleanLanguage(requestedLanguage));

  if (/\p{Script=Hiragana}|\p{Script=Katakana}/u.test(text)) addIfSupported(selected, "ja");
  if (/\p{Script=Han}/u.test(text)) addIfSupported(selected, "zh");
  if (/\p{Script=Hangul}/u.test(text)) addIfSupported(selected, "ko");
  if (/\p{Script=Arabic}/u.test(text)) {
    addIfSupported(selected, "ar");
    addIfSupported(selected, "fa");
  }
  if (/\p{Script=Cyrillic}/u.test(text) && selected.length === 0) {
    addIfSupported(selected, "ru");
    addIfSupported(selected, "uk");
  }
  if (/\p{Script=Devanagari}/u.test(text)) {
    addIfSupported(selected, "hi");
    addIfSupported(selected, "mr");
  }
  if (/\p{Script=Tamil}/u.test(text)) addIfSupported(selected, "ta");
  if (/\p{Script=Telugu}/u.test(text)) addIfSupported(selected, "te");
  if (/\p{Script=Thai}/u.test(text)) addIfSupported(selected, "th");
  if (/\p{Script=Armenian}/u.test(text)) addIfSupported(selected, "hy");
  if (/\p{Script=Ethiopic}/u.test(text)) addIfSupported(selected, "am");
  if (/\p{Script=Myanmar}/u.test(text)) addIfSupported(selected, "my");
  if (/\p{Script=Khmer}/u.test(text)) addIfSupported(selected, "kh");

  if (selected.length === 0) addIfSupported(selected, "en");
  return selected;
}

function collectDictionaryRanges(text, tokens, dictionary, ranges) {
  for (const tokenCount of dictionary.tokenCounts) {
    if (tokenCount > tokens.length) break;
    const terms = dictionary.termsByTokenCount.get(tokenCount);

    for (let index = 0; index <= tokens.length - tokenCount; index++) {
      const phrase = tokens
        .slice(index, index + tokenCount)
        .map((token) => token.value)
        .join(" ");
      if (!terms.has(phrase)) continue;

      ranges.push({
        start: tokens[index].start,
        end: tokens[index + tokenCount - 1].end
      });
    }
  }

  if (dictionary.wildcardPatterns.length > 0) {
    for (const token of tokens) {
      if (dictionary.wildcardPatterns.some((pattern) => pattern.test(token.value))) {
        ranges.push({ start: token.start, end: token.end });
      }
    }
  }

  for (const pattern of dictionary.continuousPatterns) {
    pattern.lastIndex = 0;
    let match;
    while ((match = pattern.exec(text)) !== null) {
      ranges.push({ start: match.index, end: match.index + match[0].length });
      if (match[0].length === 0) pattern.lastIndex++;
    }
  }
}

function applyRanges(text, ranges) {
  if (ranges.length === 0) return text;

  const ordered = ranges
    .filter((range) => range.start >= 0 && range.end > range.start && range.end <= text.length)
    .sort((left, right) => left.start - right.start || right.end - left.end);
  const merged = [];

  for (const range of ordered) {
    const previous = merged[merged.length - 1];
    if (previous && range.start <= previous.end) {
      previous.end = Math.max(previous.end, range.end);
    } else {
      merged.push({ ...range });
    }
  }

  let output = "";
  let cursor = 0;
  for (const range of merged) {
    output += text.slice(cursor, range.start);
    output += "*".repeat(Array.from(text.slice(range.start, range.end)).length);
    cursor = range.end;
  }
  return output + text.slice(cursor);
}

function filterMultilingualProfanity(text, requestedLanguage) {
  const source = String(text || "");
  if (!source || dictionaries.size === 0) return source;

  const tokens = tokenize(source);
  const ranges = [];
  for (const language of selectedLanguages(source, requestedLanguage)) {
    collectDictionaryRanges(source, tokens, dictionaries.get(language), ranges);
  }

  return applyRanges(source, ranges);
}

function multilingualModerationStatus() {
  return {
    source: "Wiqaya 0.2.4",
    license: "MIT",
    languages: dictionaries.size + 1,
    dictionaryLanguages: dictionaries.size,
    dictionaryEntries: totalEntries,
    sourceLanguages: SOURCE_LANGUAGE_COUNT,
    sourceEntries: SOURCE_ENTRY_COUNT,
    curatedEnglishList: true,
    localFallbackLayers: 4
  };
}

loadDictionaries();

module.exports = {
  cleanModerationLanguage: cleanLanguage,
  filterMultilingualProfanity,
  multilingualModerationStatus
};