Spaces:
Running
Running
| /** | |
| * Converts HTML email to plain text, conservatively removing signatures | |
| * and quoted reply chains. | |
| * | |
| * Public API: | |
| * htmlEmailToText(html, requesterName?) | |
| * | |
| * Important: | |
| * Email signature extraction is heuristic. Preserve the original HTML | |
| * separately; this output should not be the authoritative ticket record. | |
| */ | |
| const cheerio = require('cheerio'); | |
| const BLOCK_TAGS = new Set([ | |
| 'address', 'article', 'aside', 'blockquote', 'caption', | |
| 'dd', 'div', 'dl', 'dt', 'fieldset', 'figcaption', 'figure', | |
| 'footer', 'form', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', | |
| 'header', 'li', 'main', 'nav', 'ol', 'p', 'pre', | |
| 'section', 'table', 'tr', 'ul', | |
| ]); | |
| // Do not include divtagdefaultwrapper: it can wrap the entire message. | |
| const SIGNATURE_MARKERS = [ | |
| 'signature', | |
| 'gmail_signature', | |
| 'email-signature', | |
| 'mail-signature', | |
| 'x_signature', | |
| 'moz-signature', | |
| ]; | |
| const QUOTE_MARKERS = [ | |
| 'gmail_quote', | |
| 'gmail_quote_container', | |
| 'freshdesk_quote', | |
| 'appendonsend', | |
| // divrplyfwdmsg deliberately excluded: often header-only. | |
| 'yahoo_quoted', | |
| 'protonmail_quote', | |
| 'zmail_extra', | |
| ]; | |
| const ATTRIBUTION_RE = | |
| /^(?:on\b.{3,500}\bwrote|le\b.{3,500}\ba\s*[ée]crit|el\b.{3,500}\bescribi[óo]|am\b.{1,200}\bschrieb\b.{1,300}|il\b.{3,500}\bha\s+scritto|em\b.{3,500}\bescreveu)\s*:\s*$/i; | |
| const ORIGINAL_MESSAGE_RE = | |
| /^[-–—]{2,}\s*(?:original message|forwarded message)\s*[-–—]{2,}\s*$/i; | |
| const FORWARDED_MESSAGE_RE = | |
| /^begin forwarded message\s*:\s*$/i; | |
| const UNDERSCORE_SEPARATOR_RE = /^_{5,}$/; | |
| const HEADER_LINE_RE = | |
| /^(from|sent|date|to|subject|cc|reply-to|de|envoyé|envoye|à|a|objet|para|fecha|asunto|von|gesendet|an|betreff)\s*:\s*(.*)$/i; | |
| const HEADER_NAMES = { | |
| from: 'from', | |
| de: 'from', | |
| von: 'from', | |
| sent: 'date', | |
| date: 'date', | |
| envoyé: 'date', | |
| envoye: 'date', | |
| fecha: 'date', | |
| gesendet: 'date', | |
| to: 'to', | |
| à: 'to', | |
| a: 'to', | |
| para: 'to', | |
| an: 'to', | |
| subject: 'subject', | |
| objet: 'subject', | |
| asunto: 'subject', | |
| betreff: 'subject', | |
| cc: 'cc', | |
| 'reply-to': 'reply-to', | |
| }; | |
| const SIGNOFF_RE = | |
| /^(?:thanks(?:\s+(?:(?:and|&)\s+)?regards)?|thank\s+you(?:\s+so\s+much)?|many\s+thanks|best(?:\s+(?:regards|wishes))?|kind\s+regards|warm\s+regards|regards|sincerely|cheers|yours(?:\s+(?:truly|sincerely))?|with\s+appreciation|cordially|respectfully)[,!.\s]*$/i; | |
| const SIGNATURE_SEPARATOR_RE = /^--\s*$/; | |
| const MOBILE_SIGNATURE_RE = | |
| /^(?:sent from (?:my|a) (?:iphone|ipad|ipod|android|samsung|galaxy|blackberry|mobile|phone)\b.*|sent from (?:outlook|mail) for (?:ios|android|windows)\b.*|get outlook for (?:ios|android)\b.*)$/i; | |
| const DISCLAIMER_RE = | |
| /^(?:confidentiality\s+(?:notice|disclaimer)|(?:legal\s+)?disclaimer\s*:\s*(?:this|the|please|if|confidential|privileged)|privileged\s+(?:and\s+)?confidential|this\s+(?:e-?mail|message|communication)(?:\s+and\s+(?:any\s+)?attachments?)?\s+(?:is|are|may\s+be)\s+(?:confidential|privileged|intended)|the\s+information\s+contained\s+in\s+this\s+(?:e-?mail|message|communication)\s+is\s+(?:confidential|privileged|intended)|if\s+you\s+(?:are\s+not\s+the\s+intended\s+recipient|have\s+received\s+this\s+(?:e-?mail|message)\s+in\s+error)|please\s+consider\s+the\s+environment\s+before\s+printing)/i; | |
| const LEGAL_CONTINUATION_RE = | |
| /(?:confidential|privileged|intended recipient|unauthori[sz]ed|disclos(?:e|ure)|distribution|dissemination|attachments?|received.{0,40}error|delete.{0,40}(?:message|email|e-mail)|notify.{0,40}sender|liability|virus|printing this|all rights reserved|copyright)/i; | |
| // Used to prevent a disclaimer/sign-off from eating an obvious new request. | |
| const REQUEST_LINE_RE = | |
| /^(?:p\.?\s*s\.?(?=\s|[:.]|$)\s*[:.]?|please\b|can you\b|could you\b|would you\b|i (?:need|want|have|am|cannot|can't)\b|we (?:need|want|have|are|cannot|can't)\b|also\b|however\b|but\b|the (?:issue|problem|error)\b|it (?:still|does|doesn't|is|isn't)\b)/i; | |
| const FORM_FIELD_RE = | |
| /^(?:first\s*name|last\s*name|full\s*name|name|e-?mail|organization|organisation|company|products?|timezone|time\s*zone|phone|telephone|mobile|subject|message|comments?|description|department|account(?:\s+(?:id|number))?|ticket(?:\s+(?:id|number))?)\s*[:\t]/i; | |
| const EMAIL_RE = /[\w.!#$%&'*+/=?^`{|}~-]+@[\w-]+(?:\.[\w-]+)+/i; | |
| const EMAIL_LINE_RE = | |
| /^(?:(?:e-?mail|e)\s*[:|]\s*)?[^\s<>@]+@[^\s<>@]+\.[^\s<>@]+[.,;]?$/i; | |
| const URL_LINE_RE = | |
| /^(?:(?:web(?:site)?|url|linkedin)\s*[:|]\s*)?(?:https?:\/\/|www\.)\S+$/i; | |
| const PHONE_LINE_RE = | |
| /^(?:(?:tel(?:ephone)?|phone|mobile|cell|fax|direct|office|work|t|m|p|f)\s*[:.]?\s*)?(?:\+?\d|\(\d)[\d\s()./-]*(?:\s*(?:(?:internal|int\.?)\s*[:.]?\s*(?:ex\.?|ext\.?)?|ext\.?|extension|ex\.?|x)\s*[:.]?\s*\d+)?$/i; | |
| const TITLE_LINE_RE = | |
| /^(?:(?:senior|junior|principal|staff|lead|chief|vice president|vp|head of|assistant|associate|regional|global|technical|customer success)\s+)?(?:software|systems?|network|support|sales|security|data|product|project|account|marketing|operations?|business|solutions?)?\s*(?:engineer|architect|developer|manager|director|specialist|consultant|coordinator|analyst|administrator|representative|officer|executive|president|founder|owner|ceo|cto|coo|cfo|designer)(?:\s+of\s+[A-Za-z][A-Za-z0-9&/ -]{1,40})?(?:\s*[,–-]\s*[A-Za-z0-9&/.-]{1,30})?$/i; | |
| const CREDENTIALS_LINE_RE = | |
| /^[A-Z][A-Z0-9/&.-]{1,10}(?:\s+[IVX]{1,4})?(?:,\s*[A-Z][A-Z0-9/&.-]{1,10}(?:\s+[IVX]{1,4})?)+$/; | |
| const SIGNATURE_BOOKING_LINE_RE = | |
| /^(?:(?:schedule|book)\s+(?:a\s+)?(?:meeting|time|call|appointment)(?:\s+with\s+me)?\s*:(?:\s*\S.*)?|calendar)$/i; | |
| const SIGNATURE_STATUS_LINE_RE = | |
| /^(?:out\s+of\s+(?:the\s+)?office|ooo|upcoming\s+(?:pto|out\s+of\s+office|leave)|travel(?:l?ing)?(?:\s+for\s+work)?|working\s+hours)\s*:\s*\S.*$/i; | |
| const COMPANY_LINE_RE = | |
| /(?:\b(?:inc|incorporated|llc|l\.?l\.?c|ltd|limited|corp|corporation|gmbh|plc|llp|p\.?c|s\.?a|b\.?v|pty\.?\s*ltd)\b\.?|\b(?:university|college|institute|hospital|foundation)\b)$/i; | |
| // Corporate suffix followed by a short comma clause (contractor / division / client relationship). | |
| // Example: "Infosys Limited, Contracted to Pfizer Business Technology" | |
| const COMPANY_WITH_CLAUSE_RE = | |
| /\b(?:inc\.?|incorporated|llc|l\.?l\.?c\.?|ltd\.?|limited|corp\.?|corporation|gmbh|plc|llp|p\.?c\.?|s\.?a\.?|b\.?v\.?|pty\.?\s*ltd\.?)\b,\s*[\p{L}\d&/().' -]{2,70}$/iu; | |
| // Title-cased org-unit / department line with a unit keyword, optional (ACRONYM). | |
| // Example: "Cloud Transformation, Infrastructure and Enterprise Solutions(C-TIES)" | |
| const ORG_UNIT_KEYWORD_RE = | |
| /\b(?:solutions?|services|technology|technologies|infrastructure|transformation|operations|engineering|department|division|group|team|practice|unit|cent(?:er|re)|office|business)\b/i; | |
| const ADDRESS_RE = | |
| /^(?:\d{1,6}\s+.{1,70}\b(?:street|st|avenue|ave|road|rd|parkway|pkwy|boulevard|blvd|drive|dr|lane|ln|way|court|ct|suite|ste|floor|fl)\b.*|p\.?\s*o\.?\s*box\s+\d+.*|[\p{L} .'-]+,\s*[\p{L} .'-]+\s+\d{5}(?:-\d{4})?(?:[\s,]+(?:USA|U\.S\.A\.?|US|U\.S\.?|United States(?:\s+of\s+America)?|Canada|UK|U\.K\.?))?)$/iu; | |
| const SIGNATURE_LOOKBACK = 15; | |
| function normalizeLine(value) { | |
| return value | |
| .replace(/\u00a0/g, ' ') | |
| .replace(/[\u200b\u200e\u200f\ufeff]/g, '') | |
| .replace(/[^\S\t\r\n]+/g, ' ') | |
| .trim(); | |
| } | |
| function normalizeName(value) { | |
| return normalizeLine(value || '') | |
| .normalize('NFKC') | |
| .toLowerCase() | |
| .replace(/\s+/g, ' '); | |
| } | |
| function attrHitsMarker(value, markers) { | |
| if (!value) return false; | |
| const lower = value.toLowerCase(); | |
| return markers.some(marker => { | |
| const escaped = marker.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); | |
| return new RegExp( | |
| `(?:^|[\\s_-])${escaped}(?:$|[\\s_-])` | |
| ).test(lower); | |
| }); | |
| } | |
| function hasMarker($el, markers) { | |
| return ( | |
| attrHitsMarker($el.attr('class'), markers) || | |
| attrHitsMarker($el.attr('id'), markers) | |
| ); | |
| } | |
| function isAttribution(text) { | |
| return ATTRIBUTION_RE.test(text); | |
| } | |
| const DATE_OR_TIME_EVIDENCE_RE = | |
| /(?:\b(?:mon(?:day)?|tue(?:sday)?|wed(?:nesday)?|thu(?:rsday)?|fri(?:day)?|sat(?:urday)?|sun(?:day)?)\b|\b(?:jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|jun(?:e)?|jul(?:y)?|aug(?:ust)?|sep(?:tember)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)\b|\b\d{1,2}[/-]\d{1,2}(?:[/-]\d{2,4})?\b|\b\d{4}[/-]\d{1,2}[/-]\d{1,2}\b|\b\d{1,2}:\d{2}(?:\s*[ap]\.?m\.?)?\b)/i; | |
| function isStrongAttribution(text) { | |
| if (!isAttribution(text)) return false; | |
| return ( | |
| EMAIL_RE.test(text) || | |
| DATE_OR_TIME_EVIDENCE_RE.test(text) | |
| ); | |
| } | |
| function isExplicitTextBoundary(text) { | |
| return ( | |
| ORIGINAL_MESSAGE_RE.test(text) || | |
| FORWARDED_MESSAGE_RE.test(text) || | |
| isStrongAttribution(text) | |
| ); | |
| } | |
| /** | |
| * Plain-text rendering used to inspect a possible signature card. | |
| * Includes the root node and handles text nodes, blocks, and <br>. | |
| */ | |
| function visibleText(node) { | |
| if (!node) return ''; | |
| if (node.type === 'text') return node.data || ''; | |
| const tag = (node.tagName || '').toLowerCase(); | |
| if (tag === 'script' || tag === 'style') return ''; | |
| if (tag === 'br') return '\n'; | |
| const text = (node.children || []).map(visibleText).join(''); | |
| if (tag === 'td' || tag === 'th') return `${text}\t`; | |
| return BLOCK_TAGS.has(tag) ? `\n${text}\n` : text; | |
| } | |
| function isPhoneLine(line) { | |
| if (!PHONE_LINE_RE.test(line)) return false; | |
| const digits = line.replace(/\D/g, ''); | |
| return digits.length >= 7 && digits.length <= 18; | |
| } | |
| function isContactLine(line) { | |
| return EMAIL_LINE_RE.test(line) || URL_LINE_RE.test(line) || isPhoneLine(line); | |
| } | |
| function looksLikeName(line, requesterName) { | |
| if (requesterName && normalizeName(line) === normalizeName(requesterName)) { | |
| return true; | |
| } | |
| if (line.length > 70 || REQUEST_LINE_RE.test(line)) return false; | |
| if (isTitleLine(line) || isCompanyLine(line)) return false; | |
| // Deliberately conservative without a known sender name. | |
| // Lowercase/non-Latin display names still work through requesterName. | |
| return /^(?:\p{Lu}[\p{L}'’.-]*)(?:\s+(?:(?:de|del|van|von|da|di|la)\s+)?\p{Lu}[\p{L}'’.-]*){1,3}$/u.test(line); | |
| } | |
| function isTitleLine(line) { | |
| return ( | |
| line.length <= 90 && | |
| !/[?!]/.test(line) && | |
| !/[.]\s*$/.test(line) && | |
| !REQUEST_LINE_RE.test(line) && | |
| !/\b(?:i(?!\.)|we|you|your|our|need|please|escalate|send|contact|call|update|fix|change)\b/i.test(line) && | |
| TITLE_LINE_RE.test(line) | |
| ); | |
| } | |
| function isCompanyLine(line) { | |
| return ( | |
| line.length <= 90 && | |
| !/[?!]/.test(line) && | |
| !REQUEST_LINE_RE.test(line) && | |
| !/\b(?:i(?!\.)|we|you|your|our|need|please|escalate|send|contact|call|update|fix|change)\b/i.test(line) && | |
| (COMPANY_LINE_RE.test(line) || COMPANY_WITH_CLAUSE_RE.test(line)) | |
| ); | |
| } | |
| function isOrgUnitLine(line) { | |
| if (line.length > 90 || /[?!]/.test(line) || /[.]\s*$/.test(line)) return false; | |
| if (REQUEST_LINE_RE.test(line)) return false; | |
| if (/\b(?:i(?!\.)|we|you|your|our|need|please|escalate|send|contact|call|update|fix|change|is|are|was|has|have|will)\b/i.test(line)) return false; | |
| if (!ORG_UNIT_KEYWORD_RE.test(line)) return false; | |
| // Strip an optional trailing "(C-TIES)" acronym, then require Title Case | |
| // words (small connectors like "and"/"of"/"for" allowed). | |
| const body = line.replace(/\s*\([A-Z][A-Z0-9&/-]{1,12}\)\s*$/, ''); | |
| const words = body.split(/[\s,/&-]+/).filter(Boolean); | |
| return words.length >= 2 && words.every(w => | |
| /^(?:and|of|for|the|to|in|on)$/i.test(w) || | |
| /^\p{Lu}[\p{L}\d'’.-]*$/u.test(w) | |
| ); | |
| } | |
| function isSignatureDetail(line, requesterName) { | |
| return ( | |
| looksLikeName(line, requesterName) || | |
| isTitleLine(line) || | |
| isCompanyLine(line) || | |
| isOrgUnitLine(line) || | |
| isContactLine(line) || | |
| ADDRESS_RE.test(line) || | |
| MOBILE_SIGNATURE_RE.test(line) || | |
| SIGNATURE_SEPARATOR_RE.test(line) || | |
| SIGNOFF_RE.test(line) || | |
| CREDENTIALS_LINE_RE.test(line) || | |
| SIGNATURE_BOOKING_LINE_RE.test(line) || | |
| SIGNATURE_STATUS_LINE_RE.test(line) | |
| ); | |
| } | |
| // Examples: | |
| // DPC, Donnelly 115 | |
| // MIT, Building 32 | |
| // | |
| // Deliberately requires an uppercase campus/site code, a comma, | |
| // a capitalized building name, and a room/building number. | |
| const CAMPUS_LOCATION_RE = | |
| /^[A-Z][A-Z0-9&.-]{1,11},\s*(?:\p{Lu}[\p{L}.'’-]*\s+){1,4}\d{1,5}[A-Za-z]?$/u; | |
| // Examples: | |
| // Room 115 | |
| // Office: B-204 | |
| // Suite 300 | |
| // Building 32 | |
| // | |
| // Does not accept arbitrary text after the room identifier. | |
| const LABELED_ROOM_RE = | |
| /^(?:room|rm\.?|office|suite|ste\.?|building|bldg\.?)\s*:?\s*(?:[A-Za-z]{1,3}[- ]?)?\d{1,5}[A-Za-z]?(?:[-/]\d{1,5}[A-Za-z]?)?$/i; | |
| function isSignatureLocationLine(line) { | |
| return ( | |
| line.length <= 80 && | |
| !REQUEST_LINE_RE.test(line) && | |
| ( | |
| CAMPUS_LOCATION_RE.test(line) || | |
| LABELED_ROOM_RE.test(line) | |
| ) | |
| ); | |
| } | |
| /** | |
| * Validate a complete signature tail. | |
| * | |
| * A building/room line is allowed only when: | |
| * - the tail contains a recognizable contact line; | |
| * - it also contains a title or company line; | |
| * - at most one line needs the location exception. | |
| * | |
| * Every other line must satisfy the existing signature-detail rules. | |
| * Unknown prose is never silently accepted. | |
| */ | |
| function isSignatureTail(lines, requesterName) { | |
| if (hasStructuredForm(lines)) return false; | |
| const hasContact = lines.some(isContactLine); | |
| const hasProfessionalContext = lines.some(line => | |
| isTitleLine(line) || isCompanyLine(line) || isOrgUnitLine(line) | |
| ); | |
| let locationExceptions = 0; | |
| for (const line of lines) { | |
| if (isSignatureDetail(line, requesterName)) continue; | |
| if ( | |
| hasContact && | |
| hasProfessionalContext && | |
| locationExceptions === 0 && | |
| isSignatureLocationLine(line) | |
| ) { | |
| locationExceptions++; | |
| continue; | |
| } | |
| return false; | |
| } | |
| return true; | |
| } | |
| /** | |
| * A logo + website alone is not enough: it could be a real message card. | |
| * Require a compact contact card with a person/title signal and a direct | |
| * contact link, and never classify a leading layout table as a signature. | |
| */ | |
| function isSignatureCard($, el, requesterName, hasPrecedingText) { | |
| if (!hasPrecedingText) return false; | |
| const tag = (el.tagName || '').toLowerCase(); | |
| if (!['table', 'div', 'section'].includes(tag)) return false; | |
| const $el = $(el); | |
| if (!$el.find('img').length) return false; | |
| const hasDirectContact = $el.find('a[href]').toArray().some(a => | |
| /^(?:mailto|tel):/i.test(($(a).attr('href') || '').trim()) | |
| ); | |
| const lines = visibleText(el) | |
| .split(/\r\n?|\n|\t/) | |
| .map(normalizeLine) | |
| .filter(Boolean); | |
| const hasVisibleContact = lines.some(isContactLine); | |
| if (!hasDirectContact && !hasVisibleContact) return false; | |
| if (lines.length < 2 || lines.length > 8) return false; | |
| if (lines.join(' ').length > 500) return false; | |
| return ( | |
| lines.some(line => | |
| looksLikeName(line, requesterName) || isTitleLine(line) | |
| ) && | |
| isSignatureTail(lines, requesterName) | |
| ); | |
| } | |
| /** | |
| * Render into structured lines rather than injecting magic text tokens. | |
| * | |
| * A removed quote remains represented as { kind: 'quote' }, allowing: | |
| * | |
| * On ... wrote: | |
| * <blockquote type="cite">old message</blockquote> | |
| * My response below the quote. | |
| * | |
| * to retain the response rather than truncating at the attribution. | |
| */ | |
| function renderEmailLines($, root, requesterName, options = {}) { | |
| const keepQuotes = options.keepQuotes === true; | |
| const keepSignatures = options.keepSignatures === true; | |
| const lines = []; | |
| let buffer = ''; | |
| let hasPrecedingText = false; | |
| function flush() { | |
| for (const part of buffer.split(/\r\n?|\n/)) { | |
| const text = normalizeLine(part); | |
| if (!text) continue; | |
| lines.push({ kind: 'text', text }); | |
| hasPrecedingText = true; | |
| } | |
| buffer = ''; | |
| } | |
| function marker(kind) { | |
| flush(); | |
| lines.push({ kind }); | |
| } | |
| function walk(node) { | |
| if (node.type === 'text') { | |
| buffer += node.data || ''; | |
| return; | |
| } | |
| const tag = (node.tagName || '').toLowerCase(); | |
| // Also traverse document/root nodes. | |
| if (!tag) { | |
| (node.children || []).forEach(walk); | |
| return; | |
| } | |
| if (['script', 'style', 'head', 'template'].includes(tag)) return; | |
| const $el = $(node); | |
| const style = $el.attr('style') || ''; | |
| if ( | |
| $el.attr('hidden') !== undefined || | |
| /(?:^|;)\s*display\s*:\s*none\b/i.test(style) || | |
| /(?:^|;)\s*visibility\s*:\s*hidden\b/i.test(style) | |
| ) { | |
| return; | |
| } | |
| if ($el.attr('data-skip-ai-process') !== undefined) { | |
| marker('skipped'); | |
| return; | |
| } | |
| // Document wrappers must not be classified as signatures/quotes. | |
| if (!['html', 'body'].includes(tag)) { | |
| if (!keepSignatures && hasMarker($el, SIGNATURE_MARKERS)) { | |
| marker('signature'); | |
| return; | |
| } | |
| let lastText = normalizeLine(buffer); | |
| if (!lastText) { | |
| for (let li = lines.length - 1; li >= 0; li--) { | |
| if (lines[li].kind === 'text') { | |
| lastText = lines[li].text; | |
| break; | |
| } | |
| } | |
| } | |
| const citePrecededByAttribution = | |
| tag === 'blockquote' && | |
| ($el.attr('type') || '').toLowerCase() === 'cite' && | |
| isStrongAttribution(lastText); | |
| const explicitQuote = | |
| !keepQuotes && | |
| ( | |
| hasMarker($el, QUOTE_MARKERS) || | |
| citePrecededByAttribution | |
| ); | |
| if (explicitQuote) { | |
| marker('quote'); | |
| return; | |
| } | |
| if ( | |
| !keepSignatures && | |
| isSignatureCard( | |
| $, node, requesterName, | |
| hasPrecedingText || Boolean(normalizeLine(buffer)) | |
| ) | |
| ) { | |
| marker('signature'); | |
| return; | |
| } | |
| } | |
| if (tag === 'hr') { | |
| marker('divider'); | |
| return; | |
| } | |
| if (tag === 'br') { | |
| flush(); | |
| return; | |
| } | |
| const block = BLOCK_TAGS.has(tag); | |
| if (block) flush(); | |
| (node.children || []).forEach(walk); | |
| if (tag === 'td' || tag === 'th') { | |
| buffer += '\t'; | |
| } else if (block) { | |
| flush(); | |
| } | |
| } | |
| walk(root); | |
| flush(); | |
| return lines; | |
| } | |
| /** | |
| * Recognize a contiguous reply-header cluster. | |
| * | |
| * Unlike collecting arbitrary headers from the next eight lines, this | |
| * stops at ordinary prose. "From:" in one paragraph and "Subject:" much | |
| * later cannot combine into a false reply boundary. | |
| */ | |
| function readReplyHeader(lines, start) { | |
| const fields = new Map(); | |
| let end = start; | |
| while (end < lines.length && end - start < 8) { | |
| const entry = lines[end]; | |
| if (entry.kind !== 'text') break; | |
| const match = entry.text.match(HEADER_LINE_RE); | |
| if (!match) break; | |
| const name = HEADER_NAMES[match[1].toLowerCase()]; | |
| fields.set(name, match[2].trim()); | |
| end++; | |
| } | |
| const sender = fields.get('from'); | |
| if (!sender) return null; | |
| const hasDate = fields.has('date'); | |
| const hasEmailSender = EMAIL_RE.test(sender); | |
| const confident = | |
| ( | |
| fields.size >= 2 && | |
| hasDate && | |
| (hasEmailSender || fields.size >= 3) | |
| ) || | |
| ( | |
| fields.size >= 3 && | |
| hasEmailSender && | |
| fields.has('subject') | |
| ); | |
| return confident ? { end } : null; | |
| } | |
| function nextRelevantIndex(lines, start) { | |
| let i = start; | |
| while ( | |
| i < lines.length && | |
| ['divider', 'skipped', 'signature'].includes(lines[i].kind) | |
| ) { | |
| i++; | |
| } | |
| return i; | |
| } | |
| /** | |
| * Remove explicit quote nodes and detect unmarked quoted tails. | |
| * | |
| * Explicit >-prefixed quotes can be interleaved with customer answers. | |
| * Do not truncate those answers merely because an attribution precedes | |
| * the first quoted line. | |
| */ | |
| function removeReplyNoise(entries) { | |
| const kept = []; | |
| for (let i = 0; i < entries.length; i++) { | |
| const entry = entries[i]; | |
| if (entry.kind !== 'text') continue; | |
| const text = entry.text; | |
| if (/^>/.test(text)) continue; | |
| const header = readReplyHeader(entries, i); | |
| const attribution = isAttribution(text); | |
| const boundary = isExplicitTextBoundary(text); | |
| const underscoreHeader = | |
| UNDERSCORE_SEPARATOR_RE.test(text) && | |
| readReplyHeader(entries, nextRelevantIndex(entries, i + 1)); | |
| if (!header && !attribution && !boundary && !underscoreHeader) { | |
| kept.push(text); | |
| continue; | |
| } | |
| const after = header ? header.end : i + 1; | |
| const next = nextRelevantIndex(entries, after); | |
| const nextEntry = entries[next]; | |
| const followedByExplicitQuote = | |
| nextEntry?.kind === 'quote' || | |
| ( | |
| nextEntry?.kind === 'text' && | |
| /^>/.test(nextEntry.text) | |
| ); | |
| if (followedByExplicitQuote) { | |
| // Remove only the attribution/header, not later answers. | |
| i = after - 1; | |
| continue; | |
| } | |
| if (header || boundary || underscoreHeader) { | |
| if (kept.length > 0) break; | |
| // A forward/quote at the beginning may be the ticket's entire | |
| // submitted content. Preserve its body rather than returning | |
| // an empty ticket; omit only the recognized header. | |
| i = after - 1; | |
| continue; | |
| } | |
| // Attribution-like prose without corroboration stays intact. | |
| kept.push(text); | |
| } | |
| return kept; | |
| } | |
| // Contact labels frequently occur in real signatures. They do not, by | |
| // themselves, prove that a block is a structured intake submission. | |
| const CONTACT_FIELD_LABEL_RE = | |
| /^(?:e-?mail|phone|telephone|mobile)\s*[:\t]/i; | |
| // These fields strongly suggest submitted ticket/form data. | |
| // Preserve them even when there is only one such label in the candidate. | |
| const INTAKE_ONLY_FIELD_RE = | |
| /^(?:first\s*name|last\s*name|products?|timezone|time\s*zone)\s*[:\t]/i; | |
| function hasStructuredForm(lines) { | |
| if (lines.some(line => INTAKE_ONLY_FIELD_RE.test(line))) { | |
| return true; | |
| } | |
| const fields = lines.filter(line => FORM_FIELD_RE.test(line)); | |
| // Email + Phone can be signature details. | |
| // Organization + Email, Name + Phone, etc. remain protected. | |
| return ( | |
| fields.length >= 2 && | |
| fields.some(line => !CONTACT_FIELD_LABEL_RE.test(line)) | |
| ); | |
| } | |
| // Hard-wrapped disclaimer text (a signature block or pasted legal footer | |
| // wrapped at a fixed column, using literal newlines instead of <br>) can | |
| // split a trigger phrase across two physical lines, e.g.: | |
| // "...if you have received this e-mail in" | |
| // "error please notify the sender..." | |
| // Joining a line with the one right after it lets the phrase regexes match | |
| // across that wrap point. REQUEST_LINE_RE is still tested against the | |
| // unjoined line only, so a genuine new paragraph still breaks the boundary. | |
| function withNextLine(lines, i) { | |
| return i + 1 < lines.length ? `${lines[i]} ${lines[i + 1]}` : lines[i]; | |
| } | |
| function findTrailingDisclaimerBoundary(lines, end) { | |
| for (let i = Math.max(1, end - 20); i < end; i++) { | |
| const anchor = lines[i]; | |
| const anchorJoined = withNextLine(lines, i); | |
| if (!DISCLAIMER_RE.test(anchor) && !DISCLAIMER_RE.test(anchorJoined)) continue; | |
| const rest = lines.slice(i + 1, end); | |
| // Conservative: an unrelated paragraph after the notice makes | |
| // its extent ambiguous, so do not delete the whole tail. | |
| if (rest.every((line, j) => { | |
| const joined = withNextLine(rest, j); | |
| return ( | |
| DISCLAIMER_RE.test(line) || | |
| DISCLAIMER_RE.test(joined) || | |
| ( | |
| !REQUEST_LINE_RE.test(line) && | |
| ( | |
| LEGAL_CONTINUATION_RE.test(line) || | |
| LEGAL_CONTINUATION_RE.test(joined) || | |
| isContactLine(line) || | |
| /^©|^copyright\b/i.test(line) | |
| ) | |
| ) | |
| ); | |
| })) { | |
| return i; | |
| } | |
| } | |
| return -1; | |
| } | |
| function findSignatureBoundary(lines, end, requesterName) { | |
| const floor = Math.max(1, end - SIGNATURE_LOOKBACK); | |
| for (let i = floor; i < end; i++) { | |
| const line = lines[i]; | |
| const isBoundary = | |
| SIGNOFF_RE.test(line) || | |
| SIGNATURE_SEPARATOR_RE.test(line) || | |
| MOBILE_SIGNATURE_RE.test(line); | |
| if (!isBoundary) continue; | |
| const tail = lines.slice(i + 1, end); | |
| if (isSignatureTail(tail, requesterName)) { | |
| return i; | |
| } | |
| } | |
| return -1; | |
| } | |
| function findUnmarkedSignatureBoundary(lines, end, requesterName) { | |
| const floor = Math.max(1, end - SIGNATURE_LOOKBACK); | |
| const window = lines.slice(floor, end); | |
| // Protect structured submitted data before considering any cut point. | |
| if (hasStructuredForm(window)) return -1; | |
| for (let i = floor; i < end - 1; i++) { | |
| const name = lines[i]; | |
| if (!looksLikeName(name, requesterName)) continue; | |
| const tail = lines.slice(i + 1, end); | |
| if (!tail.length) continue; | |
| if (!isSignatureTail(tail, requesterName)) continue; | |
| const knownName = Boolean( | |
| requesterName && | |
| normalizeName(name) === normalizeName(requesterName) | |
| ); | |
| const hasContact = tail.some(isContactLine); | |
| const hasTitle = tail.some(isTitleLine); | |
| const hasCompany = tail.some(isCompanyLine); | |
| if ( | |
| (knownName && (hasContact || hasTitle)) || | |
| (hasContact && (hasTitle || hasCompany)) | |
| ) { | |
| let cut = i; | |
| while (cut > 1 && SIGNATURE_STATUS_LINE_RE.test(lines[cut - 1])) { | |
| cut--; | |
| } | |
| return cut; | |
| } | |
| } | |
| return -1; | |
| } | |
| function stripTrailingEmailNoise(lines, requesterName) { | |
| if (!lines.length) return ''; | |
| let end = lines.length; | |
| const disclaimer = findTrailingDisclaimerBoundary(lines, end); | |
| if (disclaimer >= 0) end = disclaimer; | |
| let signature = findSignatureBoundary(lines, end, requesterName); | |
| if (signature < 0) { | |
| signature = findUnmarkedSignatureBoundary(lines, end, requesterName); | |
| } | |
| if (signature >= 0) end = signature; | |
| return lines.slice(0, end).join('\n'); | |
| } | |
| /** | |
| * @param {string} html | |
| * @param {string} [requesterName] | |
| * @returns {string} | |
| */ | |
| function htmlEmailToText(html, requesterName) { | |
| if (html == null || html === '') return ''; | |
| // Fail explicitly on unexpected API payloads instead of converting an | |
| // object to "[object Object]" or silently producing an empty message. | |
| if (typeof html !== 'string') { | |
| throw new TypeError('htmlEmailToText: html must be a string'); | |
| } | |
| const name = typeof requesterName === 'string' ? requesterName : undefined; | |
| const $ = cheerio.load(html); | |
| const root = $('body').get(0) || $.root().get(0); | |
| const entries = renderEmailLines($, root, name); | |
| const lines = removeReplyNoise(entries); | |
| const cleaned = stripTrailingEmailNoise(lines, name); | |
| if (cleaned) return cleaned; | |
| // Never let the heuristics turn a non-empty message into an empty one. | |
| // Re-render with quote/signature containers kept. Hidden, | |
| // display:none, and data-skip-ai-process content stays excluded, | |
| // because those checks run above the options gates. | |
| const fallbackEntries = renderEmailLines($, root, name, { | |
| keepQuotes: true, | |
| keepSignatures: true, | |
| }); | |
| const fallbackLines = removeReplyNoise(fallbackEntries); | |
| const fallbackText = stripTrailingEmailNoise(fallbackLines, name); | |
| if (fallbackText) return fallbackText; | |
| // Last resort: return everything textual rather than an empty ticket. | |
| return fallbackEntries | |
| .filter(entry => entry.kind === 'text') | |
| .map(entry => entry.text) | |
| .filter(Boolean) | |
| .join('\n'); | |
| } | |
| module.exports = { htmlEmailToText }; |