Spaces:
Paused
Paused
unknown
Claude Opus 5.5
Sentences with citation fields (Zotero / Mendeley / EndNote), links or superscript citations inside are now rewritten around them: the language model sees markers ([REF1] ...), a version is used only if it keeps every marker once and in order, and its words go into the plain text around the untouched fields (citations stay live); choosing, the faithfulness check and the report read the real text. In a thesis a third of the highlighted sentences were locked before (FAST_AROUND_FIELDS=0 = old behaviour)
343161f | """Sentences with citation fields (Zotero / Mendeley / EndNote), links or superscript citations inside: rewritten | |
| around them; the fields stay exactly as they were (live, never retyped).""" | |
| import os | |
| import docx | |
| import pytest | |
| from docx.oxml import parse_xml | |
| os.environ.setdefault("SKIP_MODEL_LOAD", "1") | |
| import fast as F # noqa: E402 | |
| import smart as S # noqa: E402 | |
| W = 'xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"' | |
| def _run(text, extra=""): | |
| return f'<w:r {W}>{extra}<w:t xml:space="preserve">{text}</w:t></w:r>' | |
| def _zotero(result): | |
| return [f'<w:r {W}><w:fldChar w:fldCharType="begin"/></w:r>', | |
| f'<w:r {W}><w:instrText xml:space="preserve"> ADDIN ZOTERO_ITEM CSL_CITATION {{"id":1}} </w:instrText></w:r>', | |
| f'<w:r {W}><w:fldChar w:fldCharType="separate"/></w:r>', _run(result), | |
| f'<w:r {W}><w:fldChar w:fldCharType="end"/></w:r>'] | |
| def _para(doc, xml_runs): | |
| p = doc.add_paragraph() | |
| for x in xml_runs: | |
| p._p.append(parse_xml(x)) | |
| return p | |
| class _Block: | |
| """What fast._jobs gets for one paragraph (one sentence spanning it).""" | |
| kind = "para" | |
| def __init__(self, p): | |
| self.el = p._p | |
| import docx_report as DX | |
| self.raw = DX.para_text(p._p) | |
| self.sents, self.cspans, self.scores = [self.raw.strip()], [(0, len(self.raw.rstrip()))], [0.99] | |
| def _doc_with(xml_runs): | |
| d = docx.Document() | |
| return d, _para(d, xml_runs) | |
| def test_a_zotero_citation_in_the_middle_stays_live_and_the_words_around_it_change(): | |
| d, p = _doc_with([_run("As reported by "), *_zotero("(Smith et al., 2020)"), | |
| _run(" the proposed method improves fault detection accuracy considerably.")]) | |
| b = _Block(p) | |
| around = {} | |
| jobs, locked = F._jobs([b], 0.5, around=around) | |
| assert locked == 0 and len(jobs) == 1 | |
| text = jobs[0][2] | |
| assert text == "As reported by [REF1] the proposed method improves fault detection accuracy considerably." | |
| segs, refs = around[(id(b), 0)] | |
| assert refs == ["(Smith et al., 2020)"] | |
| assert F._replace_around(b, segs, "Smith and colleagues [REF1] show the method lifts fault detection accuracy a lot.") | |
| xml = p._p.xml | |
| assert "ADDIN ZOTERO_ITEM" in xml and xml.count('w:fldCharType="begin"') == 1 # the field is untouched | |
| import docx_report as DX | |
| assert DX.para_text(p._p) == ("Smith and colleagues (Smith et al., 2020) show the method lifts fault detection " | |
| "accuracy a lot.") | |
| def test_versions_that_move_drop_or_repeat_a_citation_are_not_used(): | |
| d, p = _doc_with([*_zotero("(Lee, 2019)"), _run(" found that training on augmented data improves recall by 7%.")]) | |
| b = _Block(p) | |
| around = {} | |
| jobs, _ = F._jobs([b], 0.5, around=around) | |
| segs = around[(id(b), 0)][0] | |
| assert F._fits(segs, "[REF1] reported that augmented training data raise recall by 7%.") is not None | |
| assert F._fits(segs, "Augmented training data raise recall by 7% [REF1].") is None # words before the field | |
| assert F._fits(segs, "Augmented training data raise recall by 7%.") is None # citation dropped | |
| assert F._fits(segs, "[REF1] and [REF1] reported a 7% gain.") is None # repeated | |
| def test_links_and_superscript_citations_are_kept_too(): | |
| link = (f'<w:hyperlink {W} xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" ' | |
| f'r:id="rId99">' + _run("WHO report") + '</w:hyperlink>') | |
| sup = _run("12", '<w:rPr><w:vertAlign w:val="superscript"/></w:rPr>') | |
| d, p = _doc_with([_run("The "), f'{link}', _run(" shows that incidence rose sharply"), sup, | |
| _run(" across all regions during the last decade.")]) | |
| b = _Block(p) | |
| around = {} | |
| jobs, locked = F._jobs([b], 0.5, around=around) | |
| assert locked == 0 | |
| segs, refs = around[(id(b), 0)] | |
| assert refs == ["WHO report", "12"] | |
| assert F._replace_around(b, segs, "The [REF1] indicates a steep rise in incidence[REF2] in every region over " | |
| "the past ten years.") | |
| xml = p._p.xml | |
| assert "w:hyperlink" in xml and 'w:vertAlign w:val="superscript"' in xml | |
| import docx_report as DX | |
| assert DX.para_text(p._p).startswith("The WHO report indicates a steep rise in incidence12 in every region") | |
| def test_sentences_with_equations_stay_locked(): | |
| d, p = _doc_with([_run("The loss is defined as "), | |
| '<m:oMath xmlns:m="http://schemas.openxmlformats.org/officeDocument/2006/math"><m:r>' | |
| '<m:t>L=x</m:t></m:r></m:oMath>', _run(" and it is minimised with gradient descent in training.")]) | |
| b = _Block(p) | |
| around = {} | |
| jobs, locked = F._jobs([b], 0.5, around=around) | |
| assert jobs == [] or all((id(b), 0) not in around for _ in jobs) | |