schemer / harness /string_trim.json
pveugen's picture
release: harness/string_trim.json
bbdaefc verified
Raw
History Blame Contribute Delete
1.21 kB
{
"doc": "String span boundary cleanup, two tiers. Tier 1 (always): never end on a function word, particle, or dangling punctuation. Tier 2 (only when the schema contains a datetime field): temporal expressions at the span tail are cross-field leakage; trim via the relative-date lexicon. Interior words never touched.",
"trailing_function_words": [
"a",
"aan",
"af",
"alla",
"am",
"an",
"and",
"at",
"att",
"au",
"aux",
"av",
"avec",
"bij",
"com",
"con",
"da",
"dal",
"das",
"de",
"der",
"des",
"di",
"die",
"dla",
"do",
"du",
"e",
"ed",
"een",
"el",
"en",
"et",
"for",
"fra",
"från",
"für",
"gli",
"het",
"hos",
"i",
"il",
"im",
"in",
"la",
"las",
"le",
"les",
"lo",
"los",
"med",
"met",
"mit",
"na",
"o",
"och",
"of",
"og",
"on",
"op",
"or",
"para",
"per",
"pour",
"przy",
"på",
"the",
"til",
"to",
"um",
"uma",
"und",
"van",
"von",
"voor",
"w",
"with",
"y",
"z",
"zu",
"à"
],
"trailing_particles": [
"の",
"に",
"で",
"と",
"へ",
"が",
"を",
"は",
"も"
],
"trailing_punctuation": " \t,.;:!?、。·-–"
}