Spaces:
Sleeping
Sleeping
File size: 5,383 Bytes
d3295cf | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 | """Normalisation of free-text values into the shared vocabularies.
Used by the Excel importer and by the admin editor (to pre-check boxes from a
stored value). The rule is conservative: a piece of text is converted to
canonical vocabulary items only when every word in it is accounted for by the
vocabulary; otherwise the piece is kept verbatim (whitespace collapsed) so
nothing a faculty member wrote is lost or guessed.
"""
from __future__ import annotations
import re
from typing import Iterable, Optional
from .fields import LIST_SEPARATOR
_WS = re.compile(r"\s+")
_HARD_SEPARATORS = re.compile(r"[\n\r\t;]|,|\s{2,}")
# Characters that Excel stores from the Windows "Symbol" font as private-use
# code points; mapped to the real characters they stand for.
_SYMBOL_FONT = str.maketrans({
"\uf0a2": "\u2032", # prime
"\uf0b2": "\u2033", # double prime
"\uf0b7": "\u2022", # bullet
"\uf0b0": "\u00b0", # degree
"\uf0a0": " ",
})
def clean_text(value, keep_newlines: bool = True) -> Optional[str]:
"""Trim, drop non-breaking spaces, collapse runs of blanks. None if empty."""
if value is None:
return None
s = str(value).translate(_SYMBOL_FONT).replace("\xa0", " ").replace("\r\n", "\n").replace("\r", "\n")
if keep_newlines:
lines = [re.sub(r"[ \t]+", " ", ln).strip() for ln in s.split("\n")]
# Collapse 3+ blank lines to a single blank line.
out, blank = [], 0
for ln in lines:
if ln == "":
blank += 1
if blank > 1:
continue
else:
blank = 0
out.append(ln)
s = "\n".join(out).strip()
else:
s = _WS.sub(" ", s).strip()
return s or None
def _build_lookup(vocab: dict[str, list[str]]) -> tuple[dict[tuple[str, ...], str], int]:
lookup: dict[tuple[str, ...], str] = {}
longest = 1
for canonical, variants in vocab.items():
for v in [canonical] + list(variants):
words = tuple(_WS.sub(" ", v.lower()).strip().split(" "))
lookup[words] = canonical
longest = max(longest, len(words))
return lookup, longest
def _tokenize_piece(piece: str, lookup: dict, longest: int) -> Optional[list[str]]:
"""Greedy longest-match of vocabulary phrases. None if any word is unknown."""
words = piece.lower().strip(" .").split(" ")
words = [w.strip(".,;:") for w in words if w.strip(".,;:")]
if not words:
return []
out: list[str] = []
i = 0
while i < len(words):
matched = False
for n in range(min(longest, len(words) - i), 0, -1):
cand = tuple(words[i:i + n])
if cand in lookup:
out.append(lookup[cand])
i += n
matched = True
break
if not matched:
return None
return out
def normalize_choices(value, vocab: dict[str, list[str]]) -> Optional[str]:
"""Turn a messy multi-value cell into 'Item, Item, free text' form.
Pieces separated by newlines, tabs, commas, semicolons or runs of spaces are
examined one by one. Pieces made only of vocabulary phrases become canonical
items (duplicates removed, vocabulary order preserved). Any other piece is
kept verbatim after the vocabulary items.
"""
if value is None:
return None
# Split on the raw text so tabs and line breaks still act as separators.
text = str(value).replace("\xa0", " ").strip()
if not text:
return None
lookup, longest = _build_lookup(vocab)
canonical_order = list(vocab.keys())
found: list[str] = []
extras: list[str] = []
for raw in _HARD_SEPARATORS.split(text):
piece = _WS.sub(" ", raw or "").strip()
if not piece:
continue
tokens = _tokenize_piece(piece, lookup, longest)
if tokens is None:
extras.append(piece.rstrip(","))
else:
for t in tokens:
if t not in found:
found.append(t)
found.sort(key=canonical_order.index)
parts = found + extras
return LIST_SEPARATOR.join(parts) if parts else None
def split_known(value: Optional[str], vocab: dict[str, list[str]]) -> tuple[list[str], str]:
"""Split a stored list value into (canonical items present, leftover text).
Used by the admin editor to pre-check boxes and fill the free-text box.
"""
if not value:
return [], ""
lookup, longest = _build_lookup(vocab)
known: list[str] = []
other: list[str] = []
for piece in value.split(","):
piece = piece.strip()
if not piece:
continue
tokens = _tokenize_piece(piece, lookup, longest)
if tokens is None:
other.append(piece)
else:
for t in tokens:
if t not in known:
known.append(t)
return known, ", ".join(other)
def join_choices(selected: Iterable[str], other: str, vocab: dict[str, list[str]]) -> Optional[str]:
"""Combine checkbox selections and free text into the stored list form."""
order = list(vocab.keys())
chosen = [s for s in order if s in set(selected)]
extra = clean_text(other, keep_newlines=False)
parts = chosen + ([extra] if extra else [])
return LIST_SEPARATOR.join(parts) if parts else None
|