Spaces:
Sleeping
Sleeping
| """Normalisation of free-text values into the shared vocabularies. | |
| Used by the Excel importer and by the admin editor (to pre-check boxes from a | |
| stored value). The rule is conservative: a piece of text is converted to | |
| canonical vocabulary items only when every word in it is accounted for by the | |
| vocabulary; otherwise the piece is kept verbatim (whitespace collapsed) so | |
| nothing a faculty member wrote is lost or guessed. | |
| """ | |
| from __future__ import annotations | |
| import re | |
| from typing import Iterable, Optional | |
| from .fields import LIST_SEPARATOR | |
| _WS = re.compile(r"\s+") | |
| _HARD_SEPARATORS = re.compile(r"[\n\r\t;]|,|\s{2,}") | |
| # Characters that Excel stores from the Windows "Symbol" font as private-use | |
| # code points; mapped to the real characters they stand for. | |
| _SYMBOL_FONT = str.maketrans({ | |
| "\uf0a2": "\u2032", # prime | |
| "\uf0b2": "\u2033", # double prime | |
| "\uf0b7": "\u2022", # bullet | |
| "\uf0b0": "\u00b0", # degree | |
| "\uf0a0": " ", | |
| }) | |
| def clean_text(value, keep_newlines: bool = True) -> Optional[str]: | |
| """Trim, drop non-breaking spaces, collapse runs of blanks. None if empty.""" | |
| if value is None: | |
| return None | |
| s = str(value).translate(_SYMBOL_FONT).replace("\xa0", " ").replace("\r\n", "\n").replace("\r", "\n") | |
| if keep_newlines: | |
| lines = [re.sub(r"[ \t]+", " ", ln).strip() for ln in s.split("\n")] | |
| # Collapse 3+ blank lines to a single blank line. | |
| out, blank = [], 0 | |
| for ln in lines: | |
| if ln == "": | |
| blank += 1 | |
| if blank > 1: | |
| continue | |
| else: | |
| blank = 0 | |
| out.append(ln) | |
| s = "\n".join(out).strip() | |
| else: | |
| s = _WS.sub(" ", s).strip() | |
| return s or None | |
| def _build_lookup(vocab: dict[str, list[str]]) -> tuple[dict[tuple[str, ...], str], int]: | |
| lookup: dict[tuple[str, ...], str] = {} | |
| longest = 1 | |
| for canonical, variants in vocab.items(): | |
| for v in [canonical] + list(variants): | |
| words = tuple(_WS.sub(" ", v.lower()).strip().split(" ")) | |
| lookup[words] = canonical | |
| longest = max(longest, len(words)) | |
| return lookup, longest | |
| def _tokenize_piece(piece: str, lookup: dict, longest: int) -> Optional[list[str]]: | |
| """Greedy longest-match of vocabulary phrases. None if any word is unknown.""" | |
| words = piece.lower().strip(" .").split(" ") | |
| words = [w.strip(".,;:") for w in words if w.strip(".,;:")] | |
| if not words: | |
| return [] | |
| out: list[str] = [] | |
| i = 0 | |
| while i < len(words): | |
| matched = False | |
| for n in range(min(longest, len(words) - i), 0, -1): | |
| cand = tuple(words[i:i + n]) | |
| if cand in lookup: | |
| out.append(lookup[cand]) | |
| i += n | |
| matched = True | |
| break | |
| if not matched: | |
| return None | |
| return out | |
| def normalize_choices(value, vocab: dict[str, list[str]]) -> Optional[str]: | |
| """Turn a messy multi-value cell into 'Item, Item, free text' form. | |
| Pieces separated by newlines, tabs, commas, semicolons or runs of spaces are | |
| examined one by one. Pieces made only of vocabulary phrases become canonical | |
| items (duplicates removed, vocabulary order preserved). Any other piece is | |
| kept verbatim after the vocabulary items. | |
| """ | |
| if value is None: | |
| return None | |
| # Split on the raw text so tabs and line breaks still act as separators. | |
| text = str(value).replace("\xa0", " ").strip() | |
| if not text: | |
| return None | |
| lookup, longest = _build_lookup(vocab) | |
| canonical_order = list(vocab.keys()) | |
| found: list[str] = [] | |
| extras: list[str] = [] | |
| for raw in _HARD_SEPARATORS.split(text): | |
| piece = _WS.sub(" ", raw or "").strip() | |
| if not piece: | |
| continue | |
| tokens = _tokenize_piece(piece, lookup, longest) | |
| if tokens is None: | |
| extras.append(piece.rstrip(",")) | |
| else: | |
| for t in tokens: | |
| if t not in found: | |
| found.append(t) | |
| found.sort(key=canonical_order.index) | |
| parts = found + extras | |
| return LIST_SEPARATOR.join(parts) if parts else None | |
| def split_known(value: Optional[str], vocab: dict[str, list[str]]) -> tuple[list[str], str]: | |
| """Split a stored list value into (canonical items present, leftover text). | |
| Used by the admin editor to pre-check boxes and fill the free-text box. | |
| """ | |
| if not value: | |
| return [], "" | |
| lookup, longest = _build_lookup(vocab) | |
| known: list[str] = [] | |
| other: list[str] = [] | |
| for piece in value.split(","): | |
| piece = piece.strip() | |
| if not piece: | |
| continue | |
| tokens = _tokenize_piece(piece, lookup, longest) | |
| if tokens is None: | |
| other.append(piece) | |
| else: | |
| for t in tokens: | |
| if t not in known: | |
| known.append(t) | |
| return known, ", ".join(other) | |
| def join_choices(selected: Iterable[str], other: str, vocab: dict[str, list[str]]) -> Optional[str]: | |
| """Combine checkbox selections and free text into the stored list form.""" | |
| order = list(vocab.keys()) | |
| chosen = [s for s in order if s in set(selected)] | |
| extra = clean_text(other, keep_newlines=False) | |
| parts = chosen + ([extra] if extra else []) | |
| return LIST_SEPARATOR.join(parts) if parts else None | |