|
|
|
|
|
|
|
|
|
|
| '''
|
| cd d:\TextSimplificationDemo
|
|
|
| # Delete old venv (if needed)
|
| Remove-Item -Recurse -Force .\venv
|
|
|
| # Create venv with Python 3.11
|
| py -3.11 -m venv venv
|
|
|
| # Activate it
|
| .\venv\Scripts\Activate.ps1
|
|
|
| # Upgrade pip
|
| python -m pip install --upgrade pip
|
|
|
| # Install packages
|
| pip install -r requirements.txt
|
|
|
| # Install spacy-alignments safely
|
| pip install spacy-alignments --only-binary :all:
|
|
|
| # Download spaCy model
|
| python -m spacy download en_core_web_sm
|
|
|
| # Run the app
|
| python app.py
|
|
|
| # Interface at http://127.0.0.1:7860/
|
|
|
| EXAMPLE SENTENCE:
|
| John is one test, Mary, who lives in London, loves exams, and Bill eats bread.
|
|
|
| John went to Wolverhampton, Bill set out for Coventry, and Mary returned from Birmingham.
|
|
|
| The Environment Agency said record winter rainfall had helped to replenish reservoirs and groundwater, but reservoir storage has now fallen to 7.4% below the average for this time of year.
|
|
|
| The Environment Agency said record winter rainfall had helped to replenish reservoirs and groundwater, but reservoir storage has now fallen to 7.4 below the average for this time of year.
|
|
|
| Reservoir storage of 8.2 has been recorded and Mary went home.
|
|
|
| The demo loops infinitely when more than 1 sentence is included on the line.
|
| '''
|
|
|
| import os
|
| os.environ["CUDA_VISIBLE_DEVICES"] = ""
|
| os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
| import re
|
| import sys
|
| import spaces
|
| import torch
|
| from transformers import AutoModelForTokenClassification, AutoTokenizer
|
| import spacy
|
| import xml.etree.ElementTree as ET
|
| import spacy_alignments as tokenizations
|
| import xml.dom.minidom
|
| import pandas as pd
|
| import gradio as gr
|
|
|
| from collections import deque
|
|
|
| seen = set()
|
|
|
| import io
|
|
|
|
|
| if sys.platform == "win32":
|
| sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace')
|
|
|
|
|
| test_on_unrestricted_text = True
|
|
|
| import spacy
|
| from spacy.cli import download as spacy_download
|
|
|
| try:
|
| nlp = spacy.load("en_core_web_sm")
|
| except OSError:
|
| spacy_download("en_core_web_sm")
|
| nlp = spacy.load("en_core_web_sm")
|
|
|
|
|
| nlp = spacy.load("en_core_web_sm")
|
|
|
|
|
| device = "cuda" if torch.cuda.is_available() else "cpu"
|
|
|
| sign_tagging_model_name = "RJ3vans/SignTagger"
|
| SignTaggingTokenizer = AutoTokenizer.from_pretrained(sign_tagging_model_name)
|
| SignTaggingModel = AutoModelForTokenClassification.from_pretrained(
|
| sign_tagging_model_name
|
| ).to(device)
|
|
|
| ssccv_span_tagging_model_name = "RJ3vans/SSCCVspanTagger"
|
| SSCCVspanTaggingTokenizer = AutoTokenizer.from_pretrained(ssccv_span_tagging_model_name)
|
| SSCCVspanTaggingModel = AutoModelForTokenClassification.from_pretrained(
|
| ssccv_span_tagging_model_name
|
| ).to(device)
|
|
|
| ccv_span_tagging_model_name = "RJ3vans/CCVspanTagger"
|
| CCVspanTaggingTokenizer = AutoTokenizer.from_pretrained(ccv_span_tagging_model_name)
|
| CCVspanTaggingModel = AutoModelForTokenClassification.from_pretrained(
|
| ccv_span_tagging_model_name
|
| ).to(device)
|
|
|
|
|
| sign_label_list = [
|
| "M:N_CCV",
|
| "M:N_CIN",
|
| "M:N_CLA",
|
| "M:N_CLAdv",
|
| "M:N_CLN",
|
| "M:N_CLP",
|
| "M:N_CLQ",
|
| "M:N_CLV",
|
| "M:N_CMA1",
|
| "M:N_CMAdv",
|
| "M:N_CMN1",
|
| "M:N_CMN2",
|
| "M:N_CMN3",
|
| "M:N_CMN4",
|
| "M:N_CMP",
|
| "M:N_CMP2",
|
| "M:N_CMV1",
|
| "M:N_CMV2",
|
| "M:N_CMV3",
|
| "M:N_COMBINATORY",
|
| "M:N_CPA",
|
| "M:N_ESAdvP",
|
| "M:N_ESCCV",
|
| "M:N_ESCM",
|
| "M:N_ESMA",
|
| "M:N_ESMAdvP",
|
| "M:N_ESMI",
|
| "M:N_ESMN",
|
| "M:N_ESMP",
|
| "M:N_ESMV",
|
| "M:N_HELP",
|
| "M:N_SPECIAL",
|
| "M:N_SSCCV",
|
| "M:N_SSCM",
|
| "M:N_SSMA",
|
| "M:N_SSMAdvP",
|
| "M:N_SSMI",
|
| "M:N_SSMN",
|
| "M:N_SSMP",
|
| "M:N_SSMV",
|
| "M:N_STQ",
|
| "M:N_V",
|
| "M:N_nan",
|
| "M:Y_CCV",
|
| "M:Y_CIN",
|
| "M:Y_CLA",
|
| "M:Y_CLAdv",
|
| "M:Y_CLN",
|
| "M:Y_CLP",
|
| "M:Y_CLQ",
|
| "M:Y_CLV",
|
| "M:Y_CMA1",
|
| "M:Y_CMAdv",
|
| "M:Y_CMN1",
|
| "M:Y_CMN2",
|
| "M:Y_CMN4",
|
| "M:Y_CMP",
|
| "M:Y_CMP2",
|
| "M:Y_CMV1",
|
| "M:Y_CMV2",
|
| "M:Y_CMV3",
|
| "M:Y_COMBINATORY",
|
| "M:Y_CPA",
|
| "M:Y_ESAdvP",
|
| "M:Y_ESCCV",
|
| "M:Y_ESCM",
|
| "M:Y_ESMA",
|
| "M:Y_ESMAdvP",
|
| "M:Y_ESMI",
|
| "M:Y_ESMN",
|
| "M:Y_ESMP",
|
| "M:Y_ESMV",
|
| "M:Y_HELP",
|
| "M:Y_SPECIAL",
|
| "M:Y_SSCCV",
|
| "M:Y_SSCM",
|
| "M:Y_SSMA",
|
| "M:Y_SSMAdvP",
|
| "M:Y_SSMI",
|
| "M:Y_SSMN",
|
| "M:Y_SSMP",
|
| "M:Y_SSMV",
|
| "M:Y_STQ",
|
| ]
|
|
|
| SSCCVspan_label_list = [
|
| "AFTER_ADJECTIVAL",
|
| "AFTER_ADVERBIAL",
|
| "AFTER_CLEFT_CLAUSE",
|
| "AFTER_COGNITIVE_COMMUNICATIVE_VP",
|
| "AFTER_COMPLEX_NP",
|
| "AFTER_COMPLEX_PHRASE",
|
| "AFTER_FREE_RELATIVE_CLAUSE",
|
| "AFTER_INTENSIFYING_CLAUSE",
|
| "AFTER_REPORTING_CLAUSE",
|
| "AFTER_RESTRICTIVE_CLAUSE_GENERIC_HEAD",
|
| "AFTER_WH_PHRASE",
|
| "BEFORE_ADJECTIVAL",
|
| "BEFORE_ADVERBIAL",
|
| "BEFORE_CLEFT_CLAUSE",
|
| "BEFORE_COGNITIVE_COMMUNICATIVE_VP",
|
| "BEFORE_COMPLEX_NP",
|
| "BEFORE_COMPLEX_PHRASE",
|
| "BEFORE_FREE_RELATIVE_CLAUSE",
|
| "BEFORE_INTENSIFYING_CLAUSE",
|
| "BEFORE_REPORTING_CLAUSE",
|
| "BEFORE_RESTRICTIVE_CLAUSE_GENERIC_HEAD",
|
| "BEFORE_WH_PHRASE",
|
| "ERROR",
|
| "IN_ADJECTIVAL",
|
| "IN_ADJECTIVAL_AFTERSIGN",
|
| "IN_ADJECTIVAL_BEFORESIGN",
|
| "IN_ADVERBIAL",
|
| "IN_ADVERBIAL_AFTERSIGN",
|
| "IN_ADVERBIAL_BEFORESIGN",
|
| "IN_CLEFT_CLAUSE",
|
| "IN_CLEFT_CLAUSE_AFTERSIGN",
|
| "IN_CLEFT_CLAUSE_BEFORESIGN",
|
| "IN_COGNITIVE_COMMUNICATIVE_VP",
|
| "IN_COGNITIVE_COMMUNICATIVE_VP_AFTERSIGN",
|
| "IN_COGNITIVE_COMMUNICATIVE_VP_BEFORESIGN",
|
| "IN_COMPLEX_NP",
|
| "IN_COMPLEX_NP_AFTERSIGN",
|
| "IN_COMPLEX_NP_BEFORESIGN",
|
| "IN_COMPLEX_PHRASE",
|
| "IN_COMPLEX_PHRASE_AFTERSIGN",
|
| "IN_COMPLEX_PHRASE_BEFORESIGN",
|
| "IN_FREE_RELATIVE_CLAUSE",
|
| "IN_FREE_RELATIVE_CLAUSE_AFTERSIGN",
|
| "IN_FREE_RELATIVE_CLAUSE_BEFORESIGN",
|
| "IN_INTENSIFYING_CLAUSE",
|
| "IN_INTENSIFYING_CLAUSE_AFTERSIGN",
|
| "IN_INTENSIFYING_CLAUSE_BEFORESIGN",
|
| "IN_REPORTING_CLAUSE_AFTERSIGN",
|
| "IN_REPORTING_CLAUSE_BEFORESIGN",
|
| "IN_RESTRICTIVE_CLAUSE_GENERIC_HEAD",
|
| "IN_RESTRICTIVE_CLAUSE_GENERIC_HEAD_AFTERSIGN",
|
| "IN_RESTRICTIVE_CLAUSE_GENERIC_HEAD_BEFORESIGN",
|
| "IN_WH_PHRASE",
|
| "IN_WH_PHRASE_AFTERSIGN",
|
| "IN_WH_PHRASE_BEFORESIGN",
|
| "UNKNOWN",
|
| ]
|
|
|
| CCVspan_label_list = [
|
| "AFTER_COMPOUND",
|
| "BEFORE_COMPOUND",
|
| "IN_COMPOUND",
|
| "IN_COMPOUND_AFTERSIGN",
|
| "IN_COMPOUND_BEFORESIGN",
|
| "NOT_CLAUSE_COORDINATOR",
|
| "UNKNOWN",
|
| ]
|
|
|
| required_sign_tags = [
|
| "M:Y_SSCCV",
|
| "M:Y_CCV",
|
| "M:Y_CMV1",
|
| "M:Y_CMP",
|
| "M:Y_CMN1",
|
| "M:Y_CLN",
|
| "M:Y_CLV",
|
| "M:Y_CLP",
|
| ]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| def split_into_sentences(text):
|
| """
|
| Split plain text into sentences using spaCy.
|
| Falls back to a conservative regex if spaCy returns a single
|
| span that still contains '. ' + capital letter.
|
| """
|
| text = (text or "").strip()
|
| if not text:
|
| return []
|
|
|
| blocks = [b.strip() for b in text.split("\n") if b.strip()]
|
| sents = []
|
|
|
| for block in blocks:
|
| doc = nlp(block)
|
| piece = [s.text.strip() for s in doc.sents if s.text.strip()]
|
|
|
|
|
| expanded = []
|
| for p in piece:
|
| parts = re.split(r'(?<=[.!?])\s+(?=[A-Z])', p)
|
| expanded.extend(x.strip() for x in parts if x.strip())
|
| sents.extend(expanded)
|
|
|
|
|
| seen = set()
|
| unique = []
|
| for s in sents:
|
| key = re.sub(r"\s+", " ", s.lower())
|
| if key not in seen:
|
| seen.add(key)
|
| unique.append(s)
|
| return unique
|
|
|
|
|
|
|
| def strip_xml_tags(text):
|
| """Remove all <M:Y_...> and </M:Y_...> tags."""
|
| text = re.sub(r'<M:Y_[^>]+>', '', text)
|
| text = re.sub(r'</M:Y_[^>]+>', '', text)
|
| text = re.sub(r'\s+', ' ', text).strip()
|
| return text
|
|
|
|
|
|
|
| def clean_for_spaCy(sent):
|
| print("CLEANING sent BEFORE spaCy processes it", sent)
|
|
|
|
|
|
|
| m = re.search(
|
| r"^(.*?)"
|
| r"\["
|
| r"("
|
| r"[,;:(]|"
|
| r"and|but|or|that|what|when|where|which|while|who"
|
| r")"
|
| r"("
|
| r"_(?:and|but|or|that|what|when|where|which|while|who)|"
|
| r"\s+(?:and|but|or|that|what|when|where|which|while|who)"
|
| r")?"
|
| r"\]"
|
| r"(.*)$",
|
| sent,
|
| flags=re.IGNORECASE | re.DOTALL,
|
| )
|
|
|
| if m:
|
| prefix = m.group(1)
|
| head = m.group(2)
|
| extra = m.group(3) or ""
|
| suffix = m.group(4)
|
|
|
| if extra.startswith("_"):
|
|
|
| inside = head + extra
|
| elif extra:
|
|
|
| inside = f"{head} {extra.strip()}"
|
| else:
|
| inside = head
|
|
|
| clean_sent = f"{prefix}[{inside}]{suffix}"
|
| else:
|
| clean_sent = sent
|
|
|
| clean_sent = re.sub(r"\s+", " ", clean_sent).strip()
|
| print("clean_sent is\n" + clean_sent)
|
| return clean_sent
|
|
|
|
|
|
|
|
|
|
|
|
|
| def sign_cleaning(
|
| aligned_tokens, aligned_pos_tags, aligned_lemmas, aligned_predictions
|
| ):
|
|
|
| clean_tokens = []
|
| clean_pos_tags = []
|
| clean_lemmas = []
|
| clean_predictions = []
|
|
|
| x = 0
|
| while x < len(aligned_tokens):
|
|
|
|
|
| if aligned_tokens[x] == "[" and aligned_tokens[x + 1] == ":]":
|
| clean_predictions.append(aligned_predictions[x])
|
|
|
| clean_tokens.append(":")
|
| clean_pos_tags.append(":")
|
| clean_lemmas.append(":")
|
|
|
| x += 1
|
|
|
| elif aligned_tokens[x] == "[" and aligned_tokens[x + 2] == "]":
|
| clean_predictions.append(aligned_predictions[x])
|
|
|
| clean_tokens.append(aligned_tokens[x + 1])
|
| clean_pos_tags.append(aligned_pos_tags[x + 1])
|
| clean_lemmas.append(aligned_lemmas[x + 1])
|
|
|
| x += 1
|
|
|
|
|
| elif aligned_tokens[x] == "[" and aligned_tokens[x + 4] == "]":
|
| clean_predictions.append(aligned_predictions[x])
|
| clean_predictions.append(aligned_predictions[x])
|
| clean_predictions.append(aligned_predictions[x])
|
|
|
| clean_pos_tags.append(aligned_pos_tags[x + 1])
|
| clean_pos_tags.append(aligned_pos_tags[x + 2])
|
| clean_pos_tags.append(aligned_pos_tags[x + 3])
|
|
|
| clean_lemmas.append(aligned_lemmas[x + 1])
|
| clean_lemmas.append(aligned_lemmas[x + 2])
|
| clean_lemmas.append(aligned_lemmas[x + 3])
|
|
|
| clean_tokens.append(aligned_tokens[x + 1])
|
| clean_tokens.append(aligned_tokens[x + 2])
|
| clean_tokens.append(aligned_tokens[x + 3])
|
|
|
| x += 3
|
|
|
| elif aligned_tokens[x] == "]" and aligned_tokens[x - 2] == "[":
|
| True
|
|
|
| elif aligned_tokens[x] == "]" and aligned_tokens[x - 4] == "[":
|
| True
|
|
|
| else:
|
| clean_tokens.append(aligned_tokens[x])
|
| clean_pos_tags.append(aligned_pos_tags[x])
|
| clean_lemmas.append(aligned_lemmas[x])
|
| clean_predictions.append(aligned_predictions[x])
|
|
|
| x += 1
|
|
|
| new_clean_predictions = improve_predicted_tag_sequence(clean_predictions)
|
| clean_predictions = new_clean_predictions
|
|
|
| return (clean_tokens, clean_pos_tags, clean_lemmas, clean_predictions)
|
|
|
|
|
|
|
|
|
|
|
|
|
| def improve_predicted_tag_sequence(predictions):
|
| new_predictions = []
|
|
|
|
|
|
|
|
|
| core_tags = []
|
| x = 0
|
| sign_position = False
|
| while x < len(predictions):
|
| px = predictions[x]
|
| core_tag = re.sub(
|
| "(IN_|BEFORE_|AFTER_|_AFTERSIGN|_BEFORESIGN)",
|
| "",
|
| px,
|
| )
|
| core_tags.append(core_tag)
|
| x += 1
|
|
|
| most_frequent_core_tag = max(set(core_tags), key=core_tags.count)
|
|
|
| sign_tag = re.compile(r"IN_" + most_frequent_core_tag + "$")
|
|
|
| x = 0
|
| while x < len(predictions):
|
| px = predictions[x]
|
|
|
| if re.match(
|
| sign_tag,
|
| px,
|
| ):
|
| sign_position = x
|
| break
|
| x += 1
|
|
|
| improved_predicted_tag_sequence = []
|
| x = 0
|
| while x < len(predictions):
|
| px = predictions[x]
|
|
|
| if x < sign_position:
|
| if px == "UNKNOWN" or px == "NOT_CLAUSE_COORDINATOR":
|
| px = "IN_" + core_tag + "_BEFORESIGN"
|
|
|
| improved_predicted_tag_sequence.append(px)
|
| x += 1
|
|
|
| predictions = improved_predicted_tag_sequence
|
|
|
|
|
|
|
| x = 0
|
| previous_prediction = ""
|
| while x < len(predictions):
|
| if x > 0:
|
| if (
|
| predictions[x] == "UNKNOWN"
|
| or predictions[x] == "NOT_CLAUSE_COORDINATOR"
|
| ):
|
|
|
| try:
|
| new_predictions.append(new_predictions[-1])
|
| except:
|
| new_predictions.append(predictions[x])
|
| else:
|
| new_predictions.append(predictions[x])
|
| else:
|
| new_predictions.append(predictions[x])
|
|
|
| x += 1
|
|
|
|
|
| return new_predictions
|
|
|
|
|
|
|
|
|
|
|
|
|
| def tag_SSCCV_span(sentence):
|
| if re.search("\[\:\]\s*$", sentence):
|
| sentence = sentence + " blah blah blah."
|
|
|
| tokens = SSCCVspanTaggingTokenizer.tokenize(
|
| SSCCVspanTaggingTokenizer.decode(SSCCVspanTaggingTokenizer.encode(sentence))
|
| )
|
| inputs = SSCCVspanTaggingTokenizer.encode(sentence, return_tensors="pt")
|
| inputs = inputs.to(device)
|
|
|
| outputs = SSCCVspanTaggingModel(inputs)[0]
|
| predictions = torch.argmax(outputs, dim=2)
|
|
|
|
|
|
|
| SSCCV_span_tagged_tokens = [
|
| (token, SSCCVspan_label_list[prediction])
|
| for token, prediction in zip(tokens, predictions[0].tolist())
|
| ]
|
|
|
| SSCCV_span_tagged_predictions = [p for t, p in SSCCV_span_tagged_tokens]
|
|
|
| aligned_tokens = []
|
| aligned_predictions = []
|
| aligned_pos_tags = []
|
| aligned_lemmas = []
|
|
|
| sc_sentence = (
|
| clean_for_spaCy(sentence) if "[" in sentence and "]" in sentence else sentence
|
| )
|
| doc = nlp(sc_sentence)
|
| spacy_pos_tags = [t.tag_ for t in doc]
|
| spacy_tokens = [t.text for t in doc]
|
| spacy_lemmas = [t.lemma_ for t in doc]
|
|
|
| print('Checking proper nouns after building doc')
|
| for t in doc:
|
| if 'NNP' in t.tag_:
|
| print(t.tag_, t.text)
|
|
|
| a2b, b2a = tokenizations.get_alignments(spacy_tokens, tokens)
|
|
|
| for i in range(len(spacy_tokens)):
|
| for j in a2b[i]:
|
| aligned_tokens.append(spacy_tokens[i])
|
| aligned_pos_tags.append(spacy_pos_tags[i])
|
| aligned_lemmas.append(spacy_lemmas[i])
|
| aligned_predictions.append(SSCCV_span_tagged_predictions[j])
|
| break
|
|
|
| clean_tokens, clean_pos_tags, clean_lemmas, clean_predictions = sign_cleaning(
|
| aligned_tokens, aligned_pos_tags, aligned_lemmas, aligned_predictions
|
| )
|
|
|
| simplifiable = all("COMPLEX_NP" in p for p in clean_predictions)
|
|
|
| if not simplifiable:
|
| True
|
|
|
| return (
|
| clean_tokens,
|
| clean_pos_tags,
|
| clean_lemmas,
|
| clean_predictions,
|
| )
|
|
|
|
|
|
|
|
|
|
|
|
|
| def tag_CCV_span(sentence):
|
| if re.search("\[\:\]\s*$", sentence):
|
| sentence = sentence + " blah blah blah."
|
|
|
| tokens = CCVspanTaggingTokenizer.tokenize(
|
| CCVspanTaggingTokenizer.decode(CCVspanTaggingTokenizer.encode(sentence))
|
| )
|
| inputs = CCVspanTaggingTokenizer.encode(sentence, return_tensors="pt")
|
| inputs = inputs.to(device)
|
|
|
| outputs = CCVspanTaggingModel(inputs)[0]
|
| predictions = torch.argmax(outputs, dim=2)
|
|
|
|
|
| CCV_span_tagged_tokens = [
|
| (token, CCVspan_label_list[prediction])
|
| for token, prediction in zip(tokens, predictions[0].tolist())
|
| ]
|
|
|
| CCV_span_tagged_predictions = [p for t, p in CCV_span_tagged_tokens]
|
|
|
| aligned_tokens = []
|
| aligned_predictions = []
|
| aligned_pos_tags = []
|
| aligned_lemmas = []
|
|
|
| sc_sentence = (
|
| clean_for_spaCy(sentence) if "[" in sentence and "]" in sentence else sentence
|
| )
|
| doc = nlp(sc_sentence)
|
| spacy_pos_tags = [t.tag_ for t in doc]
|
| spacy_tokens = [t.text for t in doc]
|
| spacy_lemmas = [t.lemma_ for t in doc]
|
|
|
| a2b, b2a = tokenizations.get_alignments(spacy_tokens, tokens)
|
|
|
| for i in range(len(spacy_tokens)):
|
| for j in a2b[i]:
|
| aligned_tokens.append(spacy_tokens[i])
|
| aligned_pos_tags.append(spacy_pos_tags[i])
|
| aligned_lemmas.append(spacy_lemmas[i])
|
| aligned_predictions.append(CCV_span_tagged_predictions[j])
|
| break
|
|
|
| clean_tokens, clean_pos_tags, clean_lemmas, clean_predictions = sign_cleaning(
|
| aligned_tokens, aligned_pos_tags, aligned_lemmas, aligned_predictions
|
| )
|
|
|
| simplifiable = all("COMPOUND" in p for p in clean_predictions)
|
|
|
| if not simplifiable:
|
| True
|
|
|
| return (
|
| clean_tokens,
|
| clean_pos_tags,
|
| clean_lemmas,
|
| clean_predictions,
|
| )
|
|
|
|
|
|
|
|
|
|
|
| def merge_wordpieces(sign_tagged_tokens):
|
| """
|
| Merge BERT WordPiece tokens back into whole words.
|
| Example: [('ma', 'M:N_CCV'), ('##ry', 'M:N_CCV')] → [('mary', 'M:N_CCV')]
|
| """
|
| merged = []
|
| for tok, lab in sign_tagged_tokens:
|
| if tok in ('[CLS]', '[SEP]'):
|
| continue
|
| if tok.startswith('##') and merged:
|
| prev_tok, prev_lab = merged[-1]
|
| merged[-1] = (prev_tok + tok[2:], prev_lab)
|
| else:
|
|
|
| if tok in ('#', '##'):
|
| continue
|
| merged.append((tok, lab))
|
| return merged
|
|
|
|
|
|
|
| def tag_all_signs_in_sent(sent):
|
| """Your original Colab function - returns a LIST of strings with <M:Y_...> tags inserted"""
|
| sent = re.sub("do not", "don't", sent)
|
|
|
|
|
| sent = re.sub(
|
| r"\[((,|;|:|and|but|or|that|what|when|where|which|while|who)(\_(and|but|or|that|what|when|where|which|while|who))?)\]",
|
| r"\1\2",
|
| sent,
|
| re.IGNORECASE,
|
| )
|
|
|
| tokens = SignTaggingTokenizer.tokenize(
|
| SignTaggingTokenizer.decode(SignTaggingTokenizer.encode(sent))
|
| )
|
| inputs = SignTaggingTokenizer.encode(sent, return_tensors="pt")
|
| inputs = inputs.to(device)
|
|
|
| outputs = SignTaggingModel(inputs)[0]
|
| predictions = torch.argmax(outputs, dim=2)
|
|
|
|
|
|
|
| sign_tagged_tokens = [
|
| (token, sign_label_list[prediction])
|
| for token, prediction in zip(tokens, predictions[0].tolist())
|
| ]
|
|
|
|
|
| sign_tagged_tokens = merge_wordpieces(sign_tagged_tokens)
|
|
|
| print("sign_tagged_tokens:", sign_tagged_tokens)
|
|
|
| relevant_sign_offsets = []
|
| relevant_tags = []
|
|
|
| t = 0
|
| while t < len(sign_tagged_tokens):
|
| lab = sign_tagged_tokens[t][1]
|
| if re.match("M:Y", lab):
|
| relevant_sign_offsets.append(t)
|
| relevant_tags.append(lab)
|
| t += 1
|
|
|
| tagged_sent_list = [tok for tok, _ in sign_tagged_tokens]
|
|
|
| x = 0
|
| while x < len(relevant_sign_offsets):
|
| o = relevant_sign_offsets[x]
|
| rtag = relevant_tags[x]
|
| tagged_sent_list[o] = "<" + rtag + ">" + tagged_sent_list[o] + "</" + rtag + ">"
|
| x += 1
|
|
|
|
|
| tagged_sent = []
|
| x = 0
|
| while x < len(tagged_sent_list):
|
| if sign_tagged_tokens[x][0] not in ["[CLS]", "[SEP]"]:
|
| tagged_sent.append(tagged_sent_list[x])
|
| x += 1
|
|
|
| print(f"Returning tagged_sent: result = {tagged_sent}")
|
| return tagged_sent
|
|
|
|
|
|
|
|
|
|
|
| def split_coordinated_relative_body(body):
|
| """
|
| Split 'Steven kicked and who Mary liked'
|
| -> ['Steven kicked', 'Mary liked']
|
| Also handles 'and which', bare 'and' between relative-like conjuncts.
|
| """
|
| body = body.strip()
|
|
|
| parts = re.split(
|
| r"\s+and\s+(?:who|which|whom|that)\s+",
|
| body,
|
| flags=re.IGNORECASE,
|
| )
|
| if len(parts) >= 2:
|
| return [p.strip() for p in parts if p.strip()]
|
| return [body]
|
|
|
|
|
|
|
| def _region_text(elem):
|
| if elem is None:
|
| return ""
|
| parts = []
|
| for w in elem.findall("W"):
|
| if w.text and str(w.text).strip():
|
| parts.append(str(w.text).strip())
|
| return " ".join(parts).strip()
|
|
|
|
|
|
|
| def _split_coord_relative_body(body):
|
| """
|
| 'Steven kicked and who Mary liked'
|
| -> ['Steven kicked', 'Mary liked']
|
| """
|
| body = re.sub(r"\s+", " ", (body or "")).strip()
|
| if not body:
|
| return []
|
| parts = re.split(
|
| r"\s+and\s+(?:who|which|whom|that)\s+",
|
| body,
|
| flags=re.IGNORECASE,
|
| )
|
| parts = [p.strip() for p in parts if p.strip()]
|
| return parts if parts else [body]
|
|
|
|
|
|
|
| def _tidy_plain(s):
|
| s = re.sub(r"\s+", " ", (s or "")).strip()
|
| s = re.sub(r"\s+([.,!?;:])", r"\1", s)
|
| s = re.sub(r",+\s*\.", ".", s)
|
| s = re.sub(r",+\s*$", "", s)
|
| if s and not s.endswith((".", "!", "?")):
|
| s += "."
|
| if s:
|
| s = s[0].upper() + s[1:]
|
| return s
|
|
|
|
|
|
|
|
|
| def SSCCVsimplify(xml_sent):
|
| simplification = []
|
|
|
| xml_sent2 = xml_sent
|
|
|
| xml_string = ET.tostring(xml_sent).decode("utf-8")
|
| xml_for_printing = xml.dom.minidom.parseString(xml_string)
|
| pretty_xml_as_string = xml_for_printing.toprettyxml()
|
|
|
|
|
| simp_sent1 = ET.Element("S")
|
| simp_sent2 = ET.Element("S")
|
|
|
| if xml_sent.attrib["TYPE"] == "COMPLEX_NP":
|
| complex_NP_type = get_COMPLEX_NP_type(xml_sent)
|
| print("SSCCV complex_NP_type:", complex_NP_type)
|
|
|
|
|
|
|
|
|
|
|
|
|
| if complex_NP_type == "SUBJECT_RELATIVISED":
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| before_el = xml_sent.find("BEFORE_COMPLEX_CONSTITUENT")
|
| head_el = xml_sent.find("IN_COMPLEX_CONSTITUENT_BEFORESIGN")
|
| body_el = xml_sent.find("IN_COMPLEX_CONSTITUENT_AFTERSIGN")
|
| after_el = xml_sent.find("AFTER_COMPLEX_CONSTITUENT")
|
|
|
| head = re.sub(r"[,\s]+$", "", _region_text(head_el)).strip()
|
| body = re.sub(r"^[,\s]+", "", _region_text(body_el)).strip()
|
| after = re.sub(r"^[,\s]+", "", _region_text(after_el)).strip()
|
| before = _region_text(before_el)
|
|
|
| matrix = _tidy_plain(" ".join(x for x in [before, head, after] if x))
|
| conjuncts = _split_coord_relative_body(body)
|
|
|
| out = []
|
| if matrix:
|
| out.append(matrix)
|
| for conj in conjuncts:
|
| if conj and head:
|
| out.append(_tidy_plain(f"{head} {conj}"))
|
|
|
| return out
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| elif complex_NP_type == "PREPOSITION_OBJECT_RELATIVISED":
|
|
|
| copula = ""
|
|
|
| xml_prep = ET.Element("W")
|
| for vor_sent_el in xml_sent:
|
| if vor_sent_el.tag == "IN_COMPLEX_CONSTITUENT_BEFORESIGN":
|
| for vor_sent_subel in vor_sent_el:
|
| if vor_sent_subel.attrib["POS"] == "IN":
|
| xml_prep = vor_sent_subel
|
|
|
| vor_sent_matrix_subject = ET.Element("NA")
|
| vor_sent_el_bcc = ET.Element("NA")
|
| vor_sent_prep = ET.Element("NA")
|
| vor_sent_acc = ET.Element("NA")
|
|
|
| vor_sent2_prep = ET.Element("NA")
|
| vor_sent2_matrix_subject = ET.Element("NA")
|
| vor_sent2_clause = ET.Element("NA")
|
|
|
| for vor_sent_el in xml_sent:
|
|
|
| if vor_sent_el.tag == "IN_COMPLEX_CONSTITUENT":
|
| try:
|
| if re.match("(\,|\:|\;)$", vor_sent_el[-2].text):
|
| punctuation_boundary = True
|
| except:
|
| True
|
|
|
|
|
| elif vor_sent_el.tag == "IN_COMPLEX_CONSTITUENT_BEFORESIGN":
|
| if re.match("(\,|\:|\;)$", vor_sent_el[-1].text):
|
| del vor_sent_el[-1]
|
|
|
|
|
| elif vor_sent_el.tag == "AFTER_COMPLEX_CONSTITUENT":
|
| if re.match("(\,|\:|\;)$", vor_sent_el[0].text):
|
| del vor_sent_el[0]
|
|
|
| for vor_sent_el in xml_sent:
|
|
|
| if vor_sent_el.tag == "BEFORE_COMPLEX_CONSTITUENT":
|
| vor_sent_el_bcc = vor_sent_el
|
|
|
| elif vor_sent_el.tag == "IN_COMPLEX_CONSTITUENT_BEFORESIGN":
|
| vor_sent_prep = xml_prep
|
| vor_sent2_prep = xml_prep
|
| vor_sent_matrix_subject = vor_sent_el
|
| vor_sent_matrix_subject.remove(vor_sent_matrix_subject[-1])
|
|
|
| elif vor_sent_el.tag == "AFTER_COMPLEX_CONSTITUENT":
|
| if re.match("(\,|\:|\;)$", vor_sent_el[0].text):
|
| del vor_sent_el[0]
|
| vor_sent_acc = vor_sent_el
|
|
|
| simp_sent1.append(vor_sent_el_bcc)
|
| simp_sent1.append(vor_sent_matrix_subject)
|
| simp_sent1.append(vor_sent_acc)
|
|
|
| for vor_sent2_el in xml_sent2:
|
|
|
| if vor_sent2_el.tag == "IN_COMPLEX_CONSTITUENT_BEFORESIGN":
|
| try:
|
| if vor_sent2_el[-1].attrib["POS"] == "IN":
|
| vor_sent2_prep = vor_sent2_el[-1]
|
|
|
| vor_sent2_matrix_subject = vor_sent2_el
|
|
|
| except:
|
| True
|
|
|
| elif vor_sent2_el.tag == "IN_COMPLEX_CONSTITUENT_AFTERSIGN":
|
| vor_sent2_clause = vor_sent2_el
|
|
|
| simp_sent2.append(vor_sent2_clause)
|
| simp_sent2.append(vor_sent2_prep)
|
| simp_sent2.append(vor_sent2_matrix_subject)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| elif complex_NP_type == "VERB_OBJECT_RELATIVISED":
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| before_el = xml_sent.find("BEFORE_COMPLEX_CONSTITUENT")
|
| head_el = xml_sent.find("IN_COMPLEX_CONSTITUENT_BEFORESIGN")
|
| body_el = xml_sent.find("IN_COMPLEX_CONSTITUENT_AFTERSIGN")
|
| after_el = xml_sent.find("AFTER_COMPLEX_CONSTITUENT")
|
|
|
| head = re.sub(r"[,\s]+$", "", _region_text(head_el)).strip()
|
| body = re.sub(r"^[,\s]+", "", _region_text(body_el)).strip()
|
| after = re.sub(r"^[,\s]+", "", _region_text(after_el)).strip()
|
| before = _region_text(before_el)
|
|
|
| matrix = _tidy_plain(" ".join(x for x in [before, head, after] if x))
|
| conjuncts = _split_coord_relative_body(body)
|
|
|
| out = []
|
| if matrix:
|
| out.append(matrix)
|
| for conj in conjuncts:
|
| if conj and head:
|
| out.append(_tidy_plain(f"{conj} {head}"))
|
|
|
| return out
|
|
|
|
|
| if len(simp_sent1) > 0 and len(simp_sent2) > 0:
|
|
|
| s1_xml_string = ET.tostring(simp_sent1).decode("utf-8")
|
| s1_xml_for_printing = xml.dom.minidom.parseString(s1_xml_string)
|
| s1_pretty_xml_as_string = s1_xml_for_printing.toprettyxml()
|
|
|
| s2_xml_string = ET.tostring(simp_sent2).decode("utf-8")
|
| s2_xml_for_printing = xml.dom.minidom.parseString(s2_xml_string)
|
| s2_pretty_xml_as_string = s2_xml_for_printing.toprettyxml()
|
|
|
| """
|
| Be aware that simp_sent1 and simp_sent2 have a different structure
|
| from xml_sent. This affects the xml_to_text function/process.
|
| """
|
|
|
| try:
|
| simplification.append(simplified_sent_xml_to_text(simp_sent1))
|
| simplification.append(simplified_sent_xml_to_text(simp_sent2))
|
| except Exception as e:
|
| print("SSCCVsimplify xml_to_text error:", e)
|
| print("simp_sent1:", ET.tostring(simp_sent1).decode("utf-8"))
|
| print("simp_sent2:", ET.tostring(simp_sent2).decode("utf-8"))
|
|
|
|
|
|
|
| return simplification
|
|
|
|
|
|
|
|
|
| def CompoundSimplify(xml_sent):
|
| """
|
| Reconstruct two sentences from the compound XML structure.
|
| sent1 = BEFORE + BEFORESIGN + AFTER
|
| sent2 = BEFORE + AFTERSIGN + AFTER
|
| The coordinator (IN_COMPOUND_CONSTITUENT) is discarded.
|
|
|
| Recovery: if AFTERSIGN is empty but IN_COMPOUND contains tokens
|
| after the coordinator, treat those tokens as AFTERSIGN.
|
| """
|
| try:
|
| def get_text(elem):
|
| if elem is None:
|
| return ""
|
| parts = []
|
| for w in elem.findall("W"):
|
| if w.text:
|
| t = str(w.text).strip()
|
| t = re.sub(r"[\[\]]+", "", t)
|
| if t:
|
| parts.append(t)
|
| return " ".join(parts).strip()
|
|
|
| def get_tokens(elem):
|
| if elem is None:
|
| return []
|
| parts = []
|
| for w in elem.findall("W"):
|
| if w.text:
|
| t = str(w.text).strip()
|
| t = re.sub(r"[\[\]]+", "", t)
|
| if t:
|
| parts.append(t)
|
| return parts
|
|
|
| before_elem = xml_sent.find(".//BEFORE_COMPOUND_CONSTITUENT")
|
| beforesign_elem = xml_sent.find(".//IN_COMPOUND_CONSTITUENT_BEFORESIGN")
|
| sign_elem = xml_sent.find(".//IN_COMPOUND_CONSTITUENT")
|
| aftersign_elem = xml_sent.find(".//IN_COMPOUND_CONSTITUENT_AFTERSIGN")
|
| after_elem = xml_sent.find(".//AFTER_COMPOUND_CONSTITUENT")
|
|
|
| before_text = get_text(before_elem)
|
| beforesign_text = get_text(beforesign_elem)
|
| aftersign_text = get_text(aftersign_elem)
|
| after_text = get_text(after_elem)
|
|
|
|
|
| if not aftersign_text and sign_elem is not None:
|
| sign_toks = get_tokens(sign_elem)
|
| coord = {",", ";", ":", "and", "but", "or", "nor"}
|
| i = 0
|
| while i < len(sign_toks) and sign_toks[i].lower() in coord:
|
| i += 1
|
| if i < len(sign_toks):
|
| aftersign_text = " ".join(sign_toks[i:]).strip()
|
|
|
|
|
| sent1 = " ".join(
|
| x for x in [before_text, beforesign_text, after_text] if x
|
| ).strip()
|
| sent2 = " ".join(
|
| x for x in [before_text, aftersign_text, after_text] if x
|
| ).strip()
|
|
|
| def tidy(s):
|
| s = re.sub(r"\s+", " ", s).strip()
|
| s = re.sub(r"\s+([.,!?;:])", r"\1", s)
|
| s = re.sub(r"([.,!?;:])\s+", r"\1 ", s)
|
|
|
| s = re.sub(r"(\d)\s*\.\s*(\d)", r"\1.\2", s)
|
| s = re.sub(r",+\s*\.", ".", s)
|
| s = re.sub(r",+\s*$", "", s)
|
| if s and not s.endswith((".", "!", "?")):
|
| s += "."
|
| if s:
|
| s = s[0].upper() + s[1:]
|
| return s
|
|
|
| sent1 = tidy(sent1)
|
| sent2 = tidy(sent2)
|
|
|
| result = []
|
| if sent1:
|
| result.append(sent1)
|
| if sent2 and sent2 != sent1:
|
| result.append(sent2)
|
|
|
| print(f"CompoundSimplify produced {len(result)} sentences: {result}")
|
| return result
|
|
|
| except Exception as e:
|
| print(f"CompoundSimplify error: {e}")
|
| return []
|
|
|
|
|
|
|
|
|
|
|
| def create_xml_sent(tokens, pos_tags, lemmas, predictions, sign_tag):
|
| xml_sent = ET.Element("SENT")
|
|
|
| print(
|
| "tokens:",
|
| tokens,
|
| "\npos_tags:",
|
| pos_tags,
|
| "\nlemmas:",
|
| lemmas,
|
| "\npredictions:",
|
| predictions,
|
| "\nsign_tag:",
|
| sign_tag,
|
| )
|
|
|
| before_complex_constituent = None
|
| in_complex_constituent_beforesign = None
|
| in_complex_constituent = None
|
| in_complex_constituent_aftersign = None
|
| after_complex_constituent = None
|
|
|
| if sign_tag == "M:Y_SSCCV":
|
| before_complex_constituent = ET.SubElement(
|
| xml_sent, "BEFORE_COMPLEX_CONSTITUENT"
|
| )
|
| in_complex_constituent_beforesign = ET.SubElement(
|
| xml_sent, "IN_COMPLEX_CONSTITUENT_BEFORESIGN"
|
| )
|
| in_complex_constituent = ET.SubElement(xml_sent, "IN_COMPLEX_CONSTITUENT")
|
| in_complex_constituent_aftersign = ET.SubElement(
|
| xml_sent, "IN_COMPLEX_CONSTITUENT_AFTERSIGN"
|
| )
|
| after_complex_constituent = ET.SubElement(xml_sent, "AFTER_COMPLEX_CONSTITUENT")
|
|
|
| elif sign_tag in ["M:Y_CCV"]:
|
| before_complex_constituent = ET.SubElement(
|
| xml_sent, "BEFORE_COMPOUND_CONSTITUENT"
|
| )
|
| in_complex_constituent_beforesign = ET.SubElement(
|
| xml_sent, "IN_COMPOUND_CONSTITUENT_BEFORESIGN"
|
| )
|
| in_complex_constituent = ET.SubElement(xml_sent, "IN_COMPOUND_CONSTITUENT")
|
| in_complex_constituent_aftersign = ET.SubElement(
|
| xml_sent, "IN_COMPOUND_CONSTITUENT_AFTERSIGN"
|
| )
|
| after_complex_constituent = ET.SubElement(
|
| xml_sent, "AFTER_COMPOUND_CONSTITUENT"
|
| )
|
|
|
| core_prediction = predictions[0]
|
|
|
| core_prediction = re.sub("IN_", "", core_prediction)
|
| core_prediction = re.sub("BEFORE_", "", core_prediction)
|
| core_prediction = re.sub("AFTER_", "", core_prediction)
|
| core_prediction = re.sub("_AFTERSIGN", "", core_prediction)
|
| core_prediction = re.sub("_BEFORESIGN", "", core_prediction)
|
|
|
| xml_sent.attrib["TYPE"] = core_prediction
|
|
|
| x = 0
|
| while x < len(predictions):
|
| if "_BEFORESIGN" in predictions[x]:
|
| word = ET.SubElement(in_complex_constituent_beforesign, "W")
|
| word.text = tokens[x]
|
| word.attrib["POS"] = pos_tags[x]
|
| word.attrib["LEMMA"] = lemmas[x]
|
|
|
| elif "_AFTERSIGN" in predictions[x]:
|
| if re.match(
|
| "(and|but|or|that|what|when|where|which|while|who)",
|
| tokens[x],
|
| re.IGNORECASE,
|
| ):
|
| if x > 0 and tokens[x - 1] in [",", ":", ";"]:
|
| word = ET.SubElement(in_complex_constituent, "W")
|
| word.text = tokens[x]
|
| word.attrib["POS"] = pos_tags[x]
|
| word.attrib["LEMMA"] = lemmas[x]
|
| else:
|
| word = ET.SubElement(in_complex_constituent_aftersign, "W")
|
| word.text = tokens[x]
|
| word.attrib["POS"] = pos_tags[x]
|
| word.attrib["LEMMA"] = lemmas[x]
|
| else:
|
| word = ET.SubElement(in_complex_constituent_aftersign, "W")
|
| word.text = tokens[x]
|
| word.attrib["POS"] = pos_tags[x]
|
| word.attrib["LEMMA"] = lemmas[x]
|
|
|
| elif "IN_" in predictions[x]:
|
| word = ET.SubElement(in_complex_constituent, "W")
|
| word.text = tokens[x]
|
| word.attrib["POS"] = pos_tags[x]
|
| word.attrib["LEMMA"] = lemmas[x]
|
|
|
| elif "BEFORE_" in predictions[x]:
|
| word = ET.SubElement(before_complex_constituent, "W")
|
| word.text = tokens[x]
|
| word.attrib["POS"] = pos_tags[x]
|
| word.attrib["LEMMA"] = lemmas[x]
|
|
|
| elif "AFTER_" in predictions[x]:
|
| word = ET.SubElement(after_complex_constituent, "W")
|
| word.text = tokens[x]
|
| word.attrib["POS"] = pos_tags[x]
|
| word.attrib["LEMMA"] = lemmas[x]
|
|
|
| x += 1
|
|
|
|
|
| print("BUILT XML SENTENCE:\n" + ET.tostring(xml_sent).decode("utf-8"))
|
| return xml_sent
|
|
|
|
|
|
|
|
|
|
|
|
|
| def get_COMPLEX_NP_type(xml_sent):
|
| complex_NP_type = "UNKNOWN"
|
|
|
| final_icca_pos_tag = "UNKNOWN"
|
| first_icca_pos_tag = "UNKNOWN"
|
| final_iccb_pos_tag = "UNKNOWN"
|
|
|
| punct = re.compile(r"^[\,\:\;]$")
|
|
|
| for el in xml_sent:
|
| if el.tag == "IN_COMPLEX_CONSTITUENT_AFTERSIGN":
|
| try:
|
|
|
| for w in el:
|
| if w.text and not punct.match(w.text.strip()):
|
| first_icca_pos_tag = w.attrib.get("POS", "UNKNOWN")
|
| break
|
| final_icca_pos_tag = el[-1].attrib["POS"]
|
| except Exception:
|
| pass
|
|
|
| elif el.tag == "IN_COMPLEX_CONSTITUENT_BEFORESIGN":
|
| try:
|
|
|
| for w in reversed(list(el)):
|
| if w.text and not punct.match(w.text.strip()):
|
| final_iccb_pos_tag = w.attrib.get("POS", "UNKNOWN")
|
| break
|
| except Exception:
|
| pass
|
|
|
| if final_iccb_pos_tag in ["IN"]:
|
| complex_NP_type = "PREPOSITION_OBJECT_RELATIVISED"
|
| elif first_icca_pos_tag in [
|
| "NNP",
|
| "NNPS",
|
| "NN",
|
| "DT",
|
| "CD",
|
| ] and final_icca_pos_tag in ["VB", "VBD", "VBG", "VBN", "VBZ"]:
|
| complex_NP_type = "VERB_OBJECT_RELATIVISED"
|
| elif first_icca_pos_tag in ["VB", "VBD", "VBG", "VBN", "VBZ"]:
|
| complex_NP_type = "SUBJECT_RELATIVISED"
|
|
|
| return complex_NP_type
|
|
|
|
|
|
|
|
|
|
|
| def simplified_sent_xml_to_text(xml):
|
| parts = []
|
|
|
| for el in xml:
|
| if el.tag == "W":
|
| if el.text:
|
| parts.append(el.text.strip())
|
| else:
|
| for sub_el in el:
|
| if sub_el.text:
|
| parts.append(sub_el.text.strip())
|
|
|
|
|
| text = " ".join(parts).strip()
|
|
|
|
|
| text = re.sub(r"\s+", " ", text)
|
|
|
|
|
| if text and not text.endswith((".", "!", "?")):
|
| text += "."
|
|
|
|
|
| if text:
|
| text = text[0].upper() + text[1:]
|
|
|
| return text
|
|
|
|
|
|
|
|
|
|
|
|
|
| def clean_bpe_text(text):
|
| """Remove BPE fragments, labels, brackets, and normalize whitespace."""
|
| text = re.sub(r"##", "", text)
|
| text = re.sub(r"\s*\[[^\]]+\]\s*\(M:Y_[^\)]+\)", "", text)
|
| text = re.sub(r"\s*\[[^\]]+\]", "", text)
|
| text = re.sub(r"\s*\(M:Y_[^\)]+\)", "", text)
|
| text = re.sub(r"\s*\(M:N_[^\)]+\)", "", text)
|
| text = re.sub(r"M Y _[^\s]+", "", text)
|
| text = re.sub(r"\s+", " ", text).strip()
|
| return text
|
|
|
|
|
|
|
|
|
|
|
|
|
| def clean_output(text):
|
| if not text:
|
| return ""
|
|
|
|
|
| text = re.sub(r'[\[\]]+', '', text)
|
| text = re.sub(r'\s*##\s*', '', text)
|
| text = re.sub(r'#', '', text)
|
|
|
| text = re.sub(r'\s+', ' ', text).strip()
|
|
|
|
|
| text = re.sub(r'\s+([.,!?;:])', r'\1', text)
|
| text = re.sub(r'([.,!?;:])\s+', r'\1 ', text)
|
|
|
|
|
| text = re.sub(r',+\s*\.', '.', text)
|
| text = re.sub(r',+\s*$', '', text)
|
| text = re.sub(r'\.+$', '.', text)
|
|
|
| text = text.strip()
|
|
|
| if text:
|
| text = text[0].upper() + text[1:]
|
| if not text.endswith(('.', '!', '?')):
|
| text += '.'
|
|
|
| return text.strip()
|
|
|
|
|
|
|
|
|
|
|
| def drop_bracket_tokens(tokens, pos, lemmas, preds):
|
| """Remove literal [ and ] that the tokenizer sometimes emits"""
|
| kept_tokens, kept_pos, kept_lemmas, kept_preds = [], [], [], []
|
| for t, p, l, pr in zip(tokens, pos, lemmas, preds):
|
| if t in ('[', ']'):
|
| continue
|
| kept_tokens.append(t)
|
| kept_pos.append(p)
|
| kept_lemmas.append(l)
|
| kept_preds.append(pr)
|
| return kept_tokens, kept_pos, kept_lemmas, kept_preds
|
|
|
|
|
|
|
|
|
| def build_single_sign_bracketed(tagged_list, mode="ccv"):
|
| """
|
| mode = "ccv" → bracket only the rightmost CCV sign
|
| mode = "ssccv" → bracket one relative SSCCV trigger:
|
| - nested relatives (2+ opens before next ESCCV): innermost
|
| (rightmost preferred trigger before that ESCCV)
|
| - otherwise: leftmost relative trigger
|
| - fallback: leftmost SSCCV/ESCCV (e.g. bare "that")
|
| """
|
|
|
| def surface_of(item):
|
| if item.startswith("<M:Y_") and ">" in item:
|
| start = item.find(">") + 1
|
| end = item.rfind("<")
|
| if end > start:
|
| return item[start:end]
|
| return item
|
|
|
| ccv_indices = []
|
| ssccv_indices = []
|
| esccv_indices = []
|
|
|
| for i, item in enumerate(tagged_list):
|
| s = str(item)
|
| if "<M:Y_CCV>" in s or "M:Y_CCV" in s:
|
| ccv_indices.append(i)
|
| if "M:Y_SSCCV" in s:
|
| ssccv_indices.append(i)
|
| if "M:Y_ESCCV" in s:
|
| esccv_indices.append(i)
|
|
|
| if "M:Y_ESCCV" in s and i not in ssccv_indices:
|
| ssccv_indices.append(i)
|
|
|
| target = set()
|
|
|
| if mode == "ccv" and ccv_indices:
|
| target.add(ccv_indices[-1])
|
|
|
| elif mode == "ssccv" and ssccv_indices:
|
| relative_pronouns = {"who", "which", "whom", "whose"}
|
| preferred = []
|
|
|
| for i in ssccv_indices:
|
| surface = surface_of(tagged_list[i]).strip().lower()
|
|
|
|
|
| if surface in relative_pronouns:
|
| preferred.append(i)
|
| continue
|
|
|
|
|
| if surface in {",", ";", ":"} and i + 1 < len(tagged_list):
|
| nxt = surface_of(tagged_list[i + 1]).strip().lower()
|
| if nxt in relative_pronouns or nxt == "that":
|
| preferred.append(i)
|
| continue
|
|
|
| if preferred:
|
|
|
| nested = False
|
| if esccv_indices:
|
| first_es = min(esccv_indices)
|
| opens_before_close = [i for i in preferred if i < first_es]
|
| if len(opens_before_close) >= 2:
|
| nested = True
|
| target.add(opens_before_close[-1])
|
| if not nested:
|
| target.add(preferred[0])
|
| else:
|
|
|
| pure_ssccv = [
|
| i for i, item in enumerate(tagged_list)
|
| if "M:Y_SSCCV" in str(item)
|
| ]
|
| target.add((pure_ssccv or ssccv_indices)[0])
|
|
|
| parts = []
|
| for i, item in enumerate(tagged_list):
|
| if item.startswith("<M:Y_") and item.endswith(">"):
|
| start = item.find(">") + 1
|
| end = item.rfind("<")
|
| surface = item[start:end] if end > start else item
|
| if i in target:
|
| parts.append(f"[{surface}]")
|
| else:
|
| parts.append(surface)
|
| else:
|
| parts.append(item)
|
|
|
| text = " ".join(parts)
|
|
|
|
|
| text = re.sub(r"\s+\[([,;:])", r"[\1", text)
|
| text = re.sub(r"(?<!\[)\[([A-Za-z])", r" [\1", text)
|
| text = re.sub(r"\[\s+", "[", text)
|
| text = re.sub(r"\s+\]", "]", text)
|
| text = re.sub(r"\s+", " ", text).strip()
|
|
|
|
|
| text = re.sub(
|
| r"\[([,;:])\]\s+(who|which|that|whom|whose)\b",
|
| r"[\1 \2]",
|
| text,
|
| flags=re.I,
|
| )
|
|
|
| return text
|
|
|
|
|
|
|
|
|
|
|
| @spaces.GPU
|
| def process_text(input_text):
|
| print("=== ITERATIVE SIMPLIFICATION (priority + clean tokens) ===")
|
| try:
|
| MAX_ITERS = 40
|
|
|
|
|
| input_text = re.sub(r"supposingthat", "supposing that", input_text)
|
| input_text = re.sub(r'"', "''", input_text)
|
| input_text = re.sub(r"\.\s+([a-z])", ".\n\\g<1>", input_text)
|
| input_text = re.sub(r"mainly f or", "mainly for", input_text)
|
| input_text = re.sub(r"sothat", "so that", input_text)
|
| input_text = re.sub(r"in alabama", "in Alabama", input_text)
|
| input_text = re.sub(
|
| r"LEISURE DESIGN IS IMPORTANT WHY is",
|
| "LEISURE DESIGN IS IMPORTANT\n\nWhy is",
|
| input_text,
|
| )
|
| input_text = re.sub(r"WHEN", "When", input_text)
|
| input_text = re.sub(r"WHAT'S", "What's", input_text)
|
| input_text = re.sub(r"(\s+)an'", " and", input_text)
|
| input_text = re.sub(r"An'", "And", input_text)
|
|
|
|
|
|
|
| raw_paragraphs = re.split(r"\n\s*\n", input_text)
|
| output_paragraphs = []
|
|
|
| for para in raw_paragraphs:
|
| para = para.strip()
|
| if not para:
|
| continue
|
|
|
|
|
| para = re.sub(r"\s*\n\s*", " ", para)
|
| para = re.sub(r"\s+", " ", para).strip()
|
|
|
| sents = split_into_sentences(para)
|
| if not sents:
|
| continue
|
|
|
| print(f"Paragraph → {len(sents)} sentence(s):")
|
| for i, s in enumerate(sents, 1):
|
| print(f" [{i}] {s}")
|
|
|
| para_simplified = []
|
|
|
| for original_sent in sents:
|
| working_set = deque([original_sent])
|
| seen = set()
|
| sentence_final = []
|
|
|
| iters = 0
|
| while working_set:
|
| iters += 1
|
|
|
| if iters > MAX_ITERS:
|
| print(
|
| f"ABORT: exceeded {MAX_ITERS} iterations; "
|
| f"draining {len(working_set)} item(s) to final"
|
| )
|
| while working_set:
|
| sentence_final.append(
|
| clean_output(working_set.popleft())
|
| )
|
| break
|
|
|
| current = working_set.popleft().strip()
|
| if not current:
|
| continue
|
|
|
| current = clean_output(current)
|
| if not current:
|
| continue
|
|
|
| key = re.sub(r"\s+", " ", current.lower())
|
| if key in seen:
|
| print(f"SKIP duplicate: {current[:80]}...")
|
| sentence_final.append(current)
|
| continue
|
| seen.add(key)
|
|
|
| print(
|
| f"\n--- iter {iters} | queue={len(working_set)} | "
|
| f"{current[:100]}..."
|
| )
|
|
|
| tagged_list = tag_all_signs_in_sent(current)
|
|
|
| has_ccv = any(
|
| "<M:Y_CCV>" in t or "M:Y_CCV" in t for t in tagged_list
|
| )
|
| has_ssccv = any(
|
| any(x in t for x in ["M:Y_SSCCV", "M:Y_ESCCV"])
|
| for t in tagged_list
|
| )
|
|
|
| if has_ssccv:
|
| has_ccv = False
|
|
|
| if not has_ccv and not has_ssccv:
|
| sentence_final.append(current)
|
| continue
|
|
|
| if has_ccv:
|
| mode = "ccv"
|
| xml_tag = "M:Y_CCV"
|
| simplifier = CompoundSimplify
|
| span_fn = tag_CCV_span
|
| else:
|
| mode = "ssccv"
|
| xml_tag = "M:Y_SSCCV"
|
| simplifier = SSCCVsimplify
|
| span_fn = tag_SSCCV_span
|
|
|
| bracketed_version = build_single_sign_bracketed(
|
| tagged_list, mode=mode
|
| )
|
|
|
| print(f"\nProcessing: {current}")
|
| print(f"Bracketed version ({mode}): {bracketed_version}")
|
|
|
| cleaned = clean_for_spaCy(bracketed_version)
|
| tokens, pos, lemmas, preds = span_fn(cleaned)
|
|
|
| tokens, pos, lemmas, preds = drop_bracket_tokens(
|
| tokens, pos, lemmas, preds
|
| )
|
|
|
| cleaned_tokens, clean_pos, clean_lemmas, cleaned_preds = (
|
| sign_cleaning(tokens, pos, lemmas, preds)
|
| )
|
| improved_preds = improve_predicted_tag_sequence(
|
| cleaned_preds
|
| )
|
|
|
| xml_sent = create_xml_sent(
|
| cleaned_tokens,
|
| clean_pos,
|
| clean_lemmas,
|
| improved_preds,
|
| xml_tag,
|
| )
|
| simps = simplifier(xml_sent)
|
|
|
| if not simps:
|
| print(
|
| "Simplifier returned []; forcing final (no re-queue)"
|
| )
|
| sentence_final.append(current)
|
| continue
|
|
|
| queued_any = False
|
| for s in simps:
|
| plain_s = clean_output(s)
|
| if not plain_s:
|
| continue
|
| plain_key = re.sub(r"\s+", " ", plain_s.lower())
|
| if plain_key == key:
|
| print(
|
| f" No progress (same as input): "
|
| f"{plain_s[:80]}..."
|
| )
|
| sentence_final.append(plain_s)
|
| continue
|
| if plain_key in seen:
|
| print(
|
| f" Skip already-seen result: "
|
| f"{plain_s[:80]}..."
|
| )
|
| sentence_final.append(plain_s)
|
| continue
|
| working_set.append(plain_s)
|
| queued_any = True
|
| print(f" → Queued: {plain_s}")
|
|
|
| if not queued_any:
|
| sentence_final.append(current)
|
|
|
| for s in sentence_final:
|
| s = clean_output(s)
|
| if s:
|
| para_simplified.append(s)
|
|
|
| if para_simplified:
|
|
|
| output_paragraphs.append(" ".join(para_simplified))
|
|
|
| final_output = "\n\n".join(output_paragraphs)
|
| print(f"\n=== FINAL OUTPUT ===\n{final_output}")
|
| return final_output
|
|
|
| except Exception as e:
|
| import traceback
|
|
|
| print(traceback.format_exc())
|
| return f"ERROR: {str(e)}"
|
|
|
|
|
|
|
|
|
|
|
|
|
| '''
|
| # ================== NEW SAFE GRADIO BLOCK (2026 version) ===================
|
| print(" All three models loaded successfully!")
|
| print("Creating Gradio interface...")
|
|
|
| with gr.Blocks(title="Syntactic Sentence Simplifier") as demo:
|
| gr.Markdown("# Syntactic Sentence Simplifier")
|
| gr.Markdown(
|
| "Paste or type your text below. Uses my original Hugging Face models + simplification logic. Will simplify sentences containing compound clauses and complex noun phrases. Explore other versions of this tool for simplification of other compound phrase or detection of other types of complex constituent."
|
| )
|
|
|
| input_text = gr.Textbox(
|
| lines=10, label="Input Text", placeholder="Paste your text here..."
|
| )
|
|
|
| output_text = gr.Textbox(lines=10, label="Simplified Output")
|
|
|
| btn = gr.Button("Process & Simplify", variant="primary")
|
|
|
| btn.click(
|
| fn=process_text, inputs=input_text, outputs=output_text # your main function
|
| )
|
|
|
| print(" Gradio interface ready. Starting web server...")
|
|
|
| if __name__ == "__main__":
|
|
|
| demo.launch(
|
| show_error=True,
|
| ssr_mode=False,
|
| )
|
|
|
|
|
|
|
|
|
| test = (
|
| "Joshua told me that John, who lives in London, is going to Wolverhampton, Jack loves Norwich, and Mary, who is a bit nervous, adores Birmingham, just as Fred predicted."
|
| )
|
| print(process_text(test))
|
|
|
|
|
|
|
| '''
|
|
|
| print(" All three models loaded successfully!")
|
| print("Creating Gradio interface...")
|
|
|
| custom_css = """
|
| /* Slightly larger base font */
|
| .gradio-container, .gradio-container * {
|
| font-size: 16px !important;
|
| }
|
|
|
| /* Input = original text → bright pink */
|
| .input-text textarea {
|
| font-size: 17px !important;
|
| color: #ffd0d0 !important; /* DeepPink */
|
| font-weight: 500 !important;
|
| line-height: 1.45 !important;
|
| }
|
|
|
| /* Output = simplified text → bright green */
|
| .output-text textarea {
|
| font-size: 17px !important;
|
| color: #d0ffd0 !important; /* bright green */
|
| font-weight: 500 !important;
|
| line-height: 1.45 !important;
|
| }
|
|
|
| /* Labels under the boxes */
|
| .input-text label,
|
| .output-text label {
|
| font-size: 15px !important;
|
| font-weight: 600 !important;
|
| }
|
|
|
| .input-text label {
|
| color: #ffd0d0 !important;
|
| }
|
|
|
| .output-text label {
|
| color: #d0ffd0 !important;
|
| }
|
| """
|
|
|
| with gr.Blocks(
|
| title="Syntactic Sentence Simplifier",
|
| css=custom_css,
|
| ) as demo:
|
| gr.Markdown("# Syntactic Sentence Simplifier")
|
| gr.Markdown(
|
| "Paste or type your text below. Uses Hugging Face models + simplification logic."
|
| )
|
|
|
| input_text = gr.Textbox(
|
| lines=10,
|
| label="Original text",
|
| placeholder="Paste your text here...",
|
| elem_classes=["input-text"],
|
| )
|
|
|
| output_text = gr.Textbox(
|
| lines=10,
|
| label="Simplified text",
|
| elem_classes=["output-text"],
|
| )
|
|
|
| btn = gr.Button("Process & Simplify", variant="primary")
|
|
|
| btn.click(
|
| fn=process_text,
|
| inputs=input_text,
|
| outputs=output_text,
|
| )
|
|
|
| print(" Gradio interface ready. Starting web server...")
|
|
|
| if __name__ == "__main__":
|
| demo.launch(
|
| server_name="0.0.0.0",
|
| server_port=7860,
|
| debug=True,
|
| share=False,
|
| )
|
|
|
