| | from huggingface_hub import from_pretrained_keras |
| | import tensorflow as tf |
| | import gradio as gr |
| | import nltk |
| | import json |
| | nltk.download('brown') |
| | from nltk.corpus import brown |
| | from nltk import word_tokenize |
| | nltk.download('punkt') |
| | nltk.download('stopwords') |
| | nltk.download('wordnet') |
| | from nltk.corpus import stopwords |
| | from nltk import pos_tag |
| | nltk.download('averaged_perceptron_tagger') |
| | import re |
| | import torch |
| | from transformers import BertTokenizer, BertModel, BertForMaskedLM |
| | from wordfreq import zipf_frequency |
| | import keras |
| | from keras_preprocessing.sequence import pad_sequences |
| | from tensorflow.keras.utils import to_categorical |
| | from huggingface_hub import hf_hub_download |
| | import numpy as np |
| | sent_max_length = 103 |
| |
|
| | bert_model = 'bert-large-uncased' |
| | tokenizer = BertTokenizer.from_pretrained(bert_model) |
| | model = BertForMaskedLM.from_pretrained(bert_model) |
| |
|
| | model_cwi = from_pretrained_keras("jaimin/CWI") |
| |
|
| | stop_words_ = set(stopwords.words('english')) |
| |
|
| |
|
| | with open(hf_hub_download(repo_id="jaimin/CWI", filename="word2index.json")) as outfile: |
| | word2index = json.load(outfile) |
| | with open(hf_hub_download(repo_id="jaimin/CWI", filename="index2word.json")) as indexfile: |
| | index2word = json.load(indexfile) |
| |
|
| | def cleaner(word): |
| | |
| | word = re.sub(r'((http|https)\:\/\/)?[a-zA-Z0-9\.\/\?\:@\-_=#]+\.([a-zA-Z]){2,6}([a-zA-Z0-9\.\&\/\?\:@\-_=#])*', |
| | '', word, flags=re.MULTILINE) |
| | word = re.sub('[\W]', ' ', word) |
| | word = re.sub('[^a-zA-Z]', ' ', word) |
| | return word.lower().strip() |
| | |
| | def process_input(input_text): |
| | input_text = cleaner(input_text) |
| | clean_text = [] |
| | index_list =[] |
| | input_token = [] |
| | index_list_zipf = [] |
| | for i, word in enumerate(input_text.split()): |
| | if word in word2index: |
| | clean_text.append(word) |
| | input_token.append(word2index[word]) |
| | else: |
| | index_list.append(i) |
| | input_padded = pad_sequences(maxlen=sent_max_length, sequences=[input_token], padding="post", value=0) |
| | return input_padded, index_list, len(clean_text) |
| | |
| | def complete_missing_word(pred_binary, index_list, len_list): |
| | list_cwi_predictions = list(pred_binary[0][:len_list]) |
| | for i in index_list: |
| | list_cwi_predictions.insert(i, 0) |
| | return list_cwi_predictions |
| | |
| |
|
| | def get_bert_candidates(input_text, list_cwi_predictions, numb_predictions_displayed = 10): |
| | list_candidates_bert = [] |
| | for word,pred in zip(input_text.split(), list_cwi_predictions): |
| | if (pred and (pos_tag([word])[0][1] in ['NNS', 'NN', 'VBP', 'RB', 'VBG','VBD' ])) or (zipf_frequency(word, 'en')) <3.1: |
| | replace_word_mask = input_text.replace(word, '[MASK]') |
| | text = f'[CLS]{replace_word_mask} [SEP] {input_text} [SEP] ' |
| | tokenized_text = tokenizer.tokenize(text) |
| | masked_index = [i for i, x in enumerate(tokenized_text) if x == '[MASK]'][0] |
| | indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text) |
| | segments_ids = [0]*len(tokenized_text) |
| | tokens_tensor = torch.tensor([indexed_tokens]) |
| | segments_tensors = torch.tensor([segments_ids]) |
| | |
| | with torch.no_grad(): |
| | outputs = model(tokens_tensor, token_type_ids=segments_tensors) |
| | predictions = outputs[0][0][masked_index] |
| | predicted_ids = torch.argsort(predictions, descending=True)[:numb_predictions_displayed] |
| | predicted_tokens = tokenizer.convert_ids_to_tokens(list(predicted_ids)) |
| | list_candidates_bert.append((word, predicted_tokens)) |
| | return list_candidates_bert |
| | |
| | def cwi(input_text): |
| | new_text = input_text |
| | input_padded, index_list, len_list = process_input(input_text) |
| | pred_cwi = model_cwi.predict(input_padded) |
| | pred_cwi_binary = np.argmax(pred_cwi, axis = 2) |
| | complete_cwi_predictions = complete_missing_word(pred_cwi_binary, index_list, len_list) |
| | bert_candidates = get_bert_candidates(input_text, complete_cwi_predictions) |
| | for word_to_replace, l_candidates in bert_candidates: |
| | tuples_word_zipf = [] |
| | for w in l_candidates: |
| | if w.isalpha(): |
| | tuples_word_zipf.append((w, zipf_frequency(w, 'en'))) |
| | tuples_word_zipf = sorted(tuples_word_zipf, key = lambda x: x[1], reverse=True) |
| | new_text = re.sub(word_to_replace, tuples_word_zipf[0][0], new_text) |
| | return new_text |
| | |
| | interface = gr.Interface(fn=cwi, |
| | inputs=["text"], |
| | outputs="text", |
| | title='CWI') |
| | |
| |
|
| | interface.launch(inline=False) |
