agent/target_extraction/target_extractor.py

import pandas as pd
from collections import Counter
from nltk import pos_tag
from nltk.tokenize import word_tokenize, sent_tokenize
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
import string
from gensim.models.phrases import Phrases, Phraser
from anytree import Node, RenderTree, PreOrderIter
import numpy as np
import re
from gensim.models import Word2Vec
import itertools
import pickle
from agent.target_extraction.product import Product
from agent.target_extraction.BERT.entity_extractor.entity_dataset import EntityDataset
from agent.target_extraction.BERT.entity_extractor.bert_entity_extractor import BertEntityExtractor
from agent.target_extraction.BERT.relation_extractor.pair_rel_dataset import PairRelDataset
from agent.target_extraction.BERT.relation_extractor.bert_rel_extractor import BertRelExtractor
from pathos.multiprocessing import ProcessingPool as Pool
import itertools
from time import time
import os
import streamlit as st

#to get the current working directory
directory = os.getcwd()

print(directory)
np.set_printoptions(precision=4, threshold=np.inf, suppress=True)
stop_words = stopwords.words('english')
wnl = WordNetLemmatizer()

pool = Pool(1)
sentiment_lexicon = pd.read_csv(directory+'/NRC_Emotion_Lexicon-master/NRC-Emotion-Lexicon-Wordlevel-v0.92.txt', index_col=0)
entity_extractor_path = directory+'/models/entity_model.pt'
rel_extractor_path = directory+'/models/model.pt'

def ngrams(text, phraser):
    if any(isinstance(subtext, list) for subtext in text):
        return

    tags = [tag for _, tag in pos_tag(text)]
    unfiltered = [term.split('_') for term in phraser[text]]
    tagged_unfiltered = []
    n = 0
    for term in unfiltered:
        tagged_unfiltered.append([(subterm, tags[n + idx]) for idx, subterm in enumerate(term)])
        n += len(term)

    def filter_ngram(term):
        if len(term) > 1 and (any(not re.compile('NN|JJ').match(tag) for _, tag in term)
                              or any(tag.startswith('JJ') and polar_adjective(t) for t, tag in term)):
            return [subterm for subterm, _ in term]
        return [' '.join([subterm for subterm, _ in term])]

    def polar_adjective(adj):
        print("adjjj"+str(adj))
        return adj in sentiment_lexicon.index and (sentiment_lexicon.loc[adj]['positive'] == 1 or
                                                   sentiment_lexicon.loc[adj]['negative'] == 1)

    return [subterm for term in tagged_unfiltered for subterm in filter_ngram(term)]


def get_nouns(phrase, ngrams):
    pos_tags = pos_tag(phrase)

    def is_noun(pos_tagged):
        word, tag = pos_tagged
        return tag.startswith('NN') and word not in string.punctuation and word not in stop_words

    # true if term is not a preposition and does not include special characters
    def is_valid_term(pos_tagged):
        alpha_numeric_pat = '^\w+$'
        word, tag = pos_tagged
        return tag != 'IN' and re.match(alpha_numeric_pat, word)

    nouns = []
    word_idx = 0
    for token in ngrams:
        if ' ' in token:
            words = token.split(' ')
            word_range = range(word_idx, word_idx + len(words))
            has_noun = any(is_noun(pos_tags[i]) for i in word_range)
            all_terms_valid = all(is_valid_term(pos_tags[i]) for i in word_range)
            if has_noun and all_terms_valid:
                nouns.append(token)
            word_idx += len(words)
        else:
            token_is_noun = is_noun(pos_tags[word_idx])
            is_valid = is_valid_term(pos_tags[word_idx])
            if len(token) > 1 and token_is_noun and is_valid:
                nouns.append(token)
            word_idx += 1
    return nouns


def entity_mentions_in_text(text, phrase, ngrams, entities):
    all_tokens = set().union(*[phrase, ngrams])

    entity_mention = None
    for entity in entities:
        n_mentions = sum(1 for token in all_tokens if entity == token.lower())
        if n_mentions > 1:
            # many mentions of same entity
            return None
        if n_mentions == 1:
            if entity_mention is None:
                entity_mention = entity
            elif entity_mention in entity:
                entity_mention = entity
            elif entity not in entity_mention:
                # text cannot have more than one entity mention, unless one is a subset of the other,
                # in which case the longer one is taken
                return None

    if entity_mention is not None:
        return text, [{'text': entity_mention}]

    return None


def pair_relations_for_text(text, ngrams, aspects, syn_dict):

    def overlapping_terms(ts, t):
        if len(ts) == 0:
            return False
        return any(t in t2.split(' ') if len(t) < len(t2) else t2 in t.split(' ') for t2 in ts)

    noun_ngrams = [ngram for ngram, tag in pos_tag(ngrams) if tag.startswith('NN')]

    found_aspects = []
    for aspect in aspects:
        found_form = False
        for form in syn_dict[aspect]:
            if form in noun_ngrams:
                if len(found_aspects) > 1 or found_form or overlapping_terms(found_aspects, form):
                    # cannot have more than two aspects, or two forms of the same aspect, or overlapping terms
                    return None
                found_aspects.append(form)
                found_form = True

    return (text, [{'em1Text': found_aspects[0], 'em2Text': found_aspects[1]}]) if len(found_aspects) == 2 else None


class TargetExtractor:
    N_ASPECTS = 100

    # phraser
    PHRASE_THRESHOLD = 4

    # tree
    SUBFEATURE_MULT = 1.4  # for z to be a subfeature of x, matrix(z, x) > matrix(z, f) * SUBFEATURE_MULT for all other f
    COUNT_MULT = 5
    MAX_DEPTH = 2

    # word2vec
    MIN_TERM_COUNT = 100
    SYNONYM_SIMILARITY = 0.21
    SYNONYM_SIMILARITY_PRODUCT = 0.08
    WV_SIZE = 300
    WV_WINDOW = 4

    # bert
    ENTITY_PROB_THRESHOLD = 0.65

    # parent is a TargetExtrator of a parent category, eg. > electronics > camera
    def __init__(self, product, file_path, text_column):
        self.product = product
        self.file_path = file_path
        self.sentiment_lexicon=sentiment_lexicon
        self.entity_extractor_path=entity_extractor_path
        self.rel_extractor_path=rel_extractor_path

        ts = time()

        print('tokenizing phrases...')
        st.write('tokenizing phrases...')

        print("hello")
        # tokenize and normalize phrases
        texts = TargetExtractor.obtain_texts(self.file_path, text_column, n=50)
        self.sentences = list(itertools.chain.from_iterable(map(sent_tokenize, texts)))
        self.sentences =list(map(lambda s: s.replace('_', ' ').lower(), self.sentences))
        self.phrases = list(map(word_tokenize, self.sentences))

        print('obtaining n-grams...')
        st.write('obtaining n-grams...')
        
        # train bigram map
        bigram = Phrases(self.phrases, threshold=TargetExtractor.PHRASE_THRESHOLD)
        trigram = Phrases(bigram[self.phrases], threshold=TargetExtractor.PHRASE_THRESHOLD)
        phraser = Phraser(trigram)
        self.ngram_phrases = list(map(ngrams, self.phrases, itertools.repeat(phraser, len(self.phrases))))

        print('counting terms...')
        st.write('counting terms...')

        # count terms
        self.counter = self.count_nouns()
        self.total_count = sum(self.counter.values())

        t_noun = time()
        print('Noun extraction took {} seconds'.format(t_noun - ts))
        st.write('Noun extraction took {} seconds'.format(t_noun - ts))

        print('mining aspects...')
        st.write('mining aspects...')

        # mine aspects
        self.aspects, self.counts = self.get_aspects(self.counter)

        t_feature = time()
        print('Feature extraction took {} seconds'.format(t_feature - t_noun))
        st.write('Feature extraction took {} seconds'.format(t_feature - t_noun))
        

        print('training word2vec model...')
        st.write('training word2vec model...')

        # train word2vec model
        self.wv = self.get_word2vec_model(TargetExtractor.WV_SIZE, window=TargetExtractor.WV_WINDOW,
                                          min_count=TargetExtractor.MIN_TERM_COUNT)

        print('extracting synonyms...')
        st.write('extracting synonyms...')

        # obtain synonyms
        self.syn_dict = self.get_syn_dict()

        # remove aspect synonyms and reorder list based on sum of all synonym counts
        self.aspects = [aspect for aspect in self.aspects if aspect in self.syn_dict.keys()]
        dt={}
        for aspect in self.aspects:
          summ=0
          for syn in self.syn_dict[aspect]:
            try:
              print(self.counts[syn])
              st.write(self.counts[syn])

              summ=summ+1
            except:
              print("nothing")
              st.write("nothing")

          dt[aspect]=summ
        self.counts =  dt  #{aspect: sum(self.counts[syn] for syn in self.syn_dict[aspect]) for aspect in self.aspects}
        print(self.counts)
        st.write(self.counts)

        self.aspects = sorted(self.aspects, key=self.counts.get, reverse=True)

        t_syn = time()
        print('Synonym extraction took {} seconds'.format(t_syn - t_feature))
        st.write('Synonym extraction took {} seconds'.format(t_syn - t_feature))


        print('extracting relatedness matrix...')
        st.write('extracting relatedness matrix...')

        self.relatedness_matrix = self.get_bert_relations()

        print('extracting aspect tree...')
        st.write('extracting aspect tree...')

        self.tree = self.get_product_tree3()

        te = time()
        print('Ontology extraction took {} seconds'.format(te - t_syn))
        st.write('Ontology extraction took {} seconds'.format(te - t_syn))

        print('Full process took {} seconds'.format(te - ts))
        st.write('Full process took {} seconds'.format(te - ts))


        print('saving...')
        st.write('saving...')

        self.save()

        print('done:')
        st.write('done:')


        print(self.aspects)
        st.write(self.aspects)

        print(self.syn_dict)
        st.write(self.syn_dict)

        print(self.relatedness_matrix)
        st.write(self.relatedness_matrix)

        print(self.tree)
        st.write(self.tree)
        
        print(RenderTree(self.tree))
        st.write(RenderTree(self.tree))


    def save_product_representation(self,project_dir):
        f = open(directory+ project_dir +"/"+ self.product + Product.FILE_EXTENSION, 'wb')
        p = Product(self.tree, self.syn_dict)
        pickle.dump(p, f)
        f.close()

    @staticmethod
    def obtain_texts(path, col, n=None):
        print(path)
        file = pd.read_csv(path)
        file = file[~file[col].isnull()]
        if n and n < len(file.index):
            file = file.sample(frac=1).reset_index(drop=True)
            file = file.head(n)
        texts = [text for _, text in file[col].items() if not pd.isnull(text)]
        print('Obtained {} texts'.format(len(texts)))
        st.write('Obtained {} texts'.format(len(texts)))

        return texts

    def get_bert_relations(self):
        print('  select phrases for relation extraction...')
        st.write('  select phrases for relation extraction...')
        
        pair_texts = [rel for rel in map(pair_relations_for_text, self.sentences, self.ngram_phrases,
                                              itertools.repeat(self.aspects, len(self.sentences)),
                                              itertools.repeat(self.syn_dict, len(self.sentences))) if rel is not None]
        df = pd.DataFrame(pair_texts, columns=['sentText', 'relationMentions'])

        print('  extracting relations with BERT...')
        st.write('  extracting relations with BERT...')

        dataset = PairRelDataset.from_df(df)
        bert_extractor = BertRelExtractor.load_saved(self.rel_extractor_path)
        aspect_counts = np.array([self.counts[aspect] for aspect in self.aspects])
        prob_matrix, count_matrix = bert_extractor.extract_relations(len(self.aspects), self.aspect_index_map(),
                                                                     aspect_counts, dataset=dataset)

        self.relatedness_matrix = prob_matrix / aspect_counts  # scale rows by aspect counts
        return self.relatedness_matrix

    def extract_synset(self):
        for idx, aspect in enumerate(self.aspects):
            if idx == 0:
                continue
            synset = {idx}
            aspect_dependence = self.aspect_dependence(idx)
            for syn_idx in self.get_syns(aspect):
                if syn_idx < idx and syn_idx != aspect_dependence:
                    synset.add(syn_idx)
                    self.print_relations_from(aspect)
            if len(synset) > 1:
                return synset
        return None

    def get_syns(self, aspect):
        return {idx for idx, a in enumerate(self.aspects)
                if a != aspect and self.wv.relative_cosine_similarity(a, aspect) > TargetExtractor.SYNONYM_SIMILARITY}

    def aspect_index_map(self):
        return {syn: idx for idx, aspect in enumerate(self.aspects) for syn in self.syn_dict[aspect]}

    def count_nouns(self):
        nouns = itertools.chain.from_iterable(map(get_nouns, self.phrases, self.ngram_phrases))
        return Counter(nouns)

    def get_aspects(self, counter):
        # take N_ASPECTS most common terms
        term_counts = counter.most_common()[:TargetExtractor.N_ASPECTS]
        terms = [term for term, count in term_counts]

        print('  preparing entity texts for BERT...')
        st.write('  preparing entity texts for BERT...')

        entity_texts = [t for t in map(entity_mentions_in_text, self.sentences, self.phrases, self.ngram_phrases,
                                            itertools.repeat(terms, len(self.sentences)))
                        if t is not None]
        df = pd.DataFrame(entity_texts, columns=['sentText', 'entityMentions'])

        print('  extracting entities with BERT...')
        st.write('  extracting entities with BERT...')

        dataset = EntityDataset.from_df(df)
        entity_extractor = BertEntityExtractor.load_saved(self.entity_extractor_path)
        probs = entity_extractor.extract_entity_probabilities(terms, dataset=dataset)

        aspects = [term for term in terms if probs[term] is not None and probs[term] >= TargetExtractor.ENTITY_PROB_THRESHOLD]

        # bring product to front of list
        if self.product in aspects:
            aspects.remove(self.product)
        aspects.insert(0, self.product)

        return aspects, {term: count for term, count in term_counts if term in aspects}

    def get_word2vec_model(self, size, window, min_count):
        print("phrases",str(self.ngram_phrases))
        model = Word2Vec(self.ngram_phrases, size=size, window=window, min_count=min_count).wv
        return model

    def save(self):
        f = open(directory+'/content/{}_extractor_f.pickle'.format(self.product), 'wb')
        pickle.dump(self, f)
        f.close()

    @staticmethod
    def load_saved(product):
        f = open(directory+'/content/{}_extractor_f.pickle'.format(product), 'rb')
        extractor = pickle.load(f)
        f.close()
        return extractor

    def closest_relative_for_idx(self, idx):
        return np.argmax(self.relatedness_matrix[idx])

    def aspect_dependence(self, idx):
        row = self.relatedness_matrix[idx]
        max_idx1, max_idx2 = row[1:].argsort()[-2:][::-1] + 1
        if max_idx1 < idx and row[max_idx1] >= row[max_idx2] * TargetExtractor.SUBFEATURE_MULT:
            return max_idx1
        else:
            return None

    def get_product_tree(self):
        root = Node(self.aspects[0])
        root.idx = 0

        for idx in range(1, len(self.aspects)):  # for each feature in order from highest to lowest count
            dep_idx = self.aspect_dependence(idx)
            if dep_idx is not None:
                parent = next(n for n in root.descendants if n.idx == dep_idx)
            else:
                parent = root
            node = Node(self.aspects[idx], parent=parent)
            node.idx = idx

        self.node_map = {n.idx: n for n in (root,) + root.descendants}

        return root

    def aspect_dependence_with_strength(self, idx):
        row = self.relatedness_matrix[idx]
        max_idx1, max_idx2 = row[1:].argsort()[-2:][::-1] + 1
        if (row[max_idx1] >= row[max_idx2] * TargetExtractor.SUBFEATURE_MULT and
                self.counts[self.aspects[max_idx1]] * TargetExtractor.COUNT_MULT > self.counts[self.aspects[idx]]):
            return max_idx1, row[max_idx1]
        else:
            return None

    def aspect_dependence_with_strength2(self, idx):
        row = self.relatedness_matrix[idx]
        max_idx1 = np.argmax(row[1:]) + 1
        if (row[max_idx1] >= row[0] and
                self.counts[self.aspects[max_idx1]] * TargetExtractor.COUNT_MULT > self.counts[self.aspects[idx]]):
            return max_idx1, row[max_idx1]
        else:
            return None

    def get_product_tree2(self):
        root = Node(self.aspects[0])
        root.idx = 0

        deps = {idx: self.aspect_dependence_with_strength2(idx) for idx in range(1, len(self.aspects))}

        for no_dep_idx in {idx for idx, dep in deps.items() if dep is None}:
            node = Node(self.aspects[no_dep_idx], parent=root)
            node.idx = no_dep_idx
            del deps[no_dep_idx]

        sorted_deps = sorted(deps.items(), key=lambda x: x[1][1], reverse=True)

        for idx, (dep, _) in sorted_deps:
            n = next((n for n in root.descendants if n.idx == idx), None)
            dep_n = next((n for n in root.descendants if n.idx == dep), None)
            if dep_n is None:
                dep_n = Node(self.aspects[dep], parent=root)
                dep_n.idx = dep

            if n is not None:
                if dep_n not in n.descendants and dep_n.depth + (max(c.depth for c in n.descendants) if n.descendants else 0) <= TargetExtractor.MAX_DEPTH:
                    n.parent = dep_n
            else:
                if dep_n.depth < TargetExtractor.MAX_DEPTH:
                    n = Node(self.aspects[idx], parent=dep_n)
                else:
                    n = Node(self.aspects[idx], parent=root)
                n.idx = idx

        return root

    def get_product_tree3(self):
        root = Node(self.aspects[0])
        root.idx = 0

        deps = {idx: self.aspect_dependence_with_strength2(idx) for idx in range(1, len(self.aspects))}

        for no_dep_idx in {idx for idx, dep in deps.items() if dep is None}:
            node = Node(self.aspects[no_dep_idx], parent=root)
            node.idx = no_dep_idx
            del deps[no_dep_idx]

        sorted_deps = sorted(deps.items(), key=lambda x: x[1][1], reverse=True)

        for idx, (dep_idx, _) in sorted_deps:
            if any(n for n in root.descendants if n.idx == idx):
                continue

            dep_n = next((n for n in root.descendants if n.idx == dep_idx), None)
            if dep_n:
                if dep_n.depth < 2:
                    n = Node(self.aspects[idx], parent=dep_n)
                else:
                    n = Node(self.aspects[idx], parent=dep_n.parent)
            else:
                dep_n = Node(self.aspects[dep_idx], parent=root)
                dep_n.idx = dep_idx
                n = Node(self.aspects[idx], parent=dep_n)
            n.idx = idx

        return root

    @staticmethod
    def print_relations(target_indices, dep_matrix, targets):
        idx_pairs = {frozenset((idx1, idx2)) for idx1 in target_indices for idx2 in target_indices if idx1 != idx2}
        for idx1, idx2 in idx_pairs:
            t1 = targets[idx1]
            t2 = targets[idx2]
            print('{} {:.4f} {}'.format(t1, dep_matrix[idx1][idx2], t2))
            print('{} {:.4f} {}'.format(' ' * len(t1), dep_matrix[idx2][idx1], ' ' * len(t2)))
            print('')
            st.write('{} {:.4f} {}'.format(t1, dep_matrix[idx1][idx2], t2))
            st.write('{} {:.4f} {}'.format(' ' * len(t1), dep_matrix[idx2][idx1], ' ' * len(t2)))
            st.write('')

    def print_relations_from(self, aspect):
        idx = self.aspects.index(aspect)
        rels = self.relatedness_matrix[idx].copy()
        print('  relations from {}:'.format(aspect))
        st.write('  relations from {}:'.format(aspect))

        for rel_idx in sorted(range(len(self.aspects)), key=lambda i: rels[i], reverse=True)[:20]:
            print('    {:.4f}'.format(rels[rel_idx]), self.aspects[rel_idx])
            st.write('    {:.4f}'.format(rels[rel_idx]), self.aspects[rel_idx])


    def get_syn_dict(self):
        all_pairs = {frozenset((t1, t2)) for t1 in self.aspects for t2 in self.aspects if t1 != t2}
        syn_pairs = {frozenset((t1, t2)) for t1, t2 in all_pairs if self.are_syns(t1, t2)}
        synset = Synset(self.aspects, syn_pairs, self.product)
        return synset.get_dict(self.counts)

    def are_syns(self, t1, t2):
        if wnl.lemmatize(t1) == wnl.lemmatize(t2):
            return True
        try:
          if self.product in [t1, t2]:
            print(t1,t2,self.wv.wv.n_similarity([t1], [t2]))
            st.write(t1,t2,self.wv.wv.n_similarity([t1], [t2]))

            return (self.wv.wv.n_similarity([t1], [t2]) >= TargetExtractor.SYNONYM_SIMILARITY_PRODUCT or
                    self.wv.wv.n_similarity([t2], [t1]) >= TargetExtractor.SYNONYM_SIMILARITY_PRODUCT)
          else:
            print(t1,t2)
            st.write(t1,t2)

            sim_sum = self.wv.wv.n_similarity([t1], [t2]) + self.wv.wv.n_similarity([t2], [t1])
            return sim_sum >= TargetExtractor.SYNONYM_SIMILARITY
        except:
            return False

class Synset:

    def __init__(self, aspects, syn_pairs, product):
        self.vocab = aspects
        self.syn_pairs = syn_pairs
        self.product = product

    def get_dict(self, counts):
        groups = self.get_groups()
        return {max(group, key=counts.get) if self.product not in group else self.product: group for group in groups}

    def get_groups(self):
        groups = []
        for w1, w2 in self.syn_pairs:
            if not Synset.join_groups(w1, w2, groups):
                groups.append({w1, w2})
        for word in self.vocab:
            if not Synset.group_for(word, groups):
                groups.append({word})
        return groups

    @staticmethod
    def join_groups(w1, w2, groups):
        g1 = Synset.group_for(w1, groups)
        g2 = Synset.group_for(w2, groups)
        if g1 and g2 and g1 == g2:
            return True
        if g1:
            groups.remove(g1)
        if g2:
            groups.remove(g2)
        g1 = g1 if g1 else {w1}
        g2 = g2 if g2 else {w2}
        groups.append(g1.union(g2))
        return True

    @staticmethod
    def group_for(w, groups):
        for group in groups:
            if w in group:
                return group
        return None