Duplicate from veneta/preprocessing

Co-authored-by: Veneta Kireva <veneta@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,13 @@
+---
+title: Preprocessing
+emoji: 🔥
+colorFrom: gray
+colorTo: purple
+sdk: gradio
+sdk_version: 3.32.0
+app_file: app.py
+pinned: false
+duplicated_from: veneta/preprocessing
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,87 @@
+import pandas as pd
+import gradio as gr
+
+from helper_funcs import functions, INPUT_FILE_TYPE, OUTPUT_FILE_TYPE
+
+
+def run_function(selected_function, file_obj, input_column, output_column, output_type):
+    if 'json' in file_obj.name.lower():
+        df = pd.read_json(file_obj.name)
+    if any([x in file_obj.name.lower() for x in ['csv', 'txt']]):
+        df = pd.read_csv(file_obj.name)
+
+    output_file = 'result' + output_type
+
+    if input_column not in list(df.columns):
+        raise gr.Error("Input column name: such column does not exist in dataframe!")
+
+    return functions[selected_function](df, input_column, output_column, output_file)
+
+
+app = gr.Blocks()
+
+with app:
+    gr.Markdown(
+        """
+        # Instructions
+        1. Upload CSV file to process.
+        2. Enter the name of the column containing the values to process.
+        3. Enter the name of the column in which to save the output.
+        4. Select function to process the data.
+        5. Click Process.
+        """
+    )
+    with gr.Row():
+        with gr.Column():
+            gr.Markdown(
+                """
+                # Input
+                """
+            )
+            file_obj = gr.File(
+                label="Input File",
+                file_count="single",
+                file_types=INPUT_FILE_TYPE
+            )
+            input_column = gr.Textbox(
+                label="Input column name",
+                info="Please enter the name of the column you want to process (as it appears in your dataset)",
+                lines=1,
+            )
+            output_column = gr.Textbox(
+                label="Output column name",
+                info="Please enter the name of the column you want to save the result to",
+                lines=1,
+            )
+            selected_function = gr.Dropdown(
+                list(functions.keys()),
+                label="Select processing",
+                info=""
+            )
+            output_type = gr.Dropdown(
+                list(OUTPUT_FILE_TYPE),
+                label="Select the output file type",
+                info=""
+            )
+
+        with gr.Column():
+            gr.Markdown(
+                """
+                # Output
+                """
+            )
+            output_dataframe = gr.Dataframe(
+                label="Output Data"
+            )
+            output_csv = gr.File(
+                label="Output File",
+                file_types=OUTPUT_FILE_TYPE
+            )
+
+    gr.Button("Process").click(
+        run_function,
+        inputs=[selected_function, file_obj, input_column, output_column, output_type],
+        outputs=[output_dataframe, output_csv]
+    )
+
+app.launch()

helper_funcs.py

@@ -0,0 +1,175 @@
+import ast
+import warnings
+
+import classla
+import pandas as pd
+
+classla.download('bg')
+classla_nlp = classla.Pipeline('bg')
+
+warnings.filterwarnings('ignore')
+
+INPUT_FILE_TYPE = ['.csv', '.json', '.txt']
+OUTPUT_FILE_TYPE = ['.csv', '.xlsx']
+
+
+def to_output(df, output_file):
+    if 'xlsx' in output_file:
+        df.to_excel(output_file, index=False)
+    if 'csv' in output_file:
+        df.to_csv(output_file, index=False)
+    return df.head(10), output_file
+
+
+def remove_duplicates(df, input_column, output_column, output_file):
+    df.drop_duplicates(subset=[input_column], inplace=True)
+    return to_output(df, output_file)
+
+
+def remove_links(df, input_column, output_column, output_file):
+    link_regex = r'(https?:\/\/(?:www\.|(?!www))[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]{2,}|www\.[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]{2,}|https?:\/\/(?:www\.|(?!www))[a-zA-Z0-9]+\.[^\s]{2,}|www\.[a-zA-Z0-9]+\.[^\s]{2,})'
+
+    if input_column != output_column:
+        df[output_column] = df[input_column]
+
+    df_links = df[df[output_column].str.contains(link_regex, regex=True, na=False)]
+    df = pd.concat([df, df_links, df_links]).drop_duplicates(keep=False)
+    df_links[output_column] = df_links[output_column].str.replace(link_regex, '', regex=True)
+    df = pd.concat([df, df_links])
+    return to_output(df, output_file)
+
+
+def remove_emails(df, input_column, output_column, output_file):
+    email_regex = r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b'
+
+    if input_column != output_column:
+        df[output_column] = df[input_column]
+
+    df_email = df[df[output_column].str.contains(email_regex, regex=True, na=False)]
+    df = pd.concat([df, df_email, df_email]).drop_duplicates(keep=False)
+    df_email[output_column] = df_email[output_column].str.replace(email_regex, '<EMAIL>', regex=True)
+    df = pd.concat([df, df_email])
+    return to_output(df, output_file)
+
+
+def remove_phones(df, input_column, output_column, output_file):
+    phone_regex = r'(?<!\S)(\+|0)[1-9][0-9 \-\(\)]{7,32}'
+
+    if input_column != output_column:
+        df[output_column] = df[input_column]
+
+    df_phone = df[df[output_column].str.contains(phone_regex, regex=True, na=False)]
+    df = pd.concat([df, df_phone, df_phone]).drop_duplicates(keep=False)
+    df_phone[output_column] = df_phone[output_column].str.replace(phone_regex, '<PHONE>', regex=True)
+    df = pd.concat([df, df_phone])
+    return to_output(df, output_file)
+
+
+def get_sentences(df, input_column, output_column, output_file):
+    def split_sentences(input_list=None):
+        if input_list is None:
+            input_list = []
+        temp = []
+        res = []
+        for idx in range(len(input_list)):
+            if input_list[idx][0] <= input_list[idx - 1][0]:
+                res.append(" ".join([x[1] for x in temp]))
+                temp = []
+            temp.append(input_list[idx])
+        res.append(" ".join([x[1] for x in temp]))
+        res.pop(0)  # first element is always [], so it is removed
+        return res
+
+    if input_column != output_column:
+        df[output_column] = df[input_column]
+
+    sentences_separated = []
+
+    for index in range(df.shape[0]):
+        row_nlp = classla_nlp(df.iloc[index][input_column])
+        row_result_upos = row_nlp.get('upos')
+        row_id = row_nlp.get('id')
+        row_text = row_nlp.get('text')
+
+        row_result = [[row_id[x], row_text[x]] for x in range(len(row_id)) if
+                      row_result_upos[x] != 'PUNCT']  # filter punctuation
+        row_result = split_sentences(input_list=row_result)  # splitting messages
+
+        sentences_separated.append(row_result)
+
+    df[output_column] = sentences_separated
+    return to_output(df, output_file)
+
+
+def get_classla_ner(df, input_column, output_column, output_file):
+    def sentence_classla(sentence_list):
+        result_ner = list()
+        for sentence in sentence_list:
+            current_nlp = classla_nlp(sentence).to_dict()
+            sentence_ner = [word['ner'] for word in current_nlp[0][0]]
+            result_ner.append(sentence_ner)
+        return result_ner
+
+    df[input_column] = df[input_column].apply(lambda x: ast.literal_eval(x))
+
+    if input_column != output_column:
+        df[output_column] = df[input_column]
+
+    clarin_classla_result = [sentence_classla(df.iloc[index][input_column]) for index in range(df.shape[0])]
+    df[output_column] = [clarin_classla_result[index] for index in range(df.shape[0])]
+    return to_output(df, output_file)
+
+
+def get_classla_all(df, input_column, output_column, output_file):
+    def sentence_classla(sentence_list):
+        result_all = list()
+        for sentence in sentence_list:
+            current_nlp = classla_nlp(sentence).to_dict()
+            result_all.append(current_nlp)
+        return result_all
+
+    df[input_column] = df[input_column].apply(lambda x: ast.literal_eval(x))
+
+    if input_column != output_column:
+        df[output_column] = df[input_column]
+
+    clarin_classla_result = [sentence_classla(df.iloc[index][input_column]) for index in range(df.shape[0])]
+    df[output_column] = [clarin_classla_result[index] for index in range(df.shape[0])]
+    return to_output(df, output_file)
+
+
+def run_all(df, input_column, output_column, output_file):
+    def load_file(output_file):
+        df = None
+        if 'xlsx' in output_file:
+            df = pd.read_excel(output_file)
+        if 'csv' in output_file:
+            df = pd.read_csv(output_file)
+        return df
+    
+    _, _ = remove_duplicates(df, input_column, output_column, output_file)
+    df = load_file(output_file)
+    _, _ = remove_links(df, input_column, 'removed_links', output_file)
+    df = load_file(output_file)
+    _, _ = remove_emails(df, 'removed_links', 'removed_emails', output_file)
+    df = load_file(output_file)
+    _, _ = remove_phones(df, 'removed_emails', 'removed_phones', output_file)
+    df = load_file(output_file)
+    _, _ = get_sentences(df, 'removed_phones', 'extracted_sentences', output_file)
+    df = load_file(output_file)
+    _, _ = get_classla_all(df, 'extracted_sentences', 'classla_all', output_file)
+    df = load_file(output_file)
+    _, _ = get_classla_ner(df, 'extracted_sentences', 'classla_ner', output_file)
+    return df.head(10), output_file
+
+
+functions = {
+    'remove duplicate rows': remove_duplicates,
+    'remove links': remove_links,
+    'remove e-mails': remove_emails,
+    'remove phone numbers': remove_phones,
+    'separate sentences': get_sentences,
+    'Classla NER': get_classla_ner,
+    'Classla full result': get_classla_all,
+    'run all': run_all
+}