Delete parsing.py

parsing.py

@@ -1,245 +0,0 @@
-# -*- coding: utf-8 -*-
-"""parsing.ipynb
-
-Automatically generated by Colaboratory.
-
-Original file is located at
-    https://colab.research.google.com/drive/1thvkAz498jADcaVirJG91V-3-XBhdkq1
-"""
-
-import requests
-from bs4 import BeautifulSoup
-
-import re
-import os
-
-import pandas as pd
-import numpy as np
-
-from tqdm import tqdm
-
-def get_transcripts_from_url(url):
-    # Send a GET request to the URL and retrieve the webpage content
-    response = requests.get(url)
-
-    # Parse the HTML content using Beautiful Soup
-    soup = BeautifulSoup(response.content, 'html.parser')
-
-    # Find elements by tag name
-    titles = soup.find_all('li')
-
-    # names for series
-    transcript_paths = []
-    # Extract text from elements
-    for title in titles:
-        a = title.find('a')
-
-        path = a.get("href")
-
-        transcript_paths.append("https://fangj.github.io/friends/" + path)
-
-    return transcript_paths
-
-def get_text_from_html(url):
-    path = url
-    response = requests.get(path)
-    html_content = response.text
-
-    # Parse HTML content
-    soup = BeautifulSoup(html_content, 'html.parser')
-
-    transcript = soup.find_all('p')
-
-    transcript_name = path.split("/")[-1].replace(".html", ".txt")
-
-    with open(os.path.join("friends_raw_scripts", transcript_name), 'w', encoding='utf-8') as file:
-        text = soup.get_text(strip=False).lower().replace("'", ""). replace('"', "").replace("\xa0", "")
-        file.write(text + "\n")
-
-    return transcript_name
-
-def clean_and_write_text(transcript_name):
-
-    char = []
-    texts = []
-    flag = None
-    pattern = re.compile(r'\b\w+:')
-
-    with open(os.path.join("friends_raw_scripts", transcript_name), 'r', encoding='utf-8') as file:
-        final_transcript = file.readlines()
-
-    skip_lines = 10
-    pattern = re.compile(r'\b\w+:')
-    scene_words = ["commercial break", "closing credits", "opening credits", "end"]
-    for ind in range(1, len(final_transcript) - 1):
-        final_list = []
-
-        pre_line = final_transcript[ind - 1].strip()
-        cur_line = final_transcript[ind].strip()
-        next_line = final_transcript[ind + 1].strip()
-
-        next_condition = re.sub(r"\([^()]*\)|\[[^\[\]]*\]", '', next_line).strip()
-        cur_conditon = re.sub(r"\([^()]*\)|\[[^\[\]]*\]", '', cur_line).strip()
-
-        if sum([bool(pre_line), bool(cur_line), bool(next_line)]) == 1:
-            continue
-
-        elif cur_line in scene_words:
-            continue
-
-        elif "by:" in cur_line or "note:" in cur_line:
-            continue
-
-        elif "[" in cur_line or "]" in cur_line:
-            continue
-
-        elif not cur_conditon:
-            continue
-
-        elif pattern.search(cur_line) and flag == None:
-            name, text = cur_line.split(":", maxsplit=1)
-            char.append(name)
-            text = re.sub(r'\([^)]*\)', '', text)
-            text = text.strip()
-            flag = "char"
-
-            if pattern.search(next_line) or not next_condition or next_line in scene_words or "[" in next_line:
-                texts.append(text)
-                flag = None
-
-                if len(char) != len(texts):
-                    print(ind)
-                    print(char[-1], texts[-1])
-
-        elif cur_line and flag == 'char':
-            text += " " + cur_line
-            if pattern.search(next_line) or not next_condition or next_line in scene_words or "[" in next_line:
-                text = re.sub(r"\([^()]*\)|\[[^\[\]]*\]", '', text).strip()
-                texts.append(text)
-                flag = None
-
-                if len(char) != len(texts):
-                    print(ind)
-                    print(char[-1], texts[-1])
-
-    new_name = "pre_" + transcript_name
-    with open(os.path.join("friends_preprocessed_scripts", new_name), 'w', encoding='utf-8') as file:
-        for c, d in zip(char, texts):
-            file.write(f"{c}: {d}\n")
-
-raw_texts_exists = False  # change on False to download transcripts and preprocess them
-# parse data from website to get txt transcripts
-transcript_paths = get_transcripts_from_url("https://fangj.github.io/friends/")
-
-transcript_paths[:10]
-
-os.makedirs("friends_preprocessed_scripts", exist_ok=True)
-os.makedirs("friends_raw_scripts", exist_ok=True)
-
-# define list with certain scripts from south park
-# dir_list = [file for file in os.listdir("./raw_scripts")]
-if not raw_texts_exists:
-    print("Parse all scripts from this website https://fangj.github.io/friends/")
-    for path in tqdm(transcript_paths, desc='Total'):
-        transcript_name = get_text_from_html(path)
-        clean_and_write_text(transcript_name)
-
-dir_list = [file for file in os.listdir("./friends_preprocessed_scripts")]
-
-def df_scripts(path):
-    """function take preprocessed_script.txt from dir and create dataframes"""
-    chars = []
-    texts = []
-
-    with open(os.path.join("friends_preprocessed_scripts", path), 'r', encoding="utf-8") as file:
-        for line in file:
-            char, text = line.split(":", 1)
-            chars.append(char)
-            texts.append(text.strip().lower())
-
-    df_name = path.replace("prep_SP_", "df_").replace(".txt", ".csv")
-    df = pd.DataFrame({'Characters': chars, 'Dialogs': texts})
-    df.to_csv(os.path.join("dataframes", "friends", df_name), index=False)
-
-os.makedirs("dataframes/friends", exist_ok=True)
-
-for preprocessed_script in dir_list:
-    df_scripts(preprocessed_script)
-
-def collect_df():
-    """function concatenate dataframes in one single dataframe"""
-    dfs = []
-    for file in os.listdir("dataframes/friends"):
-        dfs.append(pd.read_csv(os.path.join("dataframes", "friends", file)))
-    df = pd.concat(dfs, ignore_index=True).dropna().reset_index(drop=True)
-    # print(df["Characters"].value_counts()[:10])
-    return df
-
-"""### Which most frequent characters we can meet in the movie"""
-
-def form_df(df, char):
-    # get indices where character is favorite_character
-    favorite_character_df = df[df.Characters == char]  # .dropna()
-    favorite_character_ind = favorite_character_df.index.tolist()
-
-    # get indices where speech could be to favorite charecter
-    text_to_favorite_character_ind = (np.array(favorite_character_ind) - 1).tolist()
-
-    # form datasets with favorite charecter's dialogs and possible dialogs to favorite charecter
-    # favorite_character_dialog = df.iloc[favorite_character_ind] restore
-    favorite_character_dialog = df[df.index.isin(favorite_character_ind)]
-    # text_to_favorite_character = df.iloc[text_to_favorite_character_ind]  restore# .dropna(subset=["Dialogs"])
-    text_to_favorite_character = df[df.index.isin(text_to_favorite_character_ind)]
-    # remove from text to cartman rows where speak Cartman
-    text_to_favorite_character = text_to_favorite_character[text_to_favorite_character["Characters"] != char]
-
-    # save data for debugging. Uncomment if necessary
-    # favorite_character_dialog.to_csv("test_favotite.csv", index=favorite_character_ind)
-    # text_to_favorite_character.to_csv("test_question.csv", index=text_to_favorite_character_ind)
-
-    # find in dialog_to_cartman lines with char "?"
-    # mask = text_to_favorite_character['Dialogs'].str.contains('\?')
-
-    # question_to_favorite_character = text_to_favorite_character[mask]
-    question_to_favorite_character = text_to_favorite_character # delete if return question char ?
-    # save data for debugging. Uncomment if necessary
-    # question_to_favorite_character.to_csv("question_to_favorite_character.csv")
-
-    question_to_favorite_character_ind = question_to_favorite_character.index.tolist()
-    true_answers_ind = (np.array(question_to_favorite_character_ind) + 1).tolist()
-    # favorite_character_answer = favorite_character_dialog.loc[true_answers_ind]
-    favorite_character_answer = favorite_character_dialog[favorite_character_dialog.index.isin(true_answers_ind)]
-    # save data for debugging. Uncomment if necessary
-    # favorite_character_answer.to_csv("favorite_character_answer.csv")
-
-    # change name of columns for final dataframe
-    question_to_favorite_character = question_to_favorite_character.rename(
-        columns={"Characters": "questioner", "Dialogs": "question"})
-    favorite_character_answer = favorite_character_answer.rename(columns={"Characters": char, "Dialogs": "answer"})
-
-    question_to_favorite_character.reset_index(inplace=True, drop=True)
-    favorite_character_answer.reset_index(inplace=True, drop=True)
-
-    df = pd.concat([question_to_favorite_character, favorite_character_answer], axis=1)
-
-    return df
-
-"""## Choose your favorite character"""
-
-# concatenate data in one single dataframe
-df = collect_df()
-df.to_csv("full_trancscripts.csv", index=False)
-
-# form the final dataset for tf-idf / word2vec, which no need labels between strings
-characters = ["rachel", "ross", "chandler", "monica", "joey", "phoebe"]
-
-print()
-for char in characters:
-    print(f"Prepare data for {char} -> {char}_friends.csv")
-    df_char = form_df(df, char)
-    # create final dataframe
-    df_char.to_csv(char + "_friends.csv", index=False)
-
-print("scripts created")
-
-