Update app.py
Browse files
app.py
CHANGED
|
@@ -5,71 +5,18 @@ from gensim.models import Word2Vec
|
|
| 5 |
import gradio as gr
|
| 6 |
from sklearn.decomposition import PCA
|
| 7 |
import plotly.graph_objects as go
|
| 8 |
-
import nltk
|
| 9 |
-
from nltk.tokenize import word_tokenize
|
| 10 |
-
from nltk.corpus import stopwords
|
| 11 |
-
from nltk.stem import PorterStemmer, WordNetLemmatizer
|
| 12 |
-
from nltk.tag import pos_tag
|
| 13 |
-
|
| 14 |
-
from docs import NOVEL_TEXT
|
| 15 |
-
|
| 16 |
-
def download_nltk_library():
|
| 17 |
-
try:
|
| 18 |
-
nltk.download('punkt')
|
| 19 |
-
nltk.download('stopwords')
|
| 20 |
-
nltk.download('wordnet')
|
| 21 |
-
nltk.download('averaged_perceptron_tagger')
|
| 22 |
-
nltk.download('punkt_tab')
|
| 23 |
-
return True
|
| 24 |
-
except:
|
| 25 |
-
return False
|
| 26 |
-
|
| 27 |
-
# Function to process each sentence
|
| 28 |
-
def process_text(text):
|
| 29 |
-
|
| 30 |
-
lemmatizer = WordNetLemmatizer()
|
| 31 |
-
stop_words = set(stopwords.words('english'))
|
| 32 |
-
|
| 33 |
-
# Tokenization
|
| 34 |
-
tokens = word_tokenize(text.lower())
|
| 35 |
-
|
| 36 |
-
# Remove stop words and apply lemmatization
|
| 37 |
-
processed_tokens = [
|
| 38 |
-
lemmatizer.lemmatize(token)
|
| 39 |
-
for token in tokens if token.isalnum() and token not in stop_words
|
| 40 |
-
]
|
| 41 |
|
| 42 |
-
return processed_tokens
|
| 43 |
|
| 44 |
# Word2Vec ๋ชจ๋ธ ํ์ต ํจ์
|
| 45 |
def train_word2vec(sentences):
|
| 46 |
-
model = Word2Vec(sentences, vector_size=100, window=
|
|
|
|
| 47 |
return model
|
| 48 |
|
| 49 |
-
# def preprocess_text(file_path):
|
| 50 |
-
# with open(file_path, 'r', encoding='utf-8') as file:
|
| 51 |
-
# text = file.read()
|
| 52 |
-
|
| 53 |
-
# # ํ ํฐํ ๋ฐ ํ์ฌ ํ๊น
|
| 54 |
-
# tokens = word_tokenize(text)
|
| 55 |
-
# tagged = pos_tag(tokens)
|
| 56 |
-
|
| 57 |
-
# # ๋ช
์ฌ๋ง ์ถ์ถ (NN, NNS, NNP, NNPS)
|
| 58 |
-
# nouns = [word.lower() for word, pos in tagged if pos.startswith('NN')]
|
| 59 |
-
|
| 60 |
-
# # ์ค๋ณต ์ ๊ฑฐ ๋ฐ ์ ๋ ฌ
|
| 61 |
-
# unique_nouns = sorted(set(nouns))
|
| 62 |
-
|
| 63 |
-
# # ๊ฐ๋จํ ๋ฌธ์ฅ ์์ฑ (๊ฐ ๋ช
์ฌ๋ฅผ ๊ฐ๋ณ ๋ฌธ์ฅ์ผ๋ก ์ทจ๊ธ)
|
| 64 |
-
# sentences = [[noun] for noun in unique_nouns]
|
| 65 |
-
|
| 66 |
-
# return sentences, unique_nouns
|
| 67 |
-
|
| 68 |
def apply_pca(word_vectors):
|
| 69 |
pca = PCA(n_components=3)
|
| 70 |
return pca.fit_transform(word_vectors)
|
| 71 |
|
| 72 |
-
# def process_text(file_path, target_word):
|
| 73 |
|
| 74 |
def get_unique(model):
|
| 75 |
vocablist1=list(model.wv.index_to_key)
|
|
@@ -89,7 +36,8 @@ def train_model(sentence):
|
|
| 89 |
return model, unique_words
|
| 90 |
|
| 91 |
def process_model(target_word):
|
| 92 |
-
|
|
|
|
| 93 |
# Word2Vec ๋ชจ๋ธ ๋ก๋
|
| 94 |
model = Word2Vec.load("word2vec.model")
|
| 95 |
unique_words = get_unique(model)
|
|
@@ -122,6 +70,7 @@ def process_model(target_word):
|
|
| 122 |
for idx in dissimilar_word_indices:
|
| 123 |
colors[idx] = 'rgba(138, 43, 226, 0.8)' # ๊ฐ์ฅ ๋จผ ๋จ์ด๋ค์ ๋ณด๋ผ์์ผ๋ก ํ์
|
| 124 |
|
|
|
|
| 125 |
# Plotly๋ฅผ ์ฌ์ฉํ 3D ์ฐ์ ๋ ์์ฑ
|
| 126 |
fig = go.Figure(data=[go.Scatter3d(
|
| 127 |
x=word_vectors_3d[:, 0],
|
|
@@ -143,76 +92,60 @@ def process_model(target_word):
|
|
| 143 |
yaxis_title="Y",
|
| 144 |
zaxis_title="Z"
|
| 145 |
),
|
| 146 |
-
width=
|
| 147 |
-
height=
|
| 148 |
)
|
| 149 |
|
| 150 |
# ๊ฐ์ฅ ๊ฐ๊น์ด ๋จ์ด 10๊ฐ ๋ชฉ๋ก ์์ฑ
|
| 151 |
similar_words_text = ""
|
| 152 |
if target_word in model.wv:
|
| 153 |
-
similar_words_text = "
|
| 154 |
|
| 155 |
-
|
| 156 |
if target_word in model.wv:
|
| 157 |
-
dissimilar_words_text = "
|
| 158 |
|
| 159 |
return fig, similar_words_text, dissimilar_words_text
|
| 160 |
|
| 161 |
-
def change_button_state_true():
|
| 162 |
-
# If the first button is clicked, enable or disable the second button based on its state
|
| 163 |
-
return gr.update(interactive=True)
|
| 164 |
-
|
| 165 |
-
def change_button_state_false():
|
| 166 |
-
# If the first button is clicked, enable or disable the second button based on its state
|
| 167 |
-
return gr.update(interactive=False)
|
| 168 |
|
| 169 |
|
| 170 |
# Gradio ์ธํฐํ์ด์ค ์์
|
| 171 |
with gr.Blocks(css=".plot-box {width: 70%; height: 500px;}") as iface:
|
| 172 |
gr.Markdown("# Word Embedding 3D ์๊ฐํ")
|
| 173 |
-
gr.Markdown("
|
| 174 |
-
|
| 175 |
-
download_nltk_library()
|
| 176 |
|
| 177 |
with gr.Row():
|
| 178 |
-
word_input = gr.Textbox(label="**๊ฐ์กฐํ ๋จ์ด ์
๋ ฅ**", elem_id="input-box", placeholder="๋จ์ด๋ฅผ ์
๋ ฅํ์ธ์", lines=1, interactive=False)
|
| 179 |
-
with gr.Column(scale=1):
|
| 180 |
# ์ฌ์ฉ์ ์
๋ ฅ ๋ฐ์ค๋ฅผ ๊ฐ์กฐํ๊ธฐ ์ํด ์คํ์ผ์ ๋ณ๊ฒฝ
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
|
|
|
| 184 |
|
| 185 |
with gr.Row():
|
| 186 |
# ์๊ฐํ ํ๋ฉด์ ํฌ๊ธฐ๋ฅผ CSS๋ก ์ฆ๊ฐ
|
| 187 |
plot_output = gr.Plot(label="Word Embedding 3D ์๊ฐํ", elem_id="plot-box")
|
| 188 |
|
| 189 |
with gr.Column(scale=0.3): # ์ปฌ๋ผ์ ๋๋น๋ฅผ ์ค์ด๊ธฐ ์ํด scale ๊ฐ์ ๋ฎ์ถค
|
| 190 |
-
similar_words_output = gr.Textbox(label="
|
| 191 |
-
dissimilar_words_output = gr.Textbox(label="
|
|
|
|
| 192 |
|
| 193 |
-
load_btn.click(
|
| 194 |
-
fn=process_model,
|
| 195 |
-
inputs=[word_input],
|
| 196 |
-
outputs=[plot_output, similar_words_output, dissimilar_words_output]
|
| 197 |
-
)
|
| 198 |
-
load_btn.click(
|
| 199 |
-
fn=change_button_state_true,
|
| 200 |
-
outputs=submit_btn
|
| 201 |
-
)
|
| 202 |
-
load_btn.click(
|
| 203 |
-
fn=change_button_state_true,
|
| 204 |
-
outputs=word_input
|
| 205 |
-
)
|
| 206 |
submit_btn.click(
|
| 207 |
-
fn=
|
|
|
|
| 208 |
inputs=[word_input],
|
| 209 |
-
outputs=[plot_output, similar_words_output, dissimilar_words_output]
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
fn=change_button_state_false,
|
| 213 |
-
outputs=load_btn
|
| 214 |
-
)
|
| 215 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 216 |
|
| 217 |
if __name__ == "__main__":
|
| 218 |
-
iface.launch()
|
|
|
|
| 5 |
import gradio as gr
|
| 6 |
from sklearn.decomposition import PCA
|
| 7 |
import plotly.graph_objects as go
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
|
|
|
|
| 9 |
|
| 10 |
# Word2Vec ๋ชจ๋ธ ํ์ต ํจ์
|
| 11 |
def train_word2vec(sentences):
|
| 12 |
+
# model = Word2Vec(sentences, vector_size=100, window=4, min_count=6, workers=4, sg=0, epochs=100)
|
| 13 |
+
model = Word2Vec(sentences, vector_size=50, window=4, min_count=1, sg=0, epochs=100)
|
| 14 |
return model
|
| 15 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
def apply_pca(word_vectors):
|
| 17 |
pca = PCA(n_components=3)
|
| 18 |
return pca.fit_transform(word_vectors)
|
| 19 |
|
|
|
|
| 20 |
|
| 21 |
def get_unique(model):
|
| 22 |
vocablist1=list(model.wv.index_to_key)
|
|
|
|
| 36 |
return model, unique_words
|
| 37 |
|
| 38 |
def process_model(target_word):
|
| 39 |
+
target_word =target_word.lower() #################
|
| 40 |
+
|
| 41 |
# Word2Vec ๋ชจ๋ธ ๋ก๋
|
| 42 |
model = Word2Vec.load("word2vec.model")
|
| 43 |
unique_words = get_unique(model)
|
|
|
|
| 70 |
for idx in dissimilar_word_indices:
|
| 71 |
colors[idx] = 'rgba(138, 43, 226, 0.8)' # ๊ฐ์ฅ ๋จผ ๋จ์ด๋ค์ ๋ณด๋ผ์์ผ๋ก ํ์
|
| 72 |
|
| 73 |
+
|
| 74 |
# Plotly๋ฅผ ์ฌ์ฉํ 3D ์ฐ์ ๋ ์์ฑ
|
| 75 |
fig = go.Figure(data=[go.Scatter3d(
|
| 76 |
x=word_vectors_3d[:, 0],
|
|
|
|
| 92 |
yaxis_title="Y",
|
| 93 |
zaxis_title="Z"
|
| 94 |
),
|
| 95 |
+
width=1100,
|
| 96 |
+
height=900
|
| 97 |
)
|
| 98 |
|
| 99 |
# ๊ฐ์ฅ ๊ฐ๊น์ด ๋จ์ด 10๊ฐ ๋ชฉ๋ก ์์ฑ
|
| 100 |
similar_words_text = ""
|
| 101 |
if target_word in model.wv:
|
| 102 |
+
similar_words_text = "\n".join([f"{word}: {score:.4f}" for word, score in similar_words])
|
| 103 |
|
| 104 |
+
dissimlar_words_Text=""
|
| 105 |
if target_word in model.wv:
|
| 106 |
+
dissimilar_words_text = "\n".join([f"{word}: {score:.4f}" for word, score in dissimilar_words])
|
| 107 |
|
| 108 |
return fig, similar_words_text, dissimilar_words_text
|
| 109 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 110 |
|
| 111 |
|
| 112 |
# Gradio ์ธํฐํ์ด์ค ์์
|
| 113 |
with gr.Blocks(css=".plot-box {width: 70%; height: 500px;}") as iface:
|
| 114 |
gr.Markdown("# Word Embedding 3D ์๊ฐํ")
|
| 115 |
+
gr.Markdown("<Inside Out 2> ๋จ์ด ์๋ฏธ ์ง๋(์๋ฒ ๋ฉ ๋ฒกํฐ) 3D ์๊ฐํ ๋๊ตฌ")
|
|
|
|
|
|
|
| 116 |
|
| 117 |
with gr.Row():
|
|
|
|
|
|
|
| 118 |
# ์ฌ์ฉ์ ์
๋ ฅ ๋ฐ์ค๋ฅผ ๊ฐ์กฐํ๊ธฐ ์ํด ์คํ์ผ์ ๋ณ๊ฒฝ
|
| 119 |
+
with gr.Column():
|
| 120 |
+
word_input = gr.Textbox(label="**๋จ์ด ์
๋ ฅ**", elem_id="input-box", placeholder="ex. emotion, puberty, hockey, friend, anxiety, memory, ...", lines=1)
|
| 121 |
+
submit_btn = gr.Button("์ ์ถ", elem_id="submit-btn")
|
| 122 |
+
bulletin = gr.Textbox(label="์ฌ์ฉ๋ฒ ์๋ด", interactive=False, lines=4, value="1. ์์ค์ ๋์จ ๋จ์ด๋ฅผ ์
๋ ฅํ๊ณ [์ ์ถ] ๋ฒํผ์ด๋ [Enter]๋ฅผ ๋๋ฅด์ธ์ \n2. ์
๋ ฅ ๋จ์ด๋ ๋นจ๊ฐ์, ๊ฐ๊น์ด ๋จ์ด๋ค์ ์ฃผํฉ์, ๋จผ ๋จ์ด๋ค์ ๋ณด๋ผ์์ผ๋ก ๊ฐ์กฐ๋ฉ๋๋ค. \n3. <Error>๊ฐ ๋ํ๋๋ ๊ฒฝ์ฐ, ๋ค๋ฅธ ๋จ์ด๋ฅผ ์
๋ ฅํด๋ณด์ธ์.\n4. ๋ง์ฐ์ค ๋๋๊ทธ ๋ฐ ์คํฌ๋กค์ ํ์ฉํ์ฌ 3D ํ๋ฉด์ ์ดํด๋ณด์ธ์. \n5. ๋จ์ด ์
๋ ฅ์ฐฝ์ ๋ค๋ฅธ ๋จ์ด๋ค๋ ์
๋ ฅํด๋ณด์ธ์.")
|
| 123 |
|
| 124 |
with gr.Row():
|
| 125 |
# ์๊ฐํ ํ๋ฉด์ ํฌ๊ธฐ๋ฅผ CSS๋ก ์ฆ๊ฐ
|
| 126 |
plot_output = gr.Plot(label="Word Embedding 3D ์๊ฐํ", elem_id="plot-box")
|
| 127 |
|
| 128 |
with gr.Column(scale=0.3): # ์ปฌ๋ผ์ ๋๋น๋ฅผ ์ค์ด๊ธฐ ์ํด scale ๊ฐ์ ๋ฎ์ถค
|
| 129 |
+
similar_words_output = gr.Textbox(label="๊ฐ์ฅ ๊ฐ๊น์ด ๋จ์ด 10๊ฐ", interactive=False, lines=5)
|
| 130 |
+
dissimilar_words_output = gr.Textbox(label="๊ฐ์ฅ ๋จผ ๋จ์ด 10๊ฐ", interactive=False, lines=5)
|
| 131 |
+
gr.Image(value="https://compote.slate.com/images/8324cd2e-21f5-4b20-84d5-f08ece97ac38.jpeg?crop=1560%2C1040%2Cx0%2Cy0&width=1280", label="URL ์ด๋ฏธ์ง", interactive=False)
|
| 132 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 133 |
submit_btn.click(
|
| 134 |
+
fn=process_text,
|
| 135 |
+
# word_input = word_input.lower(),
|
| 136 |
inputs=[word_input],
|
| 137 |
+
outputs=[plot_output, similar_words_output, dissimilar_words_output],
|
| 138 |
+
# preprocess=lambda word: word.lower() if word else "" # None ์ฒดํฌ ํ ์๋ฌธ์ ๋ณํ
|
| 139 |
+
)
|
|
|
|
|
|
|
|
|
|
| 140 |
|
| 141 |
+
# "Enter" ํค ์
๋ ฅ ์ ๋์ ์ค์
|
| 142 |
+
word_input.submit(
|
| 143 |
+
fn=process_text,
|
| 144 |
+
# word_input = word_input.lower(),
|
| 145 |
+
inputs=[word_input],
|
| 146 |
+
outputs=[plot_output, similar_words_output, dissimilar_words_output],
|
| 147 |
+
preprocess=lambda word: word.lower() if word else "" # None ์ฒดํฌ ํ ์๋ฌธ์ ๋ณํ
|
| 148 |
+
)
|
| 149 |
|
| 150 |
if __name__ == "__main__":
|
| 151 |
+
iface.launch()
|