Duplicate from Bernd-Ebenhoch/Texte

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+Chatbots.pptx filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,13 @@
+---
+title: Texte
+emoji: 👁
+colorFrom: purple
+colorTo: red
+sdk: streamlit
+sdk_version: 1.17.0
+app_file: app.py
+pinned: false
+duplicated_from: Bernd-Ebenhoch/Texte
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,415 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Sun Mar 26 21:07:00 2023
+
+@author: Bernd Ebenhoch
+"""
+
+
+import tensorflow as tf
+import numpy as np
+import matplotlib.pyplot as plt
+from matplotlib import animation
+from matplotlib.animation import FuncAnimation
+import matplotlib as mpl
+import streamlit as st
+
+from matplotlib import cm
+
+import matplotlib.pyplot as plt
+from sklearn.linear_model import LinearRegression
+import mpl_toolkits.mplot3d as a3
+import matplotlib.colors as colors
+from matplotlib.colors import LightSource
+from tensorflow import keras
+import pandas as pd
+
+from transformers import pipeline
+import transformers
+
+
+# Farben definieren
+cb = [15/255, 25/255, 35/255]
+cf = [25/255*2, 35/255*2, 45/255*2]
+w = [242/255, 242/255, 242/255]
+blue = [68/255, 114/255, 196/255]
+orange = [197/255, 90/255, 17/255]
+
+
+# Pipelines definieren
+#en_de_translator = pipeline("translation_de_to_en", model='google/bert2bert_L-24_wmt_de_en')
+qa_pipeline = pipeline("question-answering", model='deepset/gelectra-base-germanquad')
+sentiment = pipeline("text-classification", model='oliverguhr/german-sentiment-bert')
+
+tab1, tab2, tab3, tab4 = st.tabs(
+    ["Künstliche Neuronale Netze", "Wortvektoren Stimmung", "Wörter Maskieren", "HuggingFace Pipelines"])
+
+
+with tab1:
+    st.markdown(
+        'Definieren Sie ein neuronales Netz und beobachten Sie wie sich die Kurve krümmen kann, um die Daten zu fitten')
+
+    col1, col2 = tab1.columns(2)
+    size = np.array([12., 27., 32., 47., 58., 56., 58., 61.,
+                    64., 67., 70., 80., 84., 88., 108.])
+    price = np.array([88., 135., 178., 216., 220., 246., 241., 275.,
+                     305., 267., 297., 310., 292., 317., 422.])
+    location = np.array([2., 2., 0., 1., 2., 0., 1., 0., 1., 2., 0., 2., 1., 1., 2.])
+    price[location == 1] = price[location == 1]*1+30
+    price[location == 2] = price[location == 2]*1+60
+
+    size_location = np.concatenate((size.reshape(-1, 1), location.reshape(-1, 1)), axis=1)
+
+    data = np.concatenate((size.reshape(-1, 1), location.reshape(-1, 1),
+                          price.reshape(-1, 1)), axis=1)
+    data = pd.DataFrame(data, columns=['Wohnungsgröße (qm)', 'Ort', 'Preis (k€)'])
+
+    col1.dataframe(data.style.format(precision=0))
+    #edited_df = st.experimental_data_editor(data)
+    edited_df = data
+
+    edited_data = edited_df.to_numpy()
+    size_location = edited_data[:, :2]
+    price = edited_data[:, 2]
+
+    string = col2.text_area(
+        'Architektur des neuronalen Netzes. Anzahl der Neuronen in den verdeckten Schichten', value='4', height=275)
+    layers = string.split('\n')
+
+    if st.button('Modell trainieren und Fit-Kurve darstellen'):
+
+        with st.spinner('Der Fit-Prozess kann einige Sekunden dauern ...'):
+
+            model = keras.models.Sequential()
+
+            if len(layers) > 0:
+                for neurons in layers:
+                    model.add(keras.layers.Dense(int(neurons), activation='tanh'))
+            model.add(keras.layers.Dense(1, activation='tanh'))
+
+            model.compile(loss='binary_crossentropy', optimizer='SGD')
+
+            lr_reduction = keras.callbacks.ReduceLROnPlateau(
+                monitor='loss', patience=1000, min_lr=0.00001)
+
+            model.fit(size_location/[120, 2], price/500, epochs=5000,
+                      batch_size=4, callbacks=lr_reduction, verbose=False)
+
+            y_pred = model.predict((size_location)/[120, 2], verbose=False).reshape(-1)*500
+
+            x = np.linspace(0, 125, 400)
+            y = np.linspace(0, 2, 400)
+            X, Y = np.meshgrid(x, y)
+
+            Z = np.concatenate([X.reshape(-1, 1)/120, Y.reshape(-1, 1)/2], axis=1)
+            Z = model.predict(Z, verbose=False)*500
+
+            Z = Z.reshape(len(y), len(x))
+
+            fig = plt.figure(facecolor=cb, figsize=(7, 7))
+            ax = fig.add_subplot(projection='3d')
+            ax.tick_params(color=w, labelcolor=w, labelsize=12)
+            ax.set_facecolor(cb)
+
+            ax.w_xaxis.set_pane_color(cf)
+            ax.w_yaxis.set_pane_color(cf)
+            ax.w_zaxis.set_pane_color(cf)
+
+            ax.set_yticks([0, 1, 2])
+            ax.view_init(25, 50)
+
+            rgb = np.tile(orange, (Z.shape[0], Z.shape[1], 1))
+
+            ls = LightSource(azdeg=315, altdeg=45, hsv_min_val=0.9,
+                             hsv_max_val=1, hsv_min_sat=1, hsv_max_sat=0)
+            illuminated_surface = ls.shade_rgb(rgb, Z)
+
+            below_price = price[price < y_pred]
+            below_location = location[price < y_pred]
+            below_size = size[price < y_pred]
+
+            ax.plot(below_size, below_location, below_price, '.', markersize=20, color=blue)
+
+            ax.plot_surface(X, Y, Z, facecolors=illuminated_surface, edgecolors=[0, 0, 0, 0],
+                            linewidth=0, antialiased=True, rcount=400, ccount=400, alpha=0.8)
+
+            above_price = price[price >= y_pred]
+            above_location = location[price >= y_pred]
+            above_size = size[price >= y_pred]
+
+            ax.plot(above_size, above_location, above_price,
+                    '.', markersize=20, color=blue, zorder=20,)
+
+            ax.set_ylim(2, 0)
+            ax.set_xlim(125, 0)
+            ax.set_zlim(0, 450)
+
+            ax.set_xlabel('Wohnungsgröße (qm)', color=w, fontsize=15, labelpad=10)
+            ax.set_ylabel('Ort', color=w, fontsize=15, labelpad=10)
+            ax.set_zlabel('Preis (k€)', color=w, fontsize=15, rotation=270, labelpad=10)
+
+            st.pyplot(fig)
+
+
+# %%
+with tab2:
+
+    st.markdown(
+        'Definieren Sie Sätze, die positiv oder negativ gestimmt sind und beobachten Sie die resultierenden Wort-Vektoren.')
+
+    text_input_2 = '1: Das schöne Allgäu\n' + \
+        '1: So toll hier im Allgäu\n' + \
+        '1: Uns gefallen die Berge und Seen\n' + \
+        '1: Wir mögen die Landschaft und die Berge\n' + \
+        '1: Ganz toll im Allgäu\n' + \
+        '1: Wir mögen das Allgäu\n' + \
+        '0: Uns gefiel es leider nicht\n' + \
+        '0: Bei Regen ist es total langweilig\n' + \
+        '0: Ganz langweilig!\n' + \
+        '0: So schade, dass es oft Regen gibt\n' + \
+        '0: Sehr schade, wir konnten gar nicht skifahren\n' + \
+        '0: Das gefiel uns überhaupt nicht'
+
+    string_2 = st.text_area('', value=text_input_2, height=275)
+    texts_2 = string_2.split('\n')
+
+    text = []
+    labels = []
+    for element in texts_2:
+        if element != '':
+            label_element, text_element = element.split(':')
+            text.append(text_element)
+            labels.append(float(label_element))
+
+    if st.button('Modell trainieren und Wort-Vektoren darstellen', key=1):
+        with st.spinner('Der Fit-Prozess kann einige Sekunden dauern ...'):
+
+            vectorizer = tf.keras.layers.TextVectorization(
+                max_tokens=1000, output_sequence_length=7)
+
+            vectorizer.adapt(text)
+
+            model = tf.keras.models.Sequential()
+            model.add(vectorizer)
+
+            model.add(tf.keras.layers.Embedding(vectorizer.vocabulary_size(), 2))
+            # model.add(tf.keras.layers.Dropout(0.6))
+            model.add(tf.keras.layers.LSTM(1, return_sequences=False, activation='sigmoid'))
+            # model.add(tf.keras.layers.Flatten())
+            #model.add(tf.keras.layers.Dense(1, activation='sigmoid', use_bias=False, trainable=True))
+
+            model.summary()
+
+            model.compile(optimizer='adam', loss='binary_crossentropy',
+                          metrics=['accuracy'])
+
+            model.fit(text, labels, epochs=2000, verbose=False)
+
+            # Word Vektoren grafisch darstellen
+            cb = [15/255, 25/255, 35/255]
+            cf = [25/255*2, 35/255*2, 45/255*2]
+            w = [242/255, 242/255, 242/255]
+            blue = [68/255, 114/255, 196/255]
+            orange = [197/255, 90/255, 17/255]
+
+            fig = plt.figure(facecolor=cb, figsize=(7, 7))
+            ax = fig.add_subplot()
+            ax.tick_params(color=w, labelcolor=w, labelsize=12)
+            ax.set_facecolor(cb)
+
+            y_pred = model.predict(np.array(vectorizer.get_vocabulary(
+                include_special_tokens=False)).reshape(-1, 1))
+
+            embed_model = tf.keras.models.Model(model.input, model.layers[1].output)
+            X_embed = embed_model(np.array(vectorizer.get_vocabulary(
+                include_special_tokens=False)).reshape(-1, 1))[:, 0, :]
+
+            # 1. Dimension der Wort-Vektoren auf X-Achse,
+            # 2. Dimension auf y-Achse, 3. auf die Z-Achse abbilden
+            ax.scatter(X_embed[:, 0], X_embed[:, 1],
+                       c=y_pred, cmap='coolwarm')
+            for i in range(vectorizer.vocabulary_size()-2):
+                ax.text(X_embed[i, 0], X_embed[i, 1],
+                        vectorizer.get_vocabulary(include_special_tokens=False)[i],
+                        color=w)
+
+            ax.set_ylim(-2, 2)
+            ax.set_xlim(-2, 2)
+
+            ax.set_xticks([-2, -1, 0, 1, 2])
+            ax.set_yticks([-2, -1, 0, 1, 2])
+
+            ax.spines['bottom'].set_color(w)
+            ax.spines['top'].set_color(w)
+            ax.spines['right'].set_color(w)
+            ax.spines['left'].set_color(w)
+
+            ax.set_xlabel('Dimension 1', color=w, fontsize=15, labelpad=10)
+            ax.set_ylabel('Dimension 2', color=w, fontsize=15, labelpad=10)
+
+            # get the mappable, the 1st and the 2nd are the x and y axes
+
+            PCM = ax.get_children()[0]
+            cbar = plt.colorbar(PCM, ax=ax, fraction=0.036, pad=0.090)
+            cbar.set_ticks([])
+
+            cbar.set_label(
+                '<- positiv           Stimmung           negativ ->', fontsize=12, color=w, rotation=270, labelpad=12)
+
+            ax.set_title('Epoche 2000', color=w, fontsize=15)
+            st.pyplot(fig)
+
+
+# %%
+with tab3:
+
+    st.markdown(
+        'Definieren Sie Sätze bei denen beliebige Wörter maskiert werden, um allgemeine Bezüge in Wort-Vektoren abzubilden.')
+
+    text_input = 'Das schöne Allgäu\n' + \
+        'Das wunderbare Allgäu\n' + \
+        'Das grüne Allgäu\n' + \
+        'Radfahren im Allgäu\n' + \
+        'Wandern im Allgäu\n' + \
+        'Radfahren in Oberschwaben\n' + \
+        'Urlaub in Oberschwaben\n' + \
+        'Künstliche Intelligenz für das Allgäu\n' + \
+        'Künstliche Intelligenz für Oberschwaben\n' + \
+        'Data Science für Oberschwaben\n' + \
+        'Data Science und Machine Learning\n' + \
+        'Machine Learning für das Allgäu'
+
+    string = st.text_area('', value=text_input, height=275)
+    text = string.split('\n')
+
+    if st.button('Modell trainieren und Wort-Vektoren darstellen', key=2):
+        with st.spinner('Der Fit-Prozess kann einige Sekunden dauern ...'):
+
+            vectorizer = tf.keras.layers.TextVectorization(
+                max_tokens=1000, output_sequence_length=7)
+
+            vectorizer.adapt(text)
+
+            def generator():
+                while True:
+                    x = vectorizer(text)
+                    mask = tf.reduce_max(x)+1
+
+                    lengths = tf.argmin(x, axis=1)
+                    lengths = tf.cast(lengths, tf.float32)
+
+                    masks = tf.random.uniform(shape=(x.shape[0],), minval=0, maxval=lengths)
+                    masks = tf.cast(masks, tf.int32)
+
+                    masks = tf.one_hot(masks, x.shape[1], dtype=tf.int32)
+                    masks = tf.cast(masks, tf.bool)
+
+                    y = x[masks]
+                    masks = tf.cast(masks, tf.int64)
+                    x = x * (1-masks) + mask * masks
+                    yield x, y
+
+            # data = tf.data.Dataset.from_tensor_slices(vectorizer(text),vectorizer(text))
+            # data = data.map(masking_generator)
+            model = tf.keras.models.Sequential()
+
+            model.add(tf.keras.layers.Embedding(vectorizer.vocabulary_size()+1, 3))
+
+            model.add(tf.keras.layers.LSTM(100, return_sequences=False, activation='sigmoid'))
+            model.add(tf.keras.layers.Dense(vectorizer.vocabulary_size(), activation='softmax'))
+
+            model.summary()
+
+            model.compile(optimizer='adam', loss='sparse_categorical_crossentropy',
+                          metrics=['accuracy'])
+
+            lr_reduce = tf.keras.callbacks.ReduceLROnPlateau(
+                monitor='loss', patience=500, min_lr=1e-6)
+            model.fit(generator(), steps_per_epoch=1,
+                      epochs=3000, callbacks=lr_reduce, verbose=False)
+
+            fig = plt.figure(facecolor=cb, figsize=(7, 7))
+            ax = fig.add_subplot()
+            ax.tick_params(color=w, labelcolor=w, labelsize=12)
+            ax.set_facecolor(cb)
+
+            embed_model = tf.keras.models.Model(model.input, model.layers[0].output)
+            X_embed = embed_model(vectorizer(vectorizer.get_vocabulary(
+                include_special_tokens=False)))[:, 0, :]
+
+            # 1. Dimension der Wort-Vektoren auf X-Achse,
+            # 2. Dimension auf y-Achse, 3. auf die Z-Achse abbilden
+            ax.scatter(X_embed[:, 0], X_embed[:, 1],
+                       color=blue)
+            for i in range(vectorizer.vocabulary_size()-2):
+                ax.text(X_embed[i, 0], X_embed[i, 1],
+                        vectorizer.get_vocabulary(include_special_tokens=False)[i],
+                        color=w)
+
+            ax.set_ylim(-2, 2)
+            ax.set_xlim(-2, 2)
+
+            ax.set_xticks([-2, -1, 0, 1, 2])
+            ax.set_yticks([-2, -1, 0, 1, 2])
+
+            ax.spines['bottom'].set_color(w)
+            ax.spines['top'].set_color(w)
+            ax.spines['right'].set_color(w)
+            ax.spines['left'].set_color(w)
+
+            ax.set_xlabel('Dimension 1', color=w, fontsize=15, labelpad=10)
+            ax.set_ylabel('Dimension 2', color=w, fontsize=15, labelpad=10)
+
+            st.pyplot(fig)
+
+
+# %%
+with tab4:
+    # st.header("Übersetzung: Deutsch --> Englisch")
+    #st.text("Übersetzung: Deutsch --> Englisch")
+    st.markdown('Probieren Sie verschiedene Pipelines der Transformer-Bibliothek von HuggingFace.')
+
+    text_input_4 = 'Was ist der Schwerpunkt?'
+
+    string_4 = st.text_area('Frage zum Kontext beantworten', value=text_input_4, height=25)
+
+    text_input_5 = 'Wir unterstützen Unternehmen bei der Datenanalyse durch individuelle Beratung und Projekte mit besonderem Fokus auf maschinelles Lernen und Deep Learning.'
+
+    string_5 = st.text_area('Kontext', value=text_input_5, height=75)
+
+    if st.button('Ein fertig trainiertes Transformer-Modell von HuggingFace anwenden', key=4):
+        with st.spinner('Die Beantwortung der Frage kann einige Sekunden dauern ...'):
+
+            a5 = qa_pipeline(question=string_4, context=string_5)
+            st.text(a5)
+
+    ############################################################
+
+    st.text('')
+    st.markdown("""<hr style="height:10px;border:none;color:#333;background-color:#333;" /> """,
+                unsafe_allow_html=True)
+    st.text('')
+
+    text_input_7 = 'Wir lieben Data Science!'
+
+    string_7 = st.text_area('Stimmungsanalyse', value=text_input_7, height=25)
+
+    if st.button('Ein fertig trainiertes Transformer-Modell von HuggingFace anwenden', key=5):
+        with st.spinner('Die Beurteilung der Stimmung kann einige Sekunden dauern ...'):
+
+            a5 = sentiment(string_7)
+            st.text(a5[0]['label'])
+
+    ############################################################
+    st.text('')
+    st.markdown("""<hr style="height:10px;border:none;color:#333;background-color:#333;" /> """,
+                unsafe_allow_html=True)
+    st.text('')
+
+    references = 'Verwendete Modelle:\n' + \
+        '\n\nFrage beantworten:\n' + \
+        'deepset/gelectra-base-germanquad, Autoren: Timo Möller, Julian Risch, Malte Pietsch' + \
+        '\n\nStimmung:\n' + \
+        'oliverguhr/german-sentiment-bert, Autoren:  Oliver Guhr, Anne-Kathrin Schumann, Frank Bahrmann, Hans Joachim Böhme'
+
+    st.markdown(references)

requirements.txt

@@ -0,0 +1,8 @@
+tensorflow==2.10.0
+streamlit>=1.10.0
+matplotlib==3.5.1
+numpy
+scikit-learn
+pandas
+transformers==4.18.0
+torch>1.11.0