functions.py

import pandas as pd
from bertopic import BERTopic
from huggingface_hub import InferenceClient
from bertopic.vectorizers import ClassTfidfTransformer
from sentence_transformers import SentenceTransformer
from sklearn import preprocessing
from sklearn.preprocessing import LabelEncoder
from tempfile import NamedTemporaryFile
import matplotlib.pyplot as plt
import plotly.express as px
import subprocess

from wordcloud import WordCloud


def process_file_bm25(file,mode,min_cluster_size,top_n_words,ngram):
    
    

    if file.name.endswith('.csv'):
        df = pd.read_csv(file)
    elif file.name.endswith('.xls') or file.name.endswith('.xlsx'):
        df = pd.read_excel(file)
    else:
        raise ValueError("Unsupported file format. Please provide a CSV or Excel file.")

    # Ensure that the 'products' column is present in the dataframe
    if 'products' not in df.columns.str.lower():
        raise ValueError("The input file must have a column named 'products'.")

    # Convert the 'products' column to a list
    sentences_list = df['products'].tolist()
    print(len(sentences_list))
    ctfidf_model = ClassTfidfTransformer(bm25_weighting=True,reduce_frequent_words=True)

    if mode=="Automated clustering":

      topic_model = BERTopic(ctfidf_model=ctfidf_model,n_gram_range =(1,ngram),top_n_words=top_n_words)

    else:

      topic_model = BERTopic(ctfidf_model=ctfidf_model,n_gram_range =(1,ngram),top_n_words=top_n_words,min_topic_size=min_cluster_size)


    # Perform topic modeling
    topics, probabilities = topic_model.fit_transform(sentences_list)

    # Visualize all graphs

    topics_info=topic_model.get_topic_info()
    df_topics_bm25= topics_info
    #print(topics)
    try:
      barchart = topic_model.visualize_barchart(top_n_topics=10)
    except:
      barchart='Error message'
    try:
      topics_plot = topic_model.visualize_topics()
    except:
      topics_plot = ' Error message'
    heatmap = topic_model.visualize_heatmap()
    hierarchy = topic_model.visualize_hierarchy()
    df['topic_number'] = topics

    # Encode the topic numbers to make them categorical
    label_encoder = LabelEncoder()
    df['topic_number_encoded'] = label_encoder.fit_transform(df['topic_number'])
    temp_file = NamedTemporaryFile(delete=False, suffix=".xlsx")
    df.to_excel(temp_file.name, index=False)
    df_bm25=df
    #print(df)

    return  df,temp_file.name,topics_info ,barchart,topics_plot, heatmap, hierarchy 
    

def process_file_bert(file,mode,min_cluster_size,top_n_words,ngram):
    # Read the Excel sheet or CSV file
    if file.name.endswith('.csv'):
        df = pd.read_csv(file)
    elif file.name.endswith('.xls') or file.name.endswith('.xlsx'):
        df = pd.read_excel(file)
    else:
        raise ValueError("Unsupported file format. Please provide a CSV or Excel file.")

    # Ensure that the 'products' column is present in the dataframe
    if 'products' not in df.columns.str.lower():
        raise ValueError("The input file must have a column named 'products'.")

    # Convert the 'products' column to a list
    sentences_list = df['products'].tolist()
    print(len(sentences_list))
    representation_model = KeyBERTInspired()
    if mode=="Automated clustering":
      # Fine-tune your topic representations

      topic_model = BERTopic(representation_model=representation_model,n_gram_range =(1,ngram),top_n_words=top_n_words)

    else:
      
      topic_model = BERTopic(representation_model=representation_model,n_gram_range =(1,ngram),top_n_words=top_n_words,min_topic_size=min_cluster_size)
      
    topics, probabilities = topic_model.fit_transform(sentences_list)

    # Visualize all graphs

    topics_info=topic_model.get_topic_info()
    state.df_topics_bert= topics_info
    #print(topics)
    try:
      barchart = topic_model.visualize_barchart(top_n_topics=10)
    except:
      barchart='Error message'
    try:
      topics_plot = topic_model.visualize_topics()
    except:
      topics_plot = ' Error message'
    heatmap = topic_model.visualize_heatmap()
    hierarchy = topic_model.visualize_hierarchy()
    df['topic_number'] = topics

    # Encode the topic numbers to make them categorical
    label_encoder = LabelEncoder()
    df['topic_number_encoded'] = label_encoder.fit_transform(df['topic_number'])
    temp_file = NamedTemporaryFile(delete=False, suffix=".xlsx")
    df.to_excel(temp_file.name, index=False)

    state.df_bert=df
    return  df, topics_info ,barchart,topics_plot, heatmap, hierarchy


client = InferenceClient(
    "mistralai/Mixtral-8x7B-Instruct-v0.1"
)

def format_prompt(message, history):
  prompt = "<s>"
  for user_prompt, bot_response in history:
    prompt += f"[INST] {user_prompt} [/INST]"
    prompt += f" {bot_response}</s> "
  prompt += f"[INST] {message} [/INST]"
  return prompt

def generate(
    prompt, history, system_prompt, temperature=0.9, max_new_tokens=4096, top_p=0.95, repetition_penalty=1.0,
):
    temperature = float(temperature)
    if temperature < 1e-2:
        temperature = 1e-2
    top_p = float(top_p)

    generate_kwargs = dict(
        temperature=temperature,
        max_new_tokens=max_new_tokens,
        top_p=top_p,
        repetition_penalty=repetition_penalty,
        do_sample=True,
        seed=42,
    )

    formatted_prompt = format_prompt(f"{system_prompt}, {prompt}", history)
    stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
    output = ""

    for response in stream:
        output += response.token.text
        yield output
    return output


# Define the function to generate the plot based on user inputs
def generate_plot(topic, x_axis_index, y_axis_index, chart_type, agg_func):
    x_axis = df.columns[1:][x_axis_index]
    y_axis = df.columns[1:][y_axis_index]
    print(x_axis,y_axis)
    filtered_df = df[df['Topic Number'] == topic]

    if chart_type == "scatter":
        fig = px.scatter(filtered_df, x=x_axis, y=y_axis)
    elif chart_type == "bar":
        print('Bar chart selected')
        if agg_func == "count_distinct":
            fig = px.bar(filtered_df, x=x_axis, y=y_axis, color=y_axis, barmode='group')
        else:
            fig = px.bar(filtered_df, x=x_axis, y=y_axis, color=y_axis)
    elif chart_type == "line":
        fig = px.line(filtered_df, x=x_axis, y=y_axis)
    elif chart_type == "box":
        fig = px.box(filtered_df, x=x_axis, y=y_axis)
    elif chart_type == "wordcloud":
        text = ' '.join(filtered_df[y_axis].astype(str))
        wordcloud = WordCloud(width=800, height=400, random_state=21, max_font_size=110).generate(text)
        plt.figure(figsize=(10, 7))
        plt.imshow(wordcloud, interpolation="bilinear")
        plt.axis('off')
        plt.show()
        return None
    elif chart_type == "pie":
        fig = px.pie(filtered_df, names=x_axis, values=y_axis)
        print('Pie chart selected')

    return fig