update

.gradio/certificate.pem

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
+WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
+ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
+MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
+h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
+0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
+A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
+T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
+B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
+B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
+KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
+OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
+jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
+qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
+rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
+HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
+hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
+ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
+3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
+NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
+ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
+TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
+jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
+oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
+4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
+mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
+emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
+-----END CERTIFICATE-----

AulSign.log

@@ -0,0 +1,21 @@
+2024-12-08 19:23:54,119 - DEBUG - load_ssl_context verify=True cert=None trust_env=True http2=False
+2024-12-08 19:23:54,119 - DEBUG - load_verify_locations cafile='/opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages/certifi/cacert.pem'
+2024-12-08 19:23:54,151 - INFO - Use pytorch device_name: mps
+2024-12-08 19:23:54,151 - INFO - Load pretrained SentenceTransformer: mixedbread-ai/mxbai-embed-large-v1
+2024-12-08 19:23:54,196 - DEBUG - Starting new HTTPS connection (1): huggingface.co:443
+2024-12-08 19:23:54,752 - DEBUG - https://huggingface.co:443 "HEAD /mixedbread-ai/mxbai-embed-large-v1/resolve/main/modules.json HTTP/1.1" 200 0
+2024-12-08 19:23:54,918 - DEBUG - https://huggingface.co:443 "HEAD /mixedbread-ai/mxbai-embed-large-v1/resolve/main/config_sentence_transformers.json HTTP/1.1" 200 0
+2024-12-08 19:23:55,369 - DEBUG - https://huggingface.co:443 "HEAD /mixedbread-ai/mxbai-embed-large-v1/resolve/main/README.md HTTP/1.1" 200 0
+2024-12-08 19:23:55,537 - DEBUG - https://huggingface.co:443 "HEAD /mixedbread-ai/mxbai-embed-large-v1/resolve/main/modules.json HTTP/1.1" 200 0
+2024-12-08 19:23:55,979 - DEBUG - https://huggingface.co:443 "HEAD /mixedbread-ai/mxbai-embed-large-v1/resolve/main/sentence_bert_config.json HTTP/1.1" 200 0
+2024-12-08 19:23:56,154 - DEBUG - https://huggingface.co:443 "HEAD /mixedbread-ai/mxbai-embed-large-v1/resolve/main/adapter_config.json HTTP/1.1" 404 0
+2024-12-08 19:23:56,403 - DEBUG - https://huggingface.co:443 "HEAD /mixedbread-ai/mxbai-embed-large-v1/resolve/main/config.json HTTP/1.1" 200 0
+2024-12-08 19:23:57,239 - DEBUG - https://huggingface.co:443 "HEAD /mixedbread-ai/mxbai-embed-large-v1/resolve/main/tokenizer_config.json HTTP/1.1" 200 0
+2024-12-08 19:23:57,440 - DEBUG - https://huggingface.co:443 "GET /api/models/mixedbread-ai/mxbai-embed-large-v1/revision/main HTTP/1.1" 200 148585
+2024-12-08 19:23:57,932 - DEBUG - https://huggingface.co:443 "GET /api/models/mixedbread-ai/mxbai-embed-large-v1 HTTP/1.1" 200 148585
+2024-12-08 19:23:59,135 - INFO - 2 prompts are loaded, with the keys: ['query', 'passage']
+2024-12-08 19:24:04,411 - DEBUG - Using selector: KqueueSelector
+2024-12-08 19:24:04,418 - DEBUG - load_ssl_context verify=True cert=None trust_env=True http2=False
+2024-12-08 19:24:04,441 - DEBUG - Starting new HTTPS connection (1): huggingface.co:443
+2024-12-08 19:24:04,441 - DEBUG - load_verify_locations cafile='/opt/homebrew/Caskroom/miniconda/base/lib/python3.10/site-packages/certifi/cacert.pem'
+2024-12-08 19:24:04,455 - DEBUG - connect_tcp.started host='api.gradio.app' port=443 local_address=None timeout=3 socket_options=None

app.py

@@ -0,0 +1,107 @@
+import gradio as gr
+import json
+import pandas as pd
+from sentence_transformers import SentenceTransformer
+# Importa le funzioni dal tuo script
+from scripts.aulsign import AulSign
+from scripts.scripts.sign2text_mapping import sign2text
+
+# Configurazione iniziale del modello e dei dati
+def load_resources():
+    # Modello di embedding
+    model_name = "mixedbread-ai/mxbai-embed-large-v1"
+    model = SentenceTransformer(model_name)
+
+    # Percorsi dei file
+    corpus_embeddings_path = 'tools/corpus_embeddings.json'
+    sentences_train_embeddings_path = 'tools/sentences_train_embeddings_filtered_01.json'
+    rules_prompt_path_text2sign = 'tools/rules_prompt_text2sign.txt'
+    rules_prompt_path_sign2text = 'tools/rules_prompt_sign2text.txt'
+
+    # Carica dati
+    with open(corpus_embeddings_path, 'r') as file:
+        corpus_embeddings = pd.DataFrame(json.load(file))
+
+    with open(sentences_train_embeddings_path, 'r') as file:
+        sentences_train_embeddings = pd.DataFrame(json.load(file))
+
+    return model, corpus_embeddings_path, corpus_embeddings, sentences_train_embeddings, rules_prompt_path_text2sign, rules_prompt_path_sign2text
+
+# Funzione per la modalità text2sign
+def text_to_sign(sentence_to_analyse):
+    try:
+        pseudo_can, fsw_seq, can_desc_association_seq, _ = AulSign(
+            input=sentence_to_analyse,
+            rules_prompt_path=rules_prompt_path_text2sign,
+            train_sentences=sentences_train_embeddings,
+            vocabulary=corpus_embeddings,
+            model=model,
+            ollama=False,
+            modality="text2sign"
+        )
+        return fsw_seq,can_desc_association_seq
+    except Exception as e:
+        return f"Error: {str(e)}"
+# Funzione per la modalità sign2text
+def sign_to_text(fsw_to_analyse):
+    try:
+        mapped_input = sign2text(fsw_to_analyse,corpus_embeddings_path)
+        print(mapped_input)
+    except Exception as e:
+        return f"Error on mapping : {str(e)}"
+
+    try:
+        translation = AulSign(
+            input=mapped_input,
+            rules_prompt_path=rules_prompt_path_sign2text,
+            train_sentences=sentences_train_embeddings,
+            vocabulary=corpus_embeddings,
+            model=model,
+            ollama=False,
+            modality="sign2text"
+        )
+        return translation, mapped_input
+    except Exception as e:
+        return f"Error on AulSign: {str(e)}"
+
+# Carica le risorse
+model, corpus_embeddings_path, corpus_embeddings, sentences_train_embeddings, rules_prompt_path_text2sign, rules_prompt_path_sign2text = load_resources()
+
+# Interfaccia Gradio
+with gr.Blocks() as demo:
+    gr.Markdown("# AulSign Translator")
+    gr.Markdown("Translate from Natural Language to Formal SignWriting (FSW) or viceversa.")
+
+    with gr.Tab("Text to Sign"):
+        text_input = gr.Textbox(label="Insert a sentence", placeholder="Digit here your sentence...")
+        fsw_output = gr.Textbox(label="Output")
+        intermediate_output = gr.Textbox(label="Intermediate Output")
+
+        translate_button = gr.Button("Translate")
+        translate_button.click(text_to_sign, inputs=text_input, outputs=[fsw_output,intermediate_output])
+
+        gr.Examples(
+            examples=["This is a new ASL translator"],
+            inputs=text_input,
+            outputs=[fsw_output,intermediate_output],
+            fn=text_to_sign
+        )
+
+    with gr.Tab("Sign to Text"):
+        sign_input = gr.Textbox(label="Insert a FSW sequence", placeholder="Digit here your FSW sequence...")
+        text_output = gr.Textbox(label="Output")
+        intermediate_output = gr.Textbox(label="Intermediate Output")
+
+        reconstruct_button = gr.Button("Translate")
+        reconstruct_button.click(sign_to_text, inputs=sign_input, outputs=[text_output,intermediate_output])
+
+        gr.Examples(
+            examples=["M518x584S10004492x534S22a04493x569S30a00482x483 AS33b00S19210S20500S26504M519x547S33b00482x482S20500466x512S26504464x532S19210498x511 M530x522S15a36502x510S1813e501x503S2890f470x478 M512x535S1f720492x466S20320497x485S1dc20488x505 M528x595S10009483x405S10021473x422S2e024488x453S10001491x488S10029493x504S15a48477x548S15a40515x548S22a14476x580S22a04515x580"],
+            inputs=sign_input,
+            outputs=[text_output,intermediate_output],
+            fn=sign_to_text
+        )
+
+# Avvia l'app
+if __name__ == "__main__":
+    demo.launch(share=True)

scripts/aulsign.py

@@ -0,0 +1,588 @@
+import os
+import json
+import numpy as np
+import pandas as pd
+import logging
+from collections import Counter
+from sentence_transformers import SentenceTransformer
+import warnings
+from datetime import datetime
+from sklearn.preprocessing import normalize
+import requests
+import json
+import argparse
+from openai import OpenAI
+
+from scripts.scripts.sign2text_mapping import sign2text
+
+warnings.filterwarnings("ignore", category=FutureWarning)
+
+
+# Set up logging configuration
+logging.basicConfig(
+    filename='AulSign.log',  # Log to a file
+    level=logging.DEBUG,         # Log everything, including debug info
+    format='%(asctime)s - %(levelname)s - %(message)s',  # Log format
+    filemode='w'                 # Overwrite the log file each run
+)
+
+
+
+client = OpenAI(
+  organization=os.getenv("OPENAI_ORGANIZATION"),
+  project=os.getenv("OPENAI_PROJECT"),
+  api_key=os.getenv("OPENAI_API_KEY")
+)
+
+print('Inference started...')
+
+def query_ollama(messages, model="mistral:7b-instruct-fp16"):
+    url = "http://localhost:11434/api/chat"
+
+    options = {"seed": 42,"temperature": 0.1}
+
+
+    payload = {
+        "model": model,
+        "messages": messages,
+        "options": options,
+        "stream": False
+    }
+
+    response = requests.post(url, json=payload)
+
+    if response.status_code == 200:
+        return response.json()["message"]["content"]
+    else:
+        return f"Error: {response.status_code}, {response.text}"
+
+def check_repetition(text, threshold=0.2):
+    if not text:
+        return False
+    
+    words = [word.strip for word in text.split('#')]
+
+    unique_words = len(set(words))
+    total_words = len(words)
+
+    if "<unk>" in words:
+        logging.debug(f"Check repetition: '<unk>' was generated in the answer")
+        return True
+
+    
+    is_repetitive = unique_words < total_words * threshold
+    logging.debug(f"Check repetition: {is_repetitive} (Unique: {unique_words}, Total: {total_words})")
+    return is_repetitive
+
+
+# Function to merge predictions with gold data and compute metrics
+def prepare_dataset(prediction: pd.DataFrame, validation: pd.DataFrame, modality:str):
+    if modality=='text2sign':
+        validation = validation.rename(columns={'fsw':'gold_fsw_seq','symbol': 'gold_symbol_seq', 'word': 'gold_cd'}) 
+        metrics = prediction.merge(validation[['gold_symbol_seq','gold_cd', 'sentence','gold_fsw_seq']], on=['sentence'])
+    elif modality=='sign2text':
+        validation = validation.rename(columns={'word': 'gold_cd'}) 
+        metrics = prediction.merge(validation[['sentence','gold_cd']], on=['gold_cd'])
+    return metrics
+
+# Define cosine similarity function if it's missing
+def cos_sim(a, b):
+    return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))
+
+def find_most_similar_sentence(user_embedding, train_sentences: pd.DataFrame, n=3, unk_threshold=7):
+    # Estrai gli embedding, le decomposizioni e le frasi dal DataFrame
+    sentence_embeddings = np.vstack(train_sentences["embedding_sentence"].values)  # Matrix of sentence embeddings
+    decompositions = train_sentences["decomposition"].values
+    sentences = train_sentences["sentence"].values
+    
+    # Normalizza gli embedding delle frasi e l'embedding utente
+    sentence_embeddings = normalize(sentence_embeddings, axis=1)
+    user_embedding = normalize(user_embedding.reshape(1, -1), axis=1)
+    
+    # Calcola le similarità usando un'unica moltiplicazione matrice-vettore
+    similarities = np.dot(sentence_embeddings, user_embedding.T).flatten()  # Shape (num_sentences,)
+    
+    # Imposta la similarità a zero per le frasi con troppi "<unk>"
+    unk_counts = np.array([d.count("<unk>") for d in decompositions])
+    similarities[unk_counts > unk_threshold] = 0  # Penalizza le frasi con troppi "<unk>"
+    
+    # Ottieni gli indici delle top-n frasi più simili
+    top_n_indices = np.argsort(similarities)[-n:][::-1]
+    
+    # Ritorna le decomposizioni e le frasi corrispondenti alle top-n similitudini
+    return [decompositions[i] for i in top_n_indices], [sentences[i] for i in top_n_indices]
+
+
+def find_most_similar_canonical_entry(user_embedding, vocabulary: pd.DataFrame, n=30):
+    # Extract embeddings and words from the vocabulary
+    vocabulary_embeddings = np.vstack(vocabulary["embedding"].values)  # Matrix of embeddings
+    vocabulary_words = vocabulary["word"].values
+    
+    # Normalize vocabulary embeddings and user embedding
+    vocabulary_embeddings = normalize(vocabulary_embeddings, axis=1)
+    user_embedding = normalize(user_embedding.reshape(1, -1), axis=1)
+    
+    # Compute cosine similarities for all entries in one matrix multiplication
+    similarities = np.dot(vocabulary_embeddings, user_embedding.T).flatten()  # Shape (vocabulary_size,)
+    
+    # Get a sorted list of indices based on similarity scores
+    sorted_indices = np.argsort(similarities)[::-1]  # Sort in descending order
+    
+    # Initialize lists for canonical entries and similarities
+    canonical_list = []
+    canonical_similarities = []
+    
+    for idx in sorted_indices:
+        if len(canonical_list) >= n:  # Stop once we have n entries
+            break
+        
+        # Get canonical entry for the current word
+        canonical_entry = get_most_freq(vocabulary_words[idx])
+        
+        # Check for duplicates in canonical entries
+        if canonical_entry not in canonical_list:
+            canonical_list.append(canonical_entry)
+            canonical_similarities.append(similarities[idx])
+    
+    # Return the top n canonical entries and their similarities
+    return canonical_list#, canonical_similarities
+
+
+def get_most_freq(lista:list):
+    lista_cleaned = []
+    for segno in lista:
+        segno_pulito = segno.lower().strip()
+        if segno_pulito not in lista_cleaned:
+            lista_cleaned.append(segno_pulito)
+
+    frequency_count = Counter(lista_cleaned)
+    #print(frequency_count)
+    top_two_words = frequency_count.most_common(2)
+
+    if len(top_two_words) >= 2:
+        first_word = top_two_words[0][0]
+        second_word = top_two_words[1][0]
+
+        return first_word+'|'+second_word
+    else:
+        first_word = top_two_words[0][0]
+        return first_word
+
+def get_most_freq_fsw(lista_fsw):
+    if isinstance(lista_fsw,str):
+        return lista_fsw
+    else:
+        frequency_count = Counter(lista_fsw)
+        max_freq_word = frequency_count.most_common(1)[0][0]
+        return max_freq_word
+
+
+def get_fsw_exact(vocabulary: pd.DataFrame, can_desc_answer, model, top_k=10):
+    # Extract vocabulary embeddings and words
+    vocabulary_embeddings = np.vstack(vocabulary["embedding"].values)  # Create a matrix of all embeddings
+    vocabulary_words = vocabulary["word"].values
+    vocabulary_fsw = vocabulary["fsw"].values
+
+    # Normalize vocabulary embeddings for cosine similarity
+    vocabulary_embeddings = normalize(vocabulary_embeddings, axis=1)
+
+    fsw_seq = []
+    can_desc_association_seq = []
+    joint_prob = 1
+
+    for can_d in can_desc_answer:
+        # Encode the candidate description and normalize
+        can_d_emb = model.encode(can_d, normalize_embeddings=True).reshape(1, -1)  # Shape (1, embedding_dim)
+
+        # Compute cosine similarities using matrix multiplication
+        similarities = np.dot(vocabulary_embeddings, can_d_emb.T).flatten()  # Shape (vocabulary_size,)
+
+        # Get the indices of the top_k most similar elements
+        top_k_indices = np.argsort(similarities)[-top_k:][::-1]  # Indices of top-k elements
+        top_k_words = vocabulary_words[top_k_indices]
+        top_k_fsws = vocabulary_fsw[top_k_indices]
+        top_k_similarities = similarities[top_k_indices]
+
+        # Check for an exact match in the top_k elements
+        exact_match_index = next((i for i, word in enumerate(top_k_words) if get_most_freq(word) == can_d.strip()), None)
+
+        if exact_match_index is not None:
+            # Exact match found
+            most_similar_word = get_most_freq(top_k_words[exact_match_index])
+            fsw = top_k_fsws[exact_match_index]
+            max_similarity = 1  # Assign maximum similarity for an exact match
+        else:
+            # If no exact match, use the most similar word semantically
+            max_index = 0  # First element in the sorted top_k (highest similarity)
+            most_similar_word = get_most_freq(top_k_words[max_index])
+            fsw = top_k_fsws[max_index]
+            max_similarity = top_k_similarities[max_index]
+
+        # Append the result
+        logging.info(fsw)
+        fsw_seq.append(get_most_freq_fsw(fsw))  # Append to fsw sequence
+        joint_prob *= max_similarity  # Multiply joint probability
+        can_desc_association_seq.append(most_similar_word)
+
+        # Logging
+        logging.debug(f"Word: {can_d}")
+        logging.debug(f"Most similar word in vocabulary: {most_similar_word}")
+        logging.debug(f"Similarity: {max_similarity}")
+        logging.debug(f"Fsw_seq: {' '.join(fsw_seq)}")
+        logging.debug("---")
+
+    # Compute geometric mean of joint probability
+    joint_prob = pow(joint_prob, 1 / len(can_desc_association_seq))
+    
+    return ' '.join(fsw_seq), ' # '.join(can_desc_association_seq), np.round(joint_prob, 3)
+
+# Process input sentence through retrieval-augmented generation (RAG)
+def AulSign(input:str, rules_prompt_path:str, train_sentences:pd.DataFrame, vocabulary:pd.DataFrame, model, ollama:bool, modality:str):
+    """
+AulSign: A function for translating between text and Formal SignWriting (FSW) or vice versa.
+
+This function leverages embeddings, similarity matching, and language models to facilitate
+translations based on the specified modality (`text2sign` or `sign2text`).
+
+Args:
+    input (str): 
+        The sentence or sign sequence to be analyzed and translated.
+    rules_prompt_path (str): 
+        Path to a file containing predefined prompts and rules to guide the language model.
+    train_sentences (pd.DataFrame): 
+        A dataset containing sentences and their embeddings for training or similarity matching.
+    vocabulary (pd.DataFrame): 
+        A table of vocabulary entries with canonical descriptions and embeddings, used for matching.
+    model: 
+        The embedding model used to convert sentences or sign sequences into vector representations.
+    ollama (bool): 
+        Specifies whether to use the `query_ollama` method for querying the language model.
+    modality (str): 
+        The translation mode:
+        - `'text2sign'`: Converts text to Formal SignWriting sequences.
+        - `'sign2text'`: Converts Formal SignWriting to textual sentences.
+
+Returns:
+    For `modality == "text2sign"`:
+        tuple:
+            - answer (str): 
+                The translated text or decomposition provided by the language model.
+            - fsw (list): 
+                A list of Formal SignWriting sequences associated with the translation.
+            - can_desc_association_seq (list): 
+                A list of canonical descriptions associated with the FSW sequences.
+            - joint_prob (float): 
+                The joint probability of the most likely translation path.
+
+    For `modality == "sign2text"`:
+        str: 
+            The reconstructed textual sentence translated from the input sign sequence.
+
+    If an invalid modality is provided:
+        str: 
+            Returns 'error' to indicate invalid input.
+
+Raises:
+    Exception: 
+        Logs and raises errors encountered during API calls or message construction.
+    """
+   
+    sent_embedding = model.encode(input, normalize_embeddings=True)
+
+    if modality =='text2sign':
+    
+        similar_canonical = find_most_similar_canonical_entry(sent_embedding, vocabulary, n=100)
+        #print(similar_canonical)
+
+        
+        similar_canonical_str = ' # '.join(similar_canonical)
+
+        # Load the rules prompt from the file
+        with open(rules_prompt_path, 'r') as file:
+            rules_prompt = file.read().format(similar_canonical=similar_canonical_str)
+
+        # Find the most similar sentences from training set
+        decomposition, sentences = find_most_similar_sentence(
+            user_embedding=sent_embedding, 
+            train_sentences=train_sentences, 
+            n=20
+        )
+
+        messages = [{"role": "system", "content": rules_prompt}]
+        for sentence, decomposition in zip(sentences, decomposition):
+            # Ensure each message has 'role' and 'content' keys
+            if sentence and decomposition:
+                messages.append({"role": "user", "content": sentence})
+                messages.append({"role": "assistant", "content": decomposition})#.replace(' | ',' # ')})
+            else:
+                logging.warning("Missing 'sentence' or 'decomposition' in messages.")
+
+        messages.append({"role": "user", "content": "decompose the following sentence as shown in the previous examples"})
+        messages.append({"role": "user", "content": input})
+        
+        # Validate the constructed messages before converting to prompt text
+        valid_messages = []
+        for message in messages:
+            if 'role' in message and 'content' in message:
+                valid_messages.append(message)
+                logging.debug(message)
+            else:
+                logging.error(f"Invalid message format detected: {message}")
+
+        if ollama:
+            # Query the LLM using query_ollama instead of llm_pipeline
+            answer = query_ollama(messages)#, model="mistral:7b-instruct-fp16")
+
+            logging.info("\n[LOG] MISTRAL Answer:")
+            logging.info(answer)
+
+            can_description_answer = answer.split('#')
+        else:
+            try:
+                # Initial API call
+                completion = client.chat.completions.create(
+                    model="gpt-3.5-turbo",
+                    messages=messages,
+                    temperature=0
+                )
+                answer = completion.choices[0].message.content
+
+                if check_repetition(answer):
+                # Optional: Repetition check
+                    presence_penalty = 0.6
+                    completion = client.chat.completions.create(
+                        model="gpt-3.5-turbo",
+                        messages=messages,
+                        presence_penalty=presence_penalty,
+                        temperature=0
+                    )
+                    logging.info(f"presence_penalty: {presence_penalty}")
+                    answer = completion.choices[0].message.content
+                    logging.info('ANSWER: GPT')
+                    logging.info(answer + '\n\n')
+
+                    # Update parsed answer
+                    can_description_answer = answer.split('#')
+                    
+                else:
+                    logging.info('ANSWER: GPT')
+                    logging.info(answer + '\n\n')
+
+                    # Split for further processing
+                    can_description_answer = answer.split('#')
+
+
+            except Exception as e:
+                logging.error(f"Error during GPT API call: {e}")
+
+        # Map canonical descriptions to most similar words in vocabulary
+        fsw, can_desc_association_seq, joint_prob = get_fsw_exact(
+            vocabulary=vocabulary, 
+            can_desc_answer=can_description_answer, 
+            model=model
+        )
+
+        return answer, fsw, can_desc_association_seq, joint_prob
+    
+    elif modality =='sign2text':
+
+       # Load the rules prompt from the file
+        with open(rules_prompt_path, 'r') as file:
+            rules_prompt = file.read()
+
+
+        # Find the most similar sentences from training set
+        decomposition, sentences = find_most_similar_sentence(
+            user_embedding=sent_embedding, 
+            train_sentences=train_sentences, 
+            n=30
+        )
+
+        messages = [{"role": "system", "content": rules_prompt}]
+        for sentence, decomposition in zip(sentences, decomposition):
+            # Ensure each message has 'role' and 'content' keys
+            if sentence and decomposition:
+                messages.append({"role": "user", "content": decomposition})
+                messages.append({"role": "assistant", "content": sentence}) # qui stiamo invertendo il task! dalla decomposition vogliamo che l'assistant ci dia la sentence
+            else:
+                logging.warning("Missing 'sentence' or 'decomposition' in messages.")
+
+        messages.append({"role": "user", "content": "reconstruct the sentence as shown on the examples above"})
+        messages.append({"role": "user", "content": input})
+        
+        # Validate the constructed messages before converting to prompt text
+        valid_messages = []
+        for message in messages:
+            if 'role' in message and 'content' in message:
+                valid_messages.append(message)
+                logging.debug(message)
+            else:
+                logging.error(f"Invalid message format detected: {message}")
+
+        if ollama:
+            # Query the LLM using query_ollama instead of llm_pipeline
+            answer = query_ollama(messages)#, model="mistral:7b-instruct-fp16")
+
+            logging.info("\n[LOG] MISTRAL Answer:")
+            logging.info(answer)
+
+            can_description_answer = answer.split('#')
+        else:
+            try:
+                # Initial API call
+                completion = client.chat.completions.create(
+                    model="gpt-3.5-turbo",
+                    messages=messages,
+                    temperature=0
+                )
+                answer = completion.choices[0].message.content
+                logging.info('ANSWER: GPT')
+                logging.info(answer + '\n\n')
+
+
+            except Exception as e:
+                logging.error(f"Error during GPT API call: {e}")
+
+        return answer
+    else:
+        return 'error'
+    
+
+def main(modality, setup, input=None):
+    np.random.seed(42)
+    current_time = datetime.now().strftime("%Y_%m_%d_%H_%M")
+    data_path = f"data/preprocess_output_{setup}/file_comparison"
+    corpus_embeddings_path = 'tools/corpus_embeddings.json'
+    if setup is None:
+        sentences_train_embeddings_path = f"tools/sentences_train_embeddings_filtered_01.json"
+    else:
+        sentences_train_embeddings_path = f"tools/sentences_train_embeddings_{setup}.json"
+    rules_prompt_path_text2sign = 'tools/rules_prompt_text2sign.txt'
+    rules_prompt_path_sign2text = 'tools/rules_prompt_sign2text.txt'
+
+    # Model to use for sentence embeddings
+    model_name = "mixedbread-ai/mxbai-embed-large-v1"
+    model = SentenceTransformer(model_name)
+
+    # Load embeddings
+    with open(corpus_embeddings_path, 'r') as file:
+        corpus_embeddings = pd.DataFrame(json.load(file))
+
+    with open(sentences_train_embeddings_path, 'r') as file:
+        sentences_train_embeddings = pd.DataFrame(json.load(file))
+
+    if input:  # Se è fornita una frase personalizzata
+        if modality == 'text2sign':
+            answer, fsw_seq, can_desc_association_seq, joint_prob = AulSign(
+                input=input,
+                rules_prompt_path=rules_prompt_path_text2sign,
+                train_sentences=sentences_train_embeddings,
+                vocabulary=corpus_embeddings,
+                model=model,
+                ollama=False,
+                modality=modality
+            )
+            #print(f"Input Sentence: {input}")
+            print(f"Canonical Descriptions: {can_desc_association_seq}")
+            print(f"Translation (FSW): {fsw_seq}")
+            #print(f"Canonical Descriptions: {can_desc_association_seq}")
+            #print(f"Joint Probability: {joint_prob}")
+        
+        elif modality == 'sign2text': #qui l'input è una FSW seq, che deve essere mappata in canonicals
+            mapped_input = sign2text(input,corpus_embeddings_path)
+            logging.info(f"\nReconstructed Sentence via Vocaboulary: {mapped_input}")
+            answer= AulSign(
+                input=mapped_input,
+                rules_prompt_path=rules_prompt_path_sign2text,
+                train_sentences=sentences_train_embeddings,
+                vocabulary=corpus_embeddings,
+                model=model,
+                ollama=False,
+                modality=modality
+            )
+            print(f"Input Sign Voucaboualry Mapping: {input}")
+            print(f"Translation (Text): {answer}")
+
+    else:  # Flusso standard con testset
+        test_path = os.path.join(data_path, f"test.csv")
+        test = pd.read_csv(test_path)
+        test = test.head(1)
+
+        if modality == 'text2sign':
+            list_sentence = []
+            list_answer = []
+            list_fsw_seq = []
+            can_desc_association_list = []
+            prob_of_association_list = []
+
+            for index, row in test.iterrows():
+                sentence = row['sentence']
+                answer, fsw_seq, can_desc_association_seq, joint_prob = AulSign(
+                    input=sentence,
+                    rules_prompt_path=rules_prompt_path_text2sign,
+                    train_sentences=sentences_train_embeddings,
+                    vocabulary=corpus_embeddings,
+                    model=model,
+                    ollama=False,
+                    modality=modality
+                )
+
+                list_sentence.append(sentence)
+                list_answer.append(answer)
+                list_fsw_seq.append(fsw_seq)
+                can_desc_association_list.append(can_desc_association_seq)
+                prob_of_association_list.append(joint_prob)
+            
+            df_pred = pd.DataFrame({
+                'sentence': list_sentence,
+                'pseudo_cd': list_answer,
+                'pred_cd': can_desc_association_list,
+                'joint_prob': prob_of_association_list,
+                'pred_fsw_seq': list_fsw_seq
+            })
+            output_path = os.path.join('result', f"{modality}_{current_time}")
+            os.makedirs(output_path, exist_ok=True)
+            df_pred = prepare_dataset(df_pred,test,modality)
+            df_pred.to_csv(os.path.join(output_path, f'result_{current_time}.csv'), index=False)
+
+        elif modality == 'sign2text':
+
+            list_answer = []
+            list_gold_cd = []
+
+            for index, row in test.iterrows():
+                dec_sentence = row['word']
+                answer = AulSign(
+                    input=dec_sentence,
+                    rules_prompt_path=rules_prompt_path_sign2text,
+                    train_sentences=sentences_train_embeddings,
+                    vocabulary=corpus_embeddings,
+                    model=model,
+                    ollama=False,
+                    modality=modality
+                )
+                list_gold_cd.append(dec_sentence)
+                list_answer.append(answer)
+            
+            df_pred = pd.DataFrame({
+                'pseudo_sentence': list_answer,
+                'gold_cd': list_gold_cd,
+            })
+            output_path = os.path.join('result', f"{modality}_{current_time}")
+            os.makedirs(output_path, exist_ok=True)
+            df_pred = prepare_dataset(df_pred,test,modality)
+            df_pred.to_csv(os.path.join(output_path, f'result_{current_time}.csv'), index=False)
+
+if __name__ == "__main__":
+    
+    #sentence_to_analyze = "This is a new ASL translator"
+    #main(modality='text2sign', setup="filtered_01", input=sentence_to_analyze)
+    #main(modality='text2sign', setup="filtered_01")
+
+ 
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--mode", required=True, help="Mode of operation: text2sign or sign2text")
+    parser.add_argument("--input", help="Input text or sign sequence")
+    args = parser.parse_args()
+
+    main(args.mode, setup=None, input=args.input)

scripts/scripts/clean.py

@@ -0,0 +1,77 @@
+import re
+from signwriting.formats.fsw_to_sign import fsw_to_sign
+
+def clean_sign(fsw_list: list, glue=' ', sort = False):
+    list_factor_x, list_factor_y = [], []
+    list_symbols = []
+
+    for item in fsw_list:
+        sign = fsw_to_sign(item)  
+
+        # Aggiungi il simbolo principale con posizione se esiste
+        if 'box' in sign and 'symbol' in sign['box'] and 'position' in sign['box']:
+            #head del simbolo
+            box_symbol = sign['box']['symbol']
+            box_factor_x = sign['box']['position'][0]
+            box_factor_y = sign['box']['position'][1]
+            list_symbols.append(str(box_symbol))
+            list_factor_x.append(str(box_factor_x))
+            list_factor_y.append(str(box_factor_y))
+            if sort:
+            # Sort symbols alphabetically by 'symbol' key
+                symbols = sorted(sign['symbols'], key=lambda el: el['symbol'])
+            else:
+                symbols = sign['symbols']
+
+            for el in symbols:
+                symbol = el['symbol']
+                factor_x = el['position'][0]
+                factor_y = el['position'][1]
+                list_factor_x.append(str(factor_x))
+                list_factor_y.append(str(factor_y))
+                list_symbols.append(symbol)
+
+    return glue.join(fsw_list), glue.join(list_symbols), glue.join(list_factor_x), glue.join(list_factor_y)
+
+
+
+def replace_match(match):
+    return "|" + match.group(0)  # Helper function to add '|' separator
+
+def clean_symbol(symbols, pattern=r'\b(M|L|R|B)\b', glue=' ', order=False, add_box_symbol=False):
+    # Aggiungo i separatori ai simboli tramite il pattern
+    symbols = re.sub(pattern, replace_match, symbols)
+   
+    
+    # Rimuovo il primo carattere '|' se presente
+    if symbols.startswith('|'):
+        symbols = symbols[1:]
+   
+
+    symbols_cleaned_list = []
+    
+    # Itero sui simboli separati dal carattere '|'
+    for symbol in symbols.split('|'):
+        # Controllo se il simbolo inizia con uno dei caratteri target (M, L, R, B)
+      
+        if symbol.startswith(('M', 'L', 'R', 'B')):
+            box_symbol = symbol[0]  # Assegno la lettera come box_symbol
+            symbol_pure = symbol[1:].strip()  # Rimuovo la lettera dal simbolo
+     
+        if order:
+            symbol_pure = ' '.join(sorted(symbol_pure.split()))
+
+        else:
+            symbol_pure = ' '.join(symbol_pure.split())
+
+        # Aggiungo box_symbol al simbolo epurato
+        if add_box_symbol:
+            symbols_cleaned_list.append(box_symbol + ' ' + symbol_pure)
+        else:
+            symbols_cleaned_list.append(symbol_pure)
+        
+            
+    # Unisco tutti i simboli epurati con `glue`
+    return glue.join(symbols_cleaned_list)
+
+        

scripts/scripts/sign2text_mapping.py

@@ -0,0 +1,60 @@
+
+import pandas as pd
+import json
+from collections import Counter
+from scripts.scripts.clean import clean_sign, clean_symbol
+
+def get_most_freq(lista):
+    lista_cleaned = [item.lower().strip() for item in lista]
+    frequency_count = Counter(lista_cleaned)
+    top_two_words = frequency_count.most_common(2)
+
+    if len(top_two_words) >= 2:
+        return top_two_words[0][0] + '|' + top_two_words[1][0]
+    elif len(top_two_words) == 1:
+        return top_two_words[0][0]
+    else:
+        return ''
+
+def sign2text(fsw_seq:str, vocab_path:str):
+
+    df = pd.DataFrame({'fsw': [fsw_seq]})
+
+    df['symbol'] = df['fsw'].apply(lambda x: clean_sign(x.split())[1])
+    df['symbol'] = df['symbol'].apply(lambda x: clean_symbol(x,glue='|',order=False))
+
+    #print('\n') #deubg
+    #print(df.loc[0,'fsw']) #deubg
+    #print('\n')  #deubg
+    #print('\n') #deubg
+    #print(df.loc[0,'symbol']) #deubg
+
+
+    with open(vocab_path, 'r') as file:
+        content = file.read()
+        vocab = json.loads(content)
+
+    list_word = []
+
+    fsw, symbols = df.loc[0,'fsw'], df.loc[0,'symbol']
+    for symbol in symbols.split('|'):
+
+        #print(symbol) #debug
+        temp_ordered_symbol = ' '.join(sorted(symbol.split()))
+        #print(temp_ordered_symbol) #debug
+
+        mapped_words = [
+            ', '.join(entry['word'])
+            for entry in vocab
+            if 'symbol' in entry
+            and any(temp_ordered_symbol == s for s in (entry['symbol'] if isinstance(entry['symbol'], list) else [entry['symbol']]))
+        ]
+
+        if mapped_words:
+            canonical = get_most_freq(' '.join(mapped_words).split(','))
+            list_word.append(canonical)
+        else:
+            list_word.append("<unk>")
+        
+   
+    return ' # '.join(list_word)

tools/rules_prompt_sign2text.txt

@@ -0,0 +1,8 @@
+You will be provided with a decomposition of a sentence. Each component of the decomposition represents a lexical or semantic unit, followed by its interpretation or contextual synonym. Based on this decomposition, reconstruct the original sentence, ensuring grammatical correctness and coherence with the overall meaning. Follow these rules:
+
+1. Respect the logical order: Reconstruct the sentence by following the order of elements in the decomposition.
+2. Use synonyms or interpretations: Replace each term or symbol with its appropriate meaning or synonym indicated in the decomposition.
+3. Include punctuation: Use the indicated punctuation (e.g., full stop|. represents a period, comma|pause represents a comma, etc.).
+4. Infer the semantic units of the sentence even if corrrispondend <unk> are present.
+4. Adapt the structure: Rewrite the sentence so that it is grammatical and fluent, even if minor adjustments to the terms provided are necessary.
+5. Preserve the overall meaning: Ensure that the meaning of the reconstructed sentence aligns with the context suggested by the decomposition.

tools/rules_prompt_text2sign.txt

@@ -0,0 +1,11 @@
+Your task is to simplify each sentence by breaking it down into basic components such as ("{similar_canonical}"), using the following structure:
+
+1. Use the symbol " # " as separator between units
+    e.g "Jesus said" becames: "Jesus # said|proclamed"
+2. For each unit, provide only an additional related term, separated by a vertical bar (|).
+    e.g. "12" becames: "twelve|12"
+3. Identify and label all punctuation marks (e.g., "comma," "end of sentence," "colen").
+4. Repeat proper nouns as they are.
+    eg. "Jesus" becames: "Jesus"
+5. For numbers, include both the numeric form and its word equivalent.
+6. Avoid adding any extra explanation or commentary beyond the simplified structure.