app.py

import numpy as np
import tensorflow as tf
import tensorflow_hub as hub
import tf_keras as keras
import pandas as pd
from tensorflow.keras.models import load_model
import classifier_data_lib
import tokenization
import joblib
from deep_translator import GoogleTranslator
import sys
import os
import gradio as gr

model = load_model('ISCO-Coder-BERT.h5', custom_objects={'KerasLayer': hub.KerasLayer})

bert_layer = hub.KerasLayer("https://kaggle.com/models/tensorflow/bert/TensorFlow2/en-uncased-l-12-h-768-a-12/1",trainable=True)

vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file,do_lower_case)

# Parameters
max_seq_length = 128
dummy_label = 100
label_list = list(pd.read_excel('label_list.xlsx')['label_list'])

map_data = pd.read_excel("ISCO-08 EN Structure and definitions.xlsx")

label_encoder = joblib.load('label_encoder.joblib')

# Define a function to preprocess the new data
def get_feature_new(text, max_seq_length, tokenizer, dummy_label):
    example = classifier_data_lib.InputExample(guid=None,
                                               text_a=text.numpy().decode('utf-8'),
                                               text_b=None,
                                               label=dummy_label)  # Use a valid dummy label
    feature = classifier_data_lib.convert_single_example(0, example, label_list, max_seq_length, tokenizer)
    return feature.input_ids, feature.input_mask, feature.segment_ids

def get_feature_map_new(text):
    input_ids, input_mask, segment_ids = tf.py_function(
        lambda text: get_feature_new(text, max_seq_length, tokenizer, dummy_label),
        inp=[text],
        Tout=[tf.int32, tf.int32, tf.int32]
    )
    input_ids.set_shape([max_seq_length])
    input_mask.set_shape([max_seq_length])
    segment_ids.set_shape([max_seq_length])
    
    x = {'input_word_ids': input_ids,
         'input_mask': input_mask,
         'input_type_ids': segment_ids}
    
    return x

def preprocess_new_data(texts):
    dataset = tf.data.Dataset.from_tensor_slices((texts,))
    dataset = dataset.map(get_feature_map_new,
                          num_parallel_calls=tf.data.experimental.AUTOTUNE)
    dataset = dataset.batch(32, drop_remainder=False)
    dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
    
    return dataset
def launch(text_input):
    # Load the label encoder
    #label_encoder = joblib.load('label_encoderV2.joblib')

    # Preprocess the new data
    try:
        text_input = GoogleTranslator(source = 'auto',target = 'en').translate(text_input)
    except:
        text_input = text_input
        
    sample_example = [text_input]
    new_data_dataset = preprocess_new_data(sample_example)

    # Assuming you have a model already loaded (add model loading code if needed)
    # Make predictions on the new data
    predictions = model.predict(new_data_dataset)

    # Decode the predictions
    predicted_classes = [label_list[np.argmax(pred)] for pred in predictions]

    # Calculate the highest probabilities
    highest_probabilities = [max(instance) for instance in predictions]

    # Decode labels using the label encoder
    decoded_labels = label_encoder.inverse_transform(predicted_classes)


    # Retrieve the ISCO description based on the decoded label
    isco_description = map_data[map_data['ISCO 08 Code'] == decoded_labels[0]]['Title EN'].values
    
    # Print for debugging (optional)
    print(f"Most likely ISCO code is {decoded_labels[0]} and probability is {highest_probabilities[0]}")
    print(text_input)
    
    # Create descriptive text for the output
    result_text = (
        f"Predicted ISCO Code: {decoded_labels[0]}\n"
        f"Probability: {highest_probabilities[0]:.2f}\n"
        f"ISCO Description: {isco_description[0] if len(isco_description) > 0 else 'Description not found'}"
    )
    
    return result_text

# Define the Gradio interface
iface = gr.Interface(
    fn=launch,
    inputs=gr.Textbox(
        lines=2, 
        placeholder="Enter job title in any language (e.g., Software Engineer) AND/OR description here (e.g., Develops and maintains software applications)..."
    ),
    outputs=gr.Textbox(
        lines=4, 
        placeholder="Predicted ISCO Code: <result>\nProbability: <result>\nISCO Description: <result>"
    )
)

iface.launch()