Upload 5 files

app/database_build.py

@@ -0,0 +1,552 @@
+from sentence_transformers import SentenceTransformer, util
+import json
+import time
+import pandas as pd
+import numpy as np
+import pickle
+
+import chromadb
+from chromadb.config import Settings
+from chromadb.utils import embedding_functions
+from chromadb.db.clickhouse import NoDatapointsException
+
+
+def prepare_cd(conceptDescriptions):
+    df_cd = pd.DataFrame(
+        columns=["SemanticId", "Definition", "PreferredName", "Datatype", "Unit"]
+    )
+    # In den leeren DF werden alle Concept Descriptions eingelesen
+    for cd in conceptDescriptions:
+        semantic_id = cd["identification"]["id"]
+        data_spec = cd["embeddedDataSpecifications"][0]["dataSpecificationContent"]
+        preferred_name = data_spec["preferredName"]
+        short_name = data_spec["shortName"]
+        if len(preferred_name) > 1:
+            for name_variant in preferred_name:
+                if (
+                    name_variant["language"] == "EN"
+                    or name_variant["language"] == "en"
+                    or name_variant["language"] == "EN?"
+                ):
+                    name = name_variant["text"]
+        elif len(preferred_name) == 1:
+            name = preferred_name[0]["text"]
+        elif len(preferred_name) == 0:
+            short_name = data_spec["shortName"]
+            if len(short_name) == 0:
+                name = "NaN"
+            else:
+                name = short_name[0]["text"]
+
+        definition = data_spec["definition"]
+        if len(definition) > 1:
+            for definition_variant in definition:
+                if (
+                    definition_variant["language"] == "EN"
+                    or definition_variant["language"] == "en"
+                    or definition_variant["language"] == "EN?"
+                ):
+                    chosen_def = definition_variant["text"]
+        elif len(definition) == 1:
+            chosen_def = definition[0]["text"]
+        elif len(definition) == 0:
+            chosen_def = "NaN"
+
+        if data_spec["dataType"] == "":
+            datatype = "NaN"
+        else:
+            datatype = data_spec["dataType"]
+
+        if data_spec["unit"] == "":
+            unit = "NaN"
+        else:
+            unit = data_spec["unit"]
+
+        new_entry = pd.DataFrame(
+            {
+                "SemanticId": semantic_id,
+                "Definition": chosen_def,
+                "PreferredName": name,
+                "Datatype": datatype,
+                "Unit": unit,
+            },
+            index=[0],
+        )
+        df_cd = pd.concat([df_cd, new_entry], ignore_index=True)
+    return df_cd
+
+
+def get_values(submodel_element):
+    # Auslesen der Submodel Element Werte
+    se_type = submodel_element["modelType"]["name"]
+    se_semantic_id = submodel_element["semanticId"]["keys"][0]["value"]
+    se_semantic_id_local = submodel_element["semanticId"]["keys"][0]["local"]
+    se_id_short = submodel_element["idShort"]
+    value = []
+    se_value = submodel_element["value"]
+    value.append(se_value)
+
+    return se_type, se_semantic_id, se_semantic_id_local, se_id_short, value
+
+
+def get_concept_description(semantic_id, df_cd):
+    cd_content = df_cd.loc[df_cd["SemanticId"] == semantic_id]
+
+    if cd_content.empty:
+        cd_content = pd.DataFrame(
+            {
+                "SemanticId": semantic_id,
+                "Definition": "NaN",
+                "PreferredName": "NaN",
+                "Datatype": "NaN",
+                "Unit": "NaN",
+            },
+            index=[0],
+        )
+
+    cd_content = cd_content.iloc[0]
+
+    return cd_content
+
+
+def get_values_sec(
+    df_cd,
+    content,
+    df,
+    aas_id,
+    aas_name,
+    submodel_id,
+    submodel_name,
+    submodel_semantic_id,
+):
+    collection_values = content[0]["value"]
+    for element in collection_values:
+        content = []
+        content.append(element)
+
+        se_type, se_semantic_id, se_semantic_id_local, se_id_short, value = get_values(
+            element
+        )
+        if se_type == "SubmodelElementCollection":
+            if se_semantic_id_local == True:
+                cd_content = get_concept_description(se_semantic_id, df_cd)
+                definition = cd_content["Definition"]
+                preferred_name = cd_content["PreferredName"]
+                datatype = cd_content["Datatype"]
+                unit = cd_content["Unit"]
+
+            else:
+                definition = "NaN"
+                preferred_name = "NaN"
+                datatype = "NaN"
+                unit = "NaN"
+
+            new_row = pd.DataFrame(
+                {
+                    "AASId": aas_id,
+                    "AASIdShort": aas_name,
+                    "SubmodelId": submodel_id,
+                    "SubmodelName": submodel_name,
+                    "SubmodelSemanticId": submodel_semantic_id,
+                    "SEContent": content,
+                    "SESemanticId": se_semantic_id,
+                    "SEModelType": se_type,
+                    "SEIdShort": se_id_short,
+                    "SEValue": value,
+                    "Definition": definition,
+                    "PreferredName": preferred_name,
+                    "Datatype": datatype,
+                    "Unit": unit,
+                }
+            )
+            df = pd.concat([df, new_row], ignore_index=True)
+
+            content = []
+            content.append(element)
+            # Rekursive Funktion -> so oft durchlaufen bis unterste Ebene der Collections erreicht ist, so werden verschachteltet SECs bis zum Ende ausgelesen
+            df = get_values_sec(
+                df_cd,
+                content,
+                df,
+                aas_id,
+                aas_name,
+                submodel_id,
+                submodel_name,
+                submodel_semantic_id,
+            )
+
+        else:
+            if se_semantic_id_local == True:
+                cd_content = get_concept_description(se_semantic_id, df_cd)
+                definition = cd_content["Definition"]
+                preferred_name = cd_content["PreferredName"]
+                datatype = cd_content["Datatype"]
+                unit = cd_content["Unit"]
+
+            else:
+                definition = "NaN"
+                preferred_name = "NaN"
+                datatype = "NaN"
+                unit = "NaN"
+
+            new_row = pd.DataFrame(
+                {
+                    "AASId": aas_id,
+                    "AASIdShort": aas_name,
+                    "SubmodelId": submodel_id,
+                    "SubmodelName": submodel_name,
+                    "SubmodelSemanticId": submodel_semantic_id,
+                    "SEContent": content,
+                    "SESemanticId": se_semantic_id,
+                    "SEModelType": se_type,
+                    "SEIdShort": se_id_short,
+                    "SEValue": value,
+                    "Definition": definition,
+                    "PreferredName": preferred_name,
+                    "Datatype": datatype,
+                    "Unit": unit,
+                }
+            )
+            df = pd.concat([df, new_row], ignore_index=True)
+
+    return df
+
+
+def set_up_metadata(metalabel, df):
+    datatype_mapping = {
+        "boolean": "BOOLEAN",
+        "string": "STRING",
+        "string_translatable": "STRING",
+        "translatable_string": "STRING",
+        "non_translatable_string": "STRING",
+        "date": "DATE",
+        "data_time": "DATE",
+        "uri": "URI",
+        "int": "INT",
+        "int_measure": "INT",
+        "int_currency": "INT",
+        "integer": "INT",
+        "real": "REAL",
+        "real_measure": "REAL",
+        "real_currency": "REAL",
+        "enum_code": "ENUM_CODE",
+        "enum_int": "ENUM_CODE",
+        "ENUM_REAL": "ENUM_CODE",
+        "ENUM_RATIONAL": "ENUM_CODE",
+        "ENUM_BOOLEAN": "ENUM_CODE",
+        "ENUM_STRING": "ENUM_CODE",
+        "enum_reference": "ENUM_CODE",
+        "enum_instance": "ENUM_CODE",
+        "set(b1,b2)": "SET",
+        "constrained_set(b1,b2,cmn,cmx)": "SET",
+        "set [0,?]": "SET",
+        "set [1,?]": "SET",
+        "set [1, ?]": "SET",
+        "nan": "NaN",
+        "media_type": "LARGE_OBJECT_TYPE",
+    }
+
+    unit_mapping = {
+        "nan": "NaN",
+        "hertz": "FREQUENCY",
+        "hz": "FREQUENCY",
+        "pa": "PRESSURE",
+        "pascal": "PRESSURE",
+        "n/m²": "PRESSURE",
+        "bar": "PRESSURE",
+        "%": "SCALARS_PERC",
+        "w": "POWER",
+        "watt": "POWER",
+        "kw": "POWER",
+        "kg/m³": "CHEMISTRY",
+        "m²/s": "CHEMISTRY",
+        "pa*s": "CHEMISTRY",
+        "v": "ELECTRICAL",
+        "volt": "ELECTRICAL",
+        "db": "ACOUSTICS",
+        "db(a)": "ACOUSTICS",
+        "k": "TEMPERATURE",
+        "°c": "TEMPERATURE",
+        "n": "MECHANICS",
+        "newton": "MECHANICS",
+        "kg/s": "FLOW",
+        "kg/h": "FLOW",
+        "m³/s": "FLOW",
+        "m³/h": "FLOW",
+        "l/s": "FLOW",
+        "l/h": "FLOW",
+        "µm": "LENGTH",
+        "mm": "LENGTH",
+        "cm": "LENGTH",
+        "dm": "LENGTH",
+        "m": "LENGTH",
+        "meter": "LENGTH",
+        "m/s": "SPEED",
+        "km/h": "SPEED",
+        "s^(-1)": "FREQUENCY",
+        "1/s": "FREQUENCY",
+        "s": "TIME",
+        "h": "TIME",
+        "min": "TIME",
+        "d": "TIME",
+        "hours": "TIME",
+        "a": "ELECTRICAL",
+        "m³": "VOLUME",
+        "m²": "AREA",
+        "rpm": "FLOW",
+        "nm": "MECHANICS",
+        "m/m": "MECHANICS",
+        "m³/m²s": "MECHANICS",
+        "w(m²*K)": "HEAT_TRANSFER",
+        "kwh": "ELECTRICAL",
+        "kg/(s*m²)": "FLOW",
+        "kg": "MASS",
+        "w/(m*k)": "HEAT_TRANSFER",
+        "m²*k/w": "HEAT_TRANSFER",
+        "j/s": "POWER",
+    }
+
+    dataset = df
+    dataset["unit_lowercase"] = dataset["Unit"]
+    dataset["unit_lowercase"] = dataset["unit_lowercase"].str.lower()
+    dataset["unit_categ"] = dataset["unit_lowercase"].map(unit_mapping)
+
+    dataset["datatype_lowercase"] = dataset["Datatype"]
+    dataset["datatype_lowercase"] = dataset["datatype_lowercase"].str.lower()
+    dataset["datatype_categ"] = dataset["datatype_lowercase"].map(datatype_mapping)
+
+    dataset = dataset.fillna("NaN")
+    dataset["index"] = dataset.index
+
+    # uni_datatype=dataset['datatype_categ'].unique()
+    # uni_unit=dataset['unit_categ'].unique()
+    unique_labels_set = set()
+
+    dataset["Metalabel"] = ""
+    for i in range(0, len(dataset["Metalabel"])):
+        concat = (str(dataset["unit_categ"][i]), str(dataset["datatype_categ"][i]))
+        keys = [k for k, v in metalabel.items() if v == concat]
+        dataset["Metalabel"][i] = keys[0]
+        unique_labels_set.add(keys[0])
+    unique_label = list(unique_labels_set)
+    print(unique_label)
+
+    return dataset
+
+
+def encode(aas_df, model):
+    # Einsatz von Sentence Bert um Embeddings zu kreieren
+    aas_df["PreferredName"] = "Name: " + aas_df["PreferredName"].astype(str)
+    aas_df["Definition"] = "Description: " + aas_df["Definition"].astype(str) + "; "
+    corpus_names = aas_df.loc[:, "PreferredName"]
+    corpus_definitions = aas_df.loc[:, "Definition"]
+    embeddings_definitions = model.encode(corpus_definitions, show_progress_bar=True)
+    embeddings_names = model.encode(corpus_names, show_progress_bar=True)
+    concat_name_def_emb = np.concatenate(
+        (embeddings_definitions, embeddings_names), axis=1
+    )
+    # aas_df['EmbeddingDefinition'] = embeddings_definitions.tolist()
+    # aas_df['EmbeddingName'] = embeddings_names.tolist()
+    aas_df["EmbeddingNameDefinition"] = concat_name_def_emb.tolist()
+    return aas_df
+
+
+def convert_to_list(aas_df):
+    # Für die Datenbank werden teilweise Listen gebraucht
+    aas_index = aas_df.index.tolist()
+    aas_index_str = [str(r) for r in aas_index]
+    se_content = aas_df["SEContent"].tolist()
+    se_embedding_name_definition = aas_df["EmbeddingNameDefinition"].tolist()
+
+    aas_df_dropped = aas_df.drop(
+        ["EmbeddingNameDefinition", "SEContent", "SEValue"], axis=1
+    )
+
+    metadata = aas_df_dropped.to_dict("records")
+
+    return metadata, aas_index_str, se_content, se_embedding_name_definition
+
+
+def set_up_chroma(
+    metadata, aas_index_str, se_content, se_embedding_name_definition, aas_name, client
+):
+    aas_name = aas_name.lower()
+    # Kein Großbuchstaben in Datenbank erlaubt
+    print(aas_name)
+    # client = chromadb.Client(Settings(
+    #    chroma_db_impl="duckdb+parquet",
+    #    persist_directory="./drive/My Drive/Colab/NLP/SemantischeInteroperabilität/Deployment" # Optional, defaults to .chromadb/ in the current directory
+    # ))
+    emb_fn = embedding_functions.SentenceTransformerEmbeddingFunction(
+        model_name="gart-labor/eng-distilBERT-se-eclass"
+    )
+    collection = client.get_or_create_collection(
+        name=aas_name, embedding_function=emb_fn
+    )
+
+    aas_content_string = []
+    # Umwandeln in Json damit es in db geschrieben werden kann
+    for element in se_content:
+        content = json.dumps(element)
+        aas_content_string.append(content)
+
+    items = collection.count()  # returns the number of items in the collection
+    print(collection)
+    print("Datenbank erstellt, Anzahl Items:")
+    print(items)
+    if items == 0: 
+        # Hinzufügen der SE Inhalte, der Embeddings und weiterer Metadaten in collection der Datenbank
+        collection.add(
+            documents=aas_content_string,
+            embeddings=se_embedding_name_definition,
+            metadatas=metadata,
+            ids=aas_index_str,
+        )
+        items = collection.count()  # returns the number of items in the collection
+        print("------------")
+        print("Datenbank befüllt, Anzahl items:")
+        print(items)
+    else:
+        print("-----------")
+        print("AAS schon vorhanden")
+
+    return collection
+
+
+def read_aas(aas, submodels, assets, conceptDescriptions, submodels_ids, metalabel):
+    df = pd.DataFrame(
+        columns=[
+            "AASId",
+            "AASIdShort",
+            "SubmodelId",
+            "SubmodelName",
+            "SubmodelSemanticId",
+            "SEContent",
+            "SESemanticId",
+            "SEModelType",
+            "SEIdShort",
+            "SEValue",
+            "Definition",
+            "PreferredName",
+            "Datatype",
+            "Unit",
+        ]
+    )
+
+    aas_id = aas[0]["identification"]["id"]
+    aas_name = aas[0]["idShort"]
+    # Aufbereiten aller Concept descriptions als pandas dataframe, damit diese nachher einfacher untersucht werden können
+    df_cd = prepare_cd(conceptDescriptions)
+    # Auslesen der Teilmodelle
+    for submodel in submodels:
+        submodel_name = submodel["idShort"]
+        submodel_id = submodel["identification"]["id"]
+        # Muss gemacht werden, da Anzahl der Teilmodelle innerhalb der AAS und des Env nicht immer übereisntimmen
+        if submodel_id in submodels_ids:
+            semantic_id_existing = submodel["semanticId"]["keys"]
+            if not semantic_id_existing:
+                submodel_semantic_id = "Not defined"
+            else:
+                submodel_semantic_id = semantic_id_existing[0]["value"]
+            submodel_elements = submodel["submodelElements"]
+            # Auslesen Submodel Elements
+            for submodel_element in submodel_elements:
+                content = []
+                content.append(submodel_element)
+
+                (
+                    se_type,
+                    se_semantic_id,
+                    se_semantic_id_local,
+                    se_id_short,
+                    value,
+                ) = get_values(submodel_element)
+
+                # When Concept Description local dann auslesen der Concept Description
+                if se_semantic_id_local == True:
+                    cd_content = get_concept_description(se_semantic_id, df_cd)
+                    definition = cd_content["Definition"]
+                    preferred_name = cd_content["PreferredName"]
+                    datatype = cd_content["Datatype"]
+                    unit = cd_content["Unit"]
+
+                else:
+                    definition = "NaN"
+                    preferred_name = "NaN"
+                    datatype = "NaN"
+                    unit = "NaN"
+
+                new_row = pd.DataFrame(
+                    {
+                        "AASId": aas_id,
+                        "AASIdShort": aas_name,
+                        "SubmodelId": submodel_id,
+                        "SubmodelName": submodel_name,
+                        "SubmodelSemanticId": submodel_semantic_id,
+                        "SEContent": content,
+                        "SESemanticId": se_semantic_id,
+                        "SEModelType": se_type,
+                        "SEIdShort": se_id_short,
+                        "SEValue": value,
+                        "Definition": definition,
+                        "PreferredName": preferred_name,
+                        "Datatype": datatype,
+                        "Unit": unit,
+                    }
+                )
+                df = pd.concat([df, new_row], ignore_index=True)
+
+                # Wenn Submodel Element Collection dann diese Werte auch auslesen
+                if se_type == "SubmodelElementCollection":
+                    df = get_values_sec(
+                        df_cd,
+                        content,
+                        df,
+                        aas_id,
+                        aas_name,
+                        submodel_id,
+                        submodel_name,
+                        submodel_semantic_id,
+                    )
+        else:
+            continue
+
+    df = set_up_metadata(metalabel, df)
+
+    return df, aas_name
+
+
+def index_corpus(data, model, metalabel, client_chroma):
+    # Start Punkt
+
+    aas = data["assetAdministrationShells"]
+    aas_submodels = aas[0]["submodels"]
+    submodels_ids = []
+    for submodel in aas_submodels:
+        submodels_ids.append(submodel["keys"][0]["value"])
+    submodels = data["submodels"]
+    conceptDescriptions = data["conceptDescriptions"]
+    assets = data["assets"]
+
+    aas_df, aas_name = read_aas(
+        aas, submodels, assets, conceptDescriptions, submodels_ids, metalabel
+    )
+    # aas_df_embeddings = encode(aas_df, model)
+    aas_df = encode(aas_df, model)
+    metadata, aas_index_str, se_content, se_embedding_name_definition = convert_to_list(
+        aas_df
+    )
+    collection = set_up_chroma(
+        metadata,
+        aas_index_str,
+        se_content,
+        se_embedding_name_definition,
+        aas_name,
+        client_chroma,
+    )
+
+    return collection
+
+
+# if __name__ == '__main__':
+#    create_database = index_corpus(aas = 'festo_switch.json')

app/main.py

@@ -0,0 +1,110 @@
+from sentence_transformers import SentenceTransformer, util
+
+# from transformers import MBartForConditionalGeneration, MBart50TokenizerFast
+import time
+import os
+import json
+import pandas as pd
+import numpy as np
+import category_encoders as ce
+import string
+import pickle
+import tqdm.autonotebook
+from fastapi import FastAPI, Request, UploadFile, File
+from joblib import dump, load
+from pydantic import BaseModel
+import sys
+from database_build import index_corpus
+from predict_different_aas import ask_database
+from predict_one_aas import query_specific_aas
+from typing import Any, Dict, AnyStr, List, Union
+import chromadb
+from chromadb.config import Settings
+from typing import Union
+
+app = FastAPI(title="Interface Semantic Matching")
+
+JSONObject = Dict[AnyStr, Any]
+JSONArray = List[Any]
+JSONStructure = Union[JSONArray, JSONObject]
+
+
+class submodelElement(BaseModel):
+    datatype: str 
+    definition: str
+    name: str
+    semantic_id: str
+    unit: str 
+    return_matches: int
+    aas_id: str
+    number_aas_returned: int
+
+@app.on_event("startup")
+def load_hf_model():
+    global model
+    # Altes Modell
+    # model = SentenceTransformer('mboth/distil-eng-quora-sentence')
+
+    # Fine Tuned Modell
+    model = SentenceTransformer("gart-labor/eng-distilBERT-se-eclass")
+
+    # global model_translate
+    # model_translate = MBartForConditionalGeneration.from_pretrained("facebook/mbart-large-50-many-to-many-mmt")
+    # global tokenizer_translate
+    # tokenizer_translate = MBart50TokenizerFast.from_pretrained("facebook/mbart-large-50-many-to-many-mmt")
+
+    with open("app/metadata.pickle", "rb") as handle:
+        global metalabel
+        metalabel = pickle.load(handle)
+    global client_chroma
+    client_chroma = chromadb.Client(
+        Settings(
+            chroma_api_impl="rest",
+            # chroma_server_host muss angepasst werden nach jedem Neustart AWS
+            chroma_server_host="3.67.80.82",
+            chroma_server_http_port=8000,
+        )
+    )
+
+
+@app.post("/PostAssetAdministrationShellEmbeddings")
+async def index_aas(aas: UploadFile = File(...)):
+    data = json.load(aas.file)
+    print(type(data))
+    # aas = new_file
+    #aas, submodels, conceptDescriptions, assets, aas_df, collection, aas_name= index_corpus(data, model, metalabel, client_chroma)
+    collection = index_corpus(data, model, metalabel, client_chroma)
+    ready = 'AAS ready'
+    return ready
+
+
+@app.post("/GetSubmodelElementsFromDifferentAASBySemanticIdAndSemanticInformation")
+def predict_different_aas(name: str, definition: str, number_aas_returned: Union[int, None] = 1, semantic_id: Union[str, None] = "NaN", unit: Union[str, None] = "NaN", datatype: Union[str, None] = "NaN"):
+    collections = client_chroma.list_collections()
+    query = {
+        "Name": name,
+        "Definition": definition,
+        "Unit": unit,
+        "Datatype": datatype,
+        "SemanticId": semantic_id,
+        "NumberAASReturned": number_aas_returned
+    }
+    results = ask_database(query, metalabel, model, collections, client_chroma)
+
+    return results
+
+@app.post("/GetSubmodelElementsFromSpecificAASBySemanticIdAndSemanticInformation")
+def predict_specific_aas(name: str, definition: str, aas_id: str, return_matches:  Union[int, None] = 2,  semantic_id: Union[str, None] = "NaN", unit: Union[str, None] = "NaN", datatype: Union[str, None] = "NaN"):
+    collections = client_chroma.list_collections()
+    query = {
+        "Name": name,
+        "Definition": definition,
+        "Unit": unit,
+        "Datatype": datatype,
+        "SemanticId": semantic_id,
+        "ReturnMatches": return_matches,
+        "AASId": aas_id,
+    }
+    result = query_specific_aas(query, metalabel, model, collections, client_chroma)
+
+    return result

app/metadata.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b4aee0cd2ca534e4af8023bd334db591a0a46b2a37154758aa5e3873b8d4728
+size 1670

app/predict_different_aas.py

@@ -0,0 +1,291 @@
+from sentence_transformers import SentenceTransformer, util
+import json
+import time
+import pandas as pd
+import numpy as np
+import pickle
+
+import chromadb
+from chromadb.config import Settings
+from chromadb.utils import embedding_functions
+from chromadb.db.clickhouse import NoDatapointsException
+
+
+def query_aas(query_json, collection, model, metalabel):
+    query = json.loads(query_json)
+    name = query["Name"]
+    definition = query["Definition"]
+    unit = query["Unit"]
+    datatype = query["Datatype"]
+    semantic_id = query["SemanticId"]
+    numberAAS = query["NumberAASReturned"]
+
+    #model = SentenceTransformer("gart-labor/eng-distilBERT-se-eclass")
+
+    datatype_mapping = {
+        "boolean": "BOOLEAN",
+        "string": "STRING",
+        "string_translatable": "STRING",
+        "translatable_string": "STRING",
+        "non_translatable_string": "STRING",
+        "date": "DATE",
+        "data_time": "DATE",
+        "uri": "URI",
+        "int": "INT",
+        "int_measure": "INT",
+        "int_currency": "INT",
+        "integer": "INT",
+        "real": "REAL",
+        "real_measure": "REAL",
+        "real_currency": "REAL",
+        "enum_code": "ENUM_CODE",
+        "enum_int": "ENUM_CODE",
+        "ENUM_REAL": "ENUM_CODE",
+        "ENUM_RATIONAL": "ENUM_CODE",
+        "ENUM_BOOLEAN": "ENUM_CODE",
+        "ENUM_STRING": "ENUM_CODE",
+        "enum_reference": "ENUM_CODE",
+        "enum_instance": "ENUM_CODE",
+        "set(b1,b2)": "SET",
+        "constrained_set(b1,b2,cmn,cmx)": "SET",
+        "set [0,?]": "SET",
+        "set [1,?]": "SET",
+        "set [1, ?]": "SET",
+        "nan": "NaN",
+        "media_type": "LARGE_OBJECT_TYPE",
+    }
+
+    unit_mapping = {
+        "nan": "NaN",
+        "hertz": "FREQUENCY",
+        "hz": "FREQUENCY",
+        "pa": "PRESSURE",
+        "pascal": "PRESSURE",
+        "n/m²": "PRESSURE",
+        "bar": "PRESSURE",
+        "%": "SCALARS_PERC",
+        "w": "POWER",
+        "watt": "POWER",
+        "kw": "POWER",
+        "kg/m³": "CHEMISTRY",
+        "m²/s": "CHEMISTRY",
+        "pa*s": "CHEMISTRY",
+        "v": "ELECTRICAL",
+        "volt": "ELECTRICAL",
+        "db": "ACOUSTICS",
+        "db(a)": "ACOUSTICS",
+        "k": "TEMPERATURE",
+        "°c": "TEMPERATURE",
+        "n": "MECHANICS",
+        "newton": "MECHANICS",
+        "kg/s": "FLOW",
+        "kg/h": "FLOW",
+        "m³/s": "FLOW",
+        "m³/h": "FLOW",
+        "l/s": "FLOW",
+        "l/h": "FLOW",
+        "µm": "LENGTH",
+        "mm": "LENGTH",
+        "cm": "LENGTH",
+        "dm": "LENGTH",
+        "m": "LENGTH",
+        "meter": "LENGTH",
+        "m/s": "SPEED",
+        "km/h": "SPEED",
+        "s^(-1)": "FREQUENCY",
+        "1/s": "FREQUENCY",
+        "s": "TIME",
+        "h": "TIME",
+        "min": "TIME",
+        "d": "TIME",
+        "hours": "TIME",
+        "a": "ELECTRICAL",
+        "m³": "VOLUME",
+        "m²": "AREA",
+        "rpm": "FLOW",
+        "nm": "MECHANICS",
+        "m/m": "MECHANICS",
+        "m³/m²s": "MECHANICS",
+        "w(m²*K)": "HEAT_TRANSFER",
+        "kwh": "ELECTRICAL",
+        "kg/(s*m²)": "FLOW",
+        "kg": "MASS",
+        "w/(m*k)": "HEAT_TRANSFER",
+        "m²*k/w": "HEAT_TRANSFER",
+        "j/s": "POWER",
+    }
+
+    #with open(
+    #    "./drive/My Drive/Colab/NLP/SemantischeInteroperabilität/Deployment/metadata.pickle",
+    #    "rb",
+    #) as handle:
+    #    metalabel = pickle.load(handle)
+
+    unit_lower = unit.lower()
+    datatype_lower = datatype.lower()
+
+    unit_categ = unit_mapping.get(unit_lower)
+    datatype_categ = datatype_mapping.get(datatype_lower)
+
+    if unit_categ == None:
+        unit_categ = "NaN"
+    if datatype_categ == None:
+        datatype_categ = "NaN"
+
+    concat = (unit_categ, datatype_categ)
+    keys = [k for k, v in metalabel.items() if v == concat]
+    metadata = keys[0]
+
+    name_embedding = model.encode(name)
+    definition_embedding = model.encode(definition)
+    concat_name_def_query = np.concatenate(
+        (definition_embedding, name_embedding), axis=0
+    )
+    concat_name_def_query = concat_name_def_query.tolist()
+
+    queries = [concat_name_def_query]
+    print(type(queries))
+
+    # Query wird mit Semantic Search, k-nearest-neighbor durchgeführt
+    # Chroma verwendet hierfür hnswlib https://github.com/nmslib/hnswlib
+    # Dort kann als Distanz Cosine, Squared L2 oder Inner Product eingestellt werden
+    # In Chroma ist L2 als Distanz eingestellt, vgl. https://github.com/chroma-core/chroma/blob/4463d13f951a4d28ade1f7e777d07302ff09069b/chromadb/db/index/hnswlib.py -> suche nach l2
+
+    # Homogener fall, untersuchen nach Semant Ids, wenn welche gefunden werden, ist homgen erfolgreich
+    try:
+        homogen = collection.query(
+            query_embeddings=queries, n_results=1, where={"SESemanticId": semantic_id}
+        )
+    # except NoDatapointsException:
+    #  homogen = 'Nix'
+
+    except Exception:
+        homogen = "Nix"
+
+    if homogen != "Nix":
+        result = homogen
+        result["matching_method"] = "Semantic equivalent , same semantic Id"
+        result["matching_algorithm"] = "None"
+        result["distances"] = [[0]]
+
+        value = result['documents'][0][0]
+        value_dict = json.loads(value)
+
+        final_result = {
+        "matching_method": result['matching_method'],
+        "matching_algorithm": result['matching_algorithm'],
+        "matching_distance": result['distances'][0][0],
+        "aas_id": result['metadatas'][0][0]['AASId'],
+        "aas_id_short": result['metadatas'][0][0]['AASIdShort'],
+        "submodel_id_short": result['metadatas'][0][0]['SubmodelName'],
+        "submodel_id": result['metadatas'][0][0]['SubmodelId'],
+        "matched_object": value_dict,
+        }
+        #final_results = [final_result]
+    # Wenn keine passende semantic id gefunden, dann weiter mit NLP mit und ohne Metadaten
+    elif homogen == "Nix":
+        try:
+            with_metadata = collection.query(
+                query_embeddings=queries,
+                n_results=1,
+                where={"Metalabel": metadata},
+            )
+
+        # except NoDatapointsException:
+        #  with_metadata = 'Nix'
+
+        except Exception:
+            with_metadata = "Nix"
+
+        without_metadata = collection.query(
+            query_embeddings=queries,
+            n_results=1,
+        )
+
+        if with_metadata == "Nix":
+            result = without_metadata
+            result[
+                "matching_method"
+            ] = "Semantically not equivalent, NLP without Metadata"
+            result[
+                "matching_algorithm"
+            ] = "Semantic search, k-nearest-neighbor with squared L2 distance (euclidean distance), with model gart-labor/eng-distilBERT-se-eclass"
+
+        elif with_metadata != "Nix":
+            distance_with_meta = with_metadata["distances"][0][0]
+            distance_without_meta = without_metadata["distances"][0][0]
+            print(distance_with_meta)
+            print(distance_without_meta)
+            # Vergleich der Abstände von mit und ohne Metadaten
+            if distance_without_meta <= distance_with_meta:
+                result = without_metadata
+                result[
+                    "matching_method"
+                ] = "Semantically not equivalent, NLP without Metadata"
+                result[
+                    "matching_algorithm"
+                ] = "Semantic search, k-nearest-neighbor with squared L2 distance (euclidean distance), with model gart-labor/eng-distilBERT-se-eclass"
+
+            else:
+                result = with_metadata
+                result[
+                    "matching_method"
+                ] = "Semantically not equivalent, NLP without Metadata"
+                result[
+                    "matching_algorithm"
+                ] = "Semantic search, k-nearest-neighbor with squared L2 distance (euclidean distance), with model gart-labor/eng-distilBERT-se-eclass"
+        # Aufbereiten des passenden finalen Ergebnisses
+        """
+        final_results = []
+        for i in range(0, return_matches):
+            value = result['documents'][0][i]
+            value_dict = json.loads(value)
+            final_result = {
+            "matching_method": result['matching_method'],
+            "matching_algorithm": result['matching_algorithm'],
+            "matching_distance": result['distances'][0][i],
+            "aas_id": result['metadatas'][0][i]['AASId'],
+            "aas_id_short": result['metadatas'][0][i]['AASIdShort'],
+            "submodel_id_short": result['metadatas'][0][i]['SubmodelName'],
+            "submodel_id": result['metadatas'][0][i]['SubmodelId'],
+            #"matched_object": result['documents'][0][i]
+            "matched_object": value_dict
+            }
+            final_results.append(final_result)
+        """
+        value = result['documents'][0][0]
+        value_dict = json.loads(value)
+        final_result = {
+            "matching_method": result['matching_method'],
+            "matching_algorithm": result['matching_algorithm'],
+            "matching_distance": result['distances'][0][0],
+            "aas_id": result['metadatas'][0][0]['AASId'],
+            "aas_id_short": result['metadatas'][0][0]['AASIdShort'],
+            "submodel_id_short": result['metadatas'][0][0]['SubmodelName'],
+            "submodel_id": result['metadatas'][0][0]['SubmodelId'],
+            "matched_object": value_dict
+        }
+    return final_result
+
+def get_best_results(json_query, results):
+  query = json.loads(json_query) 
+  numberAAS = query["NumberAASReturned"]
+  sorted_results = sorted(results, key=lambda aas: aas['matching_distance']) 
+  numberAAS_count = numberAAS-1
+  best_results = sorted_results[0:numberAAS]
+  
+  return best_results
+
+
+def ask_database(query, metalabel, model, collections, client_chroma):
+    # Alle AAS werden nacheinaner abgefragt
+    json_query = json.dumps(query, indent=4)
+    results = []
+    for collection in collections:
+        print(collection.name)
+        collection = client_chroma.get_collection(collection.name)
+        result = query_aas(json_query, collection, model, metalabel)
+        results.append(result)
+    #results_json = json.dumps(results)
+    best_results = get_best_results(json_query, results)
+    return best_results

app/predict_one_aas.py

@@ -0,0 +1,188 @@
+from sentence_transformers import SentenceTransformer, util
+import json
+import time
+import pandas as pd
+import numpy as np
+import pickle
+
+import chromadb
+from chromadb.config import Settings
+from chromadb.utils import embedding_functions
+from chromadb.db.clickhouse import NoDatapointsException
+
+def query_right_aas(json_query, collection, metalabel, model):
+  query = json.loads(json_query) 
+  name = query['Name']
+  definition = query["Definition"]
+  unit = query["Unit"]
+  datatype = query["Datatype"]
+  semantic_id = query["SemanticId"]
+  return_matches = query["ReturnMatches"]
+
+  datatype_mapping = {'boolean': 'BOOLEAN', 'string': 'STRING', 'string_translatable':'STRING', 'translatable_string': 'STRING', 'non_translatable_string':'STRING',
+      'date':'DATE', 'data_time':'DATE', 'uri':'URI', 'int':'INT', 'int_measure':'INT', 'int_currency':'INT', 'integer': 'INT',
+      'real':'REAL', 'real_measure': 'REAL', 'real_currency':'REAL', 'enum_code': 'ENUM_CODE', 'enum_int':'ENUM_CODE',
+      'ENUM_REAL': 'ENUM_CODE', 'ENUM_RATIONAL': 'ENUM_CODE', 'ENUM_BOOLEAN': 'ENUM_CODE', 'ENUM_STRING': 'ENUM_CODE',
+      'enum_reference': 'ENUM_CODE', 'enum_instance': 'ENUM_CODE', 'set(b1,b2)': 'SET',
+      'constrained_set(b1,b2,cmn,cmx)': 'SET', 'set [0,?]': 'SET', 'set [1,?]': 'SET','set [1, ?]': 'SET', 'nan': 'NaN',
+        'media_type':'LARGE_OBJECT_TYPE'}
+      
+  unit_mapping = {'nan': 'NaN', 'hertz': 'FREQUENCY', 'hz': 'FREQUENCY', 'pa': 'PRESSURE', 'pascal': 'PRESSURE', 'n/m²':'PRESSURE',
+        'bar': 'PRESSURE', '%': 'SCALARS_PERC', 'w': 'POWER', 'watt': 'POWER', 'kw': 'POWER', 'kg/m³':'CHEMISTRY',
+        'm²/s': 'CHEMISTRY', 'pa*s': 'CHEMISTRY', 'v':'ELECTRICAL', 'volt': 'ELECTRICAL', 'db': 'ACOUSTICS',
+        'db(a)': 'ACOUSTICS','k': 'TEMPERATURE', '°c': 'TEMPERATURE', 'n': 'MECHANICS', 'newton':'MECHANICS', 'kg/s':'FLOW',
+        'kg/h':'FLOW', 'm³/s': 'FLOW', 'm³/h': 'FLOW', 'l/s':'FLOW', 'l/h':'FLOW', 'µm': 'LENGTH', 'mm':'LENGTH', 'cm':'LENGTH',
+        'dm':'LENGTH', 'm':'LENGTH' ,'meter': 'LENGTH', 'm/s':'SPEED', 'km/h': 'SPEED', 's^(-1)':'FREQUENCY', '1/s':'FREQUENCY',
+        's':'TIME', 'h':'TIME', 'min':'TIME', 'd': 'TIME', 'hours': 'TIME', 'a': 'ELECTRICAL', 'm³': 'VOLUME',
+        'm²': 'AREA', 'rpm': 'FLOW', 'nm': 'MECHANICS', 'm/m': 'MECHANICS', 'm³/m²s': 'MECHANICS', 'w(m²*K)': 'HEAT_TRANSFER',
+        'kwh': 'ELECTRICAL', 'kg/(s*m²)': 'FLOW', 'kg': 'MASS', 'w/(m*k)': 'HEAT_TRANSFER', 'm²*k/w': 'HEAT_TRANSFER',
+        'j/s': 'POWER'}
+  
+  unit_lower = unit.lower() 
+  datatype_lower = datatype.lower()
+
+  unit_categ = unit_mapping.get(unit_lower)
+  datatype_categ = datatype_mapping.get(datatype_lower)
+
+  if unit_categ == None:
+    unit_categ = 'NaN'
+  if datatype_categ == None:
+    datatype_categ = 'NaN'
+
+  concat= (unit_categ, datatype_categ)
+  keys = [k for k, v in metalabel.items() if v == concat]
+  metadata = keys[0]
+  
+  name_embedding = model.encode(name)
+  definition_embedding = model.encode(definition)
+  concat_name_def_query = np.concatenate((definition_embedding, name_embedding), axis = 0)
+  concat_name_def_query = concat_name_def_query.tolist()
+
+  queries = [concat_name_def_query]
+  #print(type(queries))
+
+  # Query wird mit Semantic Search, k-nearest-neighbor durchgeführt
+  # Chroma verwendet hierfür hnswlib https://github.com/nmslib/hnswlib
+  # Dort kann als Distanz Cosine, Squared L2 oder Inner Product eingestellt werden
+  # In Chroma ist L2 als Distanz eingestellt, vgl. https://github.com/chroma-core/chroma/blob/4463d13f951a4d28ade1f7e777d07302ff09069b/chromadb/db/index/hnswlib.py -> suche nach l2
+
+  # Homogener fall, untersuchen nach Semant Ids, wenn welche gefunden werden, ist homgen erfolgreich
+  try:
+    homogen = collection.query(
+        query_embeddings=queries,
+        n_results=1,
+        where={"SESemanticId": semantic_id}
+    )
+  #except NoDatapointsException:
+  #  homogen = 'Nix'
+  
+  except Exception: 
+    homogen = 'Nix'
+
+  if homogen != 'Nix':
+    result = homogen
+    result['matching_method']= 'Semantic equivalent , same semantic Id'
+    result['matching_algorithm'] = 'None'
+    result['distances'] = [[0]]
+    value = result['documents'][0][0]
+    value_dict = json.loads(value)
+
+    final_result = {
+    "matching_method": result['matching_method'],
+    "matching_algorithm": result['matching_algorithm'],
+    "matching_distance": result['distances'][0][0],
+    "aas_id": result['metadatas'][0][0]['AASId'],
+    "aas_id_short": result['metadatas'][0][0]['AASIdShort'],
+    "submodel_id_short": result['metadatas'][0][0]['SubmodelName'],
+    "submodel_id": result['metadatas'][0][0]['SubmodelId'],
+    "matched_object": value_dict,
+    }
+    final_results = [final_result]
+  # Wenn keine passende semantic id gefunden, dann weiter mit NLP mit und ohne Metadaten
+  elif homogen == 'Nix':
+    try:
+      with_metadata = collection.query(
+        query_embeddings=queries,
+        n_results=return_matches,
+        where={"Metalabel": metadata},
+      )
+
+    #except NoDatapointsException:
+    #  with_metadata = 'Nix'
+
+    except Exception: 
+      with_metadata = 'Nix'
+
+    without_metadata = collection.query(
+      query_embeddings=queries,
+      n_results=return_matches,
+    )
+    print(without_metadata)
+
+    if with_metadata == 'Nix':
+      result = without_metadata
+      result['matching_method']= 'Semantically not equivalent, NLP without Metadata'
+      result['matching_algorithm'] = 'Semantic search, k-nearest-neighbor with squared L2 distance (euclidean distance), with model gart-labor/eng-distilBERT-se-eclass'
+    
+    elif with_metadata != 'Nix': 
+      distance_with_meta = with_metadata['distances'][0][0]
+      distance_without_meta = without_metadata['distances'][0][0]
+      #print(distance_with_meta)
+      #print(distance_without_meta)
+      # Vergleich der Abstände von mit und ohne Metadaten
+      if distance_without_meta <= distance_with_meta:
+        result = without_metadata
+        result['matching_method']= 'Semantically not equivalent, NLP without Metadata'
+        result['matching_algorithm'] = 'Semantic search, k-nearest-neighbor with squared L2 distance (euclidean distance), with model gart-labor/eng-distilBERT-se-eclass'
+      
+      else:
+        result = with_metadata
+        result['matching_method']= 'Semantically not equivalent, NLP without Metadata'
+        result['matching_algorithm'] = 'Semantic search, k-nearest-neighbor with squared L2 distance (euclidean distance), with model gart-labor/eng-distilBERT-se-eclass'
+    # Aufbereiten des passenden finalen Ergebnisses
+    final_results = []
+    print(result)
+    for i in range(0, return_matches):
+        value = result['documents'][0][i]
+        value_dict = json.loads(value)
+        final_result = {
+        "matching_method": result['matching_method'],
+        "matching_algorithm": result['matching_algorithm'],
+        "matching_distance": result['distances'][0][i],
+        #"aas_id": result['metadatas'][0][i]['AASId'],
+        #"aas_id_short": result['metadatas'][0][i]['AASIdShort'],
+        "submodel_id_short": result['metadatas'][0][i]['SubmodelName'],
+        "submodel_id": result['metadatas'][0][i]['SubmodelId'],
+        "matched_object": value_dict
+        }
+        #final_result = json.dumps(final_result, indent = 4)
+        final_results.append(final_result)
+
+  return final_results
+
+def get_right_collection(collections, aas_id): 
+  right_collection = []
+  for collection in collections:
+      try_collection = collection.get(where={'AASId': aas_id})
+      try:
+        collection_aas_id = try_collection['metadatas'][0]['AASId']
+        right_collection.append(collection)
+      except:
+        print('Nix')
+  if(right_collection == []):
+    right_collection = ['AAS not in database']
+
+  return right_collection
+
+# Eine spezifische AAS
+def query_specific_aas(query, metalabel, model, collections, client_chroma):
+  json_query = json.dumps(query, indent = 4) 
+  aas_id = query['AASId']
+  right_collection = get_right_collection(collections, aas_id)
+  if right_collection == ['AAS not in database']:
+    result = right_collection
+  else: 
+    collection = client_chroma.get_collection(right_collection[0].name)
+    result = query_right_aas(json_query, collection, metalabel, model)
+  
+  return result