Create logic.py

logic.py

@@ -0,0 +1,146 @@
+#from google.colab import userdata
+import kuzu
+import logging
+import sys
+import os
+from llama_index.graph_stores.kuzu import KuzuGraphStore
+from llama_index.core import (
+    SimpleDirectoryReader,
+    ServiceContext,
+    KnowledgeGraphIndex,
+)
+from llama_index.readers.web import SimpleWebPageReader
+
+
+from llama_index.llms.openai import OpenAI
+from IPython.display import Markdown, display
+from llama_index.core.storage.storage_context import StorageContext
+
+from pyvis.network import Network
+import pandas as pd
+import numpy as np
+import plotly.express as px
+import umap
+
+def get_index(links):
+    os.environ["OPENAI_API_KEY"] = userdata.get('oai')
+    logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+    
+    db = kuzu.Database("kg1")
+    graph_store = KuzuGraphStore(db)
+    
+    
+    documents = SimpleWebPageReader(html_to_text=True).load_data(
+        links
+    )
+    
+    llm = OpenAI(temperature=0, model="gpt-3.5-turbo",api_key='')
+    service_context = ServiceContext.from_defaults(llm=llm, chunk_size=512)
+    
+    storage_context = StorageContext.from_defaults(graph_store=graph_store)
+    
+    # NOTE: can take a while!
+    index = KnowledgeGraphIndex.from_documents(documents=documents,
+                                               max_triplets_per_chunk=5,
+                                               storage_context=storage_context,
+                                               service_context=service_context,
+                                               show_progress=True,
+                                               include_embeddings=True)
+
+    return index
+
+def get_network_graph(index):
+    g = index.get_networkx_graph()
+    net = Network(notebook=True, cdn_resources="in_line", directed=True)
+    net.from_nx(g)
+    net.show("kuzugraph_draw3.html")
+    net.save_graph("kuzugraph_draw3.html")
+
+
+def get_embeddings(index):
+    embeddings = index.index_struct.to_dict()
+    embeddings_df = pd.DataFrame.from_dict(embeddings)['embedding_dict']
+    embeddings_df = embeddings_df.dropna()
+    return embeddings_df
+
+
+def get_visualize_embeddings(embedding_series, n_neighbors=15, min_dist=0.1, n_components=2):
+    # Convert Series to DataFrame
+    embedding_df = pd.DataFrame(embedding_series.tolist(), columns=[f'dim_{i+1}' for i in range(len(embedding_series[0]))])
+
+    # Perform UMAP dimensionality reduction
+    umap_embedded = umap.UMAP(
+        n_neighbors=n_neighbors,
+        min_dist=min_dist,
+        n_components=n_components,
+        random_state=42,
+    ).fit_transform(embedding_df.values)
+
+    # Plot the UMAP embedding
+    umap_df = pd.DataFrame(umap_embedded, columns=['UMAP Dimension 1', 'UMAP Dimension 2'])
+    umap_df['Label'] = embedding_series.index
+    # Plot the UMAP embedding using Plotly Express
+    fig = px.scatter(umap_df, x='UMAP Dimension 1', y='UMAP Dimension 2',hover_data=['Label'], title='UMAP Visualization of Embeddings')
+    return fig
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+