Create app.py
#2
by
aslan-ng
- opened
app.py
ADDED
|
@@ -0,0 +1,657 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Import dependencies
|
| 2 |
+
|
| 3 |
+
from langchain.prompts import PromptTemplate
|
| 4 |
+
from langchain.chains import LLMChain
|
| 5 |
+
from pyvis.network import Network
|
| 6 |
+
from pprint import pprint
|
| 7 |
+
import networkx as nx
|
| 8 |
+
import gradio as gr
|
| 9 |
+
import re
|
| 10 |
+
import datasets
|
| 11 |
+
from huggingface_hub import login, HfApi
|
| 12 |
+
from datasets import Dataset, load_dataset
|
| 13 |
+
from rapidfuzz import fuzz, process
|
| 14 |
+
import math
|
| 15 |
+
import pandas as pd
|
| 16 |
+
import gspread
|
| 17 |
+
from google.colab import auth
|
| 18 |
+
import torch
|
| 19 |
+
import json
|
| 20 |
+
from typing import Callable, Optional
|
| 21 |
+
from dataclasses import dataclass
|
| 22 |
+
from datasets import load_dataset
|
| 23 |
+
from transformers import (
|
| 24 |
+
AutoModelForSequenceClassification,
|
| 25 |
+
TrainingArguments,
|
| 26 |
+
Trainer,
|
| 27 |
+
AutoModelForCausalLM,
|
| 28 |
+
AutoTokenizer,
|
| 29 |
+
BitsAndBytesConfig,
|
| 30 |
+
pipeline
|
| 31 |
+
)
|
| 32 |
+
from peft import PeftModel, LoraConfig, get_peft_model, TaskType
|
| 33 |
+
|
| 34 |
+
# Setup
|
| 35 |
+
|
| 36 |
+
#token_public = ""
|
| 37 |
+
#login(token)
|
| 38 |
+
|
| 39 |
+
REPO_ID_NEAR_FIELD_RAW = "milistu/AMAZON-Products-2023"
|
| 40 |
+
REPO_ID_NEAR_FIELD = "aslan-ng/amazon_products_2023"
|
| 41 |
+
REPO_ID_FAR_FIELD = "aslan-ng/amazon_products_2025"
|
| 42 |
+
|
| 43 |
+
def product_quality_score(average_rating: float, rating_number: int):
|
| 44 |
+
"""
|
| 45 |
+
Bayesian Average (Amazon-style)
|
| 46 |
+
|
| 47 |
+
Args:
|
| 48 |
+
avg_rating: product's average rating
|
| 49 |
+
rating_number: number of reviews
|
| 50 |
+
"""
|
| 51 |
+
m = 1 # Minimum number of reviews required (tunable)
|
| 52 |
+
C = 3.5 # Global average rating (baseline)
|
| 53 |
+
if rating_number <= 0 or average_rating is None:
|
| 54 |
+
return C # fallback to global mean
|
| 55 |
+
return (rating_number / (rating_number + m)) * average_rating + (m / (rating_number + m)) * C
|
| 56 |
+
|
| 57 |
+
# Example
|
| 58 |
+
print("Product 1: ", product_quality_score(average_rating=4.25, rating_number=10000))
|
| 59 |
+
print("Product 2: ", product_quality_score(average_rating=5.0, rating_number=1))
|
| 60 |
+
|
| 61 |
+
def load_near_field_raw_from_huggingface():
|
| 62 |
+
"""
|
| 63 |
+
Load the raw near-field dataset from HuggingFace.
|
| 64 |
+
"""
|
| 65 |
+
ds = datasets.load_dataset(REPO_ID_NEAR_FIELD_RAW, split="train")
|
| 66 |
+
print("Initial size: ", len(ds))
|
| 67 |
+
|
| 68 |
+
# Drop the extra categories
|
| 69 |
+
main_categories_to_remove = ["meta_Books", "meta_CDs_and_Vinyl", "meta_Digital_Music", "meta_Gift_Cards", "meta_Grocery_and_Gourmet_Food",
|
| 70 |
+
"meta_Magazine_Subscriptions", "meta_Software", "meta_Video_Games"]
|
| 71 |
+
ds = ds.filter(lambda row: row["filename"] not in main_categories_to_remove) ###
|
| 72 |
+
|
| 73 |
+
# Keep only the columns we care about
|
| 74 |
+
cols_to_keep = ["title", "description", "main_category", "average_rating", "rating_number"]
|
| 75 |
+
ds = ds.remove_columns([c for c in ds.column_names if c not in cols_to_keep])
|
| 76 |
+
|
| 77 |
+
# Add product quality score column
|
| 78 |
+
def add_quality_score(batch):
|
| 79 |
+
return {
|
| 80 |
+
"product_quality_score": [
|
| 81 |
+
product_quality_score(r, n)
|
| 82 |
+
for r, n in zip(batch["average_rating"], batch["rating_number"])
|
| 83 |
+
]
|
| 84 |
+
}
|
| 85 |
+
ds = ds.map(add_quality_score, batched=True)
|
| 86 |
+
|
| 87 |
+
# Only keep rows with valid values
|
| 88 |
+
def is_valid(v):
|
| 89 |
+
"""
|
| 90 |
+
Must have valid values in the row. Will be used for filtering.
|
| 91 |
+
"""
|
| 92 |
+
if v is None:
|
| 93 |
+
return False
|
| 94 |
+
if isinstance(v, str):
|
| 95 |
+
if v.strip() == "":
|
| 96 |
+
return False
|
| 97 |
+
return True
|
| 98 |
+
|
| 99 |
+
def keep_row(row):
|
| 100 |
+
"""
|
| 101 |
+
Keep only the columns with valid data
|
| 102 |
+
"""
|
| 103 |
+
if is_valid(row.get("title")) and \
|
| 104 |
+
is_valid(row.get("description")) and \
|
| 105 |
+
is_valid(row.get("main_category")) and \
|
| 106 |
+
is_valid(row.get("average_rating")) and \
|
| 107 |
+
is_valid(row.get("rating_number")):
|
| 108 |
+
return True
|
| 109 |
+
return False
|
| 110 |
+
|
| 111 |
+
ds = ds.filter(keep_row)
|
| 112 |
+
|
| 113 |
+
return ds.to_pandas()
|
| 114 |
+
|
| 115 |
+
def load_near_field_from_huggingface():
|
| 116 |
+
"""
|
| 117 |
+
Load the near-field dataset from HuggingFace.
|
| 118 |
+
"""
|
| 119 |
+
ds = load_dataset(REPO_ID_NEAR_FIELD, split="train")
|
| 120 |
+
return ds.to_pandas()
|
| 121 |
+
|
| 122 |
+
def save_near_field_to_huggingface():
|
| 123 |
+
"""
|
| 124 |
+
Save the near-field dataset from HuggingFace.
|
| 125 |
+
"""
|
| 126 |
+
df = load_near_field_raw_from_huggingface()
|
| 127 |
+
ds = Dataset.from_pandas(df)
|
| 128 |
+
ds.push_to_hub(REPO_ID_NEAR_FIELD)
|
| 129 |
+
print(f"✅ Pushed {len(ds)} rows to {REPO_ID_NEAR_FIELD}")
|
| 130 |
+
|
| 131 |
+
#save_near_field_to_huggingface() # Run it once
|
| 132 |
+
dataset_near_field = load_near_field_from_huggingface()
|
| 133 |
+
print("Near-Field Length: ", len(dataset_near_field))
|
| 134 |
+
#print(dataset_near_field.head())
|
| 135 |
+
|
| 136 |
+
def load_far_field_from_sheet():
|
| 137 |
+
"""
|
| 138 |
+
Load the far-field dataset from Google Sheets.
|
| 139 |
+
"""
|
| 140 |
+
auth.authenticate_user()
|
| 141 |
+
from google.auth import default
|
| 142 |
+
COLS = ["title", "description", "average_rating", "rating_number"]
|
| 143 |
+
categories = ["Home & Kitchen", "Beauty & Personal Care", "Sports & Outdoors", "Clothing, Shoes & Jewelry", "Industrial & Scientific",
|
| 144 |
+
"Appliances", "Arts, Crafts & Sewing", "Electronics"]
|
| 145 |
+
sh = gspread.authorize(default()[0]).open_by_key(SHEET_ID_FAR_FIELD)
|
| 146 |
+
frames = []
|
| 147 |
+
for ws in sh.worksheets(): # iterate ALL sheets/tabs
|
| 148 |
+
rows = ws.get_all_records()
|
| 149 |
+
if not rows:
|
| 150 |
+
continue
|
| 151 |
+
df = pd.DataFrame(rows)
|
| 152 |
+
# Keep only the exact columns you want
|
| 153 |
+
df = df[COLS].copy()
|
| 154 |
+
# Add the tab name as main_category
|
| 155 |
+
df["main_category"] = ws.title
|
| 156 |
+
frames.append(df)
|
| 157 |
+
df = pd.concat(frames, ignore_index=True) if frames else pd.DataFrame(columns=COLS + ["main_category"])
|
| 158 |
+
|
| 159 |
+
# Add product quality score column
|
| 160 |
+
def _safe_pqs(row):
|
| 161 |
+
ar, n = row["average_rating"], row["rating_number"]
|
| 162 |
+
if pd.notna(ar) and pd.notna(n):
|
| 163 |
+
return product_quality_score(ar, n)
|
| 164 |
+
return float("nan")
|
| 165 |
+
|
| 166 |
+
df["product_quality_score"] = df.apply(_safe_pqs, axis=1)
|
| 167 |
+
|
| 168 |
+
return df
|
| 169 |
+
|
| 170 |
+
def load_far_field_from_huggingface():
|
| 171 |
+
"""
|
| 172 |
+
Load the far-field dataset from HuggingFace.
|
| 173 |
+
"""
|
| 174 |
+
ds = load_dataset(REPO_ID_FAR_FIELD, split="train")
|
| 175 |
+
return ds.to_pandas()
|
| 176 |
+
|
| 177 |
+
def save_far_field_to_huggingface():
|
| 178 |
+
"""
|
| 179 |
+
Save the far-field dataset from HuggingFace.
|
| 180 |
+
"""
|
| 181 |
+
df = load_far_field_from_sheet()
|
| 182 |
+
ds = Dataset.from_pandas(df)
|
| 183 |
+
ds.push_to_hub(REPO_ID_FAR_FIELD)
|
| 184 |
+
print(f"✅ Pushed {len(ds)} rows to {REPO_ID_FAR_FIELD}")
|
| 185 |
+
|
| 186 |
+
#save_far_field_to_huggingface() # Run it once
|
| 187 |
+
dataset_far_field = load_far_field_from_huggingface()
|
| 188 |
+
print("Far-Field Length: ",len(dataset_far_field))
|
| 189 |
+
#print(dataset_far_field.head())
|
| 190 |
+
|
| 191 |
+
def product_score(product_quality_score: float, fuzzy_score: float):
|
| 192 |
+
"""
|
| 193 |
+
Combine product score and fuzzy score into a single score.
|
| 194 |
+
"""
|
| 195 |
+
return math.sqrt(product_quality_score * fuzzy_score)
|
| 196 |
+
|
| 197 |
+
# Example
|
| 198 |
+
print("Product 1: ", product_score(product_quality_score=3.2, fuzzy_score=100))
|
| 199 |
+
print("Product 2: ", product_score(product_quality_score=4.5, fuzzy_score=70))
|
| 200 |
+
|
| 201 |
+
def query_near_field(input: str, top_k: int=1):
|
| 202 |
+
"""
|
| 203 |
+
Return top_k fuzzy matches for query against dataset titles as a pandas DataFrame.
|
| 204 |
+
Always returns exactly top_k rows (if available).
|
| 205 |
+
"""
|
| 206 |
+
if top_k <= 0:
|
| 207 |
+
raise ValueError
|
| 208 |
+
|
| 209 |
+
n = len(dataset_near_field)
|
| 210 |
+
if top_k > n:
|
| 211 |
+
print(f"Warning: top_k ({top_k}) is greater than the number of examples in the near-field dataset ({n}). Returning all examples.")
|
| 212 |
+
return dataset_near_field.reset_index(drop=True)
|
| 213 |
+
|
| 214 |
+
matches = process.extract(
|
| 215 |
+
input,
|
| 216 |
+
dataset_near_field["title"].fillna("").astype(str).tolist(),
|
| 217 |
+
scorer=fuzz.token_set_ratio,
|
| 218 |
+
limit=n
|
| 219 |
+
)
|
| 220 |
+
|
| 221 |
+
rows = []
|
| 222 |
+
for _text, fuzzy_score, idx in matches:
|
| 223 |
+
row = dataset_near_field.iloc[idx].to_dict() # pandas way
|
| 224 |
+
row["data_source"] = "near_field"
|
| 225 |
+
row["fuzzy_score"] = fuzzy_score
|
| 226 |
+
product_quality_score = row.get("product_quality_score")
|
| 227 |
+
row["score"] = product_score(product_quality_score, fuzzy_score)
|
| 228 |
+
rows.append(row)
|
| 229 |
+
|
| 230 |
+
return (
|
| 231 |
+
pd.DataFrame(rows)
|
| 232 |
+
.sort_values("score", ascending=False)
|
| 233 |
+
.head(top_k)
|
| 234 |
+
.reset_index(drop=True)
|
| 235 |
+
)
|
| 236 |
+
|
| 237 |
+
# Example
|
| 238 |
+
near_field_result = query_near_field("water bottle", top_k=5)
|
| 239 |
+
#print(near_field_result.head())
|
| 240 |
+
print("Example: ", near_field_result.iloc[0]["title"])
|
| 241 |
+
|
| 242 |
+
def query_far_field(input: str, top_k: int):
|
| 243 |
+
"""
|
| 244 |
+
Return top_k random elements from the far_field dataset as a pandas DataFrame.
|
| 245 |
+
The input string is ignored.
|
| 246 |
+
"""
|
| 247 |
+
if top_k < 0:
|
| 248 |
+
raise ValueError
|
| 249 |
+
|
| 250 |
+
n = len(dataset_far_field)
|
| 251 |
+
if top_k > n:
|
| 252 |
+
print(f"Warning: top_k ({top_k}) is greater than the number of examples in the far-field dataset ({n}). Returning all examples.")
|
| 253 |
+
return dataset_far_field.reset_index(drop=True)
|
| 254 |
+
|
| 255 |
+
# Sample random rows without replacement
|
| 256 |
+
sampled = dataset_far_field.sample(n=top_k, random_state=None).reset_index(drop=True)
|
| 257 |
+
|
| 258 |
+
# Add the rest
|
| 259 |
+
sampled["fuzzy_score"] = [
|
| 260 |
+
fuzz.token_set_ratio(str(t) if pd.notna(t) else "", input)
|
| 261 |
+
for t in sampled.get("title", "")
|
| 262 |
+
]
|
| 263 |
+
product_quality_scores = sampled.get("product_quality_score")
|
| 264 |
+
fuzzy_scores = sampled["fuzzy_score"]
|
| 265 |
+
sampled["score"] = [product_score(a, b) for a, b in zip(product_quality_scores, fuzzy_scores)]
|
| 266 |
+
sampled["data_source"] = "far_field"
|
| 267 |
+
|
| 268 |
+
return sampled
|
| 269 |
+
|
| 270 |
+
# Example usage
|
| 271 |
+
far_field_result = query_far_field("water bottle", top_k=3)
|
| 272 |
+
#print(far_field_result)
|
| 273 |
+
print("Top result title:", far_field_result.iloc[0]["title"])
|
| 274 |
+
#print("Top result title:", far_field_result)
|
| 275 |
+
|
| 276 |
+
def split_near_and_far_fields(total_examples: int, near_far_ratio: float = 0.5):
|
| 277 |
+
"""
|
| 278 |
+
Split the examples between near and far field.
|
| 279 |
+
The ratio represents the examples that will be in the near field to total (near + far).
|
| 280 |
+
"""
|
| 281 |
+
ratio = near_far_ratio
|
| 282 |
+
# Validate ratio
|
| 283 |
+
if ratio < 0 or ratio > 1:
|
| 284 |
+
raise ValueError("Ratio must be between 0 and 1")
|
| 285 |
+
if total_examples < 2:
|
| 286 |
+
raise ValueError("Total examples must be at least 2")
|
| 287 |
+
|
| 288 |
+
near_field_examples = int(total_examples * ratio)
|
| 289 |
+
far_field_examples = total_examples - near_field_examples
|
| 290 |
+
|
| 291 |
+
return near_field_examples, far_field_examples
|
| 292 |
+
|
| 293 |
+
# Example
|
| 294 |
+
print("Example: ", split_near_and_far_fields(total_examples=100, near_far_ratio=0.3)) # Expected: (30, 70)
|
| 295 |
+
|
| 296 |
+
def query(input: str, total_examples: int, near_far_ratio: float = 0.5):
|
| 297 |
+
near_field_examples, far_field_examples = split_near_and_far_fields(total_examples, near_far_ratio)
|
| 298 |
+
far_field_result = query_far_field(input, far_field_examples)
|
| 299 |
+
#print(far_field_result.head())
|
| 300 |
+
near_field_result = query_near_field(input, near_field_examples)
|
| 301 |
+
#print(near_field_result.head())
|
| 302 |
+
result = pd.concat([near_field_result, far_field_result], ignore_index=True)
|
| 303 |
+
return result
|
| 304 |
+
|
| 305 |
+
# Example
|
| 306 |
+
print("Example: ", query("water bottle", total_examples=4, near_far_ratio=0.5))
|
| 307 |
+
|
| 308 |
+
# 1. Load dataset
|
| 309 |
+
dataset = load_dataset("cwinkler/patents_green_plastics")
|
| 310 |
+
|
| 311 |
+
# Split into train/test
|
| 312 |
+
dataset = dataset["train"].train_test_split(test_size=0.2, seed=42)
|
| 313 |
+
train_dataset = dataset["train"]
|
| 314 |
+
test_dataset = dataset["test"]
|
| 315 |
+
|
| 316 |
+
# 2. Tokenizer
|
| 317 |
+
model_name = "distilbert-base-uncased"
|
| 318 |
+
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
| 319 |
+
|
| 320 |
+
def preprocess(examples):
|
| 321 |
+
return tokenizer(examples["abstract"], truncation=True, padding="max_length", max_length=128)
|
| 322 |
+
|
| 323 |
+
tokenized = dataset.map(preprocess, batched=True)
|
| 324 |
+
tokenized = tokenized.rename_column("label", "labels")
|
| 325 |
+
tokenized.set_format("torch", columns=["input_ids", "attention_mask", "labels"])
|
| 326 |
+
|
| 327 |
+
train_dataset = tokenized["train"].shuffle(seed=42).select(range(2000)) # subset for CPU
|
| 328 |
+
test_dataset = tokenized["test"]
|
| 329 |
+
|
| 330 |
+
# 3. Base model
|
| 331 |
+
base_model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=2)
|
| 332 |
+
|
| 333 |
+
# 4. LoRA config
|
| 334 |
+
lora_config = LoraConfig(
|
| 335 |
+
task_type=TaskType.SEQ_CLS,
|
| 336 |
+
r=8, lora_alpha=16, lora_dropout=0.1, target_modules=["q_lin", "v_lin"]
|
| 337 |
+
)
|
| 338 |
+
|
| 339 |
+
model = get_peft_model(base_model, lora_config)
|
| 340 |
+
|
| 341 |
+
# 5. Training setup
|
| 342 |
+
import os
|
| 343 |
+
os.environ["WANDB_DISABLED"] = "true"
|
| 344 |
+
|
| 345 |
+
args = TrainingArguments(
|
| 346 |
+
output_dir="./lora-green-patents",
|
| 347 |
+
do_eval=True, # instead of evaluation_strategy
|
| 348 |
+
eval_steps=500, # run eval every N steps
|
| 349 |
+
save_steps=500, # save checkpoint every N steps
|
| 350 |
+
learning_rate=2e-4,
|
| 351 |
+
per_device_train_batch_size=8,
|
| 352 |
+
per_device_eval_batch_size=8,
|
| 353 |
+
num_train_epochs=10,
|
| 354 |
+
logging_steps=20,
|
| 355 |
+
report_to=None
|
| 356 |
+
)
|
| 357 |
+
|
| 358 |
+
trainer = Trainer(
|
| 359 |
+
model=model,
|
| 360 |
+
args=args,
|
| 361 |
+
train_dataset=train_dataset,
|
| 362 |
+
eval_dataset=test_dataset,
|
| 363 |
+
)
|
| 364 |
+
|
| 365 |
+
# 6. Train
|
| 366 |
+
trainer.train()
|
| 367 |
+
|
| 368 |
+
# 7. Save adapter
|
| 369 |
+
model.save_pretrained("lora-green-patents")
|
| 370 |
+
tokenizer.save_pretrained("lora-green-patents")
|
| 371 |
+
|
| 372 |
+
# 8. Inference
|
| 373 |
+
|
| 374 |
+
# Load base + adapter
|
| 375 |
+
base_model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=2)
|
| 376 |
+
model = PeftModel.from_pretrained(base_model, "lora-green-patents")
|
| 377 |
+
|
| 378 |
+
clf = pipeline("text-classification", model=model, tokenizer=tokenizer)
|
| 379 |
+
|
| 380 |
+
# Examples of patents and products
|
| 381 |
+
texts = [
|
| 382 |
+
"A biodegradable plastic composition derived from renewable corn starch.",
|
| 383 |
+
"A new synthetic polymer with enhanced tensile strength."
|
| 384 |
+
"Refreshing Taste: Every bottle of Pure Life Water is enhanced with minerals for a crisp taste that makes drinking water delicious. 12 pack of 16.9 fl oz water bottles."
|
| 385 |
+
"This 18/8 stainless steel water bottle is designed to last a lifetime. Plastic free & Eco friendly water bottles are a healthier option for you & the planet! However, Water in stainless steel tastes different than plastic, make sure your taste buds are ready for this healthy switch"
|
| 386 |
+
]
|
| 387 |
+
print(clf(texts))
|
| 388 |
+
|
| 389 |
+
ex_waterbottle_text = [
|
| 390 |
+
"A single use case made with fossil fuels and gasoline.",
|
| 391 |
+
"An eco-friendly, sustainable bottle made with biodegradable plastic."
|
| 392 |
+
]
|
| 393 |
+
print(clf(ex_waterbottle_text))
|
| 394 |
+
|
| 395 |
+
def sustainability_filter(input: str, total_examples: int, near_far_ratio: float = 0.5):
|
| 396 |
+
initial_products = query(input, total_examples, near_far_ratio)
|
| 397 |
+
filtered_products = clf(initial_products['description']) # 1 for green patents, 0 otherwise
|
| 398 |
+
sustainable_products = filtered_products.filter(lambda x: x['label'] == 'LABEL_1')
|
| 399 |
+
return sustainable_products
|
| 400 |
+
|
| 401 |
+
# 👇 Your system prompt (can be empty)
|
| 402 |
+
SYSTEM_PROMPT = """
|
| 403 |
+
You are a product analyst. You'll receive product description as input, and extract some product functionality and some product values. Each functionality and value should be keywords only.
|
| 404 |
+
Product functionality refers to what the product does: its features, technical capabilities, and performance characteristics. It answers the question: “What can this product do?”
|
| 405 |
+
Product value refers to the benefit the customer gains from using the product: how it improves their life, solves their problem, or helps them achieve goals. It answers the question: “Why does this matter to the customer?”
|
| 406 |
+
Do **not** duplicate an item in both lists. Keep **functionalities** as concrete features. Keep **values** as clear user benefits.
|
| 407 |
+
|
| 408 |
+
Your Output is a dictionary. Here is the format:
|
| 409 |
+
|
| 410 |
+
# Your Input:
|
| 411 |
+
<product_description>
|
| 412 |
+
# Your Output:
|
| 413 |
+
{
|
| 414 |
+
"values": [
|
| 415 |
+
<value1>,
|
| 416 |
+
<value2>,
|
| 417 |
+
...
|
| 418 |
+
],
|
| 419 |
+
"functionalities": [
|
| 420 |
+
<function1>,
|
| 421 |
+
<function2>,
|
| 422 |
+
...
|
| 423 |
+
]
|
| 424 |
+
}
|
| 425 |
+
|
| 426 |
+
Don't return anything out of the output format.
|
| 427 |
+
"""
|
| 428 |
+
|
| 429 |
+
@dataclass
|
| 430 |
+
class LLMConfig:
|
| 431 |
+
model_id: str # e.g. "Qwen/Qwen2.5-1.5B-Instruct" or "Qwen/Qwen2.5-3B-Instruct"
|
| 432 |
+
system_prompt: str = "" # optional system prompt
|
| 433 |
+
max_new_tokens: int = 256
|
| 434 |
+
temperature: float = 0.2
|
| 435 |
+
top_p: float = 0.9
|
| 436 |
+
repetition_penalty: float = 1.05
|
| 437 |
+
use_4bit: bool = True # good default for Colab VRAM
|
| 438 |
+
|
| 439 |
+
def create_llm(
|
| 440 |
+
*,
|
| 441 |
+
model_id: str,
|
| 442 |
+
max_new_tokens: int = 256,
|
| 443 |
+
temperature: float = 0.2,
|
| 444 |
+
top_p: float = 0.9,
|
| 445 |
+
repetition_penalty: float = 1.05,
|
| 446 |
+
use_4bit: bool = True
|
| 447 |
+
) -> Callable[[str], str]:
|
| 448 |
+
"""
|
| 449 |
+
Load an off-the-shelf chat LLM and return a callable llm(prompt) -> str.
|
| 450 |
+
Pass ONLY the model parameters you want. No size mapping. No llama_cpp.
|
| 451 |
+
"""
|
| 452 |
+
|
| 453 |
+
cfg = LLMConfig(
|
| 454 |
+
model_id=model_id,
|
| 455 |
+
system_prompt=SYSTEM_PROMPT,
|
| 456 |
+
max_new_tokens=max_new_tokens,
|
| 457 |
+
temperature=temperature,
|
| 458 |
+
top_p=top_p,
|
| 459 |
+
repetition_penalty=repetition_penalty,
|
| 460 |
+
use_4bit=use_4bit,
|
| 461 |
+
)
|
| 462 |
+
|
| 463 |
+
has_cuda = torch.cuda.is_available()
|
| 464 |
+
qconfig: Optional[BitsAndBytesConfig] = None
|
| 465 |
+
if has_cuda and cfg.use_4bit:
|
| 466 |
+
qconfig = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16)
|
| 467 |
+
|
| 468 |
+
tokenizer = AutoTokenizer.from_pretrained(cfg.model_id, use_fast=True)
|
| 469 |
+
model = AutoModelForCausalLM.from_pretrained(
|
| 470 |
+
cfg.model_id,
|
| 471 |
+
device_map="auto",
|
| 472 |
+
torch_dtype=torch.bfloat16 if has_cuda else torch.float32,
|
| 473 |
+
quantization_config=qconfig,
|
| 474 |
+
).eval()
|
| 475 |
+
|
| 476 |
+
def _format_messages(user_text: str) -> str:
|
| 477 |
+
msgs = []
|
| 478 |
+
if cfg.system_prompt:
|
| 479 |
+
msgs.append({"role": "system", "content": cfg.system_prompt})
|
| 480 |
+
msgs.append({"role": "user", "content": user_text})
|
| 481 |
+
|
| 482 |
+
if hasattr(tokenizer, "apply_chat_template") and tokenizer.chat_template is not None:
|
| 483 |
+
return tokenizer.apply_chat_template(msgs, tokenize=False, add_generation_prompt=True)
|
| 484 |
+
|
| 485 |
+
# Fallback if no chat template is present
|
| 486 |
+
sys = f"System: {cfg.system_prompt}\n\n" if cfg.system_prompt else ""
|
| 487 |
+
return f"{sys}User: {user_text}\nAssistant:"
|
| 488 |
+
|
| 489 |
+
@torch.inference_mode()
|
| 490 |
+
def llm(prompt: str,
|
| 491 |
+
max_new_tokens: int = None,
|
| 492 |
+
temperature: float = None,
|
| 493 |
+
top_p: float = None,
|
| 494 |
+
repetition_penalty: float = None) -> str:
|
| 495 |
+
|
| 496 |
+
text = _format_messages(prompt)
|
| 497 |
+
inputs = tokenizer(text, return_tensors="pt").to(model.device)
|
| 498 |
+
out = model.generate(
|
| 499 |
+
**inputs,
|
| 500 |
+
max_new_tokens=max_new_tokens or cfg.max_new_tokens,
|
| 501 |
+
do_sample=(temperature or cfg.temperature) > 0.0,
|
| 502 |
+
temperature=temperature or cfg.temperature,
|
| 503 |
+
top_p=top_p or cfg.top_p,
|
| 504 |
+
repetition_penalty=repetition_penalty or cfg.repetition_penalty,
|
| 505 |
+
pad_token_id=tokenizer.eos_token_id,
|
| 506 |
+
eos_token_id=tokenizer.eos_token_id,
|
| 507 |
+
)
|
| 508 |
+
gen = out[0][inputs["input_ids"].shape[-1]:]
|
| 509 |
+
return tokenizer.decode(gen, skip_special_tokens=True).strip()
|
| 510 |
+
|
| 511 |
+
print(f"Loaded: {cfg.model_id} | 4-bit: {bool(qconfig)} | Device: {model.device}")
|
| 512 |
+
return llm
|
| 513 |
+
|
| 514 |
+
def response_to_triplets(product_title, response: str):
|
| 515 |
+
data = json.loads(response)
|
| 516 |
+
#print(data)
|
| 517 |
+
|
| 518 |
+
triples_list = []
|
| 519 |
+
|
| 520 |
+
for value in data["values"]:
|
| 521 |
+
triples_list.append(f"({product_title}, HAS_VALUE, {value})")
|
| 522 |
+
|
| 523 |
+
for func in data["functionalities"]:
|
| 524 |
+
triples_list.append(f"({product_title}, HAS_FUNCTIONALITY, {func})")
|
| 525 |
+
|
| 526 |
+
#print(triples_list)
|
| 527 |
+
return triples_list
|
| 528 |
+
|
| 529 |
+
llm = create_llm(
|
| 530 |
+
model_id="Qwen/Qwen2.5-1.5B-Instruct",
|
| 531 |
+
max_new_tokens=200,
|
| 532 |
+
temperature=0.2,
|
| 533 |
+
top_p=0.9,
|
| 534 |
+
repetition_penalty=1.05,
|
| 535 |
+
use_4bit=True, # set False if you have lots of VRAM
|
| 536 |
+
)
|
| 537 |
+
|
| 538 |
+
# Example
|
| 539 |
+
if False: # Change to true to check the example
|
| 540 |
+
title = """
|
| 541 |
+
Surge Protector Power Strip - HANYCONY 8 Outlets 4 USB (2 USB C) Charging Ports, Multi Plug Outlet Extender, 5Ft Braided Extension Cord, Flat Plug Wall Mount Desk Charging Station for Home Office ETL
|
| 542 |
+
"""
|
| 543 |
+
description = """
|
| 544 |
+
3-side design power strip surge protector with 8AC widely outlets and 4 USB (2 USB C) charging ports can power up to 12 devices simultaneously. That makes it easier to make the plugs not covering any outlet, and the 2.2 inchces widely spced in between outlets, larger than standard socket, fit big adapters without blocking each other. The compact design saves more space, suitable for the home, office, and college dorm room essentials
|
| 545 |
+
"""
|
| 546 |
+
product_description = f"{title}\n{description}"
|
| 547 |
+
response = llm(product_description)
|
| 548 |
+
print("Example: \n", response)
|
| 549 |
+
triplets_list = response_to_triplets(title, response)
|
| 550 |
+
print("Example Triplets: \n", triplets_list)
|
| 551 |
+
|
| 552 |
+
def main(input: str):
|
| 553 |
+
all_triplets_list = []
|
| 554 |
+
'''
|
| 555 |
+
sustainable_results = sustainability_filter(input, total_examples=10, near_far_ratio=0.5)
|
| 556 |
+
for i, product in sustainable_results.iterrows():
|
| 557 |
+
product_title = product["title"]
|
| 558 |
+
response = llm(product_title)
|
| 559 |
+
triplets_list = response_to_triplets(product_title, response)
|
| 560 |
+
for triplet in triplets_list:
|
| 561 |
+
all_triplets_list.append(triplet)
|
| 562 |
+
'''
|
| 563 |
+
all_triplets_list = [
|
| 564 |
+
'(Zojirushi Stainless Steel Mug, HAS_VALUE, temperature regulation)',
|
| 565 |
+
'(Zojirushi Stainless Steel Mug, HAS_VALUE, ease of use)',
|
| 566 |
+
'(Zojirushi Stainless Steel Mug, HAS_VALUE, portability)'
|
| 567 |
+
'(Zojirushi Stainless Steel Mug, HAS_FUNCTIONALITY, vacuum insulation)',
|
| 568 |
+
'(Zojirushi Stainless Steel Mug, HAS_FUNCTIONALITY, durability)'
|
| 569 |
+
]
|
| 570 |
+
return all_triplets_list
|
| 571 |
+
|
| 572 |
+
def create_graph_from_triplets(triplets):
|
| 573 |
+
G = nx.DiGraph()
|
| 574 |
+
for triplet in triplets:
|
| 575 |
+
line = str(triplet).strip()
|
| 576 |
+
if not line:
|
| 577 |
+
continue
|
| 578 |
+
# Try comma-delimited with max 2 splits
|
| 579 |
+
parts = [p.strip(" ()") for p in line.split(",", 2)]
|
| 580 |
+
if len(parts) != 3:
|
| 581 |
+
# Fallback: pipe-delimited
|
| 582 |
+
parts = [p.strip(" ()") for p in line.split("|")]
|
| 583 |
+
if len(parts) != 3:
|
| 584 |
+
continue # malformed, skip
|
| 585 |
+
subject, predicate, obj = parts
|
| 586 |
+
if subject and predicate and obj:
|
| 587 |
+
G.add_edge(subject, obj, label=predicate)
|
| 588 |
+
return G
|
| 589 |
+
|
| 590 |
+
def nx_to_pyvis(networkx_graph):
|
| 591 |
+
pyvis_graph = Network(notebook=True, cdn_resources='remote')
|
| 592 |
+
for node in networkx_graph.nodes():
|
| 593 |
+
pyvis_graph.add_node(node)
|
| 594 |
+
for edge in networkx_graph.edges(data=True):
|
| 595 |
+
lbl = edge[2].get("label", "") # ✅ safe access
|
| 596 |
+
pyvis_graph.add_edge(edge[0], edge[1], label=lbl, title=lbl)
|
| 597 |
+
return pyvis_graph
|
| 598 |
+
|
| 599 |
+
def generateGraph(triples_list):
|
| 600 |
+
triplets = [t.strip() for t in triples_list if t.strip()]
|
| 601 |
+
graph = create_graph_from_triplets(triplets)
|
| 602 |
+
pyvis_network = nx_to_pyvis(graph)
|
| 603 |
+
|
| 604 |
+
pyvis_network.toggle_hide_edges_on_drag(True)
|
| 605 |
+
pyvis_network.toggle_physics(False)
|
| 606 |
+
pyvis_network.set_edge_smooth('discrete')
|
| 607 |
+
|
| 608 |
+
html = pyvis_network.generate_html()
|
| 609 |
+
html = html.replace("'", "\"")
|
| 610 |
+
|
| 611 |
+
return f"""<iframe style="width: 100%; height: 600px;margin:0 auto" name="result" allow="midi; geolocation; microphone; camera;
|
| 612 |
+
display-capture; encrypted-media;" sandbox="allow-modals allow-forms
|
| 613 |
+
allow-scripts allow-same-origin allow-popups
|
| 614 |
+
allow-top-navigation-by-user-activation allow-downloads" allowfullscreen=""
|
| 615 |
+
allowpaymentrequest="" frameborder="0" srcdoc='{html}'></iframe>"""
|
| 616 |
+
|
| 617 |
+
def pipeline(user_text: str):
|
| 618 |
+
try:
|
| 619 |
+
triples = main(user_text) or [] # ✅ guard against None
|
| 620 |
+
# Normalize tuples/lists to "S, R, O" strings (keeps your existing generateGraph)
|
| 621 |
+
triples_list = []
|
| 622 |
+
for t in triples:
|
| 623 |
+
if isinstance(t, (tuple, list)) and len(t) == 3:
|
| 624 |
+
triples_list.append(f"{t[0]}, {t[1]}, {t[2]}")
|
| 625 |
+
else:
|
| 626 |
+
triples_list.append(str(t))
|
| 627 |
+
return generateGraph(triples_list)
|
| 628 |
+
except Exception:
|
| 629 |
+
return "<pre style='white-space: pre-wrap; font-size:12px; color:#b00;'>" + traceback.format_exc() + "</pre>"
|
| 630 |
+
|
| 631 |
+
demo = gr.Interface(
|
| 632 |
+
fn=pipeline,
|
| 633 |
+
inputs=gr.Textbox(label="Enter your query / text", value="", lines=6),
|
| 634 |
+
outputs=gr.HTML(),
|
| 635 |
+
title="Knowledge Graph",
|
| 636 |
+
allow_flagging="never",
|
| 637 |
+
live=False, # set True if you want it to recompute on each keystroke
|
| 638 |
+
css="""
|
| 639 |
+
#component-0, #component-1, #component-2 {
|
| 640 |
+
display: flex;
|
| 641 |
+
justify-content: center;
|
| 642 |
+
align-items: center;
|
| 643 |
+
flex-direction: column;
|
| 644 |
+
}
|
| 645 |
+
.gradio-container {
|
| 646 |
+
justify-content: center !important;
|
| 647 |
+
align-items: center !important;
|
| 648 |
+
text-align: center;
|
| 649 |
+
}
|
| 650 |
+
textarea, iframe {
|
| 651 |
+
margin: 0 auto;
|
| 652 |
+
display: block;
|
| 653 |
+
}
|
| 654 |
+
"""
|
| 655 |
+
)
|
| 656 |
+
|
| 657 |
+
demo.launch(quiet=True)
|