First PoC

demo.ipynb

@@ -0,0 +1,138 @@
+{
+ "cells": [
+  {
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2025-02-15T20:40:20.092409Z",
+     "start_time": "2025-02-15T20:40:14.408247Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "import numpy as np\n",
+    "from datasets import load_dataset\n",
+    "from bert_score import score\n",
+    "from model import MetaModel\n",
+    "import time"
+   ],
+   "id": "5d14705fffbcfb64",
+   "outputs": [],
+   "execution_count": 1
+  },
+  {
+   "metadata": {
+    "jupyter": {
+     "is_executing": true
+    },
+    "ExecuteTime": {
+     "start_time": "2025-02-15T20:40:20.098977Z"
+    }
+   },
+   "cell_type": "code",
+   "source": [
+    "dataset = load_dataset(\"cnn_dailymail\", \"3.0.0\", split=\"train\")\n",
+    "\n",
+    "train_texts = dataset[\"article\"][:100]\n",
+    "train_summaries = dataset[\"highlights\"][:100]\n",
+    "test_texts = dataset[\"article\"][100:200]\n",
+    "test_summaries = dataset[\"highlights\"][100:200]\n",
+    "\n",
+    "model_names = [\"google-t5/t5-small\", \"google-t5/t5-base\", \"google-t5/t5-large\"]\n",
+    "meta_model = MetaModel(model_names)\n",
+    "start_time = time.time()\n",
+    "meta_model.fit(train_texts, train_summaries)\n",
+    "print(\"MetaModel fitting time [sec]:\" % (time.time() - start_time))\n"
+   ],
+   "id": "6d68f234e372396d",
+   "outputs": [],
+   "execution_count": null
+  },
+  {
+   "metadata": {},
+   "cell_type": "code",
+   "outputs": [],
+   "execution_count": null,
+   "source": [
+    "# Evaluation on test set\n",
+    "meta_model_scores = []\n",
+    "meta_model_times = []\n",
+    "model_scores = {name: [] for name in model_names}\n",
+    "model_times = {name: [] for name in model_names}\n",
+    "correct_predictions = 0\n",
+    "tolerance = 0.05\n",
+    "\n",
+    "for i, text in enumerate(test_texts):\n",
+    "    predicted_summary, meta_time = meta_model.summarize(text)\n",
+    "    P, R, F1 = score([predicted_summary], [test_summaries[i]], lang=\"en\", verbose=False)\n",
+    "    meta_model_scores.append(F1.item())\n",
+    "    meta_model_times.append(meta_time)\n",
+    "\n",
+    "    model_results = []\n",
+    "    for model_name in model_names:\n",
+    "        model = meta_model.models[model_name]\n",
+    "        summary, elapsed_time = model.summarize(text)\n",
+    "        P, R, F1 = score([summary], [test_summaries[i]], lang=\"en\", verbose=False)\n",
+    "        f1_score = F1.item()\n",
+    "\n",
+    "        model_scores[model_name].append(f1_score)\n",
+    "        model_times[model_name].append(elapsed_time)\n",
+    "        model_results.append((model_name, f1_score, elapsed_time))\n",
+    "\n",
+    "    model_results.sort(key=lambda x: (-x[1], x[2]))\n",
+    "    best_model, best_score, best_time = model_results[0]\n",
+    "\n",
+    "    for model_name, f1_score, elapsed_time in model_results[1:]:\n",
+    "        if best_score - f1_score <= tolerance and elapsed_time < best_time:\n",
+    "            best_model, best_score, best_time = model_name, f1_score, elapsed_time\n",
+    "\n",
+    "    if best_model == predicted_summary:\n",
+    "        correct_predictions += 1\n",
+    "\n",
+    "def compute_avg(values):\n",
+    "    return np.mean(values)\n",
+    "\n",
+    "print(\"\\n===== Model Evaluation =====\")\n",
+    "for model_name in model_names:\n",
+    "    avg_score = compute_avg(model_scores[model_name])\n",
+    "    avg_time = compute_avg(model_times[model_name])\n",
+    "    print(f\"{model_name}: BERTScore={avg_score:.4f}, Time={avg_time:.4f}s\")\n",
+    "\n",
+    "print(\n",
+    "    f\"\\nMeta-Model: Accuracy={correct_predictions / len(test_texts):.2%}, \"\n",
+    "    f\"BERTScore={compute_avg(meta_model_scores):.4f}, \"\n",
+    "    f\"Time={compute_avg(meta_model_times):.4f}s\"\n",
+    ")\n"
+   ],
+   "id": "6fd91b97e4b6e588"
+  },
+  {
+   "metadata": {},
+   "cell_type": "code",
+   "outputs": [],
+   "execution_count": null,
+   "source": "",
+   "id": "204e55cee1ee63e4"
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

model.py

@@ -0,0 +1,91 @@
+import time
+from collections import Counter
+
+import numpy as np
+import spacy
+import torch
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+from datasets import load_dataset
+from bert_score import score
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.model_selection import train_test_split
+from sklearn.preprocessing import StandardScaler
+from scipy.stats import entropy
+
+def compute_entropy(text):
+    words = text.split()
+    word_freq = Counter(words)
+    probs = np.array(list(word_freq.values())) / sum(word_freq.values())
+    return entropy(probs)
+
+
+def compute_syntactic_complexity(text):
+    nlp = spacy.load("en_core_web_sm")
+    doc = nlp(text)
+    depths = [token.head.i - token.i for token in doc if token.head != token]
+    return np.mean(depths) if depths else 0
+
+
+class T5Model:
+    def __init__(self, model_name):
+        self.model_name = model_name
+        self.tokenizer = AutoTokenizer.from_pretrained(model_name)
+        self.model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
+
+    def summarize(self, text):
+        inputs = self.tokenizer(text, return_tensors="pt", max_length=512, truncation=True)
+        start_time = time.time()
+        outputs = self.model.generate(**inputs, max_length=150, num_beams=4, early_stopping=True)
+        end_time = time.time()
+        summary = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
+        return summary, end_time - start_time
+
+
+class MetaModel:
+    def __init__(self, model_names):
+        self.models = {name: T5Model(name) for name in model_names}
+        self.classifier = RandomForestClassifier(n_estimators=100, random_state=42)
+
+    def extract_features(self, text):
+        words = text.split()
+        num_words = len(words)
+        avg_word_length = np.mean([len(w) for w in words]) if words else 0
+        complexity = compute_syntactic_complexity(text)
+        entropy = compute_entropy(text)
+        return [num_words, avg_word_length, complexity, entropy]
+
+    def fit(self, texts, summaries):
+        X = np.array([self.extract_features(text) for text in texts])
+
+        best_model_labels = []
+        tolerance = 0.05  # BERTScore tolerance
+
+        for i, text in enumerate(texts):
+            model_results = []
+            for model_name, model in self.models.items():
+                summary, elapsed_time = model.summarize(text)
+                P, R, F1 = score([summary], [summaries[i]], lang="en", verbose=False)
+                f1_score = F1.item()
+                model_results.append((model_name, f1_score, elapsed_time))
+
+            # Sort models by BERTScore (desc) and then by time (asc)
+            model_results.sort(key=lambda x: (-x[1], x[2]))
+
+            # Select best model based on tolerance rule
+            best_model, best_score, best_time = model_results[0]
+            for model_name, f1_score, elapsed_time in model_results[1:]:
+                if best_score - f1_score <= tolerance and elapsed_time < best_time:
+                    best_model, best_score, best_time = model_name, f1_score, elapsed_time
+
+            best_model_labels.append(best_model)
+
+        y = np.array([list(self.models.keys()).index(m) for m in best_model_labels])
+        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
+        self.classifier.fit(X_train, y_train)
+
+    def summarize(self, text):
+        features = np.array([self.extract_features(text)])
+        predicted_model_index = self.classifier.predict(features)[0]
+        predicted_model_name = list(self.models.keys())[predicted_model_index]
+        return self.models[predicted_model_name].summarize(text)
+

packages.txt

@@ -0,0 +1 @@
+python -m spacy download en_core_web_lg

requirements.txt

@@ -0,0 +1,10 @@
+torch
+transformers
+datasets
+spacy
+numpy
+scipy
+rouge_score
+bert_score
+ipywidgets
+scikit-learn