Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -11,11 +11,22 @@ PKG_PATH = "neuro_semantic_package.pt"
|
|
| 11 |
|
| 12 |
print("๐ System Startup: Loading Artifacts...")
|
| 13 |
if not os.path.exists(PKG_PATH):
|
| 14 |
-
#
|
| 15 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
|
| 17 |
# Load the "Black Box" package
|
| 18 |
-
|
|
|
|
| 19 |
DATA = PKG['data']
|
| 20 |
MODELS = PKG['models'] # The Projectors
|
| 21 |
MATRIX = PKG['matrix'] # Pre-calculated Accuracy Table
|
|
@@ -111,19 +122,20 @@ def decode_neuro_semantics(subject, projector_alias, text):
|
|
| 111 |
"Match": match_icon
|
| 112 |
}])
|
| 113 |
|
| 114 |
-
return df
|
| 115 |
|
| 116 |
def run_batch_analysis(subject, projector_alias):
|
| 117 |
# Runs 5 random samples for robust demo
|
| 118 |
subject_data = DATA[subject]
|
| 119 |
total_indices = list(range(len(subject_data['Text'])))
|
|
|
|
| 120 |
selected_indices = random.sample(total_indices, min(5, len(total_indices)))
|
| 121 |
|
| 122 |
results = []
|
| 123 |
|
| 124 |
for idx in selected_indices:
|
| 125 |
txt = subject_data['Text'][idx]
|
| 126 |
-
df
|
| 127 |
results.append(df)
|
| 128 |
|
| 129 |
final_df = pd.concat(results)
|
|
@@ -134,61 +146,81 @@ def run_batch_analysis(subject, projector_alias):
|
|
| 134 |
|
| 135 |
# --- 3. UI LAYOUT ---
|
| 136 |
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 140 |
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
|
| 146 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 147 |
"""
|
| 148 |
|
| 149 |
-
with gr.Blocks(theme=gr.themes.Soft()) as demo:
|
| 150 |
gr.Markdown("# ๐ง Neuro-Semantic Alignment: Zero-Shot Decoding")
|
| 151 |
|
| 152 |
-
with gr.
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 158 |
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
proj_dropdown = gr.Dropdown(choices=list(MODELS.keys()), value="Projector A", label="Select Projector (Decoding Model)")
|
| 162 |
|
| 163 |
-
#
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
with gr.Column(scale=2):
|
| 170 |
-
gr.Markdown("### ๐ Decoding Results")
|
| 171 |
-
|
| 172 |
-
# Output Table
|
| 173 |
-
result_table = gr.Dataframe(
|
| 174 |
-
headers=["Sentence Stimulus", "Text Ground Truth (Top 2)", "Brain Decoding (Top 3)", "Match"],
|
| 175 |
-
wrap=True
|
| 176 |
)
|
| 177 |
-
batch_accuracy_box = gr.Markdown("**Batch Accuracy:** -")
|
| 178 |
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
proj_dropdown.change(fn=get_warning_status, inputs=[sub_dropdown, proj_dropdown], outputs=warning_box)
|
| 184 |
-
proj_dropdown.change(fn=get_historical_accuracy, inputs=[sub_dropdown, proj_dropdown], outputs=history_box)
|
| 185 |
-
|
| 186 |
-
# Run
|
| 187 |
-
btn.click(
|
| 188 |
-
fn=run_batch_analysis,
|
| 189 |
-
inputs=[sub_dropdown, proj_dropdown],
|
| 190 |
-
outputs=[result_table, batch_accuracy_box]
|
| 191 |
-
)
|
| 192 |
|
| 193 |
if __name__ == "__main__":
|
| 194 |
demo.launch()
|
|
|
|
| 11 |
|
| 12 |
print("๐ System Startup: Loading Artifacts...")
|
| 13 |
if not os.path.exists(PKG_PATH):
|
| 14 |
+
# Fallback for local testing if file isn't in root
|
| 15 |
+
POSSIBLE_PATHS = [
|
| 16 |
+
"neuro_semantic_package.pt",
|
| 17 |
+
"/content/drive/MyDrive/Brain2Text_Project/demo_research_v2/neuro_semantic_package.pt"
|
| 18 |
+
]
|
| 19 |
+
for p in POSSIBLE_PATHS:
|
| 20 |
+
if os.path.exists(p):
|
| 21 |
+
PKG_PATH = p
|
| 22 |
+
break
|
| 23 |
+
|
| 24 |
+
if not os.path.exists(PKG_PATH):
|
| 25 |
+
raise FileNotFoundError(f"CRITICAL: '{PKG_PATH}' missing. Please upload the .pt file.")
|
| 26 |
|
| 27 |
# Load the "Black Box" package
|
| 28 |
+
# map_location='cpu' ensures it runs on basic HF spaces without GPU if needed
|
| 29 |
+
PKG = torch.load(PKG_PATH, map_location="cpu", weights_only=False)
|
| 30 |
DATA = PKG['data']
|
| 31 |
MODELS = PKG['models'] # The Projectors
|
| 32 |
MATRIX = PKG['matrix'] # Pre-calculated Accuracy Table
|
|
|
|
| 122 |
"Match": match_icon
|
| 123 |
}])
|
| 124 |
|
| 125 |
+
return df
|
| 126 |
|
| 127 |
def run_batch_analysis(subject, projector_alias):
|
| 128 |
# Runs 5 random samples for robust demo
|
| 129 |
subject_data = DATA[subject]
|
| 130 |
total_indices = list(range(len(subject_data['Text'])))
|
| 131 |
+
# Sample up to 5 sentences
|
| 132 |
selected_indices = random.sample(total_indices, min(5, len(total_indices)))
|
| 133 |
|
| 134 |
results = []
|
| 135 |
|
| 136 |
for idx in selected_indices:
|
| 137 |
txt = subject_data['Text'][idx]
|
| 138 |
+
df = decode_neuro_semantics(subject, projector_alias, txt)
|
| 139 |
results.append(df)
|
| 140 |
|
| 141 |
final_df = pd.concat(results)
|
|
|
|
| 146 |
|
| 147 |
# --- 3. UI LAYOUT ---
|
| 148 |
|
| 149 |
+
# Formatted Report Text
|
| 150 |
+
REPORT_TEXT = """
|
| 151 |
+
### 1. Abstract
|
| 152 |
+
This interface demonstrates a **Brain-Computer Interface (BCI)** capable of decoding high-level semantic information directly from non-invasive EEG signals. By aligning biological neural activity with the latent space of Large Language Models (LLMs), we show that it is possible to reconstruct the **emotional sentiment** of a sentence a user is reading, even if the model has **never seen that user's brain data before**.
|
| 153 |
+
|
| 154 |
+
### 2. The Dataset: ZuCo (Zurich Cognitive Language Processing Corpus)
|
| 155 |
+
This project utilizes the **ZuCo 2.0 dataset**, a benchmark for cognitive modeling.
|
| 156 |
+
* **Protocol:** Subjects read movie reviews naturally while their brain activity (EEG) and eye movements were recorded.
|
| 157 |
+
* **The Challenge:** Unlike synthetic tasks, natural reading involves rapid, complex cognitive processing, making signal decoding significantly harder.
|
| 158 |
+
|
| 159 |
+
### 3. Methodology: Latent Space Projection
|
| 160 |
+
Instead of training a simple classifier to predict "Positive" or "Negative" from brain waves, we employ a **Neuro-Semantic Projector**.
|
| 161 |
+
* **The Goal:** To learn a mapping function `f(EEG) โ R^768` that transforms raw brain signals into the high-dimensional embedding space of **RoBERTa**.
|
| 162 |
+
* **The Mechanism:** The system projects the EEG signal into a vector. This vector is then fed into a frozen, pre-trained LLM (`roberta-base-go_emotions`) to generate a probability distribution over **28 distinct emotional states** (e.g., *Admiration, Annoyance, Gratitude, Remorse*).
|
| 163 |
|
| 164 |
+
### 4. Experimental Setup: Strict Zero-Shot Evaluation
|
| 165 |
+
To ensure scientific rigor, this demo adheres to a **Strict Leave-One-Group-Out** protocol.
|
| 166 |
+
* **Disjoint Training:** The "Projectors" available in this demo were trained on a subset of subjects and validated on **completely different subjects**.
|
| 167 |
+
* **No Calibration:** The model does not receive any calibration data from the target subject. It must rely on universal neural patterns shared across humans.
|
| 168 |
|
| 169 |
+
### 5. Interpretation of Results
|
| 170 |
+
The demo compares two probability distributions for every sentence:
|
| 171 |
+
1. **Text Ground Truth:** What the AI model thinks the sentence means based on the text alone.
|
| 172 |
+
2. **Brain Prediction:** What the AI model thinks the sentence means based **only** on the user's brain waves.
|
| 173 |
+
|
| 174 |
+
**Accuracy Metric:** A prediction is considered correct if the **Top-1 Emotion** predicted from the Brain Signal matches either the **#1 or #2 Emotion** predicted from the Text.
|
| 175 |
"""
|
| 176 |
|
| 177 |
+
with gr.Blocks(theme=gr.themes.Soft(), title="Neuro-Semantic Decoder") as demo:
|
| 178 |
gr.Markdown("# ๐ง Neuro-Semantic Alignment: Zero-Shot Decoding")
|
| 179 |
|
| 180 |
+
with gr.Tabs():
|
| 181 |
+
# --- TAB 1: INTERACTIVE DEMO ---
|
| 182 |
+
with gr.TabItem("๐ฎ Interactive Demo"):
|
| 183 |
+
with gr.Row():
|
| 184 |
+
with gr.Column(scale=1):
|
| 185 |
+
gr.Markdown("### โ๏ธ Configuration")
|
| 186 |
+
|
| 187 |
+
# Selectors
|
| 188 |
+
sub_dropdown = gr.Dropdown(choices=list(DATA.keys()), value="ZKB", label="Select Target Subject (Data Source)")
|
| 189 |
+
proj_dropdown = gr.Dropdown(choices=list(MODELS.keys()), value="Projector A", label="Select Projector (Decoding Model)")
|
| 190 |
+
|
| 191 |
+
# Dynamic Info Boxes
|
| 192 |
+
warning_box = gr.Markdown("โ
**VALID ZERO-SHOT CONFIGURATION**\n\nTarget Subject was NOT seen during Projector training.")
|
| 193 |
+
history_box = gr.Markdown("**Historical Compatibility:** 40.0%")
|
| 194 |
+
|
| 195 |
+
btn = gr.Button("๐ฎ Run Batch Analysis (5 Samples)", variant="primary")
|
| 196 |
+
|
| 197 |
+
with gr.Column(scale=2):
|
| 198 |
+
gr.Markdown("### ๐ Decoding Results")
|
| 199 |
+
|
| 200 |
+
# Output Table
|
| 201 |
+
result_table = gr.Dataframe(
|
| 202 |
+
headers=["Sentence Stimulus", "Text Ground Truth (Top 2)", "Brain Decoding (Top 3)", "Match"],
|
| 203 |
+
wrap=True
|
| 204 |
+
)
|
| 205 |
+
batch_accuracy_box = gr.Markdown("**Batch Accuracy:** -")
|
| 206 |
+
|
| 207 |
+
# Interactivity
|
| 208 |
+
sub_dropdown.change(fn=get_warning_status, inputs=[sub_dropdown, proj_dropdown], outputs=warning_box)
|
| 209 |
+
sub_dropdown.change(fn=get_historical_accuracy, inputs=[sub_dropdown, proj_dropdown], outputs=history_box)
|
| 210 |
|
| 211 |
+
proj_dropdown.change(fn=get_warning_status, inputs=[sub_dropdown, proj_dropdown], outputs=warning_box)
|
| 212 |
+
proj_dropdown.change(fn=get_historical_accuracy, inputs=[sub_dropdown, proj_dropdown], outputs=history_box)
|
|
|
|
| 213 |
|
| 214 |
+
# Run
|
| 215 |
+
btn.click(
|
| 216 |
+
fn=run_batch_analysis,
|
| 217 |
+
inputs=[sub_dropdown, proj_dropdown],
|
| 218 |
+
outputs=[result_table, batch_accuracy_box]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 219 |
)
|
|
|
|
| 220 |
|
| 221 |
+
# --- TAB 2: REPORT ---
|
| 222 |
+
with gr.TabItem("๐ Project Report"):
|
| 223 |
+
gr.Markdown(REPORT_TEXT)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 224 |
|
| 225 |
if __name__ == "__main__":
|
| 226 |
demo.launch()
|