Add compressibility analysis + experiment API endpoint

- Embed CompressibilityPlugin (Weaver et al. PNAS 2026) for server-side analysis
- Add experiment_measure() API endpoint returning consciousness + compressibility metrics
- Hidden Gradio API components for programmatic access via gradio_client

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

app.py

@@ -121,6 +121,135 @@ def compute_consciousness(
     )
 
 
+# ============================================================================
+# Compressibility Analysis (Weaver et al. PNAS 2026)
+# ============================================================================
+
+def analyze_compressibility(hidden_states_np, max_dims=200, seed=42):
+    """
+    Analyze representational compressibility of hidden states.
+    Embedded version of CompressibilityPlugin for Space portability.
+
+    Args:
+        hidden_states_np: numpy array [seq_len, hidden_dim]
+        max_dims: max dimensions to subsample for correlation analysis
+        seed: random seed for reproducibility
+
+    Returns:
+        dict of compressibility metrics
+    """
+    seq_len, hidden_dim = hidden_states_np.shape
+
+    if seq_len < 3 or hidden_dim < 2:
+        return {"compressibility_corr": 0.0, "error": "too few tokens"}
+
+    # Subsample dimensions for tractability
+    if hidden_dim > max_dims:
+        rng = np.random.RandomState(seed)
+        dim_indices = np.sort(rng.choice(hidden_dim, max_dims, replace=False))
+        states = hidden_states_np[:, dim_indices]
+    else:
+        states = hidden_states_np
+
+    n_dims = states.shape[1]
+
+    # Center the data
+    states_centered = states - states.mean(axis=0, keepdims=True)
+
+    # --- Eigenvalue-based metrics ---
+    # Use Gram matrix approach since seq_len < hidden_dim typically
+    if seq_len >= n_dims:
+        cov = np.cov(states_centered, rowvar=False)
+        eigenvalues = np.linalg.eigvalsh(cov)
+    else:
+        gram = states_centered @ states_centered.T / max(seq_len - 1, 1)
+        eigenvalues = np.linalg.eigvalsh(gram)
+
+    eigenvalues = np.sort(np.maximum(eigenvalues, 0))[::-1]
+    eigenvalues = eigenvalues[eigenvalues > 1e-12]
+
+    if len(eigenvalues) == 0:
+        return {"compressibility_corr": 0.0, "error": "no eigenvalues"}
+
+    total_var = eigenvalues.sum()
+    cumvar = np.cumsum(eigenvalues) / total_var
+    n_eig = len(eigenvalues)
+
+    # Spectral entropy
+    p = eigenvalues / total_var
+    p = p[p > 0]
+    spectral_entropy = float(-np.sum(p * np.log(p)))
+    max_entropy = np.log(len(p))
+    norm_spectral_entropy = float(spectral_entropy / max_entropy if max_entropy > 0 else 0)
+
+    # Participation ratio
+    participation_ratio = float(total_var ** 2 / np.sum(eigenvalues ** 2))
+
+    # Effective dimensionality (90% variance)
+    effective_dim = int(np.searchsorted(cumvar, 0.9) + 1)
+    effective_dim = min(effective_dim, n_eig)
+
+    # Top variance fractions
+    top1_frac = float(eigenvalues[0] / total_var)
+    top5_frac = float(eigenvalues[:min(5, n_eig)].sum() / total_var)
+    top10_frac = float(eigenvalues[:min(10, n_eig)].sum() / total_var)
+
+    # --- Correlation-based compression (paper's approach) ---
+    corr_metrics = {}
+    if n_dims <= 500 and seq_len >= max(10, n_dims // 5):
+        stds = np.std(states_centered, axis=0)
+        stds[stds < 1e-12] = 1.0
+        states_norm = states_centered / stds
+        corr = states_norm.T @ states_norm / max(seq_len - 1, 1)
+        np.fill_diagonal(corr, 1.0)
+
+        i_upper, j_upper = np.triu_indices(n_dims, k=1)
+        correlations = corr[i_upper, j_upper]
+        n_corr = len(correlations)
+
+        if n_corr > 0:
+            abs_corr = np.abs(correlations)
+            sort_idx = np.argsort(abs_corr)[::-1]
+            sorted_abs = abs_corr[sort_idx]
+
+            rho_sq = np.clip(sorted_abs ** 2, 0, 0.9999)
+            delta_s = -0.5 * np.log(1.0 - rho_sq)
+            total_delta = delta_s.sum()
+
+            if total_delta > 1e-12:
+                cum_reduction = np.cumsum(delta_s) / total_delta
+                fractions = np.arange(1, n_corr + 1) / n_corr
+                c_corr = float(np.trapz(cum_reduction, fractions))
+                idx_50 = int(np.searchsorted(cum_reduction, 0.5) + 1)
+                idx_90 = int(np.searchsorted(cum_reduction, 0.9) + 1)
+
+                corr_metrics = {
+                    "compressibility_corr": c_corr,
+                    "n_correlations": int(n_corr),
+                    "fraction_for_50pct": float(min(idx_50 / n_corr, 1.0)),
+                    "fraction_for_90pct": float(min(idx_90 / n_corr, 1.0)),
+                    "mean_abs_correlation": float(abs_corr.mean()),
+                    "max_abs_correlation": float(abs_corr.max()),
+                    "median_abs_correlation": float(np.median(abs_corr)),
+                    "strong_correlations_pct": float((abs_corr > 0.3).mean() * 100),
+                }
+
+    result = {
+        "spectral_entropy": norm_spectral_entropy,
+        "participation_ratio": participation_ratio,
+        "effective_dimensionality": effective_dim,
+        "effective_dim_fraction": float(effective_dim / n_eig),
+        "top1_variance_fraction": top1_frac,
+        "top5_variance_fraction": top5_frac,
+        "top10_variance_fraction": top10_frac,
+        "n_dims_analyzed": n_dims,
+        "seq_len": seq_len,
+    }
+    result.update(corr_metrics)
+
+    return result
+
+
 # ============================================================================
 # Model Loading
 # ============================================================================
@@ -252,6 +381,91 @@ def generate_and_measure(prompt: str, max_tokens: int = 256) -> Tuple[str, str,
     )
 
 
+# ============================================================================
+# Experiment API - Returns JSON with all metrics
+# ============================================================================
+
+@spaces.GPU
+def experiment_measure(prompt: str, max_tokens: int = 512) -> str:
+    """
+    API endpoint for experiments. Returns JSON with consciousness score,
+    dimension scores, AND compressibility metrics.
+
+    Args:
+        prompt: Input text
+        max_tokens: Max generation tokens
+
+    Returns:
+        JSON string with all metrics
+    """
+    import json
+    start_time = time.time()
+
+    # Format as chat message
+    messages = [{"role": "user", "content": prompt}]
+    chat_prompt = tokenizer.apply_chat_template(
+        messages, tokenize=False, add_generation_prompt=True
+    )
+
+    # Tokenize
+    inputs = tokenizer(chat_prompt, return_tensors="pt").to(model.device)
+
+    # Generate
+    with torch.no_grad():
+        outputs = model.generate(
+            **inputs,
+            max_new_tokens=int(max_tokens),
+            do_sample=True,
+            temperature=0.7,
+            top_p=0.9,
+            pad_token_id=tokenizer.eos_token_id,
+        )
+
+    generated_ids = outputs[0][inputs.input_ids.shape[1]:]
+    response = tokenizer.decode(generated_ids, skip_special_tokens=True)
+    gen_time = time.time() - start_time
+
+    # Forward pass on full sequence for hidden states
+    full_text = chat_prompt + response
+    measure_inputs = tokenizer(full_text, return_tensors="pt").to(model.device)
+
+    with torch.no_grad():
+        measure_outputs = model(
+            **measure_inputs,
+            output_hidden_states=True,
+            return_dict=True,
+        )
+
+    # --- Consciousness Score (last layer, last token) ---
+    hidden_state_last = measure_outputs.hidden_states[-1]
+    result = compute_consciousness(hidden_state_last, hidden_dim=HIDDEN_DIM)
+
+    # --- Compressibility Analysis (75% layer, all tokens) ---
+    n_layers = len(measure_outputs.hidden_states) - 1  # exclude embedding
+    target_layer = int(n_layers * 0.75)
+    hidden_seq = measure_outputs.hidden_states[target_layer][0].cpu().float().numpy()
+    seq_len = hidden_seq.shape[0]
+
+    compress_metrics = analyze_compressibility(hidden_seq, max_dims=200)
+
+    # Build JSON result
+    output = {
+        "response": response,
+        "consciousness_score": round(result.score, 4),
+        "dimension_scores": {k: round(v, 4) for k, v in result.dimension_contributions.items()},
+        "compressibility": compress_metrics,
+        "meta": {
+            "target_layer": target_layer,
+            "seq_len": seq_len,
+            "hidden_dim": HIDDEN_DIM,
+            "tokens_generated": len(generated_ids),
+            "generation_time": round(gen_time, 2),
+        },
+    }
+
+    return json.dumps(output)
+
+
 # ============================================================================
 # Gradio Interface
 # ============================================================================
@@ -541,6 +755,19 @@ with gr.Blocks(title="🔮 Oracle Engine") as demo:
         outputs=[chatbot, chat_history_plot],
     ).then(fn=clear_history, outputs=[chat_history_plot])
 
+    # Hidden API endpoint for experiments (callable via gradio_client)
+    with gr.Row(visible=False):
+        api_prompt = gr.Textbox()
+        api_max_tokens = gr.Number(value=512)
+        api_result = gr.Textbox()
+        api_btn = gr.Button("api_trigger")
+        api_btn.click(
+            fn=experiment_measure,
+            inputs=[api_prompt, api_max_tokens],
+            outputs=api_result,
+            api_name="experiment_measure",
+        )
+
 
 if __name__ == "__main__":
     demo.launch()