Update app.py

app.py

@@ -1,167 +1,223 @@
 # ============================================================================
-# ФИНАЛЬНЫЙ ТЕСТ BDH (ИСПРАВЛЕННЫЕ РАЗМЕРНОСТИ)
+# ЧЕСТНЫЙ ПЕРЕНОС: 32 СЛОЯ AURA -> 6 ЦИКЛОВ BDH (БЕЗ УСРЕДНЕНИЯ)
 # ============================================================================
-
-import gc
-import torch
-import torch.nn.functional as F
-from safetensors.torch import load_file
-from huggingface_hub import hf_hub_download
-import json
-import sys
-import numpy as np
-import time
-
-gc.collect()
-if torch.cuda.is_available():
-    torch.cuda.empty_cache()
-
-sys.path.append('.')
-from bdh import BDH, BDHConfig
-
-print("=" * 80)
-print("🧪 ФИНАЛЬНЫЙ ТЕСТ BDH (ИСПРАВЛЕННЫЕ РАЗМЕРНОСТИ)")
-print("=" * 80)
-
-device = "cpu"
-
-# Загрузка модели
-config_path = hf_hub_download(repo_id="Andrewstivan/AURA", filename="bdh_plasticity/bdh_config.json", repo_type="model")
-with open(config_path, 'r') as f:
-    config_dict = json.load(f)
-
-TARGET_N_HEAD = 16
-TARGET_N_EMBD = 1024
-TARGET_LATENT = 1024 // 16  # 64
-
-config = BDHConfig(
-    n_layer=6, 
-    n_embd=TARGET_N_EMBD, 
-    n_head=TARGET_N_HEAD, 
-    mlp_internal_dim_multiplier=1, 
-    vocab_size=256, 
-    dropout=0.1
-)
-bdh_model = BDH(config).to(device)
-
-# Выводим ожидаемые размерности
-print(f"\n📐 Ожидаемые размерности BDH:")
-print(f"   encoder:   {bdh_model.encoder.weight_fp32.shape}")   # [16, 64, 1024]
-print(f"   encoder_v: {bdh_model.encoder_v.weight_fp32.shape}") # [16, 64, 1024]
-print(f"   decoder:   {bdh_model.decoder.weight_fp32.shape}")   # [1024, 1024]
-
-weights_path = hf_hub_download(repo_id="Andrewstivan/AURA", filename="bdh_plasticity/bdh_plasticity.safetensors", repo_type="model")
-weights = load_file(weights_path)
-
-print(f"\n📦 Загруженные размерности:")
-print(f"   encoder:   {weights['encoder'].shape}")   # [32, 4096, 128] или [32, 128, 4096]
-print(f"   encoder_v: {weights['encoder_v'].shape}") # [32, 4096, 128] или [32, 1024, 128]
-print(f"   decoder:   {weights['decoder'].shape}")   # [4096, 4096]
-
-with torch.no_grad():
-    # --- ENCODER ---
-    enc_w = weights['encoder'].to(device)
-    # Исходный: [32, 4096, 128] -> транспонируем в [32, 128, 4096] -> обрезаем до [16, 64, 1024]
-    if enc_w.shape == (32, 4096, 128):
-        enc_w = enc_w.transpose(1, 2)  # [32, 128, 4096]
-    enc_w = enc_w[:TARGET_N_HEAD, :TARGET_LATENT, :TARGET_N_EMBD]
-    print(f"   encoder после обрезания: {enc_w.shape}")
-    bdh_model.encoder.weight_fp32.data = enc_w
-    
-    # --- ENCODER_V ---
-    encv_w = weights['encoder_v'].to(device)
-    print(f"   encoder_v исходный: {encv_w.shape}")
-    # Нужно привести к [16, 64, 1024]
-    if encv_w.dim() == 3:
-        if encv_w.shape[1] == 4096 and encv_w.shape[2] == 128:
-            encv_w = encv_w.transpose(1, 2)  # [32, 128, 4096]
-        elif encv_w.shape[1] == 1024 and encv_w.shape[2] == 128:
-            encv_w = encv_w.transpose(1, 2)  # [32, 128, 1024]
-    
-    # Обрезаем до нужного размера
-    fixed_encv = torch.zeros(bdh_model.encoder_v.weight_fp32.shape, device=device)
-    h = min(encv_w.shape[0], fixed_encv.shape[0])
-    n = min(encv_w.shape[1], fixed_encv.shape[1])
-    d = min(encv_w.shape[2], fixed_encv.shape[2])
-    fixed_encv[:h, :n, :d] = encv_w[:h, :n, :d]
-    print(f"   encoder_v после обрезания: {fixed_encv.shape}")
-    bdh_model.encoder_v.weight_fp32.data = fixed_encv
+import json, torch, os, gc, sys
+from tqdm import tqdm
+from safetensors.torch import save_file, safe_open
+from huggingface_hub import hf_hub_download, HfApi
+
+class Plasticity:
+    def __init__(self, n_neurons):
+        self.n_neurons = n_neurons
+        self.w = torch.zeros(n_neurons, n_neurons)
+        self.long_term_w = torch.zeros(n_neurons, n_neurons)
+        self.lr = 0.01
+        self.consolidation_rate = 0.01
+        self.forget_rate = 0.1
+        self.acc_pre = torch.zeros(n_neurons)
+        self.acc_post = torch.zeros(n_neurons)
+        self.threshold = 0.5
+        self.bcm_theta = torch.zeros(n_neurons)
+        self.lr_bcm = 0.001
+        self.target_activity = 0.5
+        self.step_count = 0
+
+    def adapt_weights(self, weight_matrix):
+        """Адаптирует матрицу весов через пластичность."""
+        original_shape = weight_matrix.shape
+        
+        # Приводим к 2D
+        if weight_matrix.dim() == 3:
+            wm_2d = weight_matrix.reshape(-1, weight_matrix.shape[-1])
+        else:
+            wm_2d = weight_matrix
+        
+        # Вычисляем a_pre и a_post
+        a_pre_raw = wm_2d.mean(dim=1)
+        a_post_raw = wm_2d.mean(dim=0)
+        
+        # Приведение a_pre к n_neurons
+        if a_pre_raw.shape[0] > self.n_neurons:
+            a_pre = a_pre_raw[:self.n_neurons]
+        elif a_pre_raw.shape[0] < self.n_neurons:
+            repeat_factor = (self.n_neurons + a_pre_raw.shape[0] - 1) // a_pre_raw.shape[0]
+            a_pre = a_pre_raw.repeat(repeat_factor)[:self.n_neurons]
+        else:
+            a_pre = a_pre_raw
+        
+        # Приведение a_post к n_neurons
+        if a_post_raw.shape[0] > self.n_neurons:
+            a_post = a_post_raw[:self.n_neurons]
+        elif a_post_raw.shape[0] < self.n_neurons:
+            repeat_factor = (self.n_neurons + a_post_raw.shape[0] - 1) // a_post_raw.shape[0]
+            a_post = a_post_raw.repeat(repeat_factor)[:self.n_neurons]
+        else:
+            a_post = a_post_raw
+        
+        # Псевдоспайки
+        self.acc_pre += a_pre
+        self.acc_post += a_post
+        
+        spike_pre = (self.acc_pre >= self.threshold).float()
+        spike_post = (self.acc_post >= self.threshold).float()
+        
+        self.acc_pre -= spike_pre * self.threshold
+        self.acc_post -= spike_post * self.threshold
+        
+        # Хеббовское обновление
+        delta = self.lr * torch.outer(spike_pre, spike_post)
+        self.w += delta
+        
+        # Применяем адаптацию к ИСХОДНОЙ матрице
+        with torch.no_grad():
+            # БЕРЁМ СРЕЗ ОТ self.w
+            update_slice = self.w[:wm_2d.shape[0], :wm_2d.shape[1]]
+            
+            # ЕСЛИ СРЕЗ МЕНЬШЕ, ЧЕМ НУЖНО, ПОВТОРЯЕМ ЕГО
+            if update_slice.shape[0] < wm_2d.shape[0] or update_slice.shape[1] < wm_2d.shape[1]:
+                repeat_rows = (wm_2d.shape[0] + update_slice.shape[0] - 1) // update_slice.shape[0]
+                repeat_cols = (wm_2d.shape[1] + update_slice.shape[1] - 1) // update_slice.shape[1]
+                update = update_slice.repeat(repeat_rows, repeat_cols)[:wm_2d.shape[0], :wm_2d.shape[1]]
+            else:
+                update = update_slice
+                
+            update = update * 0.01
+            
+            if weight_matrix.dim() == 3:
+                update = update.reshape(original_shape)
+            weight_matrix = weight_matrix + update
+        
+        self.step_count += 1
+        if self.step_count % 10 == 0:
+            self.consolidate()
+            
+        return weight_matrix
+
+    def consolidate(self):
+        self.long_term_w += self.consolidation_rate * self.w
+        self.w = self.w * (1 - self.forget_rate)
+
+# ------------------------------------------------------------
+# Загрузка shard'ов
+# ------------------------------------------------------------
+class ShardCache:
+    def __init__(self, repo_id, weight_map):
+        self.repo_id = repo_id
+        self.weight_map = weight_map
+        self.cached_shards = {}
+
+    def get_layer_weights(self, layer_idx):
+        prefix = f"model.layers.{layer_idx}."
+        layer_tensors = {n: s for n, s in self.weight_map.items() if n.startswith(prefix)}
+        if not layer_tensors:
+            return {}
+
+        for shard_file in set(layer_tensors.values()):
+            if shard_file not in self.cached_shards:
+                shard_path = hf_hub_download(repo_id=self.repo_id, filename=shard_file)
+                self.cached_shards[shard_file] = safe_open(shard_path, framework="pt", device="cpu")
+
+        weights = {}
+        for name, shard_file in layer_tensors.items():
+            param_name = name.split('.')[-2]
+            weights[param_name] = self.cached_shards[shard_file].get_tensor(name)
+        return weights
+
+    def close(self):
+        for s in self.cached_shards.values():
+            s.__exit__(None, None, None)
+# -----------------------------------------------------------------------------
+# ЗАГРУЗКА ВЕСОВ
+# -----------------------------------------------------------------------------
+repo_id = "ResplendentAI/Aura_v3_7B"
+index_path = hf_hub_download(repo_id=repo_id, filename="model.safetensors.index.json")
+with open(index_path, 'r') as f: weight_map = json.load(f)['weight_map']
+cache = ShardCache(repo_id, weight_map)
+
+# Конфиг BDH - ЧЕСТНЫЙ (32 слоя)
+config = BDHConfig(n_layer=32, n_embd=4096, n_head=32, mlp_internal_dim_multiplier=1, vocab_size=256, dropout=0.1)
+D, nh = config.n_embd, config.n_head
+N = config.mlp_internal_dim_multiplier * D // nh
+
+plasticity_enc = Plasticity(n_neurons=D)
+plasticity_enc_v = Plasticity(n_neurons=D)
+plasticity_dec = Plasticity(n_neurons=D)
+
+# Начинаем с нуля
+encoder = torch.zeros(nh, N, D)
+encoder_v = torch.zeros(nh, N, D)
+decoder = torch.zeros(nh * N, D)
+
+print(f"Начало: encoder={encoder.shape}, encoder_v={encoder_v.shape}, decoder={decoder.shape}")
+
+for layer_idx in tqdm(range(32)):
+    w = cache.get_layer_weights(layer_idx)
+    if not w: continue
+
+    # Извлекаем полные матрицы
+    q_proj = w['q_proj'].float()
+    v_proj = w['v_proj'].float()
+    o_proj = w['o_proj'].float()
+
+    # Приводим к нужной фо��ме БЕЗ УРЕЗАНИЯ
+    # Aura: [4096, 4096] -> BDH: [32, 128, 4096]
+    q_reshaped = q_proj.reshape(nh, N, D)
+    v_reshaped = v_proj.reshape(nh, N, D)
+    o_reshaped = o_proj.reshape(nh * N, D)
+
+    # Пластичность адаптирует БЕЗ УСРЕДНЕНИЯ
+    encoder = plasticity_enc.adapt_weights(q_reshaped)
+    encoder_v = plasticity_enc_v.adapt_weights(v_reshaped)
+    decoder = plasticity_dec.adapt_weights(o_reshaped)
+
+# Финальная консолидация
+plasticity_enc.consolidate()
+plasticity_enc_v.consolidate()
+plasticity_dec.consolidate()
+
+# Сохраняем честные веса
+bdh_weights = {'encoder': encoder, 'encoder_v': encoder_v, 'decoder': decoder, 'lm_head': torch.randn(D, 256) * 0.02}
+save_file(bdh_weights, "bdh_full_weights.safetensors")
+
+# Создаём временную папку
+os.makedirs("bdh_weights_upload", exist_ok=True)
+
+# Сохраняем веса в формате safetensors
+bdh_weights_safe = {k: v.cpu() for k, v in bdh_weights.items()}
+save_file(bdh_weights_safe, "bdh_weights_upload/bdh_plasticity.safetensors")
+
+# Сохраняем конфиг BDH
+config_dict = {
+    "model_type": "bdh",
+    "n_layer": config.n_layer,
+    "n_embd": config.n_embd,
+    "n_head": config.n_head,
+    "mlp_internal_dim_multiplier": config.mlp_internal_dim_multiplier,
+    "vocab_size": config.vocab_size,
+    "dropout": config.dropout
+}
+with open("bdh_weights_upload/bdh_config.json", "w") as f:
+    json.dump(config_dict, f, indent=2)
+
+# Загружаем в существующий репозиторий Andrewstivan/AURA
+token = os.environ.get('HF_TOKEN')
+if token:
+    api = HfApi(token=token)
+    repo_id = "Andrewstivan/AURA"  # <-- Ваш существующий репозиторий
     
-    # --- DECODER ---
-    dec_w = weights['decoder'].to(device)
-    target_dec = bdh_model.decoder.weight_fp32.shape
-    fixed_dec = torch.zeros(target_dec, device=device)
-    copy_0 = min(dec_w.shape[0], target_dec[0])
-    copy_1 = min(dec_w.shape[1], target_dec[1])
-    fixed_dec[:copy_0, :copy_1] = dec_w[:copy_0, :copy_1]
-    print(f"   decoder после обрезания: {fixed_dec.shape}")
-    bdh_model.decoder.weight_fp32.data = fixed_dec
+    api.upload_folder(
+        folder_path="bdh_weights_upload",
+        repo_id=repo_id,
+        repo_type="model",
+        path_in_repo="bdh_plasticity",  # Файлы будут в папке bdh_plasticity/
+        commit_message="🧠 Добавлены веса BDH, полученные через пластичность из Aura"
+    )
     
-    # Обновляем тернарные веса
-    bdh_model.encoder.update_ternary_weights()
-    bdh_model.encoder_v.update_ternary_weights()
-    bdh_model.decoder.update_ternary_weights()
-
-bdh_model.eval()
-print(f"\n✅ Модель загружена. Параметров: {sum(p.numel() for p in bdh_model.parameters()):,}")
-
-# ТЕСТ 1: РАСПРЕДЕЛЕНИЕ ВЕСОВ
-print("\n" + "=" * 80)
-print("📊 ТЕСТ 1/3: РАСПРЕДЕЛЕНИЕ ТЕРНАРНЫХ ВЕСОВ")
-print("=" * 80)
-print(f"{'Слой':<12} {'-1':<10} {'0':<10} {'+1':<10} {'Энтропия':<12} {'Использование':<12}")
-print("-" * 60)
-
-for name, param in [('encoder', bdh_model.encoder.weight_ternary), ('encoder_v', bdh_model.encoder_v.weight_ternary), ('decoder', bdh_model.decoder.weight_ternary)]:
-    total = param.numel()
-    minus1 = (param == -1).sum().item()
-    zero = (param == 0).sum().item()
-    plus1 = (param == 1).sum().item()
-    p = np.array([minus1, zero, plus1]) / total
-    p = p[p > 0]
-    entropy = -np.sum(p * np.log2(p))
-    max_entropy = np.log2(3)
-    print(f"{name:<12} {minus1/total:>8.1%} {zero/total:>9.1%} {plus1/total:>9.1%} {entropy:>10.4f} {entropy/max_entropy:>11.1%}")
-
-# ТЕСТ 2: ЖИВУЧЕСТЬ
-print("\n" + "=" * 80)
-print("📊 ТЕСТ 2/3: ЖИВУЧЕСТЬ МОДЕЛИ")
-print("=" * 80)
-
-def text_to_bytes(text):
-    return torch.tensor(list(text.encode('utf-8')), dtype=torch.long).unsqueeze(0).to(device)
-
-for text in ["Hello world", "The quick brown fox", "A" * 100]:
-    bytes_tensor = text_to_bytes(text)
-    try:
-        with torch.no_grad():
-            logits, states = bdh_model.forward_with_states(bytes_tensor)
-        print(f"✅ '{text[:30]}...' -> OK (вход: {bytes_tensor.shape[1]}, выход: {logits.shape})")
-    except Exception as e:
-        print(f"❌ '{text[:30]}...' -> {str(e)[:60]}")
-
-# ТЕСТ 3: ПРОИЗВОДИТЕЛЬНОСТЬ
-print("\n" + "=" * 80)
-print("📊 ТЕСТ 3/3: ПРОИЗВОДИТЕЛЬНОСТЬ")
-print("=" * 80)
-print(f"{'Токенов':<10} {'Время (мс)':<12} {'Токенов/с':<12}")
-print("-" * 50)
-
-for length in [10, 50, 100]:
-    bytes_tensor = text_to_bytes("A" * length)
-    try:
-        for _ in range(3): 
-            _, _ = bdh_model.forward_with_states(bytes_tensor)
-        start = time.time()
-        for _ in range(10): 
-            _, _ = bdh_model.forward_with_states(bytes_tensor)
-        elapsed = time.time() - start
-        tokens = bytes_tensor.shape[1]
-        print(f"{tokens:<10} {elapsed*1000/10:<12.2f} {tokens*10/elapsed:<12.0f}")
-    except Exception as e:
-        print(f"{length:<10} ОШИБКА: {str(e)[:40]}")
-
-# ИТОГ
-print("\n" + "=" * 80)
-print("🎉 ВСЕ ТЕСТЫ ПРОЙДЕНЫ!")
-print("=" * 80)
+    print("✅ Веса загружены в Andrewstivan/AURA/bdh_plasticity/")
+    print("   - bdh_plasticity.safetensors")
+    print("   - bdh_config.json")
+else:
+    print("⚠️ HF_TOKEN не найден! Файлы сохранены локально в папке 'bdh_weights_upload'")
+    print("   Загрузите их вручную в репозиторий Andrewstivan/AURA")
+
+print("\n🎉 ГОТОВО!")