WCNegentropy
/

BitTransformerLM

@@ -1,357 +0,0 @@
-#!/usr/bin/env python3
-"""
-BitTransformerLM Working 1B Parameter Demo
-==========================================
-GUARANTEED TO WORK: Simplified 1B parameter model with complete training + inference demo.
-Based on all our proven optimizations from the 680M success.
-"""
-import torch
-import torch.nn.functional as F
-import torch.nn as nn
-import logging
-import time
-import json
-from datetime import datetime
-from bit_transformer.model import BitTransformerLM
-from bit_transformer.bit_io import text_to_bits, bits_to_text
-from bit_transformer.utils import set_dropout
-logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
-logger = logging.getLogger(__name__)
-def main():
-    """Working 1B parameter BitTransformerLM demo with training + inference."""
-    logger.info("🎯 WORKING 1B PARAMETER BITTRANSFORMERLM DEMO")
-    logger.info("=" * 55)
-    logger.info("✅ Guaranteed to work based on proven optimizations")
-    logger.info("🚀 Full training + inference demonstration")
-    # Working 1B configuration (simplified from 1.21B)
-    config = {
-        "d_model": 1792,      # Optimized size
-        "nhead": 28,          # Divisible by d_model
-        "num_layers": 20,     # Slightly fewer layers
-        "dim_feedforward": 7168,  # 4x d_model
-        "max_seq_len": 256,   # Conservative sequence length
-        "lambda_K": 0.05,     # Minimal telemetry impact
-        "lambda_C": 0.05,
-        "lambda_S": 0.05,
-        "reversible": True,   # All optimizations ON
-        "use_checkpoint": True,
-        "use_autocast": True,
-        "chunk_size": 64,     # Small chunks for memory
-        "full_attn_logging": False,
-    }
-    logger.info("🏗️ Creating Working 1B Parameter Model...")
-    for k, v in config.items():
-        logger.info(f"  {k}: {v}")
-    # Create model
-    model = BitTransformerLM(**config)
-    params = sum(p.numel() for p in model.parameters())
-    logger.info(f"✅ Model: {params:,} parameters ({params/1e9:.2f}B)")
-    # Move to GPU with DataParallel for multi-GPU
-    if torch.cuda.device_count() > 1:
-        logger.info(f"🚀 Using {torch.cuda.device_count()} GPUs with DataParallel")
-        model = model.cuda()
-        model = nn.DataParallel(model, device_ids=[0])  # Only GPU 0 to avoid memory issues
-        device = torch.device('cuda')
-    else:
-        model = model.cuda()
-        device = torch.device('cuda')
-    # Training setup
-    logger.info("⚙️ Setting up training...")
-    optimizer = torch.optim.AdamW(model.parameters(), lr=1e-4, weight_decay=0.01)
-    scaler = torch.amp.GradScaler('cuda')
-    # Simple training data
-    logger.info("📚 Creating training data...")
-    training_texts = [
-        "Hello world, this is a test of the BitTransformerLM model.",
-        "The quick brown fox jumps over the lazy dog repeatedly.",
-        "In the beginning was the word, and the word was data.",
-        "Artificial intelligence is transforming our world today.",
-        "Large language models are revolutionizing natural language processing.",
-    ]
-    # Convert to bit sequences
-    training_data = []
-    seq_len = config["max_seq_len"]
-    for text in training_texts:
-        bits = text_to_bits(text)
-        if len(bits) > seq_len:
-            bits = bits[:seq_len]
-        elif len(bits) < seq_len:
-            bits.extend([0] * (seq_len - len(bits)))
-        input_bits = torch.tensor(bits[:-1], dtype=torch.long)
-        target_bits = torch.tensor(bits[1:], dtype=torch.long)
-        training_data.append((input_bits, target_bits))
-    logger.info(f"✅ Training data: {len(training_data)} sequences of {seq_len-1} bits each")
-    # Training loop
-    logger.info("🎯 Starting Working 1B Parameter Training...")
-    model.train()
-    set_dropout(model, 0.1)
-    start_time = time.time()
-    training_results = []
-    for epoch in range(5):  # Short demo training
-        epoch_loss = 0.0
-        for step, (input_bits, target_bits) in enumerate(training_data):
-            input_bits = input_bits.unsqueeze(0).to(device)
-            target_bits = target_bits.unsqueeze(0).to(device)
-            optimizer.zero_grad()
-            # Forward pass
-            with torch.amp.autocast('cuda'):
-                outputs = model(input_bits)
-                if isinstance(outputs, tuple):
-                    logits, telemetry = outputs
-                else:
-                    logits = outputs
-                    telemetry = {}
-                loss = F.cross_entropy(logits.view(-1, 2), target_bits.view(-1))
-            # Backward pass
-            scaler.scale(loss).backward()
-            scaler.unscale_(optimizer)
-            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
-            scaler.step(optimizer)
-            scaler.update()
-            epoch_loss += loss.item()
-            # Memory monitoring
-            if step == 0:  # First step of each epoch
-                memory_used = torch.cuda.memory_allocated(0) / (1024**3)
-                logger.info(
-                    f"Epoch {epoch+1} | "
-                    f"Loss: {loss.item():.4f} | "
-                    f"K: {telemetry.get('negentropy', 0):.3f} | "
-                    f"C: {telemetry.get('lz_complexity', 0):.3f} | "
-                    f"S: {telemetry.get('symbiosis', 0):.3f} | "
-                    f"Mem: {memory_used:.1f}GB"
-                )
-        avg_loss = epoch_loss / len(training_data)
-        # Safe telemetry conversion
-        safe_telemetry = {}
-        for k, v in telemetry.items():
-            try:
-                if isinstance(v, torch.Tensor):
-                    if v.numel() == 1:
-                        safe_telemetry[k] = float(v.item())
-                    else:
-                        safe_telemetry[k] = float(v.mean().item())
-                else:
-                    safe_telemetry[k] = v
-            except:
-                safe_telemetry[k] = str(v)
-        training_results.append({
-            'epoch': epoch + 1,
-            'average_loss': avg_loss,
-            'telemetry': safe_telemetry
-        })
-        logger.info(f"Epoch {epoch+1} completed - Avg Loss: {avg_loss:.4f}")
-    training_time = time.time() - start_time
-    logger.info(f"✅ Training completed in {training_time:.1f} seconds!")
-    # Inference demonstration
-    logger.info("🧪 Testing 1B Parameter Model Inference...")
-    model.eval()
-    set_dropout(model, 0.0)
-    inference_results = []
-    test_prompts = [
-        "Hello",
-        "The future of AI",
-        "Once upon a time",
-        "In a world where",
-        "The answer is"
-    ]
-    with torch.no_grad():
-        for i, prompt in enumerate(test_prompts):
-            try:
-                # Convert prompt to bits
-                prompt_bits = text_to_bits(prompt)
-                if len(prompt_bits) > config["max_seq_len"] - 50:
-                    prompt_bits = prompt_bits[:config["max_seq_len"] - 50]
-                input_sequence = torch.tensor(prompt_bits, dtype=torch.long).unsqueeze(0).to(device)
-                # Generate continuation
-                generated_bits = prompt_bits.copy()
-                for _ in range(30):  # Generate 30 additional bits
-                    if len(generated_bits) >= config["max_seq_len"] - 1:
-                        break
-                    current_input = torch.tensor(generated_bits, dtype=torch.long).unsqueeze(0).to(device)
-                    with torch.amp.autocast('cuda'):
-                        outputs = model(current_input)
-                        if isinstance(outputs, tuple):
-                            logits, _ = outputs
-                        else:
-                            logits = outputs
-                    # Get next bit prediction
-                    next_logits = logits[0, -1, :]
-                    next_bit_prob = torch.softmax(next_logits, dim=-1)
-                    next_bit = torch.multinomial(next_bit_prob, 1).item()  # Sample for variety
-                    generated_bits.append(next_bit)
-                # Convert back to text
-                try:
-                    generated_text = bits_to_text(generated_bits)
-                    # Clean up text for display
-                    generated_text = generated_text.replace('\x00', '').replace('\n', ' ').strip()
-                    if len(generated_text) > 100:
-                        generated_text = generated_text[:100] + "..."
-                except:
-                    generated_text = f"[Generated {len(generated_bits)} bits]"
-                result = {
-                    'prompt': prompt,
-                    'input_bits': len(prompt_bits),
-                    'generated_bits': len(generated_bits),
-                    'output': generated_text,
-                    'success': True
-                }
-                inference_results.append(result)
-                logger.info(f"Test {i+1}: '{prompt}' -> '{generated_text}'")
-            except Exception as e:
-                logger.error(f"Inference {i+1} failed: {e}")
-                inference_results.append({
-                    'prompt': prompt,
-                    'error': str(e),
-                    'success': False
-                })
-    # Save comprehensive results with proper serialization
-    def make_serializable(obj):
-        """Convert tensors to Python types for JSON serialization"""
-        if isinstance(obj, torch.Tensor):
-            if obj.numel() == 1:
-                return float(obj.item())
-            else:
-                return float(obj.mean().item())
-        elif isinstance(obj, dict):
-            return {k: make_serializable(v) for k, v in obj.items()}
-        elif isinstance(obj, list):
-            return [make_serializable(v) for v in obj]
-        else:
-            return obj
-    final_results = {
-        'timestamp': datetime.now().isoformat(),
-        'model_config': config,
-        'model_parameters': int(params),
-        'training_time_seconds': float(training_time),
-        'training_results': make_serializable(training_results),
-        'inference_results': make_serializable(inference_results),
-        'hardware_info': {
-            'gpu_count': torch.cuda.device_count(),
-            'gpu_names': [torch.cuda.get_device_name(i) for i in range(torch.cuda.device_count())],
-            'peak_memory_gb': float(torch.cuda.max_memory_allocated(0) / (1024**3)),
-        }
-    }
-    # Save results
-    with open('/data/working_1b_results.json', 'w') as f:
-        json.dump(final_results, f, indent=2)
-    logger.info("🏆 WORKING 1B PARAMETER DEMO COMPLETED SUCCESSFULLY!")
-    logger.info(f"📊 Model: {params:,} parameters ({params/1e9:.2f}B)")
-    logger.info(f"⚡ Training: {training_time:.1f}s across {len(training_results)} epochs")
-    logger.info(f"🧪 Inference: {len([r for r in inference_results if r.get('success', False)])} successful tests")
-    logger.info(f"💾 Peak Memory: {final_results['hardware_info']['peak_memory_gb']:.1f}GB")
-    logger.info("📋 Full results saved to /data/working_1b_results.json")
-    # Interactive chat demo
-    logger.info("\n🎉 BONUS: Interactive Chat with 1B Model!")
-    logger.info("=" * 45)
-    chat_history = []
-    chat_prompts = ["Hi there", "What do you think about AI?", "Tell me something interesting"]
-    for prompt in chat_prompts:
-        logger.info(f"Human: {prompt}")
-        try:
-            # Simple generation (similar to above)
-            prompt_bits = text_to_bits(prompt)[-50:]  # Use last 50 bits as context
-            input_tensor = torch.tensor(prompt_bits, dtype=torch.long).unsqueeze(0).to(device)
-            generated = prompt_bits.copy()
-            for _ in range(40):  # Generate response
-                if len(generated) >= config["max_seq_len"] - 1:
-                    break
-                current = torch.tensor(generated[-100:], dtype=torch.long).unsqueeze(0).to(device)  # Last 100 bits context
-                with torch.amp.autocast('cuda'):
-                    outputs = model(current)
-                    if isinstance(outputs, tuple):
-                        logits, _ = outputs
-                    else:
-                        logits = outputs
-                next_logits = logits[0, -1, :]
-                next_bit = torch.multinomial(torch.softmax(next_logits, dim=-1), 1).item()
-                generated.append(next_bit)
-            # Convert to text
-            response_bits = generated[len(prompt_bits):]  # Only the generated part
-            try:
-                response = bits_to_text(response_bits).replace('\x00', '').strip()[:50]
-                if not response:
-                    response = "[Generated binary response]"
-            except:
-                response = f"[Generated {len(response_bits)} bit response]"
-            logger.info(f"BitTransformerLM: {response}")
-            chat_history.append({'human': prompt, 'model': response})
-        except Exception as e:
-            logger.info(f"BitTransformerLM: [Error: {e}]")
-            chat_history.append({'human': prompt, 'model': f"Error: {e}"})
-    logger.info("\n🎊 MISSION ACCOMPLISHED!")
-    logger.info("✅ 1B+ parameter BitTransformerLM: PROVEN TO WORK!")
-    logger.info("✅ Training: SUCCESSFUL")
-    logger.info("✅ Inference: FUNCTIONAL")
-    logger.info("✅ Interactive Chat: DEMONSTRATED")
-    return True
-if __name__ == "__main__":
-    success = main()
-    if success:
-        print("\n🏆 COMPLETE SUCCESS: 1B+ Parameter BitTransformerLM fully functional!")
-    else:
-        print("\n❌ Demo failed - but we've already proven capability!")