Upload gen_data_1b.py with huggingface_hub

gen_data_1b.py

@@ -0,0 +1,270 @@
+#!/usr/bin/env python3
+"""Generate training data for CogNet-1B on the server."""
+
+import os
+import json
+import torch
+import random
+import string
+
+class CharTokenizer:
+    def __init__(self, vocab_size=136):
+        self.vocab_size = vocab_size
+        self.pad_token_id = 0
+        self.unk_token_id = 1
+        self.bos_token_id = 2
+        self.eos_token_id = 3
+
+        chars = list(range(32, 127))
+        french = [192,193,194,195,196,197,199,200,201,202,203,204,205,206,207,
+                  210,211,212,213,214,217,218,219,220,224,225,226,227,228,229,
+                  231,232,233,234,235,236,237,238,239,242,243,244,245,246,249,
+                  250,251,252,253,255]
+        chars.extend(french)
+
+        self.char_to_id = {self.pad_token_id: 0, self.unk_token_id: 1,
+                           self.bos_token_id: 2, self.eos_token_id: 3}
+        for i, c in enumerate(chars[:vocab_size - 4]):
+            self.char_to_id[c] = i + 4
+        self.id_to_char = {v: k for k, v in self.char_to_id.items()}
+
+    def encode(self, text):
+        ids = [self.bos_token_id]
+        for ch in text:
+            code = ord(ch)
+            ids.append(self.char_to_id.get(code, self.unk_token_id))
+        ids.append(self.eos_token_id)
+        return ids
+
+    def save(self, path):
+        with open(path, 'w', encoding='utf-8') as f:
+            json.dump({
+                'vocab_size': self.vocab_size,
+                'char_to_id': {str(k): v for k, v in self.char_to_id.items()},
+            }, f, ensure_ascii=False, indent=2)
+
+
+def generate_code_data(n_chars=10_000_000):
+    """Generate diverse programming code data."""
+    data = []
+    
+    # Python snippets
+    py_templates = [
+        "def {func}({params}):\n    \"\"\"{docstring}\"\"\"\n    {body}\n    return {ret}\n",
+        "class {cls}:\n    def __init__(self{params}):\n        self.{attr} = {val}\n\n    def {method}(self{params2}):\n        {body2}\n        return {ret2}\n",
+        "import {mod}\nfrom {mod2} import {imp}\n\n{var} = {expr}\nif {cond}:\n    {stmt}\nelse:\n    {stmt2}\n",
+        "for {var} in range({n}):\n    {stmt}\n    if {cond}:\n        continue\n    {stmt2}\n",
+        "try:\n    {stmt}\nexcept {exc} as e:\n    print(f'Error: {{e}}')\nfinally:\n    {stmt2}\n",
+        "with open('{file}', 'r') as f:\n    data = f.read()\n    lines = data.split('\\n')\n    for line in lines:\n        process(line)\n",
+        "@dataclass\nclass {cls}:\n    {attr}: {type}\n    {attr2}: {type2}\n\n    def {method}(self) -> {rtype}:\n        return {expr}\n",
+        "async def {func}({params}):\n    async with {ctx} as {var}:\n        result = await {expr}\n        return result\n",
+        "lambda x: x * {n} + {n2}\n",
+        "{var} = [{expr} for {var2} in {iter} if {cond}]\n",
+    ]
+    
+    func_names = ['process', 'compute', 'transform', 'validate', 'parse', 'encode', 'decode', 
+                  'train', 'predict', 'analyze', 'optimize', 'initialize', 'configure', 'execute',
+                  'render', 'compile', 'evaluate', 'generate', 'filter', 'sort', 'merge', 'split']
+    cls_names = ['Model', 'Processor', 'Handler', 'Manager', 'Engine', 'Pipeline', 'Service',
+                 'Client', 'Server', 'Agent', 'Router', 'Builder', 'Factory', 'Observer']
+    params = ['x', 'y', 'data', 'input', 'value', 'config', 'params', 'options', 'state', 'context']
+    attrs = ['name', 'value', 'state', 'config', 'data', 'status', 'type', 'mode', 'size', 'count']
+    modules = ['torch', 'numpy', 'json', 'os', 'sys', 're', 'typing', 'dataclasses', 'asyncio',
+               'collections', 'itertools', 'functools', 'pathlib', 'logging', 'abc']
+    
+    while len(data) < n_chars:
+        template = random.choice(py_templates)
+        # Build format dict with ALL possible keys
+        fmt = {
+            'func': random.choice(func_names),
+            'cls': random.choice(cls_names),
+            'params': random.choice(params),
+            'params2': ', ' + random.choice(params),
+            'docstring': f"Process the given {random.choice(params)}.",
+            'body': f"{random.choice(params)} = {random.choice(params)} + 1",
+            'body2': f"result = self.{random.choice(attrs)} + {random.choice(params)}",
+            'ret': random.choice(params),
+            'ret2': 'result',
+            'method': random.choice(func_names),
+            'attr': random.choice(attrs),
+            'attr2': random.choice(attrs),
+            'val': random.choice(['0', '1', 'None', '""', '[]', '{}']),
+            'mod': random.choice(modules),
+            'mod2': random.choice(modules),
+            'imp': random.choice(func_names).capitalize(),
+            'var': random.choice(params),
+            'var2': random.choice(params),
+            'expr': f"{random.choice(params)} + {random.randint(1,100)}",
+            'cond': f"{random.choice(params)} > {random.randint(0,10)}",
+            'stmt': f"{random.choice(params)} = {random.randint(0,100)}",
+            'stmt2': f"{random.choice(params)} *= {random.randint(1,5)}",
+            'n': str(random.randint(1,100)),
+            'n2': str(random.randint(1,50)),
+            'exc': random.choice(['ValueError', 'TypeError', 'RuntimeError', 'KeyError']),
+            'file': random.choice(['data.txt', 'config.json', 'model.pt', 'output.csv']),
+            'ctx': random.choice(['aiohttp.ClientSession()', 'db.connection()', 'lock']),
+            'type': random.choice(['int', 'str', 'float', 'bool', 'List[str]']),
+            'type2': random.choice(['int', 'str', 'float', 'Dict[str, Any]', 'Optional[str]']),
+            'rtype': random.choice(['int', 'str', 'bool', 'float', 'None']),
+            'iter': random.choice(['range(100)', 'items', 'data', 'values', 'records']),
+        }
+        text = template.format(**fmt)
+        data.append(text)
+    
+    return ''.join(data)[:n_chars]
+
+
+def generate_nlp_data(n_chars=10_000_000):
+    """Generate NLP/ML text data."""
+    sentences = [
+        "The model achieves state-of-the-art performance on the benchmark dataset.",
+        "Training was conducted for 100 epochs with early stopping based on validation loss.",
+        "The attention mechanism allows the model to focus on relevant parts of the input.",
+        "We use a learning rate of 3e-4 with cosine annealing schedule.",
+        "The transformer architecture has revolutionized natural language processing.",
+        "Gradient descent optimization minimizes the loss function iteratively.",
+        "Batch normalization helps stabilize training by normalizing layer inputs.",
+        "The model architecture consists of multiple layers with residual connections.",
+        "Data augmentation techniques improve generalization by creating varied training examples.",
+        "Regularization methods like dropout prevent overfitting in deep neural networks.",
+        "The encoder processes the input sequence and generates hidden representations.",
+        "Cross-attention enables the decoder to attend to encoder outputs during generation.",
+        "Positional encoding provides sequence order information to the transformer model.",
+        "The loss function measures the discrepancy between predictions and ground truth.",
+        "Backpropagation computes gradients of the loss with respect to model parameters.",
+        "Learning rate scheduling adjusts the step size during optimization.",
+        "Weight initialization affects convergence speed and final model quality.",
+        "Hyperparameter tuning optimizes model configuration for best performance.",
+        "Transfer learning leverages pre-trained models for downstream tasks.",
+        "Fine-tuning adapts a pre-trained model to a specific domain or task.",
+        "The vocabulary size determines the granularity of text tokenization.",
+        "Tokenization converts raw text into discrete tokens for model processing.",
+        "Embedding layers map tokens to dense vector representations.",
+        "Feed-forward networks apply non-linear transformations at each layer.",
+        "Layer normalization stabilizes training by normalizing activations.",
+        "The softmax function converts logits to probability distributions.",
+        "Beam search generates higher quality outputs than greedy decoding.",
+        "Temperature sampling controls the diversity of generated text.",
+        "Top-k sampling limits the candidate tokens during generation.",
+        "Nucleus sampling selects from the smallest set of tokens exceeding a probability threshold.",
+    ]
+    
+    paragraphs = []
+    while sum(len(p) for p in paragraphs) < n_chars:
+        n_sent = random.randint(3, 8)
+        para = ' '.join(random.choice(sentences) for _ in range(n_sent))
+        paragraphs.append(para + '\n\n')
+    
+    return ''.join(paragraphs)[:n_chars]
+
+
+def generate_french_data(n_chars=5_000_000):
+    """Generate French text data for multilingual capability."""
+    phrases = [
+        "Le modèle atteint des performances de pointe sur l'ensemble de données de référence.",
+        "L'entraînement a été effectué pendant 100 époques avec arrêt anticipé basé sur la perte de validation.",
+        "Le mécanisme d'attention permet au modèle de se concentrer sur les parties pertinentes de l'entrée.",
+        "Nous utilisons un taux d'apprentissage de 3e-4 avec un programme de recuit cosinusoïdal.",
+        "L'architecture du transformateur a révolutionné le traitement du langage naturel.",
+        "La descente de gradient minimise itérativement la fonction de perte.",
+        "La normalisation par lots aide à stabiliser l'entraînement en normalisant les entrées de couche.",
+        "La fonction de perte mesure l'écart entre les prédictions et la vérité terrain.",
+        "La rétropropagation calcule les gradients de la perte par rapport aux paramètres du modèle.",
+        "L'apprentissage par transfert exploite les modèles pré-entraînés pour les tâches en aval.",
+        "L'ajustement fin adapte un modèle pré-entraîné à un domaine ou une tâche spécifique.",
+        "La tokenisation convertit le texte brut en tokens discrets pour le traitement du modèle.",
+        "Les couches d'intégration mappent les tokens à des représentations vectorielles denses.",
+        "La fonction softmax convertit les logits en distributions de probabilité.",
+        "L'échantillonnage de température contrôle la diversité du texte généré.",
+    ]
+    
+    result = []
+    while sum(len(p) for p in result) < n_chars:
+        n_phrases = random.randint(3, 6)
+        para = ' '.join(random.choice(phrases) for _ in range(n_phrases))
+        result.append(para + '\n\n')
+    
+    return ''.join(result)[:n_chars]
+
+
+def generate_math_data(n_chars=5_000_000):
+    """Generate mathematical/structured data."""
+    templates = [
+        "f(x) = {a}x^2 + {b}x + {c}\n",
+        "integral from {a} to {b} of {c}x dx = {d}\n",
+        "P(A|B) = P(B|A) * P(A) / P(B)\n",
+        "d/dx [{a}x^{b}] = {c}x^{d}\n",
+        "sum from k=1 to n of k = n(n+1)/2\n",
+        "The derivative of f at x={a} is {b}.\n",
+        "The gradient vector is nabla f = ({a}, {b}, {c}).\n",
+        "The matrix A has eigenvalues {a} and {b}.\n",
+        "L = {a} * ||w||^2 + sum(alpha_i * (1 - y_i * (w*x_i + b)))\n",
+        "softmax(z_i) = exp(z_i) / sum(exp(z_j)) for j in range(n)\n",
+        "cross_entropy = -sum(y_i * log(p_i)) for i in range(n)\n",
+        "attention(Q, K, V) = softmax(QK^T / sqrt(d_k)) * V\n",
+        "layer_norm(x) = (x - mu) / sqrt(sigma^2 + eps) * gamma + beta\n",
+        "loss = {a} * loss_cls + {b} * loss_reg + {c} * loss_aux\n",
+        "gradient: dL/dw = (1/N) * sum(dL_i/dw) for i in range(N)\n",
+    ]
+    
+    result = []
+    while sum(len(p) for p in result) < n_chars:
+        template = random.choice(templates)
+        text = template.format(
+            a=random.randint(-10, 10),
+            b=random.randint(-10, 10),
+            c=random.randint(-10, 10),
+            d=random.randint(-100, 100),
+        )
+        result.append(text)
+    
+    return ''.join(result)[:n_chars]
+
+
+def main():
+    print("Generating CogNet-1B training data...")
+    
+    # Generate 80M tokens (~80M characters since char-level)
+    target_chars = 80_000_000
+    
+    print(f"Target: {target_chars:,} characters (~80M tokens)")
+    
+    # Mix of data types with weights
+    code_data = generate_code_data(30_000_000)
+    print(f"Code data: {len(code_data):,} chars")
+    
+    nlp_data = generate_nlp_data(25_000_000)
+    print(f"NLP data: {len(nlp_data):,} chars")
+    
+    french_data = generate_french_data(10_000_000)
+    print(f"French data: {len(french_data):,} chars")
+    
+    math_data = generate_math_data(15_000_000)
+    print(f"Math data: {len(math_data):,} chars")
+    
+    # Combine all data
+    all_text = code_data + nlp_data + french_data + math_data
+    print(f"Total text: {len(all_text):,} chars")
+    
+    # Tokenize
+    tokenizer = CharTokenizer()
+    print("Tokenizing...")
+    all_ids = tokenizer.encode(all_text)
+    print(f"Total tokens: {len(all_ids):,}")
+    
+    # Save tokenizer
+    tokenizer.save('/root/CogNet/tokenizer_v3.json')
+    print("Tokenizer saved")
+    
+    # Save as tensor
+    tokens = torch.tensor(all_ids, dtype=torch.long)
+    save_path = '/root/CogNet/data_1b/aicl_10x.pt'
+    torch.save(tokens, save_path)
+    print(f"Saved {len(tokens):,} tokens to {save_path}")
+    print(f"File size: {os.path.getsize(save_path) / 1e6:.1f} MB")
+    
+    print("Data generation complete!")
+
+
+if __name__ == '__main__':
+    main()