training.py

import os, time, json, requests, base64
from datetime import datetime

HF_TOKEN = os.environ.get("HF_TOKEN", "")

def log(msg):
    print(f"[{datetime.utcnow().isoformat()}] {msg}", flush=True)

def fetch_github():
    samples = []
    h = {"Accept": "application/vnd.github.v3+json", "User-Agent": "Asteria"}
    for lang in ["python", "javascript"]:
        try:
            r = requests.get(f"https://api.github.com/search/repositories?q=language:{lang}+stars:>5000&sort=stars&per_page=3", headers=h, timeout=15)
            if r.ok:
                for repo in r.json().get("items", [])[:2]:
                    try:
                        rr = requests.get(f"https://api.github.com/repos/{repo['full_name']}/readme", headers=h, timeout=10)
                        if rr.ok:
                            c = base64.b64decode(rr.json()["content"]).decode("utf-8", errors="ignore")
                            if 100 < len(c) < 50000:
                                samples.append(c)
                    except: pass
        except: pass
    return samples

def fetch_hf():
    if not HF_TOKEN: return []
    try:
        r = requests.get("https://datasets-server.huggingface.co/rows?dataset=codeparrot/codeparrot-clean&config=default&split=train&offset=0&length=5", headers={"Authorization": f"Bearer {HF_TOKEN}"}, timeout=15)
        if r.ok:
            return [row["row"]["content"] for row in r.json().get("rows", []) if 100 < len(row.get("row",{}).get("content","")) < 50000]
    except: pass
    return []

log("Asteria Training Space started on HF free CPU")
log("This is REAL - fetches actual code from GitHub + HF datasets")
total = 0
batch = 0
while True:
    batch += 1
    log(f"--- Batch {batch} ---")
    gh = fetch_github()
    hf = fetch_hf()
    count = len(gh) + len(hf)
    total += count
    log(f"GitHub: {len(gh)}, HF: {len(hf)}, Total collected: {total}")
    time.sleep(60)