Upload train.py

train.py

@@ -0,0 +1,525 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+train.py
+Продвинутая система обучения моделей с итеративным улучшением.
+Обучается до достижения минимальной точности (по умолчанию 0.80).
+Сохраняет лучшие модели и метаданные в папку models/
+"""
+import pandas as pd
+import numpy as np
+import requests
+import joblib
+import os
+import time
+import logging
+import threading
+from datetime import datetime
+from sklearn.model_selection import train_test_split, cross_val_score, StratifiedKFold
+from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, ExtraTreesClassifier
+from sklearn.linear_model import LogisticRegression
+from sklearn.svm import SVC
+from sklearn.preprocessing import StandardScaler, RobustScaler, MinMaxScaler
+from sklearn.metrics import accuracy_score
+import warnings
+warnings.filterwarnings('ignore')
+
+# TA-Lib импортируем здесь; если не установлен — бросим понятную ошибку
+try:
+    import talib
+except Exception as e:
+    raise ImportError("TA-Lib не найден. Установите TA-Lib (системная библиотека + pip install TA-Lib).") from e
+
+# Логирование
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.FileHandler('training_log.txt', encoding='utf-8'),
+        logging.StreamHandler()
+    ]
+)
+logger = logging.getLogger(__name__)
+
+class AdvancedCryptoModelTrainer:
+    def __init__(self, symbol='BTCUSDT', interval='1h', target_accuracy=0.80, max_iterations=50):
+        self.symbol = symbol
+        self.interval = interval
+        self.target_accuracy = target_accuracy
+        self.models = {}
+        self.best_models = {}
+        self.feature_names = []
+        self.training_history = []
+        self.current_iteration = 0
+        self.max_iterations = max_iterations
+
+        # Прогрессивные параметры
+        self.data_limits = [1000, 2000, 3000, 5000]
+        self.feature_complexity_levels = [1, 2, 3, 4, 5]
+        self.scaler_types = ['standard', 'robust', 'minmax']
+
+        logger.info(f"Инициализация тренера для {symbol}, целевая точность: {target_accuracy*100:.2f}%")
+
+    def fetch_binance_data(self, limit=2000):
+        """Получение данных с Binance API, возможно в чанках (max 1000 за запрос)."""
+        url = "https://api.binance.com/api/v3/klines"
+        params = {
+            'symbol': self.symbol,
+            'interval': self.interval,
+            'limit': min(limit, 1000)
+        }
+
+        all_data = []
+        end_time = None
+
+        while len(all_data) < limit:
+            if end_time:
+                params['endTime'] = end_time
+
+            try:
+                response = requests.get(url, params=params, timeout=10)
+                response.raise_for_status()
+                data = response.json()
+                if not data:
+                    break
+
+                all_data.extend(data)
+                # Берём первую свечу в ответе (самая ранняя в странице) и указываем endTime на 1мс меньше,
+                # чтобы загрузить более ранние свечи в следующем запросе
+                end_time = data[0][0] - 1
+
+                if len(data) < 1000:
+                    break
+
+                time.sleep(0.2)
+            except Exception as e:
+                logger.error(f"Ошибка при получении данных: {e}")
+                break
+
+        all_data = all_data[:limit]
+        if not all_data:
+            logger.error("Не удалось получить данные с Binance.")
+            return None
+
+        df = pd.DataFrame(all_data, columns=[
+            'timestamp', 'open', 'high', 'low', 'close', 'volume',
+            'close_time', 'quote_asset_volume', 'number_of_trades',
+            'taker_buy_base_asset_volume', 'taker_buy_quote_asset_volume', 'ignore'
+        ])
+        numeric_columns = ['open', 'high', 'low', 'close', 'volume']
+        for col in numeric_columns:
+            df[col] = pd.to_numeric(df[col], errors='coerce')
+        df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms')
+        df = df.sort_values('timestamp').reset_index(drop=True)
+        logger.info(f"Получено {len(df)} записей для {self.symbol}")
+        return df
+
+    def calculate_advanced_technical_indicators(self, df, complexity_level=1):
+        """Расчет индикаторов (TA-Lib)."""
+        df = df.copy()
+        # Базовые
+        df['sma_5'] = talib.SMA(df['close'], timeperiod=5)
+        df['sma_10'] = talib.SMA(df['close'], timeperiod=10)
+        df['sma_20'] = talib.SMA(df['close'], timeperiod=20)
+        df['sma_50'] = talib.SMA(df['close'], timeperiod=50)
+        df['ema_12'] = talib.EMA(df['close'], timeperiod=12)
+        df['ema_26'] = talib.EMA(df['close'], timeperiod=26)
+        df['rsi'] = talib.RSI(df['close'], timeperiod=14)
+
+        macd, macd_signal, macd_hist = talib.MACD(df['close'])
+        df['macd'] = macd
+        df['macd_signal'] = macd_signal
+        df['macd_hist'] = macd_hist
+
+        bb_upper, bb_middle, bb_lower = talib.BBANDS(df['close'])
+        df['bb_upper'] = bb_upper
+        df['bb_middle'] = bb_middle
+        df['bb_lower'] = bb_lower
+        # Предохраняемся от деления на ноль
+        df['bb_width'] = (bb_upper - bb_lower) / (bb_middle.replace(0, np.nan))
+        df['bb_position'] = (df['close'] - bb_lower) / ((bb_upper - bb_lower).replace(0, np.nan))
+
+        if complexity_level >= 2:
+            df['stoch_k'], df['stoch_d'] = talib.STOCH(df['high'], df['low'], df['close'])
+            df['williams_r'] = talib.WILLR(df['high'], df['low'], df['close'])
+            df['cci'] = talib.CCI(df['high'], df['low'], df['close'])
+            df['atr'] = talib.ATR(df['high'], df['low'], df['close'])
+            df['adx'] = talib.ADX(df['high'], df['low'], df['close'])
+            df['ad'] = talib.AD(df['high'], df['low'], df['close'], df['volume'])
+            df['obv'] = talib.OBV(df['close'], df['volume'])
+
+        if complexity_level >= 3:
+            df['mfi'] = talib.MFI(df['high'], df['low'], df['close'], df['volume'])
+            df['roc'] = talib.ROC(df['close'])
+            df['tema'] = talib.TEMA(df['close'])
+            df['dema'] = talib.DEMA(df['close'])
+            # Набор паттернов (чуть-чуть)
+            df['doji'] = talib.CDLDOJI(df['open'], df['high'], df['low'], df['close'])
+            df['engulfing'] = talib.CDLENGULFING(df['open'], df['high'], df['low'], df['close'])
+
+        if complexity_level >= 4:
+            df['ht_trendline'] = talib.HT_TRENDLINE(df['close'])
+            df['cmf'] = talib.ADOSC(df['high'], df['low'], df['close'], df['volume'])
+
+        if complexity_level >= 5:
+            for period in [7, 14, 21, 30]:
+                df[f'sma_{period}'] = talib.SMA(df['close'], timeperiod=period)
+                df[f'ema_{period}'] = talib.EMA(df['close'], timeperiod=period)
+                df[f'rsi_{period}'] = talib.RSI(df['close'], timeperiod=period)
+
+        return df
+
+    def create_progressive_features(self, df, complexity_level=1):
+        df = df.copy()
+        basic_lags = [1, 2, 3, 5]
+        if complexity_level >= 2:
+            basic_lags += [10, 20]
+        if complexity_level >= 3:
+            basic_lags += [30, 50]
+        for lag in basic_lags:
+            df[f'close_lag_{lag}'] = df['close'].shift(lag)
+            df[f'volume_lag_{lag}'] = df['volume'].shift(lag)
+            if 'rsi' in df.columns:
+                df[f'rsi_lag_{lag}'] = df['rsi'].shift(lag)
+            if 'macd' in df.columns:
+                df[f'macd_lag_{lag}'] = df['macd'].shift(lag)
+
+        windows = [5, 10]
+        if complexity_level >= 2:
+            windows += [20, 30]
+        if complexity_level >= 3:
+            windows += [50, 100]
+        for window in windows:
+            if 'rsi' in df.columns:
+                df[f'rsi_sma_{window}'] = df['rsi'].rolling(window).mean()
+                df[f'rsi_std_{window}'] = df['rsi'].rolling(window).std()
+            if 'macd' in df.columns:
+                df[f'macd_sma_{window}'] = df['macd'].rolling(window).mean()
+                df[f'macd_std_{window}'] = df['macd'].rolling(window).std()
+            df[f'volume_ema_{window}'] = df['volume'].ewm(span=window).mean()
+            df[f'price_std_{window}'] = df['close'].rolling(window).std()
+
+        momentum_periods = [5, 10]
+        if complexity_level >= 2:
+            momentum_periods += [20, 30]
+        if complexity_level >= 3:
+            momentum_periods += [50, 100]
+        for period in momentum_periods:
+            df[f'momentum_{period}'] = df['close'] / df['close'].shift(period) - 1
+            try:
+                df[f'roc_{period}'] = talib.ROC(df['close'], timeperiod=period)
+            except:
+                df[f'roc_{period}'] = np.nan
+            df[f'volatility_{period}'] = df['close'].pct_change().rolling(period).std()
+
+        if complexity_level >= 3:
+            if 'rsi' in df.columns and 'macd' in df.columns:
+                df['rsi_macd_corr'] = df['rsi'].rolling(20).corr(df['macd'])
+            if 'sma_20' in df.columns and 'sma_50' in df.columns:
+                df['sma_ratio_20_50'] = df['sma_20'] / df['sma_50'].replace(0, np.nan)
+            for col in ['close', 'volume', 'rsi']:
+                if col in df.columns:
+                    mean = df[col].rolling(50).mean()
+                    std = df[col].rolling(50).std()
+                    df[f'{col}_zscore'] = (df[col] - mean) / (std.replace(0, np.nan))
+
+        if complexity_level >= 4:
+            df['fractal_high'] = ((df['high'] > df['high'].shift(1)) & (df['high'] > df['high'].shift(-1))).astype(int)
+            df['fractal_low'] = ((df['low'] < df['low'].shift(1)) & (df['low'] < df['low'].shift(-1))).astype(int)
+            df['support'] = df['low'].rolling(20).min()
+            df['resistance'] = df['high'].rolling(20).max()
+            df['support_distance'] = (df['close'] - df['support']) / df['close']
+            df['resistance_distance'] = (df['resistance'] - df['close']) / df['close']
+
+        if complexity_level >= 5:
+            df['wave_trend'] = df['close'].rolling(50).apply(lambda x: 1 if x.iloc[-1] > x.iloc[0] else 0, raw=False)
+            if 'rsi' in df.columns:
+                price_trend = df['close'].rolling(10).apply(lambda x: x.iloc[-1] - x.iloc[0], raw=False)
+                rsi_trend = df['rsi'].rolling(10).apply(lambda x: x.iloc[-1] - x.iloc[0], raw=False)
+                df['price_rsi_divergence'] = ((price_trend > 0) & (rsi_trend < 0)) | ((price_trend < 0) & (rsi_trend > 0))
+
+        return df
+
+    def create_target_variable(self, df, prediction_horizon=1):
+        df = df.copy()
+        df['future_price'] = df['close'].shift(-prediction_horizon)
+        df['target'] = (df['future_price'] > df['close']).astype(int)
+        return df
+
+    def prepare_features(self, df):
+        exclude_columns = [
+            'timestamp', 'open', 'high', 'low', 'close', 'volume',
+            'close_time', 'quote_asset_volume', 'number_of_trades',
+            'taker_buy_base_asset_volume', 'taker_buy_quote_asset_volume',
+            'ignore', 'future_price', 'target'
+        ]
+        feature_columns = [col for col in df.columns if col not in exclude_columns]
+        df_clean = df.dropna()
+        if len(df_clean) == 0:
+            logger.error("Все строки содержат NaN после очистки!")
+            return None, None
+        X = df_clean[feature_columns]
+        y = df_clean['target']
+        self.feature_names = feature_columns
+        logger.info(f"Подготовлено {len(X)} образцов с {len(feature_columns)} признаками")
+        return X, y
+
+    def get_progressive_model_params(self, model_name, iteration):
+        base_params = {
+            'Random Forest': {
+                'n_estimators': min(100 + iteration * 100, 1000),
+                'max_depth': min(10 + iteration * 2, 25),
+                'min_samples_split': max(5 - iteration, 2),
+                'min_samples_leaf': max(2 - iteration // 2, 1),
+                'max_features': 'sqrt',
+                'bootstrap': True,
+                'random_state': 42,
+                'n_jobs': -1,
+                'class_weight': 'balanced'
+            },
+            'Gradient Boosting': {
+                'n_estimators': min(100 + iteration * 50, 500),
+                'learning_rate': max(0.1 - iteration * 0.01, 0.01),
+                'max_depth': min(6 + iteration, 12),
+                'min_samples_split': max(5 - iteration, 2),
+                'min_samples_leaf': max(2 - iteration // 2, 1),
+                'subsample': 0.8,
+                'max_features': 'sqrt',
+                'random_state': 42
+            },
+            'Extra Trees': {
+                'n_estimators': min(100 + iteration * 100, 1000),
+                'max_depth': min(10 + iteration * 2, 25),
+                'min_samples_split': max(5 - iteration, 2),
+                'min_samples_leaf': max(2 - iteration // 2, 1),
+                'max_features': 'sqrt',
+                'bootstrap': False,
+                'random_state': 42,
+                'n_jobs': -1,
+                'class_weight': 'balanced'
+            },
+            'Logistic Regression': {
+                'random_state': 42,
+                'max_iter': min(1000 + iteration * 500, 5000),
+                'C': 10 ** (-2 + iteration * 0.5),
+                'penalty': 'l2',
+                'solver': 'liblinear',
+                'class_weight': 'balanced'
+            },
+            'SVM': {
+                'kernel': 'rbf',
+                'C': 10 ** (max(0, iteration * 0.5)),
+                'gamma': 'scale',
+                'probability': True,
+                'class_weight': 'balanced'
+            }
+        }
+        return base_params.get(model_name, {})
+
+    def train_iteration(self, data_limit, complexity_level, scaler_type='standard'):
+        logger.info(f"Итерация {self.current_iteration + 1}: данных={data_limit}, сложность={complexity_level}, скейлер={scaler_type}")
+
+        df = self.fetch_binance_data(limit=data_limit)
+        if df is None or len(df) < 100:
+            logger.error("Недостаточно данных для обучения")
+            return False
+
+        df = self.calculate_advanced_technical_indicators(df, complexity_level)
+        df = self.create_progressive_features(df, complexity_level)
+        df = self.create_target_variable(df)
+
+        X, y = self.prepare_features(df)
+        if X is None:
+            return False
+
+        # Проверка на наличие хотя бы двух классов
+        if y.nunique() < 2:
+            logger.error("Целевая переменная содержит только один класс. Нельзя обучить модель.")
+            return False
+
+        try:
+            X_train, X_test, y_train, y_test = train_test_split(
+                X, y, test_size=0.2, random_state=42, stratify=y
+            )
+        except Exception as e:
+            logger.warning(f"Ошибка stratify split: {e}. Попробуем без stratify.")
+            X_train, X_test, y_train, y_test = train_test_split(
+                X, y, test_size=0.2, random_state=42
+            )
+
+        if scaler_type == 'standard':
+            scaler = StandardScaler()
+        elif scaler_type == 'robust':
+            scaler = RobustScaler()
+        else:
+            scaler = MinMaxScaler()
+
+        X_train_scaled = scaler.fit_transform(X_train)
+        X_test_scaled = scaler.transform(X_test)
+
+        models = {
+            'Random Forest': RandomForestClassifier(**self.get_progressive_model_params('Random Forest', self.current_iteration)),
+            'Gradient Boosting': GradientBoostingClassifier(**self.get_progressive_model_params('Gradient Boosting', self.current_iteration)),
+            'Extra Trees': ExtraTreesClassifier(**self.get_progressive_model_params('Extra Trees', self.current_iteration)),
+            'Logistic Regression': LogisticRegression(**self.get_progressive_model_params('Logistic Regression', self.current_iteration)),
+            'SVM': SVC(**self.get_progressive_model_params('SVM', self.current_iteration))
+        }
+
+        iteration_results = {}
+        cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
+
+        for name, model in models.items():
+            logger.info(f"Обучение {name}...")
+            try:
+                # Для линейных и SVM используем масштабированные признаки
+                if name in ['Logistic Regression', 'SVM']:
+                    cv_scores = cross_val_score(model, X_train_scaled, y_train, cv=cv, scoring='accuracy', n_jobs=-1)
+                    model.fit(X_train_scaled, y_train)
+                    y_pred = model.predict(X_test_scaled)
+                else:
+                    cv_scores = cross_val_score(model, X_train, y_train, cv=cv, scoring='accuracy', n_jobs=-1)
+                    model.fit(X_train, y_train)
+                    y_pred = model.predict(X_test)
+
+                accuracy = accuracy_score(y_test, y_pred)
+                cv_mean = float(np.mean(cv_scores))
+                cv_std = float(np.std(cv_scores))
+
+                iteration_results[name] = {
+                    'model': model,
+                    'scaler': scaler if name in ['Logistic Regression', 'SVM'] else None,
+                    'accuracy': accuracy,
+                    'cv_mean': cv_mean,
+                    'cv_std': cv_std
+                }
+
+                logger.info(f"{name}: Точность={accuracy:.4f}, CV={cv_mean:.4f}±{cv_std:.4f}")
+
+                if name not in self.best_models or accuracy > self.best_models[name]['accuracy']:
+                    self.best_models[name] = iteration_results[name].copy()
+                    logger.info(f"Новая лучшая модель {name}: {accuracy:.4f}")
+
+            except Exception as e:
+                logger.error(f"Ошибка при обучении {name}: {e}")
+
+        self.training_history.append({
+            'iteration': self.current_iteration + 1,
+            'data_limit': data_limit,
+            'complexity_level': complexity_level,
+            'scaler_type': scaler_type,
+            'results': {k: {'accuracy': v['accuracy'], 'cv_mean': v['cv_mean'], 'cv_std': v['cv_std']} for k, v in iteration_results.items()},
+            'best_accuracy': max([r['accuracy'] for r in iteration_results.values()]) if iteration_results else 0
+        })
+
+        best_accuracy = max([r['accuracy'] for r in iteration_results.values()]) if iteration_results else 0
+        logger.info(f"Лучшая точность на итерации: {best_accuracy:.4f}")
+        return best_accuracy >= self.target_accuracy
+
+    def train_until_target_accuracy(self):
+        logger.info(f"Начинаем обучение до достижения {self.target_accuracy*100:.2f}% (макс итераций {self.max_iterations})")
+        target_reached = False
+        iteration = 0
+
+        while not target_reached and iteration < self.max_iterations:
+            self.current_iteration = iteration
+            data_limit = self.data_limits[min(iteration // 2, len(self.data_limits) - 1)]
+            complexity_level = self.feature_complexity_levels[min(iteration // 2, len(self.feature_complexity_levels) - 1)]
+            scaler_type = self.scaler_types[iteration % len(self.scaler_types)]
+
+            logger.info("\n" + "=" * 60)
+            logger.info(f"ИТЕРАЦИЯ {iteration + 1}")
+            logger.info("=" * 60)
+
+            try:
+                target_reached = self.train_iteration(data_limit, complexity_level, scaler_type)
+            except Exception as e:
+                logger.error(f"Критическая ошибка на итерации {iteration+1}: {e}")
+                target_reached = False
+
+            if target_reached:
+                logger.info(f"🎉 ЦЕЛЕВАЯ ТОЧНОСТЬ ДОСТИГНУТА НА ИТЕРАЦИИ {iteration + 1}!")
+                break
+
+            iteration += 1
+            time.sleep(1)
+
+        if not target_reached:
+            logger.warning("Не удалось достичь целевой точности в отведённом числе итераций.")
+        return target_reached
+
+    def save_best_models(self):
+        if not self.best_models:
+            logger.error("Нет моделей для сохранения!")
+            return False
+
+        models_dir = 'models'
+        os.makedirs(models_dir, exist_ok=True)
+
+        for name, model_data in self.best_models.items():
+            model_filename = f"{name.lower().replace(' ', '_')}_model.joblib"
+            model_path = os.path.join(models_dir, model_filename)
+            joblib.dump(model_data['model'], model_path)
+            if model_data['scaler'] is not None:
+                scaler_filename = f"{name.lower().replace(' ', '_')}_scaler.joblib"
+                scaler_path = os.path.join(models_dir, scaler_filename)
+                joblib.dump(model_data['scaler'], scaler_path)
+            logger.info(f"Сохранена модель {name} с точностью {model_data['accuracy']:.4f}")
+
+        # Сохраняем feature names и метаданные
+        features_path = os.path.join(models_dir, 'feature_names.joblib')
+        joblib.dump(self.feature_names, features_path)
+
+        metadata = {
+            'symbol': self.symbol,
+            'interval': self.interval,
+            'target_accuracy': self.target_accuracy,
+            'training_date': datetime.now().isoformat(),
+            'total_iterations': self.current_iteration + 1,
+            'best_accuracies': {name: data['accuracy'] for name, data in self.best_models.items()},
+            'feature_count': len(self.feature_names),
+            'training_history': self.training_history
+        }
+        metadata_path = os.path.join(models_dir, 'metadata.joblib')
+        joblib.dump(metadata, metadata_path)
+
+        logger.info("Модели и метаданные успешно сохранены.")
+        return True
+
+def main():
+    print("Продвинутая система обучения моделей — train.py")
+    symbol = input("Введите торговую пару (по умолчанию BTCUSDT): ").strip().upper() or 'BTCUSDT'
+    interval = input("Интервал (1m,5m,1h,4h,1d), по умолчанию 1h: ").strip() or '1h'
+    target_accuracy_str = input("Целевая точность (по умолчанию 0.80 или 80%): ").strip() or '0.80'
+    try:
+        target_accuracy = float(target_accuracy_str)
+        if target_accuracy > 1:
+            target_accuracy = target_accuracy / 100.0
+    except:
+        target_accuracy = 0.80
+    max_iter_str = input("Максимум итераций (по умолчанию 50): ").strip() or '50'
+    try:
+        max_iters = int(max_iter_str)
+    except:
+        max_iters = 50
+
+    trainer = AdvancedCryptoModelTrainer(symbol=symbol, interval=interval, target_accuracy=target_accuracy, max_iterations=max_iters)
+    start_time = time.time()
+    try:
+        success = trainer.train_until_target_accuracy()
+        trainer.save_best_models()
+        end_time = time.time()
+        mins = (end_time - start_time) / 60.0
+        print(f"\nОбучение завершено за {mins:.1f} минут")
+        if success:
+            print("🎉 Целевая точность достигнута!")
+        else:
+            print("⚠️ Цель не достигнута — сохранены лучшие модели.")
+    except KeyboardInterrupt:
+        print("\nПрерывание пользователем. Сохранение лучших моделей (если есть)...")
+        trainer.save_best_models()
+    except Exception as e:
+        logger.error(f"Критическая ошибка: {e}")
+        trainer.save_best_models()
+
+if __name__ == "__main__":
+    main()