Spaces:
Sleeping
Sleeping
Delete utils
Browse files- utils/__init__.py +0 -0
- utils/config.py +0 -150
- utils/hf_io.py +0 -440
utils/__init__.py
DELETED
|
File without changes
|
utils/config.py
DELETED
|
@@ -1,150 +0,0 @@
|
|
| 1 |
-
"""
|
| 2 |
-
utils/config.py
|
| 3 |
-
All config is read from HuggingFace Spaces Secrets (environment variables).
|
| 4 |
-
Set these in: Space Settings → Variables and Secrets
|
| 5 |
-
No .env file needed — Spaces injects secrets automatically.
|
| 6 |
-
"""
|
| 7 |
-
|
| 8 |
-
import os
|
| 9 |
-
from pathlib import Path
|
| 10 |
-
|
| 11 |
-
# ── HF Spaces uses /tmp for ephemeral working storage ─
|
| 12 |
-
# All persistent data is written to HF Dataset via API.
|
| 13 |
-
TMP = Path("/tmp/quant")
|
| 14 |
-
TMP_PDFS = TMP / "pdfs"
|
| 15 |
-
TMP_CHUNKS = TMP / "chunks"
|
| 16 |
-
TMP_COMPILED = TMP / "compiled"
|
| 17 |
-
TMP_BACKTESTS = TMP / "backtests"
|
| 18 |
-
TMP_EXPORTS = TMP / "exports"
|
| 19 |
-
TMP_TICK_CACHE = TMP / "tick_cache"
|
| 20 |
-
TMP_KB = TMP / "knowledge_base.jsonl"
|
| 21 |
-
|
| 22 |
-
for d in [TMP_PDFS, TMP_CHUNKS, TMP_COMPILED,
|
| 23 |
-
TMP_BACKTESTS, TMP_EXPORTS, TMP_TICK_CACHE]:
|
| 24 |
-
d.mkdir(parents=True, exist_ok=True)
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
def get(key: str, default: str = "") -> str:
|
| 28 |
-
return os.environ.get(key, default)
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
# ── API Keys (set in HF Space Secrets) ────────────────
|
| 32 |
-
ANTHROPIC_API_KEY = get("ANTHROPIC_API_KEY")
|
| 33 |
-
HF_TOKEN = get("HF_TOKEN")
|
| 34 |
-
|
| 35 |
-
# ── HuggingFace Dataset repos ─────────────────────────
|
| 36 |
-
HF_DATASET_REPO = get("HF_DATASET_REPO") # knowledge base + results
|
| 37 |
-
HF_TICK_REPO = get("HF_TICK_REPO") # your tick data
|
| 38 |
-
|
| 39 |
-
# ── Extraction settings ───────────────────────────────
|
| 40 |
-
SIMILARITY_THRESHOLD = float(get("SIMILARITY_THRESHOLD", "0.85"))
|
| 41 |
-
MAX_TOKENS_PER_CHUNK = int(get("MAX_TOKENS_PER_CHUNK", "3000"))
|
| 42 |
-
OCR_DPI = int(get("OCR_DPI", "300"))
|
| 43 |
-
|
| 44 |
-
# ── Backtest settings ─────────────────────────────────
|
| 45 |
-
INITIAL_EQUITY = float(get("INITIAL_EQUITY", "10000"))
|
| 46 |
-
COMMISSION_PCT = float(get("COMMISSION_PCT", "0.0002"))
|
| 47 |
-
SLIPPAGE_PCT = float(get("SLIPPAGE_PCT", "0.0001"))
|
| 48 |
-
RISK_PER_TRADE = float(get("RISK_PER_TRADE", "0.01"))
|
| 49 |
-
WF_WINDOWS = int(get("WF_WINDOWS", "5"))
|
| 50 |
-
WF_IS_RATIO = float(get("WF_IS_RATIO", "0.70"))
|
| 51 |
-
MAX_PARAM_COMBOS = int(get("MAX_PARAM_COMBOS", "300"))
|
| 52 |
-
MIN_TRADES = int(get("MIN_TRADES", "30"))
|
| 53 |
-
MIN_SHARPE = float(get("MIN_SHARPE", "0.5"))
|
| 54 |
-
BACKTEST_TIMEFRAMES = get("BACKTEST_TIMEFRAMES", "1h,4h,1d").split(",")
|
| 55 |
-
|
| 56 |
-
# ── Strategy categories ───────────────────────────────
|
| 57 |
-
CATEGORIES = [
|
| 58 |
-
"Trend Following", "Mean Reversion", "Statistical Arbitrage",
|
| 59 |
-
"Momentum", "Breakout", "Volatility Trading", "Market Making",
|
| 60 |
-
"Pattern Recognition", "Machine Learning", "Options Strategy",
|
| 61 |
-
"High Frequency", "Pairs Trading", "Carry Trade",
|
| 62 |
-
"Seasonal / Calendar", "Risk Management", "Position Sizing",
|
| 63 |
-
"Portfolio Construction", "Market Microstructure", "Other",
|
| 64 |
-
]
|
| 65 |
-
|
| 66 |
-
# ── Extraction prompt ─────────────────────────────────
|
| 67 |
-
EXTRACTION_PROMPT = """
|
| 68 |
-
You are a quantitative finance knowledge extraction engine.
|
| 69 |
-
|
| 70 |
-
Extract ALL trading strategies, mathematical formulas, and complete trading systems
|
| 71 |
-
from the text chunk below, taken from an algorithmic trading book.
|
| 72 |
-
|
| 73 |
-
Output ONLY valid JSON — no markdown fences, no preamble:
|
| 74 |
-
{
|
| 75 |
-
"strategies": [{
|
| 76 |
-
"name": "string",
|
| 77 |
-
"category": "string",
|
| 78 |
-
"description": "string",
|
| 79 |
-
"entry_rules": ["string"],
|
| 80 |
-
"exit_rules": ["string"],
|
| 81 |
-
"filters": ["string"],
|
| 82 |
-
"timeframes": ["string"],
|
| 83 |
-
"instruments": ["string"],
|
| 84 |
-
"parameters": {"name": "description with typical value"},
|
| 85 |
-
"mathematical_basis": "string",
|
| 86 |
-
"source_context": "brief quote from text"
|
| 87 |
-
}],
|
| 88 |
-
"formulas": [{
|
| 89 |
-
"name": "string",
|
| 90 |
-
"category": "string",
|
| 91 |
-
"latex": "LaTeX string",
|
| 92 |
-
"plain_text": "string",
|
| 93 |
-
"variables": {"symbol": "description"},
|
| 94 |
-
"purpose": "string",
|
| 95 |
-
"usage_context": "string",
|
| 96 |
-
"source_context": "string"
|
| 97 |
-
}],
|
| 98 |
-
"systems": [{
|
| 99 |
-
"name": "string",
|
| 100 |
-
"components": ["string"],
|
| 101 |
-
"entry_system": "string",
|
| 102 |
-
"exit_system": "string",
|
| 103 |
-
"risk_management": "string",
|
| 104 |
-
"position_sizing": "string",
|
| 105 |
-
"backtesting_notes": "string",
|
| 106 |
-
"source_context": "string"
|
| 107 |
-
}]
|
| 108 |
-
}
|
| 109 |
-
|
| 110 |
-
Rules:
|
| 111 |
-
- Return empty arrays [] if nothing found for a category
|
| 112 |
-
- Never paraphrase formulas — preserve exact mathematical form
|
| 113 |
-
- Include LaTeX for every formula that has a standard form
|
| 114 |
-
- Be exhaustive — extract even briefly mentioned concepts
|
| 115 |
-
|
| 116 |
-
Source: {source_file} | Pages: {page_start}–{page_end}
|
| 117 |
-
--- TEXT ---
|
| 118 |
-
{text}
|
| 119 |
-
--- END ---
|
| 120 |
-
""".strip()
|
| 121 |
-
|
| 122 |
-
# ── Strategy compiler prompt ──────────────────────────
|
| 123 |
-
COMPILER_PROMPT = """
|
| 124 |
-
You are a Python algorithmic trading code generator.
|
| 125 |
-
|
| 126 |
-
Convert the strategy JSON below into a Python module.
|
| 127 |
-
Output ONLY valid Python — no markdown, no explanation.
|
| 128 |
-
|
| 129 |
-
Required functions:
|
| 130 |
-
|
| 131 |
-
def get_parameter_grid() -> dict:
|
| 132 |
-
# Returns {param_name: [list_of_values_to_try]}
|
| 133 |
-
# Provide 3-5 values per parameter covering a reasonable range
|
| 134 |
-
...
|
| 135 |
-
|
| 136 |
-
def generate_signals(df: pd.DataFrame, params: dict) -> pd.Series:
|
| 137 |
-
# df columns: open, high, low, close, volume (DatetimeIndex, UTC)
|
| 138 |
-
# Returns pd.Series of int: 1=long, -1=short, 0=flat, same index as df
|
| 139 |
-
# Use ONLY: numpy as np, pandas as pd, indicators module
|
| 140 |
-
# Available in indicators: sma, ema, rsi, atr, bbands, macd,
|
| 141 |
-
# stoch, adx, donchian, keltner, vwap, obv, cci, williams_r,
|
| 142 |
-
# momentum, roc, std_dev, zscore, crossover, crossunder
|
| 143 |
-
...
|
| 144 |
-
|
| 145 |
-
Allowed imports: numpy as np, pandas as pd, indicators (pre-imported)
|
| 146 |
-
No file I/O. No network. No print statements.
|
| 147 |
-
|
| 148 |
-
Strategy:
|
| 149 |
-
{strategy_json}
|
| 150 |
-
""".strip()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
utils/hf_io.py
DELETED
|
@@ -1,440 +0,0 @@
|
|
| 1 |
-
"""
|
| 2 |
-
utils/hf_io.py
|
| 3 |
-
─────────────────────────────────────────────────────
|
| 4 |
-
Single module for all HuggingFace Hub I/O.
|
| 5 |
-
|
| 6 |
-
The HF dataset repo is the persistent store for everything:
|
| 7 |
-
├── pdfs/ uploaded source ebooks
|
| 8 |
-
├── knowledge_base.jsonl deduplicated strategies/formulas
|
| 9 |
-
├── extracted/ individual markdown files
|
| 10 |
-
├── backtests/ per-strategy backtest reports
|
| 11 |
-
└── optimal_sets/ .set, .json, .jl files
|
| 12 |
-
|
| 13 |
-
The HF tick repo is read-only from this Space:
|
| 14 |
-
└── {SYMBOL}/ticks.parquet or {SYMBOL}/{tf}.parquet
|
| 15 |
-
"""
|
| 16 |
-
|
| 17 |
-
import json
|
| 18 |
-
import io
|
| 19 |
-
from pathlib import Path
|
| 20 |
-
from typing import Optional
|
| 21 |
-
import pandas as pd
|
| 22 |
-
from huggingface_hub import (
|
| 23 |
-
HfApi, hf_hub_download, list_repo_files, CommitOperationAdd
|
| 24 |
-
)
|
| 25 |
-
from loguru import logger
|
| 26 |
-
import utils.config as cfg
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
def _api() -> HfApi:
|
| 30 |
-
return HfApi(token=cfg.HF_TOKEN)
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
# ═══════════════════════════════════════════════════
|
| 34 |
-
# KNOWLEDGE BASE — read / write
|
| 35 |
-
# ═══════════════════════════════════════════════════
|
| 36 |
-
|
| 37 |
-
def kb_load() -> dict:
|
| 38 |
-
"""
|
| 39 |
-
Download knowledge_base.jsonl from HF dataset.
|
| 40 |
-
Returns {"strategies": [...], "formulas": [...], "systems": [...]}.
|
| 41 |
-
"""
|
| 42 |
-
empty = {"strategies": {}, "formulas": {}, "systems": {}}
|
| 43 |
-
if not cfg.HF_DATASET_REPO:
|
| 44 |
-
return empty
|
| 45 |
-
try:
|
| 46 |
-
path = hf_hub_download(
|
| 47 |
-
repo_id=cfg.HF_DATASET_REPO,
|
| 48 |
-
filename="knowledge_base.jsonl",
|
| 49 |
-
repo_type="dataset",
|
| 50 |
-
token=cfg.HF_TOKEN,
|
| 51 |
-
local_dir=str(cfg.TMP),
|
| 52 |
-
force_download=True,
|
| 53 |
-
)
|
| 54 |
-
result = {"strategies": {}, "formulas": {}, "systems": {}}
|
| 55 |
-
with open(path, encoding="utf-8") as f:
|
| 56 |
-
for line in f:
|
| 57 |
-
line = line.strip()
|
| 58 |
-
if not line:
|
| 59 |
-
continue
|
| 60 |
-
rec = json.loads(line)
|
| 61 |
-
kind = rec.get("_type", "")
|
| 62 |
-
cid = rec.get("canonical_id", "")
|
| 63 |
-
if kind in result and cid:
|
| 64 |
-
result[kind][cid] = rec
|
| 65 |
-
logger.info(
|
| 66 |
-
f"KB loaded: {len(result['strategies'])} strategies, "
|
| 67 |
-
f"{len(result['formulas'])} formulas, {len(result['systems'])} systems"
|
| 68 |
-
)
|
| 69 |
-
return result
|
| 70 |
-
except Exception as e:
|
| 71 |
-
logger.warning(f"KB load (may not exist yet): {e}")
|
| 72 |
-
return empty
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
def kb_save(kb: dict) -> bool:
|
| 76 |
-
"""Push knowledge base JSONL to HF dataset."""
|
| 77 |
-
if not cfg.HF_DATASET_REPO:
|
| 78 |
-
return False
|
| 79 |
-
try:
|
| 80 |
-
lines = []
|
| 81 |
-
for kind in ("strategies", "formulas", "systems"):
|
| 82 |
-
for rec in kb[kind].values():
|
| 83 |
-
lines.append(json.dumps({**rec, "_type": kind}))
|
| 84 |
-
content = "\n".join(lines).encode("utf-8")
|
| 85 |
-
_api().upload_file(
|
| 86 |
-
path_or_fileobj=io.BytesIO(content),
|
| 87 |
-
path_in_repo="knowledge_base.jsonl",
|
| 88 |
-
repo_id=cfg.HF_DATASET_REPO,
|
| 89 |
-
repo_type="dataset",
|
| 90 |
-
commit_message="Update knowledge base",
|
| 91 |
-
)
|
| 92 |
-
logger.info(f"KB saved: {len(lines)} records")
|
| 93 |
-
return True
|
| 94 |
-
except Exception as e:
|
| 95 |
-
logger.error(f"KB save failed: {e}")
|
| 96 |
-
return False
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
# ═══════════════════════════════════════════════════
|
| 100 |
-
# PDF — upload
|
| 101 |
-
# ═══════════════════════════════════════════════════
|
| 102 |
-
|
| 103 |
-
def pdf_upload(local_path: Path) -> str:
|
| 104 |
-
"""Upload a PDF to the HF dataset. Returns URL or ''."""
|
| 105 |
-
if not cfg.HF_DATASET_REPO:
|
| 106 |
-
return ""
|
| 107 |
-
try:
|
| 108 |
-
url = _api().upload_file(
|
| 109 |
-
path_or_fileobj=str(local_path),
|
| 110 |
-
path_in_repo=f"pdfs/{local_path.name}",
|
| 111 |
-
repo_id=cfg.HF_DATASET_REPO,
|
| 112 |
-
repo_type="dataset",
|
| 113 |
-
commit_message=f"Add PDF: {local_path.name}",
|
| 114 |
-
)
|
| 115 |
-
return str(url)
|
| 116 |
-
except Exception as e:
|
| 117 |
-
logger.warning(f"PDF upload failed {local_path.name}: {e}")
|
| 118 |
-
return ""
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
def pdf_list() -> list[str]:
|
| 122 |
-
"""List all PDFs stored in HF dataset."""
|
| 123 |
-
if not cfg.HF_DATASET_REPO:
|
| 124 |
-
return []
|
| 125 |
-
try:
|
| 126 |
-
return [
|
| 127 |
-
f for f in list_repo_files(
|
| 128 |
-
repo_id=cfg.HF_DATASET_REPO,
|
| 129 |
-
repo_type="dataset",
|
| 130 |
-
token=cfg.HF_TOKEN,
|
| 131 |
-
)
|
| 132 |
-
if f.startswith("pdfs/") and f.endswith(".pdf")
|
| 133 |
-
]
|
| 134 |
-
except Exception as e:
|
| 135 |
-
logger.warning(f"PDF list failed: {e}")
|
| 136 |
-
return []
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
# ═════════════════════════���═════════════════════════
|
| 140 |
-
# EXTRACTED MARKDOWN — push
|
| 141 |
-
# ═══════════════════════════════════════════════════
|
| 142 |
-
|
| 143 |
-
def push_markdown(content: str, remote_path: str) -> bool:
|
| 144 |
-
"""Push a single markdown file to HF dataset."""
|
| 145 |
-
if not cfg.HF_DATASET_REPO:
|
| 146 |
-
return False
|
| 147 |
-
try:
|
| 148 |
-
_api().upload_file(
|
| 149 |
-
path_or_fileobj=io.BytesIO(content.encode("utf-8")),
|
| 150 |
-
path_in_repo=remote_path,
|
| 151 |
-
repo_id=cfg.HF_DATASET_REPO,
|
| 152 |
-
repo_type="dataset",
|
| 153 |
-
commit_message=f"Add: {remote_path}",
|
| 154 |
-
)
|
| 155 |
-
return True
|
| 156 |
-
except Exception as e:
|
| 157 |
-
logger.debug(f"MD push failed {remote_path}: {e}")
|
| 158 |
-
return False
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
def push_batch(files: list[tuple[str, bytes]], commit_msg: str = "Batch update") -> int:
|
| 162 |
-
"""
|
| 163 |
-
Push multiple files in a single commit.
|
| 164 |
-
files: list of (remote_path, bytes_content)
|
| 165 |
-
Returns number of files successfully pushed.
|
| 166 |
-
"""
|
| 167 |
-
if not cfg.HF_DATASET_REPO or not files:
|
| 168 |
-
return 0
|
| 169 |
-
ops = [
|
| 170 |
-
CommitOperationAdd(
|
| 171 |
-
path_in_repo=remote_path,
|
| 172 |
-
path_or_fileobj=io.BytesIO(content),
|
| 173 |
-
)
|
| 174 |
-
for remote_path, content in files
|
| 175 |
-
]
|
| 176 |
-
# HF allows max 100 ops per commit
|
| 177 |
-
pushed = 0
|
| 178 |
-
for i in range(0, len(ops), 100):
|
| 179 |
-
batch = ops[i:i+100]
|
| 180 |
-
try:
|
| 181 |
-
_api().create_commit(
|
| 182 |
-
repo_id=cfg.HF_DATASET_REPO,
|
| 183 |
-
repo_type="dataset",
|
| 184 |
-
operations=batch,
|
| 185 |
-
commit_message=f"{commit_msg} [{i+1}–{i+len(batch)}]",
|
| 186 |
-
)
|
| 187 |
-
pushed += len(batch)
|
| 188 |
-
except Exception as e:
|
| 189 |
-
logger.error(f"Batch push failed: {e}")
|
| 190 |
-
return pushed
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
# ═══════════════════════════════════════════════════
|
| 194 |
-
# TICK DATA — read from HF tick repo
|
| 195 |
-
# ═══════════════════════════════════════════════════
|
| 196 |
-
|
| 197 |
-
def tick_list_symbols() -> list[str]:
|
| 198 |
-
"""List all symbols in the tick dataset."""
|
| 199 |
-
if not cfg.HF_TICK_REPO:
|
| 200 |
-
return []
|
| 201 |
-
try:
|
| 202 |
-
files = list(list_repo_files(
|
| 203 |
-
repo_id=cfg.HF_TICK_REPO,
|
| 204 |
-
repo_type="dataset",
|
| 205 |
-
token=cfg.HF_TOKEN,
|
| 206 |
-
))
|
| 207 |
-
seen = set()
|
| 208 |
-
symbols = []
|
| 209 |
-
for f in files:
|
| 210 |
-
parts = f.split("/")
|
| 211 |
-
if len(parts) >= 2 and parts[0] not in seen:
|
| 212 |
-
seen.add(parts[0])
|
| 213 |
-
symbols.append(parts[0])
|
| 214 |
-
return sorted(symbols)
|
| 215 |
-
except Exception as e:
|
| 216 |
-
logger.warning(f"Tick symbol list failed: {e}")
|
| 217 |
-
return []
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
def tick_load(symbol: str, timeframe: str = "1h") -> Optional[pd.DataFrame]:
|
| 221 |
-
"""
|
| 222 |
-
Load OHLCV data for a symbol at given timeframe.
|
| 223 |
-
Downloads from HF tick repo, caches locally in /tmp.
|
| 224 |
-
|
| 225 |
-
Looks for (in order):
|
| 226 |
-
{symbol}/{timeframe}.parquet
|
| 227 |
-
{symbol}/ticks.parquet → then resamples to timeframe
|
| 228 |
-
{symbol}/data.parquet
|
| 229 |
-
"""
|
| 230 |
-
cache_path = cfg.TMP_TICK_CACHE / f"{symbol}_{timeframe}.parquet"
|
| 231 |
-
if cache_path.exists():
|
| 232 |
-
return pd.read_parquet(cache_path)
|
| 233 |
-
|
| 234 |
-
if not cfg.HF_TICK_REPO:
|
| 235 |
-
return None
|
| 236 |
-
|
| 237 |
-
# Try pre-built timeframe file
|
| 238 |
-
for fname in [f"{timeframe}.parquet", f"{timeframe}.csv"]:
|
| 239 |
-
df = _try_download_tick(symbol, fname)
|
| 240 |
-
if df is not None:
|
| 241 |
-
df = _normalize_ohlcv(df)
|
| 242 |
-
if df is not None:
|
| 243 |
-
df.to_parquet(cache_path)
|
| 244 |
-
return df
|
| 245 |
-
|
| 246 |
-
# Fall back to ticks + resample
|
| 247 |
-
for fname in ["ticks.parquet", "data.parquet", "ticks.csv"]:
|
| 248 |
-
ticks = _try_download_tick(symbol, fname)
|
| 249 |
-
if ticks is not None:
|
| 250 |
-
df = _resample(ticks, timeframe)
|
| 251 |
-
if df is not None and not df.empty:
|
| 252 |
-
df.to_parquet(cache_path)
|
| 253 |
-
return df
|
| 254 |
-
|
| 255 |
-
logger.warning(f"No data found for {symbol} in tick repo")
|
| 256 |
-
return None
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
def _try_download_tick(symbol: str, filename: str) -> Optional[pd.DataFrame]:
|
| 260 |
-
try:
|
| 261 |
-
local = cfg.TMP_TICK_CACHE / symbol
|
| 262 |
-
local.mkdir(parents=True, exist_ok=True)
|
| 263 |
-
path = hf_hub_download(
|
| 264 |
-
repo_id=cfg.HF_TICK_REPO,
|
| 265 |
-
filename=f"{symbol}/{filename}",
|
| 266 |
-
repo_type="dataset",
|
| 267 |
-
token=cfg.HF_TOKEN,
|
| 268 |
-
local_dir=str(local),
|
| 269 |
-
force_download=False,
|
| 270 |
-
)
|
| 271 |
-
if filename.endswith(".parquet"):
|
| 272 |
-
return pd.read_parquet(path)
|
| 273 |
-
return pd.read_csv(path)
|
| 274 |
-
except Exception:
|
| 275 |
-
return None
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
_RESAMPLE_MAP = {
|
| 279 |
-
"1m":"1min","3m":"3min","5m":"5min","15m":"15min","30m":"30min",
|
| 280 |
-
"1h":"1h","2h":"2h","4h":"4h","6h":"6h","12h":"12h",
|
| 281 |
-
"1d":"1D","1w":"1W",
|
| 282 |
-
}
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
def _resample(ticks: pd.DataFrame, timeframe: str) -> Optional[pd.DataFrame]:
|
| 286 |
-
"""Resample raw tick data to OHLCV."""
|
| 287 |
-
import numpy as np
|
| 288 |
-
ticks = ticks.copy()
|
| 289 |
-
|
| 290 |
-
# Parse timestamp
|
| 291 |
-
ts_col = next((c for c in ticks.columns if "time" in c.lower() or "date" in c.lower()), None)
|
| 292 |
-
if ts_col:
|
| 293 |
-
ticks.index = pd.to_datetime(ticks[ts_col], utc=True)
|
| 294 |
-
ticks = ticks.drop(columns=[ts_col])
|
| 295 |
-
else:
|
| 296 |
-
try:
|
| 297 |
-
ticks.index = pd.to_datetime(ticks.index, utc=True)
|
| 298 |
-
except Exception:
|
| 299 |
-
return None
|
| 300 |
-
|
| 301 |
-
ticks.index = ticks.index.tz_convert("UTC") if ticks.index.tz else ticks.index.tz_localize("UTC")
|
| 302 |
-
|
| 303 |
-
# Derive price
|
| 304 |
-
if "bid" in ticks.columns and "ask" in ticks.columns:
|
| 305 |
-
ticks["_price"] = (ticks["bid"] + ticks["ask"]) / 2
|
| 306 |
-
elif "price" in ticks.columns:
|
| 307 |
-
ticks["_price"] = ticks["price"]
|
| 308 |
-
elif "close" in ticks.columns:
|
| 309 |
-
ticks["_price"] = ticks["close"]
|
| 310 |
-
else:
|
| 311 |
-
return None
|
| 312 |
-
|
| 313 |
-
rule = _RESAMPLE_MAP.get(timeframe, "1h")
|
| 314 |
-
ohlcv = ticks["_price"].resample(rule).ohlc()
|
| 315 |
-
ohlcv.columns = ["open", "high", "low", "close"]
|
| 316 |
-
|
| 317 |
-
vol_col = next((c for c in ticks.columns if "vol" in c.lower() or "size" in c.lower()), None)
|
| 318 |
-
ohlcv["volume"] = ticks[vol_col].resample(rule).sum() if vol_col else ticks["_price"].resample(rule).count()
|
| 319 |
-
return ohlcv.dropna()
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
def _normalize_ohlcv(df: pd.DataFrame) -> Optional[pd.DataFrame]:
|
| 323 |
-
"""Normalize column names and index for any OHLCV DataFrame."""
|
| 324 |
-
df = df.copy()
|
| 325 |
-
|
| 326 |
-
# Parse index
|
| 327 |
-
ts_col = next((c for c in df.columns if "time" in c.lower() or "date" in c.lower()), None)
|
| 328 |
-
if ts_col:
|
| 329 |
-
df.index = pd.to_datetime(df[ts_col], utc=True)
|
| 330 |
-
df = df.drop(columns=[ts_col])
|
| 331 |
-
else:
|
| 332 |
-
try:
|
| 333 |
-
df.index = pd.to_datetime(df.index, utc=True)
|
| 334 |
-
except Exception:
|
| 335 |
-
return None
|
| 336 |
-
|
| 337 |
-
df.index = df.index.tz_convert("UTC") if df.index.tz else df.index.tz_localize("UTC")
|
| 338 |
-
df.index.name = "timestamp"
|
| 339 |
-
df = df.sort_index()
|
| 340 |
-
|
| 341 |
-
# Normalize column names
|
| 342 |
-
renames = {}
|
| 343 |
-
for c in df.columns:
|
| 344 |
-
lc = c.lower().strip()
|
| 345 |
-
if lc in ("o", "open"): renames[c] = "open"
|
| 346 |
-
elif lc in ("h", "high"): renames[c] = "high"
|
| 347 |
-
elif lc in ("l", "low"): renames[c] = "low"
|
| 348 |
-
elif lc in ("c", "close"): renames[c] = "close"
|
| 349 |
-
elif lc in ("v","vol","volume","tick_volume"): renames[c] = "volume"
|
| 350 |
-
df = df.rename(columns=renames)
|
| 351 |
-
|
| 352 |
-
for col in ["open","high","low","close"]:
|
| 353 |
-
if col not in df.columns:
|
| 354 |
-
return None
|
| 355 |
-
if "volume" not in df.columns:
|
| 356 |
-
df["volume"] = 0.0
|
| 357 |
-
|
| 358 |
-
df = df[["open","high","low","close","volume"]].astype(float)
|
| 359 |
-
df = df.dropna(subset=["open","high","low","close"])
|
| 360 |
-
|
| 361 |
-
# Fix inverted high/low
|
| 362 |
-
bad = df["high"] < df["low"]
|
| 363 |
-
if bad.any():
|
| 364 |
-
df.loc[bad, ["high","low"]] = df.loc[bad, ["low","high"]].values
|
| 365 |
-
|
| 366 |
-
return df
|
| 367 |
-
|
| 368 |
-
|
| 369 |
-
# ═══════════════════════════════════════════════════
|
| 370 |
-
# BACKTEST RESULTS — push
|
| 371 |
-
# ═══════════════════════════════════════════════════
|
| 372 |
-
|
| 373 |
-
def push_backtest_result(
|
| 374 |
-
strategy_name: str,
|
| 375 |
-
symbol: str,
|
| 376 |
-
timeframe: str,
|
| 377 |
-
report_md: str,
|
| 378 |
-
optimal_json: dict,
|
| 379 |
-
mt5_set: str,
|
| 380 |
-
julia_cfg: str,
|
| 381 |
-
) -> bool:
|
| 382 |
-
"""Push all output files for one backtest result to HF dataset."""
|
| 383 |
-
from pipeline.exporter import slugify
|
| 384 |
-
slug = slugify(strategy_name)
|
| 385 |
-
prefix = f"{slug}_{symbol}_{timeframe}"
|
| 386 |
-
|
| 387 |
-
files = [
|
| 388 |
-
(f"backtests/{slug}/{prefix}_report.md", report_md.encode()),
|
| 389 |
-
(f"optimal_sets/{prefix}_optimal.json", json.dumps(optimal_json, indent=2).encode()),
|
| 390 |
-
(f"optimal_sets/{prefix}.set", mt5_set.encode()),
|
| 391 |
-
(f"optimal_sets/{prefix}_config.jl", julia_cfg.encode()),
|
| 392 |
-
]
|
| 393 |
-
pushed = push_batch(files, commit_msg=f"Backtest: {strategy_name} {symbol} {timeframe}")
|
| 394 |
-
return pushed == len(files)
|
| 395 |
-
|
| 396 |
-
|
| 397 |
-
def push_backtest_index(index_md: str, index_json: dict) -> bool:
|
| 398 |
-
"""Push master index files."""
|
| 399 |
-
files = [
|
| 400 |
-
("optimal_sets/BACKTEST_INDEX.md", index_md.encode()),
|
| 401 |
-
("optimal_sets/backtest_index.json", json.dumps(index_json, indent=2).encode()),
|
| 402 |
-
]
|
| 403 |
-
return push_batch(files, "Update backtest index") == 2
|
| 404 |
-
|
| 405 |
-
|
| 406 |
-
def fetch_backtest_index() -> dict:
|
| 407 |
-
"""Download backtest index JSON from HF dataset."""
|
| 408 |
-
if not cfg.HF_DATASET_REPO:
|
| 409 |
-
return {}
|
| 410 |
-
try:
|
| 411 |
-
path = hf_hub_download(
|
| 412 |
-
repo_id=cfg.HF_DATASET_REPO,
|
| 413 |
-
filename="optimal_sets/backtest_index.json",
|
| 414 |
-
repo_type="dataset",
|
| 415 |
-
token=cfg.HF_TOKEN,
|
| 416 |
-
local_dir=str(cfg.TMP),
|
| 417 |
-
force_download=True,
|
| 418 |
-
)
|
| 419 |
-
return json.loads(Path(path).read_text())
|
| 420 |
-
except Exception:
|
| 421 |
-
return {}
|
| 422 |
-
|
| 423 |
-
|
| 424 |
-
def fetch_file(remote_path: str) -> Optional[bytes]:
|
| 425 |
-
"""Download any file from HF dataset as bytes."""
|
| 426 |
-
if not cfg.HF_DATASET_REPO:
|
| 427 |
-
return None
|
| 428 |
-
try:
|
| 429 |
-
path = hf_hub_download(
|
| 430 |
-
repo_id=cfg.HF_DATASET_REPO,
|
| 431 |
-
filename=remote_path,
|
| 432 |
-
repo_type="dataset",
|
| 433 |
-
token=cfg.HF_TOKEN,
|
| 434 |
-
local_dir=str(cfg.TMP / "downloads"),
|
| 435 |
-
force_download=True,
|
| 436 |
-
)
|
| 437 |
-
return Path(path).read_bytes()
|
| 438 |
-
except Exception as e:
|
| 439 |
-
logger.debug(f"Fetch {remote_path}: {e}")
|
| 440 |
-
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|