cyberkyne commited on
Commit
faa30a5
·
verified ·
1 Parent(s): 40627de

Delete utils

Browse files
Files changed (3) hide show
  1. utils/__init__.py +0 -0
  2. utils/config.py +0 -150
  3. utils/hf_io.py +0 -440
utils/__init__.py DELETED
File without changes
utils/config.py DELETED
@@ -1,150 +0,0 @@
1
- """
2
- utils/config.py
3
- All config is read from HuggingFace Spaces Secrets (environment variables).
4
- Set these in: Space Settings → Variables and Secrets
5
- No .env file needed — Spaces injects secrets automatically.
6
- """
7
-
8
- import os
9
- from pathlib import Path
10
-
11
- # ── HF Spaces uses /tmp for ephemeral working storage ─
12
- # All persistent data is written to HF Dataset via API.
13
- TMP = Path("/tmp/quant")
14
- TMP_PDFS = TMP / "pdfs"
15
- TMP_CHUNKS = TMP / "chunks"
16
- TMP_COMPILED = TMP / "compiled"
17
- TMP_BACKTESTS = TMP / "backtests"
18
- TMP_EXPORTS = TMP / "exports"
19
- TMP_TICK_CACHE = TMP / "tick_cache"
20
- TMP_KB = TMP / "knowledge_base.jsonl"
21
-
22
- for d in [TMP_PDFS, TMP_CHUNKS, TMP_COMPILED,
23
- TMP_BACKTESTS, TMP_EXPORTS, TMP_TICK_CACHE]:
24
- d.mkdir(parents=True, exist_ok=True)
25
-
26
-
27
- def get(key: str, default: str = "") -> str:
28
- return os.environ.get(key, default)
29
-
30
-
31
- # ── API Keys (set in HF Space Secrets) ────────────────
32
- ANTHROPIC_API_KEY = get("ANTHROPIC_API_KEY")
33
- HF_TOKEN = get("HF_TOKEN")
34
-
35
- # ── HuggingFace Dataset repos ─────────────────────────
36
- HF_DATASET_REPO = get("HF_DATASET_REPO") # knowledge base + results
37
- HF_TICK_REPO = get("HF_TICK_REPO") # your tick data
38
-
39
- # ── Extraction settings ───────────────────────────────
40
- SIMILARITY_THRESHOLD = float(get("SIMILARITY_THRESHOLD", "0.85"))
41
- MAX_TOKENS_PER_CHUNK = int(get("MAX_TOKENS_PER_CHUNK", "3000"))
42
- OCR_DPI = int(get("OCR_DPI", "300"))
43
-
44
- # ── Backtest settings ─────────────────────────────────
45
- INITIAL_EQUITY = float(get("INITIAL_EQUITY", "10000"))
46
- COMMISSION_PCT = float(get("COMMISSION_PCT", "0.0002"))
47
- SLIPPAGE_PCT = float(get("SLIPPAGE_PCT", "0.0001"))
48
- RISK_PER_TRADE = float(get("RISK_PER_TRADE", "0.01"))
49
- WF_WINDOWS = int(get("WF_WINDOWS", "5"))
50
- WF_IS_RATIO = float(get("WF_IS_RATIO", "0.70"))
51
- MAX_PARAM_COMBOS = int(get("MAX_PARAM_COMBOS", "300"))
52
- MIN_TRADES = int(get("MIN_TRADES", "30"))
53
- MIN_SHARPE = float(get("MIN_SHARPE", "0.5"))
54
- BACKTEST_TIMEFRAMES = get("BACKTEST_TIMEFRAMES", "1h,4h,1d").split(",")
55
-
56
- # ── Strategy categories ───────────────────────────────
57
- CATEGORIES = [
58
- "Trend Following", "Mean Reversion", "Statistical Arbitrage",
59
- "Momentum", "Breakout", "Volatility Trading", "Market Making",
60
- "Pattern Recognition", "Machine Learning", "Options Strategy",
61
- "High Frequency", "Pairs Trading", "Carry Trade",
62
- "Seasonal / Calendar", "Risk Management", "Position Sizing",
63
- "Portfolio Construction", "Market Microstructure", "Other",
64
- ]
65
-
66
- # ── Extraction prompt ─────────────────────────────────
67
- EXTRACTION_PROMPT = """
68
- You are a quantitative finance knowledge extraction engine.
69
-
70
- Extract ALL trading strategies, mathematical formulas, and complete trading systems
71
- from the text chunk below, taken from an algorithmic trading book.
72
-
73
- Output ONLY valid JSON — no markdown fences, no preamble:
74
- {
75
- "strategies": [{
76
- "name": "string",
77
- "category": "string",
78
- "description": "string",
79
- "entry_rules": ["string"],
80
- "exit_rules": ["string"],
81
- "filters": ["string"],
82
- "timeframes": ["string"],
83
- "instruments": ["string"],
84
- "parameters": {"name": "description with typical value"},
85
- "mathematical_basis": "string",
86
- "source_context": "brief quote from text"
87
- }],
88
- "formulas": [{
89
- "name": "string",
90
- "category": "string",
91
- "latex": "LaTeX string",
92
- "plain_text": "string",
93
- "variables": {"symbol": "description"},
94
- "purpose": "string",
95
- "usage_context": "string",
96
- "source_context": "string"
97
- }],
98
- "systems": [{
99
- "name": "string",
100
- "components": ["string"],
101
- "entry_system": "string",
102
- "exit_system": "string",
103
- "risk_management": "string",
104
- "position_sizing": "string",
105
- "backtesting_notes": "string",
106
- "source_context": "string"
107
- }]
108
- }
109
-
110
- Rules:
111
- - Return empty arrays [] if nothing found for a category
112
- - Never paraphrase formulas — preserve exact mathematical form
113
- - Include LaTeX for every formula that has a standard form
114
- - Be exhaustive — extract even briefly mentioned concepts
115
-
116
- Source: {source_file} | Pages: {page_start}–{page_end}
117
- --- TEXT ---
118
- {text}
119
- --- END ---
120
- """.strip()
121
-
122
- # ── Strategy compiler prompt ──────────────────────────
123
- COMPILER_PROMPT = """
124
- You are a Python algorithmic trading code generator.
125
-
126
- Convert the strategy JSON below into a Python module.
127
- Output ONLY valid Python — no markdown, no explanation.
128
-
129
- Required functions:
130
-
131
- def get_parameter_grid() -> dict:
132
- # Returns {param_name: [list_of_values_to_try]}
133
- # Provide 3-5 values per parameter covering a reasonable range
134
- ...
135
-
136
- def generate_signals(df: pd.DataFrame, params: dict) -> pd.Series:
137
- # df columns: open, high, low, close, volume (DatetimeIndex, UTC)
138
- # Returns pd.Series of int: 1=long, -1=short, 0=flat, same index as df
139
- # Use ONLY: numpy as np, pandas as pd, indicators module
140
- # Available in indicators: sma, ema, rsi, atr, bbands, macd,
141
- # stoch, adx, donchian, keltner, vwap, obv, cci, williams_r,
142
- # momentum, roc, std_dev, zscore, crossover, crossunder
143
- ...
144
-
145
- Allowed imports: numpy as np, pandas as pd, indicators (pre-imported)
146
- No file I/O. No network. No print statements.
147
-
148
- Strategy:
149
- {strategy_json}
150
- """.strip()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
utils/hf_io.py DELETED
@@ -1,440 +0,0 @@
1
- """
2
- utils/hf_io.py
3
- ─────────────────────────────────────────────────────
4
- Single module for all HuggingFace Hub I/O.
5
-
6
- The HF dataset repo is the persistent store for everything:
7
- ├── pdfs/ uploaded source ebooks
8
- ├── knowledge_base.jsonl deduplicated strategies/formulas
9
- ├── extracted/ individual markdown files
10
- ├── backtests/ per-strategy backtest reports
11
- └── optimal_sets/ .set, .json, .jl files
12
-
13
- The HF tick repo is read-only from this Space:
14
- └── {SYMBOL}/ticks.parquet or {SYMBOL}/{tf}.parquet
15
- """
16
-
17
- import json
18
- import io
19
- from pathlib import Path
20
- from typing import Optional
21
- import pandas as pd
22
- from huggingface_hub import (
23
- HfApi, hf_hub_download, list_repo_files, CommitOperationAdd
24
- )
25
- from loguru import logger
26
- import utils.config as cfg
27
-
28
-
29
- def _api() -> HfApi:
30
- return HfApi(token=cfg.HF_TOKEN)
31
-
32
-
33
- # ═══════════════════════════════════════════════════
34
- # KNOWLEDGE BASE — read / write
35
- # ═══════════════════════════════════════════════════
36
-
37
- def kb_load() -> dict:
38
- """
39
- Download knowledge_base.jsonl from HF dataset.
40
- Returns {"strategies": [...], "formulas": [...], "systems": [...]}.
41
- """
42
- empty = {"strategies": {}, "formulas": {}, "systems": {}}
43
- if not cfg.HF_DATASET_REPO:
44
- return empty
45
- try:
46
- path = hf_hub_download(
47
- repo_id=cfg.HF_DATASET_REPO,
48
- filename="knowledge_base.jsonl",
49
- repo_type="dataset",
50
- token=cfg.HF_TOKEN,
51
- local_dir=str(cfg.TMP),
52
- force_download=True,
53
- )
54
- result = {"strategies": {}, "formulas": {}, "systems": {}}
55
- with open(path, encoding="utf-8") as f:
56
- for line in f:
57
- line = line.strip()
58
- if not line:
59
- continue
60
- rec = json.loads(line)
61
- kind = rec.get("_type", "")
62
- cid = rec.get("canonical_id", "")
63
- if kind in result and cid:
64
- result[kind][cid] = rec
65
- logger.info(
66
- f"KB loaded: {len(result['strategies'])} strategies, "
67
- f"{len(result['formulas'])} formulas, {len(result['systems'])} systems"
68
- )
69
- return result
70
- except Exception as e:
71
- logger.warning(f"KB load (may not exist yet): {e}")
72
- return empty
73
-
74
-
75
- def kb_save(kb: dict) -> bool:
76
- """Push knowledge base JSONL to HF dataset."""
77
- if not cfg.HF_DATASET_REPO:
78
- return False
79
- try:
80
- lines = []
81
- for kind in ("strategies", "formulas", "systems"):
82
- for rec in kb[kind].values():
83
- lines.append(json.dumps({**rec, "_type": kind}))
84
- content = "\n".join(lines).encode("utf-8")
85
- _api().upload_file(
86
- path_or_fileobj=io.BytesIO(content),
87
- path_in_repo="knowledge_base.jsonl",
88
- repo_id=cfg.HF_DATASET_REPO,
89
- repo_type="dataset",
90
- commit_message="Update knowledge base",
91
- )
92
- logger.info(f"KB saved: {len(lines)} records")
93
- return True
94
- except Exception as e:
95
- logger.error(f"KB save failed: {e}")
96
- return False
97
-
98
-
99
- # ═══════════════════════════════════════════════════
100
- # PDF — upload
101
- # ═══════════════════════════════════════════════════
102
-
103
- def pdf_upload(local_path: Path) -> str:
104
- """Upload a PDF to the HF dataset. Returns URL or ''."""
105
- if not cfg.HF_DATASET_REPO:
106
- return ""
107
- try:
108
- url = _api().upload_file(
109
- path_or_fileobj=str(local_path),
110
- path_in_repo=f"pdfs/{local_path.name}",
111
- repo_id=cfg.HF_DATASET_REPO,
112
- repo_type="dataset",
113
- commit_message=f"Add PDF: {local_path.name}",
114
- )
115
- return str(url)
116
- except Exception as e:
117
- logger.warning(f"PDF upload failed {local_path.name}: {e}")
118
- return ""
119
-
120
-
121
- def pdf_list() -> list[str]:
122
- """List all PDFs stored in HF dataset."""
123
- if not cfg.HF_DATASET_REPO:
124
- return []
125
- try:
126
- return [
127
- f for f in list_repo_files(
128
- repo_id=cfg.HF_DATASET_REPO,
129
- repo_type="dataset",
130
- token=cfg.HF_TOKEN,
131
- )
132
- if f.startswith("pdfs/") and f.endswith(".pdf")
133
- ]
134
- except Exception as e:
135
- logger.warning(f"PDF list failed: {e}")
136
- return []
137
-
138
-
139
- # ═════════════════════════���═════════════════════════
140
- # EXTRACTED MARKDOWN — push
141
- # ═══════════════════════════════════════════════════
142
-
143
- def push_markdown(content: str, remote_path: str) -> bool:
144
- """Push a single markdown file to HF dataset."""
145
- if not cfg.HF_DATASET_REPO:
146
- return False
147
- try:
148
- _api().upload_file(
149
- path_or_fileobj=io.BytesIO(content.encode("utf-8")),
150
- path_in_repo=remote_path,
151
- repo_id=cfg.HF_DATASET_REPO,
152
- repo_type="dataset",
153
- commit_message=f"Add: {remote_path}",
154
- )
155
- return True
156
- except Exception as e:
157
- logger.debug(f"MD push failed {remote_path}: {e}")
158
- return False
159
-
160
-
161
- def push_batch(files: list[tuple[str, bytes]], commit_msg: str = "Batch update") -> int:
162
- """
163
- Push multiple files in a single commit.
164
- files: list of (remote_path, bytes_content)
165
- Returns number of files successfully pushed.
166
- """
167
- if not cfg.HF_DATASET_REPO or not files:
168
- return 0
169
- ops = [
170
- CommitOperationAdd(
171
- path_in_repo=remote_path,
172
- path_or_fileobj=io.BytesIO(content),
173
- )
174
- for remote_path, content in files
175
- ]
176
- # HF allows max 100 ops per commit
177
- pushed = 0
178
- for i in range(0, len(ops), 100):
179
- batch = ops[i:i+100]
180
- try:
181
- _api().create_commit(
182
- repo_id=cfg.HF_DATASET_REPO,
183
- repo_type="dataset",
184
- operations=batch,
185
- commit_message=f"{commit_msg} [{i+1}–{i+len(batch)}]",
186
- )
187
- pushed += len(batch)
188
- except Exception as e:
189
- logger.error(f"Batch push failed: {e}")
190
- return pushed
191
-
192
-
193
- # ═══════════════════════════════════════════════════
194
- # TICK DATA — read from HF tick repo
195
- # ═══════════════════════════════════════════════════
196
-
197
- def tick_list_symbols() -> list[str]:
198
- """List all symbols in the tick dataset."""
199
- if not cfg.HF_TICK_REPO:
200
- return []
201
- try:
202
- files = list(list_repo_files(
203
- repo_id=cfg.HF_TICK_REPO,
204
- repo_type="dataset",
205
- token=cfg.HF_TOKEN,
206
- ))
207
- seen = set()
208
- symbols = []
209
- for f in files:
210
- parts = f.split("/")
211
- if len(parts) >= 2 and parts[0] not in seen:
212
- seen.add(parts[0])
213
- symbols.append(parts[0])
214
- return sorted(symbols)
215
- except Exception as e:
216
- logger.warning(f"Tick symbol list failed: {e}")
217
- return []
218
-
219
-
220
- def tick_load(symbol: str, timeframe: str = "1h") -> Optional[pd.DataFrame]:
221
- """
222
- Load OHLCV data for a symbol at given timeframe.
223
- Downloads from HF tick repo, caches locally in /tmp.
224
-
225
- Looks for (in order):
226
- {symbol}/{timeframe}.parquet
227
- {symbol}/ticks.parquet → then resamples to timeframe
228
- {symbol}/data.parquet
229
- """
230
- cache_path = cfg.TMP_TICK_CACHE / f"{symbol}_{timeframe}.parquet"
231
- if cache_path.exists():
232
- return pd.read_parquet(cache_path)
233
-
234
- if not cfg.HF_TICK_REPO:
235
- return None
236
-
237
- # Try pre-built timeframe file
238
- for fname in [f"{timeframe}.parquet", f"{timeframe}.csv"]:
239
- df = _try_download_tick(symbol, fname)
240
- if df is not None:
241
- df = _normalize_ohlcv(df)
242
- if df is not None:
243
- df.to_parquet(cache_path)
244
- return df
245
-
246
- # Fall back to ticks + resample
247
- for fname in ["ticks.parquet", "data.parquet", "ticks.csv"]:
248
- ticks = _try_download_tick(symbol, fname)
249
- if ticks is not None:
250
- df = _resample(ticks, timeframe)
251
- if df is not None and not df.empty:
252
- df.to_parquet(cache_path)
253
- return df
254
-
255
- logger.warning(f"No data found for {symbol} in tick repo")
256
- return None
257
-
258
-
259
- def _try_download_tick(symbol: str, filename: str) -> Optional[pd.DataFrame]:
260
- try:
261
- local = cfg.TMP_TICK_CACHE / symbol
262
- local.mkdir(parents=True, exist_ok=True)
263
- path = hf_hub_download(
264
- repo_id=cfg.HF_TICK_REPO,
265
- filename=f"{symbol}/{filename}",
266
- repo_type="dataset",
267
- token=cfg.HF_TOKEN,
268
- local_dir=str(local),
269
- force_download=False,
270
- )
271
- if filename.endswith(".parquet"):
272
- return pd.read_parquet(path)
273
- return pd.read_csv(path)
274
- except Exception:
275
- return None
276
-
277
-
278
- _RESAMPLE_MAP = {
279
- "1m":"1min","3m":"3min","5m":"5min","15m":"15min","30m":"30min",
280
- "1h":"1h","2h":"2h","4h":"4h","6h":"6h","12h":"12h",
281
- "1d":"1D","1w":"1W",
282
- }
283
-
284
-
285
- def _resample(ticks: pd.DataFrame, timeframe: str) -> Optional[pd.DataFrame]:
286
- """Resample raw tick data to OHLCV."""
287
- import numpy as np
288
- ticks = ticks.copy()
289
-
290
- # Parse timestamp
291
- ts_col = next((c for c in ticks.columns if "time" in c.lower() or "date" in c.lower()), None)
292
- if ts_col:
293
- ticks.index = pd.to_datetime(ticks[ts_col], utc=True)
294
- ticks = ticks.drop(columns=[ts_col])
295
- else:
296
- try:
297
- ticks.index = pd.to_datetime(ticks.index, utc=True)
298
- except Exception:
299
- return None
300
-
301
- ticks.index = ticks.index.tz_convert("UTC") if ticks.index.tz else ticks.index.tz_localize("UTC")
302
-
303
- # Derive price
304
- if "bid" in ticks.columns and "ask" in ticks.columns:
305
- ticks["_price"] = (ticks["bid"] + ticks["ask"]) / 2
306
- elif "price" in ticks.columns:
307
- ticks["_price"] = ticks["price"]
308
- elif "close" in ticks.columns:
309
- ticks["_price"] = ticks["close"]
310
- else:
311
- return None
312
-
313
- rule = _RESAMPLE_MAP.get(timeframe, "1h")
314
- ohlcv = ticks["_price"].resample(rule).ohlc()
315
- ohlcv.columns = ["open", "high", "low", "close"]
316
-
317
- vol_col = next((c for c in ticks.columns if "vol" in c.lower() or "size" in c.lower()), None)
318
- ohlcv["volume"] = ticks[vol_col].resample(rule).sum() if vol_col else ticks["_price"].resample(rule).count()
319
- return ohlcv.dropna()
320
-
321
-
322
- def _normalize_ohlcv(df: pd.DataFrame) -> Optional[pd.DataFrame]:
323
- """Normalize column names and index for any OHLCV DataFrame."""
324
- df = df.copy()
325
-
326
- # Parse index
327
- ts_col = next((c for c in df.columns if "time" in c.lower() or "date" in c.lower()), None)
328
- if ts_col:
329
- df.index = pd.to_datetime(df[ts_col], utc=True)
330
- df = df.drop(columns=[ts_col])
331
- else:
332
- try:
333
- df.index = pd.to_datetime(df.index, utc=True)
334
- except Exception:
335
- return None
336
-
337
- df.index = df.index.tz_convert("UTC") if df.index.tz else df.index.tz_localize("UTC")
338
- df.index.name = "timestamp"
339
- df = df.sort_index()
340
-
341
- # Normalize column names
342
- renames = {}
343
- for c in df.columns:
344
- lc = c.lower().strip()
345
- if lc in ("o", "open"): renames[c] = "open"
346
- elif lc in ("h", "high"): renames[c] = "high"
347
- elif lc in ("l", "low"): renames[c] = "low"
348
- elif lc in ("c", "close"): renames[c] = "close"
349
- elif lc in ("v","vol","volume","tick_volume"): renames[c] = "volume"
350
- df = df.rename(columns=renames)
351
-
352
- for col in ["open","high","low","close"]:
353
- if col not in df.columns:
354
- return None
355
- if "volume" not in df.columns:
356
- df["volume"] = 0.0
357
-
358
- df = df[["open","high","low","close","volume"]].astype(float)
359
- df = df.dropna(subset=["open","high","low","close"])
360
-
361
- # Fix inverted high/low
362
- bad = df["high"] < df["low"]
363
- if bad.any():
364
- df.loc[bad, ["high","low"]] = df.loc[bad, ["low","high"]].values
365
-
366
- return df
367
-
368
-
369
- # ═══════════════════════════════════════════════════
370
- # BACKTEST RESULTS — push
371
- # ═══════════════════════════════════════════════════
372
-
373
- def push_backtest_result(
374
- strategy_name: str,
375
- symbol: str,
376
- timeframe: str,
377
- report_md: str,
378
- optimal_json: dict,
379
- mt5_set: str,
380
- julia_cfg: str,
381
- ) -> bool:
382
- """Push all output files for one backtest result to HF dataset."""
383
- from pipeline.exporter import slugify
384
- slug = slugify(strategy_name)
385
- prefix = f"{slug}_{symbol}_{timeframe}"
386
-
387
- files = [
388
- (f"backtests/{slug}/{prefix}_report.md", report_md.encode()),
389
- (f"optimal_sets/{prefix}_optimal.json", json.dumps(optimal_json, indent=2).encode()),
390
- (f"optimal_sets/{prefix}.set", mt5_set.encode()),
391
- (f"optimal_sets/{prefix}_config.jl", julia_cfg.encode()),
392
- ]
393
- pushed = push_batch(files, commit_msg=f"Backtest: {strategy_name} {symbol} {timeframe}")
394
- return pushed == len(files)
395
-
396
-
397
- def push_backtest_index(index_md: str, index_json: dict) -> bool:
398
- """Push master index files."""
399
- files = [
400
- ("optimal_sets/BACKTEST_INDEX.md", index_md.encode()),
401
- ("optimal_sets/backtest_index.json", json.dumps(index_json, indent=2).encode()),
402
- ]
403
- return push_batch(files, "Update backtest index") == 2
404
-
405
-
406
- def fetch_backtest_index() -> dict:
407
- """Download backtest index JSON from HF dataset."""
408
- if not cfg.HF_DATASET_REPO:
409
- return {}
410
- try:
411
- path = hf_hub_download(
412
- repo_id=cfg.HF_DATASET_REPO,
413
- filename="optimal_sets/backtest_index.json",
414
- repo_type="dataset",
415
- token=cfg.HF_TOKEN,
416
- local_dir=str(cfg.TMP),
417
- force_download=True,
418
- )
419
- return json.loads(Path(path).read_text())
420
- except Exception:
421
- return {}
422
-
423
-
424
- def fetch_file(remote_path: str) -> Optional[bytes]:
425
- """Download any file from HF dataset as bytes."""
426
- if not cfg.HF_DATASET_REPO:
427
- return None
428
- try:
429
- path = hf_hub_download(
430
- repo_id=cfg.HF_DATASET_REPO,
431
- filename=remote_path,
432
- repo_type="dataset",
433
- token=cfg.HF_TOKEN,
434
- local_dir=str(cfg.TMP / "downloads"),
435
- force_download=True,
436
- )
437
- return Path(path).read_bytes()
438
- except Exception as e:
439
- logger.debug(f"Fetch {remote_path}: {e}")
440
- return None