Delete finai_xbrl_pipeline.ipynb

finai_xbrl_pipeline.ipynb

@@ -1,333 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "8b6363a9",
-   "metadata": {},
-   "source": [
-    "# FinAI — SEC XBRL 下载与解析流水线\n",
-    "\n",
-    "**说明**：本 Notebook 包含：\n",
-    "\n",
-    "- 自动下载目标公司的 XBRL（10‑K / 10‑Q）文件\n",
-    "- 解析 XBRL（.xml/.xbrl/.xsd）并导出结构化 JSON/CSV\n",
-    "- 为后续 FinGPT/LoRA 微调准备数据格式与建议\n",
-    "\n",
-    "**使用方法**：在本地或 Colab 上运行每个单元格。若在 Colab 中运行请在第一格安装依赖（已包含）。\n",
-    "\n",
-    "目标公司：AAPL, MSFT, GOOGL, JPM, GS, XOM, WMT, TSLA\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "id": "a04aab59",
-   "metadata": {
-    "jupyter": {
-     "is_executing": true
-    }
-   },
-   "source": [
-    "# 安装依赖（在本地可用 pip 安装；在 Colab 可直接运行）\n",
-    "!pip install tqdm requests lxml beautifulsoup4 pandas xbrl-parser || true\n",
-    "\n",
-    "# 小提示：在某些环境中，xbrl-parser 可能不可用或不可维护，\n",
-    "# 使用 BeautifulSoup + lxml 解析 XML/XBRL 是稳妥方案。\n"
-   ],
-   "outputs": [],
-   "execution_count": null
-  },
-  {
-   "cell_type": "code",
-   "id": "b27e39a1",
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2025-11-12T04:29:37.420996Z",
-     "start_time": "2025-11-12T04:29:36.325209Z"
-    }
-   },
-   "source": [
-    "import os\n",
-    "import requests\n",
-    "import json\n",
-    "from bs4 import BeautifulSoup\n",
-    "from tqdm import tqdm\n",
-    "import pandas as pd\n",
-    "\n",
-    "# 目标公司（CIK 保留 10 位格式）\n",
-    "companies = {\n",
-    "    \"AAPL\": {\"name\": \"Apple Inc.\", \"cik\": \"0000320193\"},\n",
-    "    \"MSFT\": {\"name\": \"Microsoft Corporation\", \"cik\": \"0000789019\"},\n",
-    "    \"GOOGL\": {\"name\": \"Alphabet Inc.\", \"cik\": \"0001652044\"},\n",
-    "    \"JPM\": {\"name\": \"JPMorgan Chase & Co.\", \"cik\": \"0000019617\"},\n",
-    "    \"GS\": {\"name\": \"Goldman Sachs Group Inc.\", \"cik\": \"0000886982\"},\n",
-    "    \"XOM\": {\"name\": \"Exxon Mobil Corporation\", \"cik\": \"0000034088\"},\n",
-    "    \"WMT\": {\"name\": \"Walmart Inc.\", \"cik\": \"0000104169\"},\n",
-    "    \"TSLA\": {\"name\": \"Tesla Inc.\", \"cik\": \"0001318605\"}\n",
-    "}\n",
-    "\n",
-    "HEADERS = {\"User-Agent\": \"FinAI-Agent/1.0 (16696065317@163.com)\"}\n",
-    "SAVE_ROOT = \"./sec_xbrl_data\"\n",
-    "os.makedirs(SAVE_ROOT, exist_ok=True)\n"
-   ],
-   "outputs": [],
-   "execution_count": 2
-  },
-  {
-   "cell_type": "code",
-   "id": "31227534",
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2025-11-12T04:30:09.569648Z",
-     "start_time": "2025-11-12T04:30:09.559109Z"
-    }
-   },
-   "source": [
-    "def download_xbrl_for_company(cik_10, ticker, max_filings=5, target_forms=('10-K','10-Q')):\n",
-    "    \"\"\"下载指定公司最近的若干 10-K/10-Q 的 XBRL 相关文件（.xml .xbrl .xsd）\n",
-    "    cik_10: 字符串，10 位 CIK（例 '0000320193'）\n",
-    "    max_filings: 最多抓取多少份对应表单（按最近排序）\n",
-    "    返回值：下载的文件路径列表\n",
-    "    \"\"\"\n",
-    "    base_submission = f\"https://data.sec.gov/submissions/CIK{cik_10}.json\"\n",
-    "    resp = requests.get(base_submission, headers=HEADERS)\n",
-    "    if resp.status_code != 200:\n",
-    "        print(f\"无法获取提交索引: {cik_10} -> HTTP {resp.status_code}\")\n",
-    "        return []\n",
-    "    data = resp.json()\n",
-    "    recent = data.get('filings', {}).get('recent', {})\n",
-    "    forms = recent.get('form', [])\n",
-    "    accessions = recent.get('accessionNumber', [])\n",
-    "\n",
-    "    selected = []\n",
-    "    for f, a in zip(forms, accessions):\n",
-    "        if f in target_forms:\n",
-    "            selected.append(a)\n",
-    "        if len(selected) >= max_filings:\n",
-    "            break\n",
-    "\n",
-    "    downloaded = []\n",
-    "    cik_num = str(int(cik_10))  # remove leading zeros for URL\n",
-    "    company_dir = os.path.join(SAVE_ROOT, ticker)\n",
-    "    os.makedirs(company_dir, exist_ok=True)\n",
-    "\n",
-    "    for acc in tqdm(selected, desc=f\"{ticker} filings\"):\n",
-    "        acc_id = acc.replace('-', '')\n",
-    "        idx_url = f\"https://www.sec.gov/Archives/edgar/data/{cik_num}/{acc_id}/index.json\"\n",
-    "        r = requests.get(idx_url, headers=HEADERS)\n",
-    "        if r.status_code != 200:\n",
-    "            continue\n",
-    "        items = r.json().get('directory', {}).get('item', [])\n",
-    "        for item in items:\n",
-    "            if item.get('name', '').lower().endswith(('.xbrl', '.xml', '.xsd')):\n",
-    "                url = f\"https://www.sec.gov/Archives/edgar/data/{cik_num}/{acc_id}/{item['name']}\"\n",
-    "                local_path = os.path.join(company_dir, f\"{acc}_{item['name']}\")\n",
-    "                if not os.path.exists(local_path):\n",
-    "                    try:\n",
-    "                        respf = requests.get(url, headers=HEADERS)\n",
-    "                        if respf.status_code == 200:\n",
-    "                            with open(local_path, 'wb') as fp:\n",
-    "                                fp.write(respf.content)\n",
-    "                            downloaded.append(local_path)\n",
-    "                    except Exception as e:\n",
-    "                        print('下载异常', e)\n",
-    "                else:\n",
-    "                    downloaded.append(local_path)\n",
-    "    return downloaded\n",
-    "\n",
-    "# 示例调用（请在有网络的环境运行）\n",
-    "# downloaded = download_xbrl_for_company('0000320193', 'AAPL', max_filings=3)\n",
-    "# print(downloaded)\n"
-   ],
-   "outputs": [],
-   "execution_count": 10
-  },
-  {
-   "cell_type": "code",
-   "id": "eaf17825",
-   "metadata": {
-    "jupyter": {
-     "is_executing": true
-    }
-   },
-   "source": [
-    "# 批量下载所有目标公司（请在有网络环境下运行）\n",
-    "all_downloads = {}\n",
-    "for ticker, info in companies.items():\n",
-    "    print('\\n---', ticker, info['name'])\n",
-    "    files = download_xbrl_for_company(info['cik'], ticker, max_filings=5)\n",
-    "    all_downloads[ticker] = files\n",
-    "\n",
-    "# 保存一份索引\n",
-    "with open(os.path.join(SAVE_ROOT, 'download_index.json'), 'w', encoding='utf-8') as f:\n",
-    "    json.dump(all_downloads, f, indent=2, ensure_ascii=False)\n",
-    "\n",
-    "print('\\n下载完成。请检查', SAVE_ROOT)\n"
-   ],
-   "outputs": [],
-   "execution_count": null
-  },
-  {
-   "cell_type": "code",
-   "id": "afafa968",
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2025-11-12T05:10:24.104088Z",
-     "start_time": "2025-11-12T05:10:24.083690Z"
-    }
-   },
-   "source": [
-    "def parse_xbrl_to_facts(filepath):\n",
-    "    \"\"\"将 XBRL/XML 文件解析为一个字典 facts，尽量提取有意义的 tag 和 value\n",
-    "    注意：XBRL 里会有命名空间和带前缀的标签，这里做简单处理以便后续使用。\n",
-    "    \"\"\"\n",
-    "    with open(filepath, 'rb') as f:\n",
-    "        content = f.read()\n",
-    "    try:\n",
-    "        soup = BeautifulSoup(content, 'lxml-xml')\n",
-    "    except Exception:\n",
-    "        soup = BeautifulSoup(content, 'xml')\n",
-    "    facts = []\n",
-    "    # 查找可能是事实（amount 或者有 contextRef 的节点）\n",
-    "    for tag in soup.find_all():\n",
-    "        name = tag.name\n",
-    "        text = (tag.get_text() or '').strip()\n",
-    "        if not text:\n",
-    "            continue\n",
-    "        # 过滤掉长文本标签（例如文档注释），保留可能的数值或简短事实\n",
-    "        if len(text) > 1000:\n",
-    "            continue\n",
-    "        # 有些 XBRL 事实带 contextRef 或 unitRef\n",
-    "        attrs = dict(tag.attrs)\n",
-    "        facts.append({\n",
-    "            'tag': name,\n",
-    "            'text': text,\n",
-    "            'attrs': attrs\n",
-    "        })\n",
-    "    return facts\n",
-    "\n",
-    "# 示例：解析单个文件（请替换为真实下载后的路径）\n",
-    "# facts = parse_xbrl_to_facts('./sec_xbrl_data/AAPL/0000320193-...-cal.xml')\n",
-    "# print(len(facts))\n"
-   ],
-   "outputs": [],
-   "execution_count": 14
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "95e236d1",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 将目录下所有下载的 XBRL 文件解析并存储为结构化 JSON/CSV\n",
-    "import glob\n",
-    "\n",
-    "all_parsed = {}\n",
-    "for ticker in companies.keys():\n",
-    "    company_dir = os.path.join(SAVE_ROOT, ticker)\n",
-    "    if not os.path.exists(company_dir):\n",
-    "        continue\n",
-    "    all_parsed[ticker] = {}\n",
-    "    for fp in glob.glob(os.path.join(company_dir, '*')):\n",
-    "        try:\n",
-    "            facts = parse_xbrl_to_facts(fp)\n",
-    "            all_parsed[ticker][os.path.basename(fp)] = facts\n",
-    "        except Exception as e:\n",
-    "            print('解析失败', fp, e)\n",
-    "\n",
-    "with open(os.path.join(SAVE_ROOT, 'parsed_facts.json'), 'w', encoding='utf-8') as f:\n",
-    "    json.dump(all_parsed, f, ensure_ascii=False, indent=2)\n",
-    "\n",
-    "print('解析并保存完成。')\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "1c339487",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 从解析结果中抽取数值型的事实示例（尝试解析数值与单位）\n",
-    "import re\n",
-    "\n",
-    "def extract_numeric_facts(facts_list):\n",
-    "    numeric = []\n",
-    "    for f in facts_list:\n",
-    "        text = f['text']\n",
-    "        # 简单判断是否为数字（包括带逗号和括号表示负数的情况）\n",
-    "        if re.match(r'^[\\(\\)\\d,\\.-]+$', text.strip()):\n",
-    "            # 清洗数字形式\n",
-    "            cleaned = text.replace('(', '-').replace(')', '').replace(',', '')\n",
-    "            try:\n",
-    "                val = float(cleaned)\n",
-    "            except:\n",
-    "                continue\n",
-    "            numeric.append({'tag': f['tag'], 'value': val, 'attrs': f['attrs']})\n",
-    "    return numeric\n",
-    "\n",
-    "# 示例读取 parsed_facts.json（如已生成）\n",
-    "# with open(os.path.join(SAVE_ROOT,'parsed_facts.json'),'r',encoding='utf-8') as f:\n",
-    "#     data = json.load(f)\n",
-    "# demo = []\n",
-    "# for k,v in data.get('AAPL',{}).items():\n",
-    "#     demo.extend(extract_numeric_facts(v))\n",
-    "# print(demo[:20])\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "facb30a9",
-   "metadata": {},
-   "source": "\n"
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "23e44084",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# 示例：从 parsed_facts.json 构造训练样本并保存为 JSONL（监督微调格式）\n",
-    "import random\n",
-    "\n",
-    "OUT_TRAIN = os.path.join(SAVE_ROOT, 'fingpt_train.jsonl')\n",
-    "\n",
-    "with open(os.path.join(SAVE_ROOT,'parsed_facts.json'),'r',encoding='utf-8') as f:\n",
-    "    parsed = json.load(f)\n",
-    "\n",
-    "samples = []\n",
-    "for ticker, files in parsed.items():\n",
-    "    for fname, facts in files.items():\n",
-    "        # 简单拼接 facts 的若干条为示例输入\n",
-    "        small = facts[:30]\n",
-    "        context = json.dumps(small, ensure_ascii=False)\n",
-    "        inst = f\"请根据下列 XBRL 提取的事实，简要总结公司的关键财务数字（最多 60 字）：\\n{context}\"\n",
-    "        resp = \"<在运行时由人工/规则生成真实标签或使用自动规则生成占位标签>\"\n",
-    "        samples.append({'instruction': inst, 'response': resp})\n",
-    "\n",
-    "# 保存前 1000 条（示例）\n",
-    "with open(OUT_TRAIN, 'w', encoding='utf-8') as fout:\n",
-    "    for s in samples[:1000]:\n",
-    "        fout.write(json.dumps(s, ensure_ascii=False) + '\\n')\n",
-    "\n",
-    "print('训练样本示例已保存：', OUT_TRAIN)\n"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "9b6f419b",
-   "metadata": {},
-   "source": "\n"
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "name": "python3",
-   "language": "python",
-   "display_name": "Python 3 (ipykernel)"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}